From c73282ba0a3eb8b367ace1a581222090c721de4b Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 22 Jul 2025 12:40:38 +0100 Subject: [PATCH 01/53] ADR: Supporting user approvals --- .../00NN-userapproval-content-types.md | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 docs/decisions/00NN-userapproval-content-types.md diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md new file mode 100644 index 0000000000..46d5282e8f --- /dev/null +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -0,0 +1,174 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +contact: westey-m +date: 2025-07-16 {YYYY-MM-DD when the decision was last updated} +deciders: sergeymenshykh, markwallace, rbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub +consulted: +informed: +--- + +# Agent User Approvals Content Types Design + +## Context and Problem Statement + +When agents are operating on behalf of a user, there may be cases where the agent requires user approval to continue an operation. +This is complicated by the fact that an agent may be remote and the user may not immediately be availale to provide the approval. + +This document aims to provide options and capture the decision on how to model this user approval interaction with the agent caller. + +## Decision Drivers + +## Considered Options + +### Return a FunctionCallContent to the agent caller, that it executes + +This introduces a manual function calling element to agents, where the caller of the agent is expected to invoke the function if the user approves it. + +This approach is problematic for a number of reasons: + +- This may not work for remote agents (e.g. via A2A), where the function that the agent wants to call does not reside on the caller's machine. +- The main value prop of an agent is to encapsulate the internal logic of the agent, but this leaks that logic to the caller, requiring the caller to know how to invoke the agent's function calls. + +### Introduce new ApprovalRequestContent and ApprovalResponseContent types + +The agent would return an `ApprovalRequestContent` to the caller, which would then be responsible for getting approval from the user in whatever way is appropriate for the application. +The caller would then invoke the agent again with an `ApprovalResponseContent` to the agent containing the user decision. + +When an agent returns an `ApprovalRequestContent`, the run is finished for the time being, and to continue, the agent must be invoked again with an `ApprovalResponseContent` on the same thread as the original request. + +The `ApprovalRequestContent` could contain an optional `FunctionCallContent` if the approval is for a function call, along with any additional information that the agent wants to provide to the user to help them make a decision. + +It is up to the agent to decide when and if a user approval is required, and therefore when to return an `ApprovalRequestContent`. + +`ApprovalRequestContent` and `ApprovalResponseContent` will not necessarily always map to a supported content type for the underlying service or agent thread storage. +Specifically, when we are deciding in the IChatClient stack to ask for approval from the user, for a function call, this does not mean that the underlying ai service or +service side thread type (where applicable) supports the concept of a function call approval request. We therefore need the ability to temporarily store the approval request in the +AgentThread, without it becoming part of the thread history. This will serve as a temporary record of the fact that there is an outstanding approval request that the agent is waiting for to continue. +There will be no long term record of an approval request in the chat history, but if the server side thread doesn't support this, there is nothing we can do to change that. + +Suggested Types: + +```csharp +class ApprovalRequestContent : TextContent // TextContent.Text may contain text to explain to the user what they are approving. This is important if the approval is not for a function call. +{ + // An ID to uniquely identify the approval request/response pair. + public string ApprovalId { get; set; } + + // Optional: If the approval is for a function call, this will contain the function call content. + public FunctionCallContent? FunctionCall { get; set; } + + public ChatMessage Approve() + { + return new ChatMessage(ChatRole.User, + [ + new ApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = true, + FunctionCall = this.FunctionCall + } + ]); + } + + public ChatMessage Reject() + { + return new ChatMessage(ChatRole.User, + [ + new ApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = false, + FunctionCall = this.FunctionCall + } + ]); + } +} + +class ApprovalResponseContent : AIContent +{ + // An ID to uniquely identify the approval request/response pair. + public string ApprovalId { get; set; } + + // Indicates whether the user approved the request. + public bool Approved { get; set; } + + // Optional: If the approval is for a function call, this will contain the function call content. + public FunctionCallContent? FunctionCall { get; set; } +} + +var response = await agent.RunAsync("Please book me a flight for Friday to Paris.", thread); +while (response is not null && response.ApprovalRequests.Count > 0) +{ + List messages = new List(); + foreach (var approvalRequest in response.ApprovalRequests) + { + // Show the approval request to the user in the appropriate format. + // The user can then approve or reject the request. + // The optional FunctionCallContent can be used to show the user what function the agent wants to call with the parameter set: + // approvalRequest.FunctionCall?.Arguments. + // The Text property of the ApprovalRequestContent can also be used to show the user any additional textual context about the request. + + // If the user approves: + var approvalMessage = approvalRequest.Approve(); + messages.Add(approvalMessage); + } + + // Get the next response from the agent. + response = await agent.RunAsync(messages, thread); +} + +class AgentThread +{ + ... + + // The thread state may need to store the approval requests and responses. + // TODO: CConsider whether we should have a more generic ActiveUserRequests list, which could include other types of user requests in the future. + // This may mean a base class for all user requests. + public List ActiveApprovalRequests { get; set; } + + ... +} +``` + +- Also see [dotnet issue 6492](https://github.com/dotnet/extensions/issues/6492), which discusses the need for a similar pattern in the context of MCP approvals. +- Also see [the openai RunToolApprovalItem](https://openai.github.io/openai-agents-js/openai/agents/classes/runtoolapprovalitem/). +- Also see [the openai human-in-the-loop guide](https://openai.github.io/openai-agents-js/guides/human-in-the-loop/#approval-requests). +- Also see [MCP Approval Requests from OpenAI](https://platform.openai.com/docs/guides/tools-remote-mcp#approvals). + +### ChatClientAgent Approval Process Flow + +1. User asks agent to perform a task and request is added to the thread. +1. Agent calls model with registered functions. +1. Model responds with function calls to make. +1. ConfirmingFunctionInvokingChatClient decorator (new feature / enhancement to FunctionInvokingChatClient) identifies any function calls that require user approval and returns an ApprovalRequestContent. + ChatClient implementations should also convert any approval requests from the service into ApprovalRequestContent. +1. Agent updates the thread with the FunctionCallContent (or this may have already been done by a service threaded agent) if the approval request is for a function call. +1. Agent stores the ApprovalRequestContent in its AgentThread under ActiveApprovalRequests, so that it knows that there is an outstanding user request. +1. Agent returns the ApprovalRequestContent to the caller which shows it to the user in the appropriate format. +1. User (via caller) invokes the agent again with ApprovalResponseContent. +1. Agent removes the ApprovalRequestContent from its AgentThread ActiveApprovalRequests. +1. Agent invokes IChatClient with ApprovalResponseContent and the ConfirmingFunctionInvokingChatClient decorator identifies the response as an approval for the function call. + If it isn't an approval for a manual function call, it can be passed through to the underlying ChatClient to be converted to the appropriate Approval content type for the service. +1. ConfirmingFunctionInvokingChatClient decorator invokes the function call and invokes the underlying IChatClient with a FunctionResultContent. +1. Model responds with the result. +1. Agent responds to caller with result message and thread is updated with the result message. + +At construction time the set of functions that require user approval will need to be registered with the `ConfirmingFunctionInvokingChatClient` decorator +so that it can identify which function calls should be returned as an `ApprovalRequestContent`. + +### CustomAgent Approval Process Flow + +1. User asks agent to perform a task and request is added to the thread. +1. Agent executes various steps. +1. Agent encounters a step for which it requires user approval to continue. +1. Agent responds with an ApprovalRequestContent. +1. Agent updates its own state with the progress that it has made up to that point and adds the ApprovalRequestContent to its AgentThread ActiveApprovalRequests. +1. User (via caller) invokes the agent again with ApprovalResponseContent. +1. Agent loads its progress from state and continues processing. +1. Agent removes its ApprovalRequestContent from its AgentThread ActiveApprovalRequests. +1. Agent responds to caller with result message and thread is updated with the result message. + +## Open Questions + +## Decision Outcome From 9e3b98b6b24997e7975740d03476e5e271c3c379 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 22 Jul 2025 12:43:57 +0100 Subject: [PATCH 02/53] Add more context. --- docs/decisions/00NN-userapproval-content-types.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index 46d5282e8f..902a59c9a5 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -13,7 +13,9 @@ informed: ## Context and Problem Statement When agents are operating on behalf of a user, there may be cases where the agent requires user approval to continue an operation. -This is complicated by the fact that an agent may be remote and the user may not immediately be availale to provide the approval. +This is complicated by the fact that an agent may be remote and the user may not immediately be available to provide the approval. + +Inference services are also increasingly supporting built-in tools or service side MCP invocation, which may require user approval before the tool can be invoked. This document aims to provide options and capture the decision on how to model this user approval interaction with the agent caller. From a9a6a97fc1b1266ddac9780d48694652e43aa928 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:57:47 +0100 Subject: [PATCH 03/53] Add further justification --- docs/decisions/00NN-userapproval-content-types.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index 902a59c9a5..339a0ffa34 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -31,6 +31,7 @@ This approach is problematic for a number of reasons: - This may not work for remote agents (e.g. via A2A), where the function that the agent wants to call does not reside on the caller's machine. - The main value prop of an agent is to encapsulate the internal logic of the agent, but this leaks that logic to the caller, requiring the caller to know how to invoke the agent's function calls. +- Inference services are introducing their own approval content types for server side tool or function invocation, and will not be addressed by this approach. ### Introduce new ApprovalRequestContent and ApprovalResponseContent types From 61063c085aa9ecbbb5a00ca51ec446be0b804263 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 22 Jul 2025 14:02:30 +0100 Subject: [PATCH 04/53] Add decision drivers. --- docs/decisions/00NN-userapproval-content-types.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index 339a0ffa34..71fb9218b1 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -21,6 +21,10 @@ This document aims to provide options and capture the decision on how to model t ## Decision Drivers +- Agents should encapsulate their internal logic and not leak it to the caller. +- We need to support approvals for local actions as well as remote actions. +- We need to support approvals for existing protocols and services, such as OpenAI's MCP and function calling. + ## Considered Options ### Return a FunctionCallContent to the agent caller, that it executes @@ -172,6 +176,6 @@ so that it can identify which function calls should be returned as an `ApprovalR 1. Agent removes its ApprovalRequestContent from its AgentThread ActiveApprovalRequests. 1. Agent responds to caller with result message and thread is updated with the result message. -## Open Questions - ## Decision Outcome + +Chosen approach: Introduce new ApprovalRequestContent and ApprovalResponseContent types. \ No newline at end of file From 3a0dabfe3dbd884dfe8bccf0d2c3dc7f4e1d3418 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 22 Jul 2025 14:04:53 +0100 Subject: [PATCH 05/53] Update with Copilot Suggestion Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/decisions/00NN-userapproval-content-types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index 71fb9218b1..24a98fa7eb 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -130,7 +130,7 @@ class AgentThread ... // The thread state may need to store the approval requests and responses. - // TODO: CConsider whether we should have a more generic ActiveUserRequests list, which could include other types of user requests in the future. + // TODO: Consider whether we should have a more generic ActiveUserRequests list, which could include other types of user requests in the future. // This may mean a base class for all user requests. public List ActiveApprovalRequests { get; set; } From eace4e74c5354274e47d3672d194f1a0a093278d Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 23 Jul 2025 11:43:44 +0100 Subject: [PATCH 06/53] Update adr with feedback. --- .../00NN-userapproval-content-types.md | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index 24a98fa7eb..d004657870 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -23,7 +23,7 @@ This document aims to provide options and capture the decision on how to model t - Agents should encapsulate their internal logic and not leak it to the caller. - We need to support approvals for local actions as well as remote actions. -- We need to support approvals for existing protocols and services, such as OpenAI's MCP and function calling. +- We need to support approvals for service-side tool use, such as remote MCP tool invocations ## Considered Options @@ -37,12 +37,43 @@ This approach is problematic for a number of reasons: - The main value prop of an agent is to encapsulate the internal logic of the agent, but this leaks that logic to the caller, requiring the caller to know how to invoke the agent's function calls. - Inference services are introducing their own approval content types for server side tool or function invocation, and will not be addressed by this approach. +### Introduce an ApprovalCallback in AgentRunOptions and ChatOptions + +This approach allows a caller to provide a callback that the agent can invoke when it requires user approval. + +This approach is easy to use when the user and agent are in the same application context, such as a desktop application, where the application can show the approval request to the user and get their response from the callback before continuing the agent run. + +This approach does not work well for cases where the agent is hosted in a remote service, and where there is no user available to provide the approval in the same application context. +For cases like this, the agent needs to be suspended, and a network response must be sent to the client app. After the user provides their approval, the client app must call the service that hosts the agent again with the user's decision, and the agent needs to be resumed. However, with a callback, the agent is deep in the call stack and cannot be suspended or resumed like this. + +```csharp +class AgentRunOptions +{ + public Func>? ApprovalCallback { get; set; } +} + +agent.RunAsync("Please book me a flight for Friday to Paris.", thread, new AgentRunOptions +{ + ApprovalCallback = async (approvalRequest) => + { + // Show the approval request to the user in the appropriate format. + // The user can then approve or reject the request. + // The optional FunctionCallContent can be used to show the user what function the agent wants to call with the parameter set: + // approvalRequest.FunctionCall?.Arguments. + // The Text property of the ApprovalRequestContent can also be used to show the user any additional textual context about the request. + + // If the user approves: + return approvalRequest.Approve(); + } +}); +``` + ### Introduce new ApprovalRequestContent and ApprovalResponseContent types The agent would return an `ApprovalRequestContent` to the caller, which would then be responsible for getting approval from the user in whatever way is appropriate for the application. The caller would then invoke the agent again with an `ApprovalResponseContent` to the agent containing the user decision. -When an agent returns an `ApprovalRequestContent`, the run is finished for the time being, and to continue, the agent must be invoked again with an `ApprovalResponseContent` on the same thread as the original request. +When an agent returns an `ApprovalRequestContent`, the run is finished for the time being, and to continue, the agent must be invoked again with an `ApprovalResponseContent` on the same thread as the original request. This doesn't of course have to be the exact same thread object, but it should have the equivalent contents as the original thread, since the agent would have stored the `ApprovalRequestContent` in its thread state. The `ApprovalRequestContent` could contain an optional `FunctionCallContent` if the approval is for a function call, along with any additional information that the agent wants to provide to the user to help them make a decision. @@ -125,6 +156,17 @@ while (response is not null && response.ApprovalRequests.Count > 0) response = await agent.RunAsync(messages, thread); } +class AgentRunResponse +{ + ... + + // A new property on AgentRunResponse to aggregate the ApprovalRequestContent items from + // the response messages (Similar to the Text property). + public IReadOnlyList ApprovalRequests { get; set; } + + ... +} + class AgentThread { ... @@ -142,6 +184,8 @@ class AgentThread - Also see [the openai RunToolApprovalItem](https://openai.github.io/openai-agents-js/openai/agents/classes/runtoolapprovalitem/). - Also see [the openai human-in-the-loop guide](https://openai.github.io/openai-agents-js/guides/human-in-the-loop/#approval-requests). - Also see [MCP Approval Requests from OpenAI](https://platform.openai.com/docs/guides/tools-remote-mcp#approvals). +- Also see [Azure AI Foundry MCP Approvals](https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/model-context-protocol-samples?pivots=rest#submit-your-approval). +- Also see [MCP Elicitation requests](https://modelcontextprotocol.io/specification/draft/client/elicitation) ### ChatClientAgent Approval Process Flow From 4076ed34766881710e397bc4c4e48e879b90b5dc Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 23 Jul 2025 18:43:25 +0100 Subject: [PATCH 07/53] Adding another generalized option for all user input. --- .../00NN-userapproval-content-types.md | 126 ++++++++++++++++-- 1 file changed, 116 insertions(+), 10 deletions(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index d004657870..78baa351c6 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -3,7 +3,7 @@ status: proposed contact: westey-m date: 2025-07-16 {YYYY-MM-DD when the decision was last updated} -deciders: sergeymenshykh, markwallace, rbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub +deciders: sergeymenshykh, markwallace, rbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub, peterychang consulted: informed: --- @@ -27,7 +27,7 @@ This document aims to provide options and capture the decision on how to model t ## Considered Options -### Return a FunctionCallContent to the agent caller, that it executes +### 1. Return a FunctionCallContent to the agent caller, that it executes This introduces a manual function calling element to agents, where the caller of the agent is expected to invoke the function if the user approves it. @@ -37,7 +37,7 @@ This approach is problematic for a number of reasons: - The main value prop of an agent is to encapsulate the internal logic of the agent, but this leaks that logic to the caller, requiring the caller to know how to invoke the agent's function calls. - Inference services are introducing their own approval content types for server side tool or function invocation, and will not be addressed by this approach. -### Introduce an ApprovalCallback in AgentRunOptions and ChatOptions +### 2. Introduce an ApprovalCallback in AgentRunOptions and ChatOptions This approach allows a caller to provide a callback that the agent can invoke when it requires user approval. @@ -68,7 +68,7 @@ agent.RunAsync("Please book me a flight for Friday to Paris.", thread, new Agent }); ``` -### Introduce new ApprovalRequestContent and ApprovalResponseContent types +### 3. Introduce new ApprovalRequestContent and ApprovalResponseContent types The agent would return an `ApprovalRequestContent` to the caller, which would then be responsible for getting approval from the user in whatever way is appropriate for the application. The caller would then invoke the agent again with an `ApprovalResponseContent` to the agent containing the user decision. @@ -136,7 +136,7 @@ class ApprovalResponseContent : AIContent } var response = await agent.RunAsync("Please book me a flight for Friday to Paris.", thread); -while (response is not null && response.ApprovalRequests.Count > 0) +while (response.ApprovalRequests.Count > 0) { List messages = new List(); foreach (var approvalRequest in response.ApprovalRequests) @@ -172,8 +172,6 @@ class AgentThread ... // The thread state may need to store the approval requests and responses. - // TODO: Consider whether we should have a more generic ActiveUserRequests list, which could include other types of user requests in the future. - // This may mean a base class for all user requests. public List ActiveApprovalRequests { get; set; } ... @@ -187,7 +185,7 @@ class AgentThread - Also see [Azure AI Foundry MCP Approvals](https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/model-context-protocol-samples?pivots=rest#submit-your-approval). - Also see [MCP Elicitation requests](https://modelcontextprotocol.io/specification/draft/client/elicitation) -### ChatClientAgent Approval Process Flow +#### ChatClientAgent Approval Process Flow 1. User asks agent to perform a task and request is added to the thread. 1. Agent calls model with registered functions. @@ -208,7 +206,7 @@ class AgentThread At construction time the set of functions that require user approval will need to be registered with the `ConfirmingFunctionInvokingChatClient` decorator so that it can identify which function calls should be returned as an `ApprovalRequestContent`. -### CustomAgent Approval Process Flow +#### CustomAgent Approval Process Flow 1. User asks agent to perform a task and request is added to the thread. 1. Agent executes various steps. @@ -220,6 +218,114 @@ so that it can identify which function calls should be returned as an `ApprovalR 1. Agent removes its ApprovalRequestContent from its AgentThread ActiveApprovalRequests. 1. Agent responds to caller with result message and thread is updated with the result message. +### 4. Introduce new UserInputRequestContent and UserInputResponseContent types + +This approach is similar to the `ApprovalRequestContent` and `ApprovalResponseContent` types, but is more generic and can be used for any type of user input request, not just approvals. + +There is some ambiguity with this approach. When using an LLM based agent the LLM may return a text response about missing user input. +E.g the LLM may need to invoke a function but the user did not supply all necessary information to fill out all arguments. +Typically an LLM would just respond with a text message asking the user for the missing information. +In this case, the message is not distinguishable from any other result message, and therefore cannot be returned to the caller as a `UserInputRequestContent`, even though it is conceptually a type of unstructured user input request. + +Open Questions: + +- Should unstructured user input requests (e.g. text messages asking for more information, with a freeform response) be modeled as `UserInputRequestContent`, and if so how do we identify these for conversion from the underlying services? +- Alternatively, should structured user input requests (e.g. schematized input, approvals, etc.) also be considered Results, similar to `TextContent`? `TextContent` is currently returned for unstructured input requests and treated as a Result. +- Why would an unstructured user input request be considered a result, but a structured user input request not be considered a result? Both mean that the run is finished, and a new run must be started with the user input requested. +- Are structured user input requests similar to unstructured when it comes to user agency? E.g. The user can choose not to answer and change the subject. + Or are they more like function calls where the caller must provide a `FunctionResponseContent` to avoid leaving the thread in an incomplete state? + +Suggested Types: + +```csharp +class UserInputRequestContent +{ + // An ID to uniquely identify the approval request/response pair. + public string ApprovalId { get; set; } + + // DecisionTarget could contain: + // FunctionCallContent: The function call that the agent wants to invoke. + // TextContent: Text that describes the question for that the user should answer. + object? DecisionTarget { get; set; } // Anything else the user may need to make a decision about. + + // Possible InputFormat subclasses: + // SchemaInputFormat: Contains a schema for the user input. + // ApprovalInputFormat: Indicates that the user needs to approve something. + // FreeformTextInputFormat: Indicates that the user can provide freeform text input. + // Other formats can be added as needed, e.g. cards when using activity protocol. + public InputFormat InputFormat { get; set; } // How the user should provide input (e.g., form, options, etc.). +} + +class UserInputResponseContent : AIContent +{ + // An ID to uniquely identify the approval request/response pair. + public string ApprovalId { get; set; } + + // Possible UserInputResult subclasses: + // SchemaInputResult: Contains the structured data provided by the user. + // ApprovalResult: Contains a bool with approved / rejected. + // FreeformTextResult: Contains the freeform text input provided by the user. + public UserInputResult Result { get; set; } // The user input. + + public object? DecisionTarget { get; set; } // A copy of the DecisionTarget from the UserInputRequestContent, if applicable. +} + +var response = await agent.RunAsync("Please book me a flight for Friday to Paris.", thread); +while (response.UserInputRequests.Count > 0) +{ + List messages = new List(); + foreach (var userInputRequest in response.UserInputRequests) + { + // Show the user input request to the user in the appropriate format. + // The DecisionTarget can be used to show the user what function the agent wants to call with the parameter set. + // The InputFormat property can be used to determine the type of UX when allowing users to provide input. + + if (userInputRequest.InputFormat is ApprovalInputFormat approvalInputFormat) + { + // Here we need to show the user an approval request. + // We can use the DecisionTarget to show e.g. the function call that the agent wants to invoke. + // The user can then approve or reject the request. + + // If the user approves: + var approvalMessage = new ChatMessage(ChatRole.User, new UserInputResponseContent { + ApprovalId = userInputRequest.ApprovalId, + Result = new ApprovalResult { Approved = true }, + DecisionTarget = userInputRequest.DecisionTarget + }); + messages.Add(approvalMessage); + } + else + { + throw new NotSupportedException("Unsupported InputFormat type."); + } + } + + // Get the next response from the agent. + response = await agent.RunAsync(messages, thread); +} + +class AgentRunResponse +{ + ... + + // A new property on AgentRunResponse to aggregate the UserInputRequestContent items from + // the response messages (Similar to the Text property). + public IReadOnlyList UserInputRequests { get; set; } + + ... +} + +class AgentThread +{ + ... + + // The thread state may need to store the user input requests. + public List ActiveUserInputRequests { get; set; } + + ... +} +``` + ## Decision Outcome -Chosen approach: Introduce new ApprovalRequestContent and ApprovalResponseContent types. \ No newline at end of file +Approach TBD. From 12817f17272e3f40f2af181a04d850e4ce0f3839 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 25 Jul 2025 14:21:00 +0100 Subject: [PATCH 08/53] Add another base class based UserInput option --- .../00NN-userapproval-content-types.md | 148 ++++++++++++++++-- 1 file changed, 136 insertions(+), 12 deletions(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index 78baa351c6..fde3e4f9f7 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -3,7 +3,7 @@ status: proposed contact: westey-m date: 2025-07-16 {YYYY-MM-DD when the decision was last updated} -deciders: sergeymenshykh, markwallace, rbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub, peterychang +deciders: sergeymenshykh, markwallace-microsoft, rogerbarreto, dmytrostruk, westey-m, eavanvalkenburg, stephentoub, peterychang consulted: informed: --- @@ -218,27 +218,19 @@ so that it can identify which function calls should be returned as an `ApprovalR 1. Agent removes its ApprovalRequestContent from its AgentThread ActiveApprovalRequests. 1. Agent responds to caller with result message and thread is updated with the result message. -### 4. Introduce new UserInputRequestContent and UserInputResponseContent types +### 4. Introduce new Container UserInputRequestContent and UserInputResponseContent types This approach is similar to the `ApprovalRequestContent` and `ApprovalResponseContent` types, but is more generic and can be used for any type of user input request, not just approvals. There is some ambiguity with this approach. When using an LLM based agent the LLM may return a text response about missing user input. E.g the LLM may need to invoke a function but the user did not supply all necessary information to fill out all arguments. Typically an LLM would just respond with a text message asking the user for the missing information. -In this case, the message is not distinguishable from any other result message, and therefore cannot be returned to the caller as a `UserInputRequestContent`, even though it is conceptually a type of unstructured user input request. - -Open Questions: - -- Should unstructured user input requests (e.g. text messages asking for more information, with a freeform response) be modeled as `UserInputRequestContent`, and if so how do we identify these for conversion from the underlying services? -- Alternatively, should structured user input requests (e.g. schematized input, approvals, etc.) also be considered Results, similar to `TextContent`? `TextContent` is currently returned for unstructured input requests and treated as a Result. -- Why would an unstructured user input request be considered a result, but a structured user input request not be considered a result? Both mean that the run is finished, and a new run must be started with the user input requested. -- Are structured user input requests similar to unstructured when it comes to user agency? E.g. The user can choose not to answer and change the subject. - Or are they more like function calls where the caller must provide a `FunctionResponseContent` to avoid leaving the thread in an incomplete state? +In this case, the message is not distinguishable from any other result message, and therefore cannot be returned to the caller as a `UserInputRequestContent`, even though it is conceptually a type of unstructured user input request. Ultimately our types are modeled to make it easy for callers to decide on the right way to represent this to users. E.g. is it just a regular message to show to users, or do we need a special UX for it. Suggested Types: ```csharp -class UserInputRequestContent +class UserInputRequestContent : AIContent { // An ID to uniquely identify the approval request/response pair. public string ApprovalId { get; set; } @@ -326,6 +318,138 @@ class AgentThread } ``` +### 5. Introduce new Base UserInputRequestContent and UserInputResponseContent types + +This approach is similar to option 4, but the `UserInputRequestContent` and `UserInputResponseContent` types are base classes rather than generic container types. + +Suggested Types: + +```csharp +class UserInputRequestContent : AIContent +{ + // An ID to uniquely identify the approval request/response pair. + public string ApprovalId { get; set; } +} + +class UserInputResponseContent : AIContent +{ + // An ID to uniquely identify the approval request/response pair. + public string ApprovalId { get; set; } +} + +// ----------------------------------- +// Used for approving a function call. +class FunctionApprovalRequestContent : UserInputRequestContent +{ + // Contains the function call that the agent wants to invoke. + public FunctionCallContent FunctionCall { get; set; } + + public ChatMessage Approve() + { + return new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = true, + FunctionCall = this.FunctionCall + } + ]); + } + + public ChatMessage Reject() + { + return new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = false, + FunctionCall = this.FunctionCall + } + ]); + } +} +class FunctionApprovalResponseContent : UserInputResponseContent +{ + // Indicates whether the user approved the request. + public bool Approved { get; set; } + + // Contains the function call that the agent wants to invoke. + public FunctionCallContent FunctionCall { get; set; } +} + +// -------------------------------------------------- +// Used for approving a request described using text. +class QuestionApprovalRequestContent : UserInputRequestContent +{ + // A user targeted message to explain what needs to be approved. + public string Text { get; set; } +} +class QuestionApprovalResponseContent : UserInputResponseContent +{ + // Indicates whether the user approved the request. + public bool Approved { get; set; } +} + +// ------------------------------------------------ +// Used for providing input in a structured format. +class StructuredDataInputRequestContent : UserInputRequestContent +{ + // A user targeted message to explain what is being requested. + public string? Text { get; set; } + + // Contains the schema for the user input. + public string JsonSchema { get; set; } +} +class StructuredDataInputResponseContent : UserInputResponseContent +{ + // Contains the structured data provided by the user. + public string StructuredData { get; set; } +} + +var response = await agent.RunAsync("Please book me a flight for Friday to Paris.", thread); +while (response.UserInputRequests.Count > 0) +{ + List messages = new List(); + foreach (var userInputRequest in response.UserInputRequests) + { + if (userInputRequest is FunctionApprovalRequestContent approvalRequest) + { + // Here we need to show the user an approval request. + // We can use the FunctionCall property to show e.g. the function call that the agent wants to invoke. + // If the user approves: + var approvalMessage = approvalRequest.Approve(); + messages.Add(approvalMessage); + } + } + + // Get the next response from the agent. + response = await agent.RunAsync(messages, thread); +} + +class AgentRunResponse +{ + ... + + // A new property on AgentRunResponse to aggregate the UserInputRequestContent items from + // the response messages (Similar to the Text property). + public IReadOnlyList UserInputRequests { get; set; } + + ... +} + +class AgentThread +{ + ... + + // The thread state may need to store the user input requests. + public List ActiveUserInputRequests { get; set; } + + ... +} +``` + ## Decision Outcome Approach TBD. From 84e2855bfb9e888d9f2352fb2ba6b93bb33744c0 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 25 Jul 2025 15:05:49 +0100 Subject: [PATCH 09/53] Remove TextContent inheritance. --- docs/decisions/00NN-userapproval-content-types.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index fde3e4f9f7..ff0f14c231 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -88,11 +88,14 @@ There will be no long term record of an approval request in the chat history, bu Suggested Types: ```csharp -class ApprovalRequestContent : TextContent // TextContent.Text may contain text to explain to the user what they are approving. This is important if the approval is not for a function call. +class ApprovalRequestContent : AIContent { // An ID to uniquely identify the approval request/response pair. public string ApprovalId { get; set; } + // An optional user targeted message to explain what needs to be approved. + public string? Text { get; set; } + // Optional: If the approval is for a function call, this will contain the function call content. public FunctionCallContent? FunctionCall { get; set; } From 632635c42e6bd7cc5e18e148ee1d4bd7c74fc484 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 25 Jul 2025 16:31:43 +0100 Subject: [PATCH 10/53] Change Schema and Strucutured input data types to JsonElement --- docs/decisions/00NN-userapproval-content-types.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index ff0f14c231..d10cc1e14a 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -403,12 +403,12 @@ class StructuredDataInputRequestContent : UserInputRequestContent public string? Text { get; set; } // Contains the schema for the user input. - public string JsonSchema { get; set; } + public JsonElement JsonSchema { get; set; } } class StructuredDataInputResponseContent : UserInputResponseContent { // Contains the structured data provided by the user. - public string StructuredData { get; set; } + public JsonElement StructuredData { get; set; } } var response = await agent.RunAsync("Please book me a flight for Friday to Paris.", thread); From 461c8ef27b268280ff8ea63367c93028eba5c65d Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 25 Jul 2025 16:32:08 +0100 Subject: [PATCH 11/53] Rename JsonSchema to Schema --- docs/decisions/00NN-userapproval-content-types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index d10cc1e14a..d8fcf30fd2 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -403,7 +403,7 @@ class StructuredDataInputRequestContent : UserInputRequestContent public string? Text { get; set; } // Contains the schema for the user input. - public JsonElement JsonSchema { get; set; } + public JsonElement Schema { get; set; } } class StructuredDataInputResponseContent : UserInputResponseContent { From ab04590428a871b4dd7360cddf914067793f8b94 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:37:16 +0100 Subject: [PATCH 12/53] Address some feedback. --- docs/decisions/00NN-userapproval-content-types.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index d8fcf30fd2..f89d238517 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -84,6 +84,7 @@ Specifically, when we are deciding in the IChatClient stack to ask for approval service side thread type (where applicable) supports the concept of a function call approval request. We therefore need the ability to temporarily store the approval request in the AgentThread, without it becoming part of the thread history. This will serve as a temporary record of the fact that there is an outstanding approval request that the agent is waiting for to continue. There will be no long term record of an approval request in the chat history, but if the server side thread doesn't support this, there is nothing we can do to change that. +We should however log approvals so that there is a trace of this for debugging and auditing purposes. Suggested Types: @@ -215,9 +216,8 @@ so that it can identify which function calls should be returned as an `ApprovalR 1. Agent executes various steps. 1. Agent encounters a step for which it requires user approval to continue. 1. Agent responds with an ApprovalRequestContent. -1. Agent updates its own state with the progress that it has made up to that point and adds the ApprovalRequestContent to its AgentThread ActiveApprovalRequests. +1. Agent adds the ApprovalRequestContent to its AgentThread ActiveApprovalRequests. 1. User (via caller) invokes the agent again with ApprovalResponseContent. -1. Agent loads its progress from state and continues processing. 1. Agent removes its ApprovalRequestContent from its AgentThread ActiveApprovalRequests. 1. Agent responds to caller with result message and thread is updated with the result message. @@ -384,12 +384,12 @@ class FunctionApprovalResponseContent : UserInputResponseContent // -------------------------------------------------- // Used for approving a request described using text. -class QuestionApprovalRequestContent : UserInputRequestContent +class TextApprovalRequestContent : UserInputRequestContent { // A user targeted message to explain what needs to be approved. public string Text { get; set; } } -class QuestionApprovalResponseContent : UserInputResponseContent +class TextApprovalResponseContent : UserInputResponseContent { // Indicates whether the user approved the request. public bool Approved { get; set; } @@ -437,7 +437,7 @@ class AgentRunResponse // A new property on AgentRunResponse to aggregate the UserInputRequestContent items from // the response messages (Similar to the Text property). - public IReadOnlyList UserInputRequests { get; set; } + public IEnumerable UserInputRequests { get; set; } ... } From 629204e8297522babe7fca93120cc61effd2d256 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Thu, 31 Jul 2025 12:19:48 +0100 Subject: [PATCH 13/53] Add base user input request and response types with FunctionApproval sub types. --- .../AIContentExtensions.cs | 6 +++ .../AgentRunResponse.cs | 7 +++ .../AgentRunResponseUpdate.cs | 7 +++ .../FunctionApprovalRequestContent.cs | 48 +++++++++++++++++++ .../FunctionApprovalResponseContent.cs | 35 ++++++++++++++ .../MEAI.Contents/UserInputRequestContent.cs | 14 ++++++ .../MEAI.Contents/UserInputResponseContent.cs | 14 ++++++ 7 files changed, 131 insertions(+) create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIContentExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIContentExtensions.cs index 22e0193e8b..73e416e764 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIContentExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIContentExtensions.cs @@ -115,4 +115,10 @@ public static string ConcatText(this IList messages) #endif } } + + public static IEnumerable EnumerateUserInputRequests(this IList messages) + => messages.SelectMany(x => x.Contents).OfType(); + + public static IEnumerable EnumerateUserInputRequests(this IList contents) + => contents.OfType(); } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs index a50448c8a6..240a1599c3 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs @@ -82,6 +82,13 @@ public IList Messages [JsonIgnore] public string Text => this._messages?.ConcatText() ?? string.Empty; + /// Gets or sets the user input requests associated with the response. + /// + /// This property concatenates all instances in the response. + /// + [JsonIgnore] + public IEnumerable UserInputRequests => this._messages?.EnumerateUserInputRequests() ?? Array.Empty(); + /// Gets or sets the ID of the agent that produced the response. public string? AgentId { get; set; } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs index 4e6ce5f22e..931a4c5724 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs @@ -92,6 +92,13 @@ public string? AuthorName [JsonIgnore] public string Text => this._contents is not null ? this._contents.ConcatText() : string.Empty; + /// Gets or sets the user input requests associated with the response. + /// + /// This property concatenates all instances in the response. + /// + [JsonIgnore] + public IEnumerable UserInputRequests => this._contents?.EnumerateUserInputRequests() ?? Array.Empty(); + /// Gets or sets the agent run response update content items. [AllowNull] public IList Contents diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs new file mode 100644 index 0000000000..0b86acf32b --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a request for user approval of a function call. +/// +public class FunctionApprovalRequestContent : UserInputRequestContent +{ + /// + /// Gets or sets the function call that pre-invoke approval is required for. + /// + public FunctionCallContent FunctionCall { get; set; } = default!; + + /// + /// Creates a representing an approval response. + /// + /// The representing the approval response. + public ChatMessage Approve() + { + return new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = true, + FunctionCall = this.FunctionCall + } + ]); + } + + /// + /// Creates a representing a rejection response. + /// + /// The representing the rejection response. + public ChatMessage Reject() + { + return new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = false, + FunctionCall = this.FunctionCall + } + ]); + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs new file mode 100644 index 0000000000..41ba57a978 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a response to a function approval request. +/// +public class FunctionApprovalResponseContent : UserInputResponseContent +{ + /// + /// Initializes a new instance of the class. + /// + public FunctionApprovalResponseContent() + { + } + + /// + /// Initializes a new instance of the class with the specified approval status. + /// + /// Indicates whether the request was approved. + public FunctionApprovalResponseContent(bool approved) + { + this.Approved = approved; + } + + /// + /// Gets or sets a value indicating whether the user approved the request. + /// + public bool Approved { get; set; } + + /// + /// Gets or sets the function call that pre-invoke approval is required for. + /// + public FunctionCallContent FunctionCall { get; set; } = default!; +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs new file mode 100644 index 0000000000..29e163bef0 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI; + +/// +/// Base class for user input request content. +/// +public abstract class UserInputRequestContent : AIContent +{ + /// + /// Gets or sets the ID to uniquely identify the user input request/response pair. + /// + public string ApprovalId { get; set; } = default!; +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs new file mode 100644 index 0000000000..140aa8a895 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI; + +/// +/// Base class for user input response content. +/// +public abstract class UserInputResponseContent : AIContent +{ + /// + /// Gets or sets the ID to uniquely identify the user input request/response pair. + /// + public string ApprovalId { get; set; } = default!; +} From 74d1f5d57193a6c3ea5ebb63f2dd99c78d1ccbcc Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:24:06 +0100 Subject: [PATCH 14/53] Add POC ApprovalGeneratingChatClient --- ...ep02_ChatClientAgent_UsingFunctionTools.cs | 80 ++ .../ChatCompletion/ChatClientExtensions.cs | 25 +- .../MEAI/ApprovalGeneratingChatClient.cs | 213 ++++ .../MEAI/LoggingHelpers.cs | 33 + .../MEAI/NewFunctionInvokingChatClient.cs | 1010 +++++++++++++++++ .../MEAI/OpenTelemetryConsts.cs | 136 +++ 6 files changed, 1496 insertions(+), 1 deletion(-) create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalGeneratingChatClient.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/LoggingHelpers.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/OpenTelemetryConsts.cs diff --git a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs index be945edfbf..5a1178546f 100644 --- a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs +++ b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs @@ -114,6 +114,86 @@ async Task RunAgentAsync(string input) await base.AgentCleanUpAsync(provider, agent, thread); } + [Theory] + [InlineData(ChatClientProviders.AzureOpenAI)] + [InlineData(ChatClientProviders.AzureAIAgentsPersistent)] + [InlineData(ChatClientProviders.OpenAIAssistant)] + [InlineData(ChatClientProviders.OpenAIChatCompletion)] + [InlineData(ChatClientProviders.OpenAIResponses)] + public async Task ApprovalsWithTools(ChatClientProviders provider) + { + // Creating a MenuTools instance to be used by the agent. + var menuTools = new MenuTools(); + + // Define the options for the chat client agent. + var agentOptions = new ChatClientAgentOptions( + name: "Host", + instructions: "Answer questions about the menu", + tools: [ + AIFunctionFactory.Create(menuTools.GetMenu), + AIFunctionFactory.Create(menuTools.GetSpecials), + AIFunctionFactory.Create(menuTools.GetItemPrice) + ]); + + // Create the server-side agent Id when applicable (depending on the provider). + agentOptions.Id = await base.AgentCreateAsync(provider, agentOptions); + + // Get the chat client to use for the agent. + using var chatClient = base.GetChatClient(provider, agentOptions); + + // Define the agent + var agent = new ChatClientAgent(chatClient, agentOptions); + + // Create the chat history thread to capture the agent interaction. + var thread = agent.GetNewThread(); + + // Respond to user input, invoking functions where appropriate. + await RunAgentAsync("What is the special soup and its price?"); + + async Task RunAgentAsync(string input) + { + this.WriteUserMessage(input); + var response = await agent.RunAsync(input, thread); + this.WriteResponseOutput(response); + + var userInputRequests = response.UserInputRequests.ToList(); + + // Loop until all user input requests are handled. + while (userInputRequests.Count > 0) + { + List nextIterationMessages = []; + + foreach (var request in userInputRequests) + { + if (request is FunctionApprovalRequestContent approvalRequest) + { + if (approvalRequest.FunctionCall.Name == "GetSpecials") + { + Console.WriteLine($"Approving the {approvalRequest.FunctionCall.Name} function call."); + nextIterationMessages.Add(approvalRequest.Approve()); + } + else + { + Console.WriteLine($"Rejecting the {approvalRequest.FunctionCall.Name} function call."); + nextIterationMessages.Add(approvalRequest.Reject()); + } + } + else + { + throw new NotSupportedException("This type of approval is not supported"); + } + } + + response = await agent.RunAsync(nextIterationMessages, thread); + this.WriteResponseOutput(response); + userInputRequests = response.UserInputRequests.ToList(); + } + } + + // Clean up the server-side agent after use when applicable (depending on the provider). + await base.AgentCleanUpAsync(provider, agent, thread); + } + private sealed class MenuTools { [Description("Get the full menu items.")] diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs index b0a61e12c0..71a5c91cfc 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs @@ -1,5 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + namespace Microsoft.Extensions.AI.Agents; internal static class ChatClientExtensions @@ -13,9 +17,28 @@ internal static IChatClient AsAgentInvokingChatClient(this IChatClient chatClien chatBuilder.UseAgentInvocation(); } - if (chatClient.GetService() is null) + if (chatClient.GetService() is null) { chatBuilder.UseFunctionInvocation(); + + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + var loggerFactory = services.GetService(); + + var newFunctionInvokingChatClient = new NewFunctionInvokingChatClient(innerClient, loggerFactory, services); + return newFunctionInvokingChatClient; + }); + } + + if (chatClient.GetService() is null) + { + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + var loggerFactory = services.GetService(); + + ApprovalGeneratingChatClient approvalGeneratingChatClient = new(innerClient, loggerFactory); + return approvalGeneratingChatClient; + }); } return chatBuilder.Build(); diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalGeneratingChatClient.cs new file mode 100644 index 0000000000..27abd56768 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalGeneratingChatClient.cs @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI.Agents; + +/// +/// Represents a chat client that seeks user approval for function calls. +/// +public class ApprovalGeneratingChatClient : DelegatingChatClient +{ + /// The logger to use for logging information about function approval. + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying , or the next instance in a chain of clients. + /// An to use for logging information about function invocation. + public ApprovalGeneratingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null) + : base(innerClient) + { + this._logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + } + + /// + public override async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + var messagesList = messages as IList ?? messages.ToList(); + + // If we got any approval responses, and we also got FunctionResultContent for those approvals, we can filter out those approval responses + // since they are already handled. + RemoveExecutedApprovedApprovalRequests(messagesList); + + // Get all the remaining approval responses. + var approvalResponses = messagesList.SelectMany(x => x.Contents).OfType().ToList(); + + if (approvalResponses.Count == 0) + { + // We have no approval responses, so we can just call the inner client. + var response = await base.GetResponseAsync(messagesList, options, cancellationToken).ConfigureAwait(false); + if (response is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); + } + + // Replace any FunctionCallContent in the response with FunctionApprovalRequestContent. + ReplaceFunctionCallsWithApprovalRequests(response.Messages); + + return response; + } + + if (approvalResponses.All(x => !x.Approved)) + { + // If we only have rejections, we can call the inner client with rejected function calls. + // Replace all rejected FunctionApprovalResponseContent with rejected FunctionResultContent. + ReplaceRejectedFunctionCallRequests(messagesList); + + var response = await base.GetResponseAsync(messagesList, options, cancellationToken).ConfigureAwait(false); + if (response is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); + } + + // Replace any FunctionCallContent in the response with FunctionApprovalRequestContent. + ReplaceFunctionCallsWithApprovalRequests(response.Messages); + + return response; + } + + // We have a mix of approvals and rejections, so we need to return the approved function calls + // to the upper layer for invocation. + // We do nothing with the rejected ones. They must be supplied by the caller again + // on the next invocation, and then we will convert them to rejected FunctionResultContent. + var approvedToolCalls = approvalResponses.Where(x => x.Approved).Select(x => x.FunctionCall).Cast().ToList(); + return new ChatResponse + { + ConversationId = options?.ConversationId, + CreatedAt = DateTimeOffset.UtcNow, + FinishReason = ChatFinishReason.ToolCalls, + ResponseId = Guid.NewGuid().ToString(), + Messages = + [ + new ChatMessage(ChatRole.Assistant, approvedToolCalls) + ] + }; + } + + /// + public override IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + return base.GetStreamingResponseAsync(messages, options, cancellationToken); + } + + private static void RemoveExecutedApprovedApprovalRequests(IList messages) + { + var functionResultCallIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.CallId).ToHashSet(); + + int messageCount = messages.Count; + for (int i = 0; i < messageCount; i++) + { + // Get any content that is not a FunctionApprovalResponseContent or is a FunctionApprovalResponseContent that has not been executed. + var content = messages[i].Contents.Where(x => x is not FunctionApprovalResponseContent || (x is FunctionApprovalResponseContent approval && !functionResultCallIds.Contains(approval.FunctionCall.CallId))).ToList(); + + // Remove the entire message if there is no content left after filtering. + if (content.Count == 0) + { + messages.RemoveAt(i); + i--; // Adjust index since we removed an item. + messageCount--; // Adjust count since we removed an item. + continue; + } + + // Replace the message contents with the filtered content. + messages[i].Contents = content; + } + } + + private static void ReplaceRejectedFunctionCallRequests(IList messages) + { + List newMessages = []; + + int messageCount = messages.Count; + for (int i = 0; i < messageCount; i++) + { + var content = messages[i].Contents; + + List replacedContent = []; + List toolCalls = []; + int contentCount = content.Count; + for (int j = 0; j < contentCount; j++) + { + // Find all responses that were rejected, and replace them with a FunctionResultContent indicating the rejection. + if (content[j] is FunctionApprovalResponseContent approval && !approval.Approved) + { + var rejectedFunctionCall = new FunctionResultContent(approval.FunctionCall.CallId, "Error: Function invocation approval was not granted."); + replacedContent.Add(rejectedFunctionCall); + content[j] = rejectedFunctionCall; + toolCalls.Add(approval.FunctionCall); + } + } + + // Since approvals are submitted as part of a user messages, we have to move the + // replaced function results to tool messages. + if (replacedContent.Count == contentCount) + { + // If all content was replaced, we can replace the entire message with a new tool message. + messages.RemoveAt(i); + i--; // Adjust index since we removed an item. + messageCount--; // Adjust count since we removed an item. + + newMessages.Add(new ChatMessage(ChatRole.Assistant, toolCalls)); + newMessages.Add(new ChatMessage(ChatRole.Tool, replacedContent)); + } + else if (replacedContent.Count > 0) + { + // If only some content was replaced, we move the updated content to a new tool message. + foreach (var replacedItem in replacedContent) + { + messages[i].Contents.Remove(replacedItem); + } + + newMessages.Add(new ChatMessage(ChatRole.Assistant, toolCalls)); + newMessages.Add(new ChatMessage(ChatRole.Tool, replacedContent)); + } + } + + if (newMessages.Count > 0) + { + // If we have new messages, we add them to the original messages. + foreach (var newMessage in newMessages) + { + messages.Add(newMessage); + } + } + } + + /// Replaces any from with . + private static void ReplaceFunctionCallsWithApprovalRequests(IList messages) + { + int count = messages.Count; + for (int i = 0; i < count; i++) + { + ReplaceFunctionCallsWithApprovalRequests(messages[i].Contents); + } + } + + /// Copies any from with . + private static void ReplaceFunctionCallsWithApprovalRequests(IList content) + { + int count = content.Count; + for (int i = 0; i < count; i++) + { + if (content[i] is FunctionCallContent functionCall) + { + content[i] = new FunctionApprovalRequestContent + { + FunctionCall = functionCall, + ApprovalId = functionCall.CallId + }; + } + } + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/LoggingHelpers.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/LoggingHelpers.cs new file mode 100644 index 0000000000..b4b9b566eb --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/LoggingHelpers.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable CA1031 // Do not catch general exception types +#pragma warning disable S108 // Nested blocks of code should not be left empty +#pragma warning disable S2486 // Generic exceptions should not be ignored + +using System.Text.Json; + +namespace Microsoft.Extensions.AI; + +/// Provides internal helpers for implementing logging. +internal static class LoggingHelpers +{ + /// Serializes as JSON for logging purposes. + public static string AsJson(T value, JsonSerializerOptions? options) + { + if (options?.TryGetTypeInfo(typeof(T), out var typeInfo) is true || + AIJsonUtilities.DefaultOptions.TryGetTypeInfo(typeof(T), out typeInfo)) + { + try + { + return JsonSerializer.Serialize(value, typeInfo); + } + catch + { + } + } + + // If we're unable to get a type info for the value, or if we fail to serialize, + // return an empty JSON object. We do not want lack of type info to disrupt application behavior with exceptions. + return "{}"; + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs new file mode 100644 index 0000000000..4ae071b716 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -0,0 +1,1010 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; + +#pragma warning disable CA2213 // Disposable fields should be disposed +#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test +#pragma warning disable SA1202 // 'protected' members should come before 'private' members +#pragma warning disable S107 // Methods should not have too many parameters + +#pragma warning disable IDE0009 // Member access should be qualified. +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task +#pragma warning disable VSTHRD111 // Use ConfigureAwait(bool) + +namespace Microsoft.Extensions.AI; + +/// +/// A delegating chat client that invokes functions defined on . +/// Include this in a chat pipeline to resolve function calls automatically. +/// +/// +/// +/// When this client receives a in a chat response, it responds +/// by calling the corresponding defined in , +/// producing a that it sends back to the inner client. This loop +/// is repeated until there are no more function calls to make, or until another stop condition is met, +/// such as hitting . +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// instances employed as part of the supplied are also safe. +/// The property can be used to control whether multiple function invocation +/// requests as part of the same request are invocable concurrently, but even with that set to +/// (the default), multiple concurrent requests to this same instance and using the same tools could result in those +/// tools being used concurrently (one per request). For example, a function that accesses the HttpContext of a specific +/// ASP.NET web request should only be used as part of a single at a time, and only with +/// set to , in case the inner client decided to issue multiple +/// invocation requests to that same function. +/// +/// +public partial class NewFunctionInvokingChatClient : DelegatingChatClient +{ + /// The for the current function invocation. + private static readonly AsyncLocal s_currentContext = new(); + + /// Gets the specified when constructing the , if any. + protected IServiceProvider? FunctionInvocationServices { get; } + + /// The logger to use for logging information about function invocation. + private readonly ILogger _logger; + + /// The to use for telemetry. + /// This component does not own the instance and should not dispose it. + private readonly ActivitySource? _activitySource; + + /// Maximum number of roundtrips allowed to the inner client. + private int _maximumIterationsPerRequest = 40; // arbitrary default to prevent runaway execution + + /// Maximum number of consecutive iterations that are allowed contain at least one exception result. If the limit is exceeded, we rethrow the exception instead of continuing. + private int _maximumConsecutiveErrorsPerRequest = 3; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying , or the next instance in a chain of clients. + /// An to use for logging information about function invocation. + /// An optional to use for resolving services required by the instances being invoked. + public NewFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) + : base(innerClient) + { + _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + _activitySource = innerClient.GetService(); + FunctionInvocationServices = functionInvocationServices; + } + + /// + /// Gets or sets the for the current function invocation. + /// + /// + /// This value flows across async calls. + /// + public static FunctionInvocationContext? CurrentContext + { + get => s_currentContext.Value; + protected set => s_currentContext.Value = value; + } + + /// + /// Gets or sets a value indicating whether detailed exception information should be included + /// in the chat history when calling the underlying . + /// + /// + /// if the full exception message is added to the chat history + /// when calling the underlying . + /// if a generic error message is included in the chat history. + /// The default value is . + /// + /// + /// + /// Setting the value to prevents the underlying language model from disclosing + /// raw exception details to the end user, since it doesn't receive that information. Even in this + /// case, the raw object is available to application code by inspecting + /// the property. + /// + /// + /// Setting the value to can help the underlying bypass problems on + /// its own, for example by retrying the function call with different arguments. However it might + /// result in disclosing the raw exception information to external users, which can be a security + /// concern depending on the application scenario. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to whether detailed errors are provided during an in-flight request. + /// + /// + public bool IncludeDetailedErrors { get; set; } + + /// + /// Gets or sets a value indicating whether to allow concurrent invocation of functions. + /// + /// + /// if multiple function calls can execute in parallel. + /// if function calls are processed serially. + /// The default value is . + /// + /// + /// An individual response from the inner client might contain multiple function call requests. + /// By default, such function calls are processed serially. Set to + /// to enable concurrent invocation such that multiple function calls can execute in parallel. + /// + public bool AllowConcurrentInvocation { get; set; } + + /// + /// Gets or sets the maximum number of iterations per request. + /// + /// + /// The maximum number of iterations per request. + /// The default value is 40. + /// + /// + /// + /// Each request to this might end up making + /// multiple requests to the inner client. Each time the inner client responds with + /// a function call request, this client might perform that invocation and send the results + /// back to the inner client in a new request. This property limits the number of times + /// such a roundtrip is performed. The value must be at least one, as it includes the initial request. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to how many iterations are allowed for an in-flight request. + /// + /// + public int MaximumIterationsPerRequest + { + get => _maximumIterationsPerRequest; + set + { + if (value < 1) + { + Throw.ArgumentOutOfRangeException(nameof(value)); + } + + _maximumIterationsPerRequest = value; + } + } + + /// + /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error. + /// + /// + /// The maximum number of consecutive iterations that are allowed to fail with an error. + /// The default value is 3. + /// + /// + /// + /// When function invocations fail with an exception, the + /// continues to make requests to the inner client, optionally supplying exception information (as + /// controlled by ). This allows the to + /// recover from errors by trying other function parameters that may succeed. + /// + /// + /// However, in case function invocations continue to produce exceptions, this property can be used to + /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be + /// rethrown to the caller. + /// + /// + /// If the value is set to zero, all function calling exceptions immediately terminate the function + /// invocation loop and the exception will be rethrown to the caller. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to how many iterations are allowed for an in-flight request. + /// + /// + public int MaximumConsecutiveErrorsPerRequest + { + get => _maximumConsecutiveErrorsPerRequest; + set => _maximumConsecutiveErrorsPerRequest = Throw.IfLessThan(value, 0); + } + + /// Gets or sets a collection of additional tools the client is able to invoke. + /// + /// These will not impact the requests sent by the , which will pass through the + /// unmodified. However, if the inner client requests the invocation of a tool + /// that was not in , this collection will also be consulted + /// to look for a corresponding tool to invoke. This is useful when the service may have been pre-configured to be aware + /// of certain tools that aren't also sent on each individual request. + /// + public IList? AdditionalTools { get; set; } + + /// Gets or sets a delegate used to invoke instances. + /// + /// By default, the protected method is called for each to be invoked, + /// invoking the instance and returning its result. If this delegate is set to a non- value, + /// will replace its normal invocation with a call to this delegate, enabling + /// this delegate to assume all invocation handling of the function. + /// + public Func>? FunctionInvoker { get; set; } + + /// + public override async Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + // A single request into this GetResponseAsync may result in multiple requests to the inner client. + // Create an activity to group them together for better observability. + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}"); + + // Copy the original messages in order to avoid enumerating the original messages multiple times. + // The IEnumerable can represent an arbitrary amount of work. + List originalMessages = [.. messages]; + messages = originalMessages; + + List? augmentedHistory = null; // the actual history of messages sent on turns other than the first + ChatResponse? response = null; // the response from the inner client, which is possibly modified and then eventually returned + List? responseMessages = null; // tracked list of messages, across multiple turns, to be used for the final response + UsageDetails? totalUsage = null; // tracked usage across all turns, to be used for the final response + List? functionCallContents = null; // function call contents that need responding to in the current turn + bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set + int consecutiveErrorCount = 0; + + for (int iteration = 0; ; iteration++) + { + functionCallContents?.Clear(); + + // Make the call to the inner client. + response = await base.GetResponseAsync(messages, options, cancellationToken); + if (response is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); + } + + // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. + bool requiresFunctionInvocation = + (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && + iteration < MaximumIterationsPerRequest && + CopyFunctionCalls(response.Messages, ref functionCallContents); + + // In a common case where we make a request and there's no function calling work required, + // fast path out by just returning the original response. + if (iteration == 0 && !requiresFunctionInvocation) + { + return response; + } + + // Track aggregate details from the response, including all of the response messages and usage details. + (responseMessages ??= []).AddRange(response.Messages); + if (response.Usage is not null) + { + if (totalUsage is not null) + { + totalUsage.Add(response.Usage); + } + else + { + totalUsage = response.Usage; + } + } + + // If there are no tools to call, or for any other reason we should stop, we're done. + // Break out of the loop and allow the handling at the end to configure the response + // with aggregated data from previous requests. + if (!requiresFunctionInvocation) + { + break; + } + + // Prepare the history for the next iteration. + FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); + + // Add the responses from the function calls into the augmented history and also into the tracked + // list of response messages. + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); + responseMessages.AddRange(modeAndMessages.MessagesAdded); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + if (modeAndMessages.ShouldTerminate) + { + break; + } + + UpdateOptionsForNextIteration(ref options, response.ConversationId); + } + + Debug.Assert(responseMessages is not null, "Expected to only be here if we have response messages."); + response.Messages = responseMessages!; + response.Usage = totalUsage; + + AddUsageTags(activity, totalUsage); + + return response; + } + + /// + public override async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client. + // Create an activity to group them together for better observability. + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}"); + UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes + + // Copy the original messages in order to avoid enumerating the original messages multiple times. + // The IEnumerable can represent an arbitrary amount of work. + List originalMessages = [.. messages]; + messages = originalMessages; + + List? augmentedHistory = null; // the actual history of messages sent on turns other than the first + List? functionCallContents = null; // function call contents that need responding to in the current turn + List? responseMessages = null; // tracked list of messages, across multiple turns, to be used in fallback cases to reconstitute history + bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set + List updates = []; // updates from the current response + int consecutiveErrorCount = 0; + + for (int iteration = 0; ; iteration++) + { + updates.Clear(); + functionCallContents?.Clear(); + + await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) + { + if (update is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}."); + } + + updates.Add(update); + + _ = CopyFunctionCalls(update.Contents, ref functionCallContents); + + if (totalUsage is not null) + { + IList contents = update.Contents; + int contentsCount = contents.Count; + for (int i = 0; i < contentsCount; i++) + { + if (contents[i] is UsageContent uc) + { + totalUsage.Add(uc.Details); + } + } + } + + yield return update; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + // If there are no tools to call, or for any other reason we should stop, return the response. + if (functionCallContents is not { Count: > 0 } || + (options?.Tools is not { Count: > 0 } && AdditionalTools is not { Count: > 0 }) || + iteration >= _maximumIterationsPerRequest) + { + break; + } + + // Reconstitute a response from the response updates. + var response = updates.ToChatResponse(); + (responseMessages ??= []).AddRange(response.Messages); + + // Prepare the history for the next iteration. + FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); + + // Process all of the functions, adding their results into the history. + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken); + responseMessages.AddRange(modeAndMessages.MessagesAdded); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + // This is a synthetic ID since we're generating the tool messages instead of getting them from + // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to + // use the same message ID for all of them within a given iteration, as this is a single logical + // message with multiple content items. We could also use different message IDs per tool content, + // but there's no benefit to doing so. + string toolResponseId = Guid.NewGuid().ToString("N"); + + // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages + // includes all activities, including generated function results. + foreach (var message in modeAndMessages.MessagesAdded) + { + var toolResultUpdate = new ChatResponseUpdate + { + AdditionalProperties = message.AdditionalProperties, + AuthorName = message.AuthorName, + ConversationId = response.ConversationId, + CreatedAt = DateTimeOffset.UtcNow, + Contents = message.Contents, + RawRepresentation = message.RawRepresentation, + ResponseId = toolResponseId, + MessageId = toolResponseId, // See above for why this can be the same as ResponseId + Role = message.Role, + }; + + yield return toolResultUpdate; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + if (modeAndMessages.ShouldTerminate) + { + break; + } + + UpdateOptionsForNextIteration(ref options, response.ConversationId); + } + + AddUsageTags(activity, totalUsage); + } + + /// Adds tags to for usage details in . + private static void AddUsageTags(Activity? activity, UsageDetails? usage) + { + if (usage is not null && activity is { IsAllDataRequested: true }) + { + if (usage.InputTokenCount is long inputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, (int)inputTokens); + } + + if (usage.OutputTokenCount is long outputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.OutputTokens, (int)outputTokens); + } + } + } + + /// Prepares the various chat message lists after a response from the inner client and before invoking functions. + /// The original messages provided by the caller. + /// The messages reference passed to the inner client. + /// The augmented history containing all the messages to be sent. + /// The most recent response being handled. + /// A list of all response messages received up until this point. + /// Whether the previous iteration's response had a conversation ID. + private static void FixupHistories( + IEnumerable originalMessages, + ref IEnumerable messages, + [NotNull] ref List? augmentedHistory, + ChatResponse response, + List allTurnsResponseMessages, + ref bool lastIterationHadConversationId) + { + // We're now going to need to augment the history with function result contents. + // That means we need a separate list to store the augmented history. + if (response.ConversationId is not null) + { + // The response indicates the inner client is tracking the history, so we don't want to send + // anything we've already sent or received. + if (augmentedHistory is not null) + { + augmentedHistory.Clear(); + } + else + { + augmentedHistory = []; + + // This is needed to allow the ApprovalGeneratingChatClient to work. + // It adds all FunctionApprovalResponseContent that were provided by the caller back into + // the downstream request messages so that they can be processed by the inner client. + // The ones that have matching FunctionCallContent are filtered out, and the ones without matching + // FunctionCallContent are converted to rejected FunctionCallContent. + var functionApprovals = originalMessages.SelectMany(x => x.Contents).OfType().Cast().ToList(); + if (functionApprovals.Count > 0) + { + augmentedHistory.Add(new ChatMessage(ChatRole.User, functionApprovals)); + } + } + + lastIterationHadConversationId = true; + } + else if (lastIterationHadConversationId) + { + // In the very rare case where the inner client returned a response with a conversation ID but then + // returned a subsequent response without one, we want to reconstitute the full history. To do that, + // we can populate the history with the original chat messages and then all of the response + // messages up until this point, which includes the most recent ones. + augmentedHistory ??= []; + augmentedHistory.Clear(); + augmentedHistory.AddRange(originalMessages); + augmentedHistory.AddRange(allTurnsResponseMessages); + + lastIterationHadConversationId = false; + } + else + { + // If augmentedHistory is already non-null, then we've already populated it with everything up + // until this point (except for the most recent response). If it's null, we need to seed it with + // the chat history provided by the caller. + augmentedHistory ??= originalMessages.ToList(); + + // Now add the most recent response messages. + augmentedHistory.AddMessages(response); + + lastIterationHadConversationId = false; + } + + // Use the augmented history as the new set of messages to send. + messages = augmentedHistory; + } + + /// Copies any from to . + private static bool CopyFunctionCalls( + IList messages, [NotNullWhen(true)] ref List? functionCalls) + { + bool any = false; + int count = messages.Count; + for (int i = 0; i < count; i++) + { + any |= CopyFunctionCalls(messages[i].Contents, ref functionCalls); + } + + return any; + } + + /// Copies any from to . + private static bool CopyFunctionCalls( + IList content, [NotNullWhen(true)] ref List? functionCalls) + { + bool any = false; + int count = content.Count; + for (int i = 0; i < count; i++) + { + if (content[i] is FunctionCallContent functionCall) + { + (functionCalls ??= []).Add(functionCall); + any = true; + } + } + + return any; + } + + private static void UpdateOptionsForNextIteration(ref ChatOptions? options, string? conversationId) + { + if (options is null) + { + if (conversationId is not null) + { + options = new() { ConversationId = conversationId }; + } + } + else if (options.ToolMode is RequiredChatToolMode) + { + // We have to reset the tool mode to be non-required after the first iteration, + // as otherwise we'll be in an infinite loop. + options = options.Clone(); + options.ToolMode = null; + options.ConversationId = conversationId; + } + else if (options.ConversationId != conversationId) + { + // As with the other modes, ensure we've propagated the chat conversation ID to the options. + // We only need to clone the options if we're actually mutating it. + options = options.Clone(); + options.ConversationId = conversationId; + } + } + + /// + /// Processes the function calls in the list. + /// + /// The current chat contents, inclusive of the function call contents being processed. + /// The options used for the response being processed. + /// The function call contents representing the functions to be invoked. + /// The iteration number of how many roundtrips have been made to the inner client. + /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors. + /// Whether the function calls are being processed in a streaming context. + /// The to monitor for cancellation requests. + /// A value indicating how the caller should proceed. + private async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync( + List messages, ChatOptions? options, List functionCallContents, int iteration, int consecutiveErrorCount, + bool isStreaming, CancellationToken cancellationToken) + { + // We must add a response for every tool call, regardless of whether we successfully executed it or not. + // If we successfully execute it, we'll add the result. If we don't, we'll add an error. + + Debug.Assert(functionCallContents.Count > 0, "Expected at least one function call."); + var shouldTerminate = false; + var captureCurrentIterationExceptions = consecutiveErrorCount < _maximumConsecutiveErrorsPerRequest; + + // Process all functions. If there's more than one and concurrent invocation is enabled, do so in parallel. + if (functionCallContents.Count == 1) + { + FunctionInvocationResult result = await ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, 0, captureCurrentIterationExceptions, isStreaming, cancellationToken); + + IList addedMessages = CreateResponseMessages([result]); + ThrowIfNoFunctionResultsAdded(addedMessages); + UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); + messages.AddRange(addedMessages); + + return (result.Terminate, consecutiveErrorCount, addedMessages); + } + else + { + List results = []; + + if (AllowConcurrentInvocation) + { + // Rather than awaiting each function before invoking the next, invoke all of them + // and then await all of them. We avoid forcibly introducing parallelism via Task.Run, + // but if a function invocation completes asynchronously, its processing can overlap + // with the processing of other the other invocation invocations. + results.AddRange(await Task.WhenAll( + from callIndex in Enumerable.Range(0, functionCallContents.Count) + select ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, callIndex, captureExceptions: true, isStreaming, cancellationToken))); + + shouldTerminate = results.Any(r => r.Terminate); + } + else + { + // Invoke each function serially. + for (int callIndex = 0; callIndex < functionCallContents.Count; callIndex++) + { + var functionResult = await ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, callIndex, captureCurrentIterationExceptions, isStreaming, cancellationToken); + + results.Add(functionResult); + + // If any function requested termination, we should stop right away. + if (functionResult.Terminate) + { + shouldTerminate = true; + break; + } + } + } + + IList addedMessages = CreateResponseMessages(results.ToArray()); + ThrowIfNoFunctionResultsAdded(addedMessages); + UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); + messages.AddRange(addedMessages); + + return (shouldTerminate, consecutiveErrorCount, addedMessages); + } + } + +#pragma warning disable CA1851 // Possible multiple enumerations of 'IEnumerable' collection + /// + /// Updates the consecutive error count, and throws an exception if the count exceeds the maximum. + /// + /// Added messages. + /// Consecutive error count. + /// Thrown if the maximum consecutive error count is exceeded. + private void UpdateConsecutiveErrorCountOrThrow(IList added, ref int consecutiveErrorCount) + { + var allExceptions = added.SelectMany(m => m.Contents.OfType()) + .Select(frc => frc.Exception!) + .Where(e => e is not null); + + if (allExceptions.Any()) + { + consecutiveErrorCount++; + if (consecutiveErrorCount > _maximumConsecutiveErrorsPerRequest) + { + var allExceptionsArray = allExceptions.ToArray(); + if (allExceptionsArray.Length == 1) + { + ExceptionDispatchInfo.Capture(allExceptionsArray[0]).Throw(); + } + else + { + throw new AggregateException(allExceptionsArray); + } + } + } + else + { + consecutiveErrorCount = 0; + } + } +#pragma warning restore CA1851 + + /// + /// Throws an exception if doesn't create any messages. + /// + private void ThrowIfNoFunctionResultsAdded(IList? messages) + { + if (messages is null || messages.Count == 0) + { + Throw.InvalidOperationException($"{GetType().Name}.{nameof(CreateResponseMessages)} returned null or an empty collection of messages."); + } + } + + /// Processes the function call described in []. + /// The current chat contents, inclusive of the function call contents being processed. + /// The options used for the response being processed. + /// The function call contents representing all the functions being invoked. + /// The iteration number of how many roundtrips have been made to the inner client. + /// The 0-based index of the function being called out of . + /// If true, handles function-invocation exceptions by returning a value with . Otherwise, rethrows. + /// Whether the function calls are being processed in a streaming context. + /// The to monitor for cancellation requests. + /// A value indicating how the caller should proceed. + private async Task ProcessFunctionCallAsync( + List messages, ChatOptions? options, List callContents, + int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, CancellationToken cancellationToken) + { + var callContent = callContents[functionCallIndex]; + + // Look up the AIFunction for the function call. If the requested function isn't available, send back an error. + AIFunction? aiFunction = FindAIFunction(options?.Tools, callContent.Name) ?? FindAIFunction(AdditionalTools, callContent.Name); + if (aiFunction is null) + { + return new(terminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null); + } + + FunctionInvocationContext context = new() + { + Function = aiFunction, + Arguments = new(callContent.Arguments) { Services = FunctionInvocationServices }, + Messages = messages, + Options = options, + CallContent = callContent, + Iteration = iteration, + FunctionCallIndex = functionCallIndex, + FunctionCount = callContents.Count, + IsStreaming = isStreaming + }; + + object? result; + try + { + result = await InstrumentedInvokeFunctionAsync(context, cancellationToken); + } + catch (Exception e) when (!cancellationToken.IsCancellationRequested) + { + if (!captureExceptions) + { + throw; + } + + return new( + terminate: false, + FunctionInvocationStatus.Exception, + callContent, + result: null, + exception: e); + } + + return new( + terminate: context.Terminate, + FunctionInvocationStatus.RanToCompletion, + callContent, + result, + exception: null); + + static AIFunction? FindAIFunction(IList? tools, string functionName) + { + if (tools is not null) + { + int count = tools.Count; + for (int i = 0; i < count; i++) + { + if (tools[i] is AIFunction function && function.Name == functionName) + { + return function; + } + } + } + + return null; + } + } + + /// Creates one or more response messages for function invocation results. + /// Information about the function call invocations and results. + /// A list of all chat messages created from . + protected virtual IList CreateResponseMessages( + ReadOnlySpan results) + { + var contents = new List(results.Length); + for (int i = 0; i < results.Length; i++) + { + contents.Add(CreateFunctionResultContent(results[i])); + } + + return [new(ChatRole.Tool, contents)]; + + FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult result) + { + _ = Throw.IfNull(result); + + object? functionResult; + if (result.Status == FunctionInvocationStatus.RanToCompletion) + { + functionResult = result.Result ?? "Success: Function completed."; + } + else + { + string message = result.Status switch + { + FunctionInvocationStatus.NotFound => $"Error: Requested function \"{result.CallContent.Name}\" not found.", + FunctionInvocationStatus.Exception => "Error: Function failed.", + _ => "Error: Unknown error.", + }; + + if (IncludeDetailedErrors && result.Exception is not null) + { + message = $"{message} Exception: {result.Exception.Message}"; + } + + functionResult = message; + } + + return new FunctionResultContent(result.CallContent.CallId, functionResult) { Exception = result.Exception }; + } + } + + /// Invokes the function asynchronously. + /// + /// The function invocation context detailing the function to be invoked and its arguments along with additional request information. + /// + /// The to monitor for cancellation requests. The default is . + /// The result of the function invocation, or if the function invocation returned . + /// is . + private async Task InstrumentedInvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + { + _ = Throw.IfNull(context); + + using Activity? activity = _activitySource?.StartActivity( + $"{OpenTelemetryConsts.GenAI.ExecuteTool} {context.Function.Name}", + ActivityKind.Internal, + default(ActivityContext), + [ + new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"), + new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId), + new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name), + new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description), + ]); + + long startingTimestamp = 0; + if (_logger.IsEnabled(LogLevel.Debug)) + { + startingTimestamp = Stopwatch.GetTimestamp(); + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogInvokingSensitive(context.Function.Name, LoggingHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions)); + } + else + { + LogInvoking(context.Function.Name); + } + } + + object? result = null; + try + { + CurrentContext = context; // doesn't need to be explicitly reset after, as that's handled automatically at async method exit + result = await InvokeFunctionAsync(context, cancellationToken); + } + catch (Exception e) + { + if (activity is not null) + { + _ = activity.SetTag("error.type", e.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, e.Message); + } + + if (e is OperationCanceledException) + { + LogInvocationCanceled(context.Function.Name); + } + else + { + LogInvocationFailed(context.Function.Name, e); + } + + throw; + } + finally + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + TimeSpan elapsed = GetElapsedTime(startingTimestamp); + + if (result is not null && _logger.IsEnabled(LogLevel.Trace)) + { + LogInvocationCompletedSensitive(context.Function.Name, elapsed, LoggingHelpers.AsJson(result, context.Function.JsonSerializerOptions)); + } + else + { + LogInvocationCompleted(context.Function.Name, elapsed); + } + } + } + + return result; + } + + /// This method will invoke the function within the try block. + /// The function invocation context. + /// Cancellation token. + /// The function result. + protected virtual ValueTask InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + { + _ = Throw.IfNull(context); + + return FunctionInvoker is { } invoker ? + invoker(context, cancellationToken) : + context.Function.InvokeAsync(context.Arguments, cancellationToken); + } + + private static TimeSpan GetElapsedTime(long startingTimestamp) => +#if NET + Stopwatch.GetElapsedTime(startingTimestamp); +#else + new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * ((double)TimeSpan.TicksPerSecond / Stopwatch.Frequency))); +#endif + + [LoggerMessage(LogLevel.Debug, "Invoking {MethodName}.", SkipEnabledCheck = true)] + private partial void LogInvoking(string methodName); + + [LoggerMessage(LogLevel.Trace, "Invoking {MethodName}({Arguments}).", SkipEnabledCheck = true)] + private partial void LogInvokingSensitive(string methodName, string arguments); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invocation completed. Duration: {Duration}", SkipEnabledCheck = true)] + private partial void LogInvocationCompleted(string methodName, TimeSpan duration); + + [LoggerMessage(LogLevel.Trace, "{MethodName} invocation completed. Duration: {Duration}. Result: {Result}", SkipEnabledCheck = true)] + private partial void LogInvocationCompletedSensitive(string methodName, TimeSpan duration, string result); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invocation canceled.")] + private partial void LogInvocationCanceled(string methodName); + + [LoggerMessage(LogLevel.Error, "{MethodName} invocation failed.")] + private partial void LogInvocationFailed(string methodName, Exception error); + + /// Provides information about the invocation of a function call. + public sealed class FunctionInvocationResult + { + /// + /// Initializes a new instance of the class. + /// + /// Indicates whether the caller should terminate the processing loop. + /// Indicates the status of the function invocation. + /// Contains information about the function call. + /// The result of the function call. + /// The exception thrown by the function call, if any. + internal FunctionInvocationResult(bool terminate, FunctionInvocationStatus status, FunctionCallContent callContent, object? result, Exception? exception) + { + Terminate = terminate; + Status = status; + CallContent = callContent; + Result = result; + Exception = exception; + } + + /// Gets status about how the function invocation completed. + public FunctionInvocationStatus Status { get; } + + /// Gets the function call content information associated with this invocation. + public FunctionCallContent CallContent { get; } + + /// Gets the result of the function call. + public object? Result { get; } + + /// Gets any exception the function call threw. + public Exception? Exception { get; } + + /// Gets a value indicating whether the caller should terminate the processing loop. + public bool Terminate { get; } + } + + /// Provides error codes for when errors occur as part of the function calling loop. + public enum FunctionInvocationStatus + { + /// The operation completed successfully. + RanToCompletion, + + /// The requested function could not be found. + NotFound, + + /// The function call failed with an exception. + Exception, + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/OpenTelemetryConsts.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/OpenTelemetryConsts.cs new file mode 100644 index 0000000000..d4a61599d8 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/OpenTelemetryConsts.cs @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI; + +#pragma warning disable CA1716 // Identifiers should not match keywords +#pragma warning disable S4041 // Type names should not match namespaces + +/// Provides constants used by various telemetry services. +internal static class OpenTelemetryConsts +{ + public const string DefaultSourceName = "Experimental.Microsoft.Extensions.AI"; + + public const string SecondsUnit = "s"; + public const string TokensUnit = "token"; + + public static class Event + { + public const string Name = "event.name"; + } + + public static class Error + { + public const string Type = "error.type"; + } + + public static class GenAI + { + public const string Choice = "gen_ai.choice"; + public const string SystemName = "gen_ai.system"; + + public const string Chat = "chat"; + public const string Embeddings = "embeddings"; + public const string ExecuteTool = "execute_tool"; + + public static class Assistant + { + public const string Message = "gen_ai.assistant.message"; + } + + public static class Client + { + public static class OperationDuration + { + public const string Description = "Measures the duration of a GenAI operation"; + public const string Name = "gen_ai.client.operation.duration"; + public static readonly double[] ExplicitBucketBoundaries = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92]; + } + + public static class TokenUsage + { + public const string Description = "Measures number of input and output tokens used"; + public const string Name = "gen_ai.client.token.usage"; + public static readonly int[] ExplicitBucketBoundaries = [1, 4, 16, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216, 67_108_864]; + } + } + + public static class Conversation + { + public const string Id = "gen_ai.conversation.id"; + } + + public static class Operation + { + public const string Name = "gen_ai.operation.name"; + } + + public static class Output + { + public const string Type = "gen_ai.output.type"; + } + + public static class Request + { + public const string EmbeddingDimensions = "gen_ai.request.embedding.dimensions"; + public const string FrequencyPenalty = "gen_ai.request.frequency_penalty"; + public const string Model = "gen_ai.request.model"; + public const string MaxTokens = "gen_ai.request.max_tokens"; + public const string PresencePenalty = "gen_ai.request.presence_penalty"; + public const string Seed = "gen_ai.request.seed"; + public const string StopSequences = "gen_ai.request.stop_sequences"; + public const string Temperature = "gen_ai.request.temperature"; + public const string TopK = "gen_ai.request.top_k"; + public const string TopP = "gen_ai.request.top_p"; + + public static string PerProvider(string providerName, string parameterName) => $"gen_ai.{providerName}.request.{parameterName}"; + } + + public static class Response + { + public const string FinishReasons = "gen_ai.response.finish_reasons"; + public const string Id = "gen_ai.response.id"; + public const string Model = "gen_ai.response.model"; + + public static string PerProvider(string providerName, string parameterName) => $"gen_ai.{providerName}.response.{parameterName}"; + } + + public static class System + { + public const string Message = "gen_ai.system.message"; + } + + public static class Token + { + public const string Type = "gen_ai.token.type"; + } + + public static class Tool + { + public const string Name = "gen_ai.tool.name"; + public const string Description = "gen_ai.tool.description"; + public const string Message = "gen_ai.tool.message"; + + public static class Call + { + public const string Id = "gen_ai.tool.call.id"; + } + } + + public static class Usage + { + public const string InputTokens = "gen_ai.usage.input_tokens"; + public const string OutputTokens = "gen_ai.usage.output_tokens"; + } + + public static class User + { + public const string Message = "gen_ai.user.message"; + } + } + + public static class Server + { + public const string Address = "server.address"; + public const string Port = "server.port"; + } +} From c0682471774f569248b06f59fd11f1465dd1a003 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:19:25 +0100 Subject: [PATCH 15/53] Add pre-FunctionInvokingChatClient ApprovalGeneratingChatClient POC --- ...ep02_ChatClientAgent_UsingFunctionTools.cs | 28 +- .../ChatCompletion/ChatClientExtensions.cs | 13 +- ...pprovalAwareFunctionInvokingChatClient.cs} | 4 +- .../MEAI/NonInvocableAIFunction.cs | 8 + ...nvocableAwareFunctionInvokingChatClient.cs | 1029 +++++++++++++++++ ...> PostFICCApprovalGeneratingChatClient.cs} | 10 +- .../PreFICCApprovalGeneratingChatClient.cs | 212 ++++ 7 files changed, 1269 insertions(+), 35 deletions(-) rename dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/{NewFunctionInvokingChatClient.cs => ApprovalAwareFunctionInvokingChatClient.cs} (99%) create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs rename dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/{ApprovalGeneratingChatClient.cs => PostFICCApprovalGeneratingChatClient.cs} (95%) create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs diff --git a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs index 5a1178546f..cd23b419da 100644 --- a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs +++ b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs @@ -163,26 +163,14 @@ async Task RunAgentAsync(string input) { List nextIterationMessages = []; - foreach (var request in userInputRequests) - { - if (request is FunctionApprovalRequestContent approvalRequest) - { - if (approvalRequest.FunctionCall.Name == "GetSpecials") - { - Console.WriteLine($"Approving the {approvalRequest.FunctionCall.Name} function call."); - nextIterationMessages.Add(approvalRequest.Approve()); - } - else - { - Console.WriteLine($"Rejecting the {approvalRequest.FunctionCall.Name} function call."); - nextIterationMessages.Add(approvalRequest.Reject()); - } - } - else - { - throw new NotSupportedException("This type of approval is not supported"); - } - } + var approvedRequests = userInputRequests.OfType().Where(x => x.FunctionCall.Name == "GetSpecials").ToList(); + var rejectedRequests = userInputRequests.OfType().Where(x => x.FunctionCall.Name != "GetSpecials").ToList(); + + approvedRequests.ForEach(x => Console.WriteLine($"Approving the {x.FunctionCall.Name} function call.")); + rejectedRequests.ForEach(x => Console.WriteLine($"Rejecting the {x.FunctionCall.Name} function call.")); + + nextIterationMessages.AddRange(approvedRequests.Select(x => x.Approve())); + nextIterationMessages.AddRange(rejectedRequests.Select(x => x.Reject())); response = await agent.RunAsync(nextIterationMessages, thread); this.WriteResponseOutput(response); diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs index 71a5c91cfc..56bf6724c6 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs @@ -17,27 +17,24 @@ internal static IChatClient AsAgentInvokingChatClient(this IChatClient chatClien chatBuilder.UseAgentInvocation(); } - if (chatClient.GetService() is null) + if (chatClient.GetService() is null) { - chatBuilder.UseFunctionInvocation(); - chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => { var loggerFactory = services.GetService(); - var newFunctionInvokingChatClient = new NewFunctionInvokingChatClient(innerClient, loggerFactory, services); - return newFunctionInvokingChatClient; + PreFICCApprovalGeneratingChatClient approvalGeneratingChatClient = new(innerClient, loggerFactory); + return approvalGeneratingChatClient; }); } - if (chatClient.GetService() is null) + if (chatClient.GetService() is null) { chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => { var loggerFactory = services.GetService(); - ApprovalGeneratingChatClient approvalGeneratingChatClient = new(innerClient, loggerFactory); - return approvalGeneratingChatClient; + return new NonInvocableAwareFunctionInvokingChatClient(innerClient, loggerFactory, services); }); } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalAwareFunctionInvokingChatClient.cs similarity index 99% rename from dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs rename to dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalAwareFunctionInvokingChatClient.cs index 4ae071b716..0e9596d509 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalAwareFunctionInvokingChatClient.cs @@ -48,7 +48,7 @@ namespace Microsoft.Extensions.AI; /// invocation requests to that same function. /// /// -public partial class NewFunctionInvokingChatClient : DelegatingChatClient +public partial class ApprovalAwareFunctionInvokingChatClient : DelegatingChatClient { /// The for the current function invocation. private static readonly AsyncLocal s_currentContext = new(); @@ -75,7 +75,7 @@ public partial class NewFunctionInvokingChatClient : DelegatingChatClient /// The underlying , or the next instance in a chain of clients. /// An to use for logging information about function invocation. /// An optional to use for resolving services required by the instances being invoked. - public NewFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) + public ApprovalAwareFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) : base(innerClient) { _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs new file mode 100644 index 0000000000..8173a699c5 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.AI.Agents.MEAI; + +/// +/// Marks an existing with additional metadata to indicate that it is not invocable. +/// +internal class NonInvocableAIFunction(AIFunction innerFunction) : DelegatingAIFunction(innerFunction); diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs new file mode 100644 index 0000000000..4abc195c3c --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs @@ -0,0 +1,1029 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI.Agents.MEAI; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; + +#pragma warning disable CA2213 // Disposable fields should be disposed +#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test +#pragma warning disable SA1202 // 'protected' members should come before 'private' members +#pragma warning disable S107 // Methods should not have too many parameters + +#pragma warning disable IDE0009 // Member access should be qualified. +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task +#pragma warning disable VSTHRD111 // Use ConfigureAwait(bool) + +namespace Microsoft.Extensions.AI; + +/// +/// A delegating chat client that invokes functions defined on . +/// Include this in a chat pipeline to resolve function calls automatically. +/// +/// +/// +/// When this client receives a in a chat response, it responds +/// by calling the corresponding defined in , +/// producing a that it sends back to the inner client. This loop +/// is repeated until there are no more function calls to make, or until another stop condition is met, +/// such as hitting . +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// instances employed as part of the supplied are also safe. +/// The property can be used to control whether multiple function invocation +/// requests as part of the same request are invocable concurrently, but even with that set to +/// (the default), multiple concurrent requests to this same instance and using the same tools could result in those +/// tools being used concurrently (one per request). For example, a function that accesses the HttpContext of a specific +/// ASP.NET web request should only be used as part of a single at a time, and only with +/// set to , in case the inner client decided to issue multiple +/// invocation requests to that same function. +/// +/// +public partial class NonInvocableAwareFunctionInvokingChatClient : DelegatingChatClient +{ + /// The for the current function invocation. + private static readonly AsyncLocal s_currentContext = new(); + + /// Gets the specified when constructing the , if any. + protected IServiceProvider? FunctionInvocationServices { get; } + + /// The logger to use for logging information about function invocation. + private readonly ILogger _logger; + + /// The to use for telemetry. + /// This component does not own the instance and should not dispose it. + private readonly ActivitySource? _activitySource; + + /// Maximum number of roundtrips allowed to the inner client. + private int _maximumIterationsPerRequest = 40; // arbitrary default to prevent runaway execution + + /// Maximum number of consecutive iterations that are allowed contain at least one exception result. If the limit is exceeded, we rethrow the exception instead of continuing. + private int _maximumConsecutiveErrorsPerRequest = 3; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying , or the next instance in a chain of clients. + /// An to use for logging information about function invocation. + /// An optional to use for resolving services required by the instances being invoked. + public NonInvocableAwareFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) + : base(innerClient) + { + _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + _activitySource = innerClient.GetService(); + FunctionInvocationServices = functionInvocationServices; + } + + /// + /// Gets or sets the for the current function invocation. + /// + /// + /// This value flows across async calls. + /// + public static FunctionInvocationContext? CurrentContext + { + get => s_currentContext.Value; + protected set => s_currentContext.Value = value; + } + + /// + /// Gets or sets a value indicating whether detailed exception information should be included + /// in the chat history when calling the underlying . + /// + /// + /// if the full exception message is added to the chat history + /// when calling the underlying . + /// if a generic error message is included in the chat history. + /// The default value is . + /// + /// + /// + /// Setting the value to prevents the underlying language model from disclosing + /// raw exception details to the end user, since it doesn't receive that information. Even in this + /// case, the raw object is available to application code by inspecting + /// the property. + /// + /// + /// Setting the value to can help the underlying bypass problems on + /// its own, for example by retrying the function call with different arguments. However it might + /// result in disclosing the raw exception information to external users, which can be a security + /// concern depending on the application scenario. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to whether detailed errors are provided during an in-flight request. + /// + /// + public bool IncludeDetailedErrors { get; set; } + + /// + /// Gets or sets a value indicating whether to allow concurrent invocation of functions. + /// + /// + /// if multiple function calls can execute in parallel. + /// if function calls are processed serially. + /// The default value is . + /// + /// + /// An individual response from the inner client might contain multiple function call requests. + /// By default, such function calls are processed serially. Set to + /// to enable concurrent invocation such that multiple function calls can execute in parallel. + /// + public bool AllowConcurrentInvocation { get; set; } + + /// + /// Gets or sets the maximum number of iterations per request. + /// + /// + /// The maximum number of iterations per request. + /// The default value is 40. + /// + /// + /// + /// Each request to this might end up making + /// multiple requests to the inner client. Each time the inner client responds with + /// a function call request, this client might perform that invocation and send the results + /// back to the inner client in a new request. This property limits the number of times + /// such a roundtrip is performed. The value must be at least one, as it includes the initial request. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to how many iterations are allowed for an in-flight request. + /// + /// + public int MaximumIterationsPerRequest + { + get => _maximumIterationsPerRequest; + set + { + if (value < 1) + { + Throw.ArgumentOutOfRangeException(nameof(value)); + } + + _maximumIterationsPerRequest = value; + } + } + + /// + /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error. + /// + /// + /// The maximum number of consecutive iterations that are allowed to fail with an error. + /// The default value is 3. + /// + /// + /// + /// When function invocations fail with an exception, the + /// continues to make requests to the inner client, optionally supplying exception information (as + /// controlled by ). This allows the to + /// recover from errors by trying other function parameters that may succeed. + /// + /// + /// However, in case function invocations continue to produce exceptions, this property can be used to + /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be + /// rethrown to the caller. + /// + /// + /// If the value is set to zero, all function calling exceptions immediately terminate the function + /// invocation loop and the exception will be rethrown to the caller. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to how many iterations are allowed for an in-flight request. + /// + /// + public int MaximumConsecutiveErrorsPerRequest + { + get => _maximumConsecutiveErrorsPerRequest; + set => _maximumConsecutiveErrorsPerRequest = Throw.IfLessThan(value, 0); + } + + /// Gets or sets a collection of additional tools the client is able to invoke. + /// + /// These will not impact the requests sent by the , which will pass through the + /// unmodified. However, if the inner client requests the invocation of a tool + /// that was not in , this collection will also be consulted + /// to look for a corresponding tool to invoke. This is useful when the service may have been pre-configured to be aware + /// of certain tools that aren't also sent on each individual request. + /// + public IList? AdditionalTools { get; set; } + + /// Gets or sets a delegate used to invoke instances. + /// + /// By default, the protected method is called for each to be invoked, + /// invoking the instance and returning its result. If this delegate is set to a non- value, + /// will replace its normal invocation with a call to this delegate, enabling + /// this delegate to assume all invocation handling of the function. + /// + public Func>? FunctionInvoker { get; set; } + + /// + public override async Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + // A single request into this GetResponseAsync may result in multiple requests to the inner client. + // Create an activity to group them together for better observability. + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}"); + + // Copy the original messages in order to avoid enumerating the original messages multiple times. + // The IEnumerable can represent an arbitrary amount of work. + List originalMessages = [.. messages]; + messages = originalMessages; + + List? augmentedHistory = null; // the actual history of messages sent on turns other than the first + ChatResponse? response = null; // the response from the inner client, which is possibly modified and then eventually returned + List? responseMessages = null; // tracked list of messages, across multiple turns, to be used for the final response + UsageDetails? totalUsage = null; // tracked usage across all turns, to be used for the final response + List? functionCallContents = null; // function call contents that need responding to in the current turn + bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set + int consecutiveErrorCount = 0; + int iteration = 0; + + NonInvocableAIFunction[] nonInvocableAIFunctions = [.. options?.Tools?.OfType() ?? [], .. AdditionalTools?.OfType() ?? []]; + HashSet nonInvocableAIFunctionNames = new(nonInvocableAIFunctions.Select(f => f.Name)); + HashSet alreadyInvokedFunctionCalls = new(originalMessages.SelectMany(m => m.Contents).OfType().Select(frc => frc.CallId)); + + // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. + bool requiresPreGetResponseFunctionInvocation = + (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && + CopyFunctionCalls(originalMessages, nonInvocableAIFunctionNames, alreadyInvokedFunctionCalls, ref functionCallContents); + + if (requiresPreGetResponseFunctionInvocation) + { + // Add the responses from the function calls into the augmented history and also into the tracked + // list of response messages. + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, functionCallContents!, iteration++, consecutiveErrorCount, isStreaming: false, cancellationToken); + responseMessages = [.. modeAndMessages.MessagesAdded]; + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + if (modeAndMessages.ShouldTerminate) + { + return new ChatResponse(responseMessages); + } + } + + for (; ; iteration++) + { + // Do pre-invocation function calls, that may be passed in. + + functionCallContents?.Clear(); + + // Make the call to the inner client. + response = await base.GetResponseAsync(messages, options, cancellationToken); + if (response is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); + } + + // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. + bool requiresFunctionInvocation = + (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && + iteration < MaximumIterationsPerRequest && + CopyFunctionCalls(response.Messages, nonInvocableAIFunctionNames, [], ref functionCallContents); + + // In a common case where we make a request and there's no function calling work required, + // fast path out by just returning the original response. + if (iteration == 0 && !requiresFunctionInvocation) + { + return response; + } + + // Track aggregate details from the response, including all of the response messages and usage details. + (responseMessages ??= []).AddRange(response.Messages); + if (response.Usage is not null) + { + if (totalUsage is not null) + { + totalUsage.Add(response.Usage); + } + else + { + totalUsage = response.Usage; + } + } + + // If there are no tools to call, or for any other reason we should stop, we're done. + // Break out of the loop and allow the handling at the end to configure the response + // with aggregated data from previous requests. + if (!requiresFunctionInvocation) + { + break; + } + + // Prepare the history for the next iteration. + FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); + + // Add the responses from the function calls into the augmented history and also into the tracked + // list of response messages. + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); + responseMessages.AddRange(modeAndMessages.MessagesAdded); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + if (modeAndMessages.ShouldTerminate) + { + break; + } + + UpdateOptionsForNextIteration(ref options, response.ConversationId); + } + + Debug.Assert(responseMessages is not null, "Expected to only be here if we have response messages."); + response.Messages = responseMessages!; + response.Usage = totalUsage; + + AddUsageTags(activity, totalUsage); + + return response; + } + + /// + public override async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client. + // Create an activity to group them together for better observability. + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}"); + UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes + + // Copy the original messages in order to avoid enumerating the original messages multiple times. + // The IEnumerable can represent an arbitrary amount of work. + List originalMessages = [.. messages]; + messages = originalMessages; + + List? augmentedHistory = null; // the actual history of messages sent on turns other than the first + List? functionCallContents = null; // function call contents that need responding to in the current turn + List? responseMessages = null; // tracked list of messages, across multiple turns, to be used in fallback cases to reconstitute history + bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set + List updates = []; // updates from the current response + int consecutiveErrorCount = 0; + NonInvocableAIFunction[] nonInvocableAIFunctions = [.. options?.Tools?.OfType() ?? [], .. AdditionalTools?.OfType() ?? []]; + HashSet nonInvocableAIFunctionNames = new(nonInvocableAIFunctions.Select(f => f.Name)); + + for (int iteration = 0; ; iteration++) + { + updates.Clear(); + functionCallContents?.Clear(); + + await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) + { + if (update is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}."); + } + + updates.Add(update); + + _ = CopyFunctionCalls(update.Contents, nonInvocableAIFunctionNames, [], ref functionCallContents); + + if (totalUsage is not null) + { + IList contents = update.Contents; + int contentsCount = contents.Count; + for (int i = 0; i < contentsCount; i++) + { + if (contents[i] is UsageContent uc) + { + totalUsage.Add(uc.Details); + } + } + } + + yield return update; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + // If there are no tools to call, or for any other reason we should stop, return the response. + if (functionCallContents is not { Count: > 0 } || + (options?.Tools is not { Count: > 0 } && AdditionalTools is not { Count: > 0 }) || + iteration >= _maximumIterationsPerRequest) + { + break; + } + + // Reconstitute a response from the response updates. + var response = updates.ToChatResponse(); + (responseMessages ??= []).AddRange(response.Messages); + + // Prepare the history for the next iteration. + FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); + + // Process all of the functions, adding their results into the history. + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken); + responseMessages.AddRange(modeAndMessages.MessagesAdded); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + // This is a synthetic ID since we're generating the tool messages instead of getting them from + // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to + // use the same message ID for all of them within a given iteration, as this is a single logical + // message with multiple content items. We could also use different message IDs per tool content, + // but there's no benefit to doing so. + string toolResponseId = Guid.NewGuid().ToString("N"); + + // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages + // includes all activities, including generated function results. + foreach (var message in modeAndMessages.MessagesAdded) + { + var toolResultUpdate = new ChatResponseUpdate + { + AdditionalProperties = message.AdditionalProperties, + AuthorName = message.AuthorName, + ConversationId = response.ConversationId, + CreatedAt = DateTimeOffset.UtcNow, + Contents = message.Contents, + RawRepresentation = message.RawRepresentation, + ResponseId = toolResponseId, + MessageId = toolResponseId, // See above for why this can be the same as ResponseId + Role = message.Role, + }; + + yield return toolResultUpdate; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + if (modeAndMessages.ShouldTerminate) + { + break; + } + + UpdateOptionsForNextIteration(ref options, response.ConversationId); + } + + AddUsageTags(activity, totalUsage); + } + + /// Adds tags to for usage details in . + private static void AddUsageTags(Activity? activity, UsageDetails? usage) + { + if (usage is not null && activity is { IsAllDataRequested: true }) + { + if (usage.InputTokenCount is long inputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, (int)inputTokens); + } + + if (usage.OutputTokenCount is long outputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.OutputTokens, (int)outputTokens); + } + } + } + + /// Prepares the various chat message lists after a response from the inner client and before invoking functions. + /// The original messages provided by the caller. + /// The messages reference passed to the inner client. + /// The augmented history containing all the messages to be sent. + /// The most recent response being handled. + /// A list of all response messages received up until this point. + /// Whether the previous iteration's response had a conversation ID. + private static void FixupHistories( + IEnumerable originalMessages, + ref IEnumerable messages, + [NotNull] ref List? augmentedHistory, + ChatResponse response, + List allTurnsResponseMessages, + ref bool lastIterationHadConversationId) + { + // We're now going to need to augment the history with function result contents. + // That means we need a separate list to store the augmented history. + if (response.ConversationId is not null) + { + // The response indicates the inner client is tracking the history, so we don't want to send + // anything we've already sent or received. + if (augmentedHistory is not null) + { + augmentedHistory.Clear(); + } + else + { + augmentedHistory = []; + } + + lastIterationHadConversationId = true; + } + else if (lastIterationHadConversationId) + { + // In the very rare case where the inner client returned a response with a conversation ID but then + // returned a subsequent response without one, we want to reconstitute the full history. To do that, + // we can populate the history with the original chat messages and then all of the response + // messages up until this point, which includes the most recent ones. + augmentedHistory ??= []; + augmentedHistory.Clear(); + augmentedHistory.AddRange(originalMessages); + augmentedHistory.AddRange(allTurnsResponseMessages); + + lastIterationHadConversationId = false; + } + else + { + // If augmentedHistory is already non-null, then we've already populated it with everything up + // until this point (except for the most recent response). If it's null, we need to seed it with + // the chat history provided by the caller. + augmentedHistory ??= originalMessages.ToList(); + + // Now add the most recent response messages. + augmentedHistory.AddMessages(response); + + lastIterationHadConversationId = false; + } + + // Use the augmented history as the new set of messages to send. + messages = augmentedHistory; + } + + /// Copies any from to . + private static bool CopyFunctionCalls( + IList messages, HashSet nonInvocableAIFunctionNames, HashSet alreadyInvokedFunctionCalls, [NotNullWhen(true)] ref List? functionCalls) + { + bool any = false; + int count = messages.Count; + for (int i = 0; i < count; i++) + { + any |= CopyFunctionCalls(messages[i].Contents, nonInvocableAIFunctionNames, alreadyInvokedFunctionCalls, ref functionCalls); + } + + return any; + } + + /// Copies any from to . + private static bool CopyFunctionCalls( + IList content, HashSet nonInvocableAIFunctionNames, HashSet alreadyInvokedFunctionCalls, [NotNullWhen(true)] ref List? functionCalls) + { + bool any = false; + int count = content.Count; + for (int i = 0; i < count; i++) + { + if (content[i] is FunctionCallContent functionCall && !nonInvocableAIFunctionNames.Contains(functionCall.Name) && !alreadyInvokedFunctionCalls.Contains(functionCall.CallId)) + { + (functionCalls ??= []).Add(functionCall); + any = true; + } + } + + return any; + } + + private static void UpdateOptionsForNextIteration(ref ChatOptions? options, string? conversationId) + { + if (options is null) + { + if (conversationId is not null) + { + options = new() { ConversationId = conversationId }; + } + } + else if (options.ToolMode is RequiredChatToolMode) + { + // We have to reset the tool mode to be non-required after the first iteration, + // as otherwise we'll be in an infinite loop. + options = options.Clone(); + options.ToolMode = null; + options.ConversationId = conversationId; + } + else if (options.ConversationId != conversationId) + { + // As with the other modes, ensure we've propagated the chat conversation ID to the options. + // We only need to clone the options if we're actually mutating it. + options = options.Clone(); + options.ConversationId = conversationId; + } + } + + /// + /// Processes the function calls in the list. + /// + /// The current chat contents, inclusive of the function call contents being processed. + /// The options used for the response being processed. + /// The function call contents representing the functions to be invoked. + /// The iteration number of how many roundtrips have been made to the inner client. + /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors. + /// Whether the function calls are being processed in a streaming context. + /// The to monitor for cancellation requests. + /// A value indicating how the caller should proceed. + private async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync( + List messages, ChatOptions? options, List functionCallContents, int iteration, int consecutiveErrorCount, + bool isStreaming, CancellationToken cancellationToken) + { + // We must add a response for every tool call, regardless of whether we successfully executed it or not. + // If we successfully execute it, we'll add the result. If we don't, we'll add an error. + + Debug.Assert(functionCallContents.Count > 0, "Expected at least one function call."); + var shouldTerminate = false; + var captureCurrentIterationExceptions = consecutiveErrorCount < _maximumConsecutiveErrorsPerRequest; + + // Process all functions. If there's more than one and concurrent invocation is enabled, do so in parallel. + if (functionCallContents.Count == 1) + { + FunctionInvocationResult result = await ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, 0, captureCurrentIterationExceptions, isStreaming, cancellationToken); + + IList addedMessages = CreateResponseMessages([result]); + ThrowIfNoFunctionResultsAdded(addedMessages); + UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); + messages.AddRange(addedMessages); + + return (result.Terminate, consecutiveErrorCount, addedMessages); + } + else + { + List results = []; + + if (AllowConcurrentInvocation) + { + // Rather than awaiting each function before invoking the next, invoke all of them + // and then await all of them. We avoid forcibly introducing parallelism via Task.Run, + // but if a function invocation completes asynchronously, its processing can overlap + // with the processing of other the other invocation invocations. + results.AddRange(await Task.WhenAll( + from callIndex in Enumerable.Range(0, functionCallContents.Count) + select ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, callIndex, captureExceptions: true, isStreaming, cancellationToken))); + + shouldTerminate = results.Any(r => r.Terminate); + } + else + { + // Invoke each function serially. + for (int callIndex = 0; callIndex < functionCallContents.Count; callIndex++) + { + var functionResult = await ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, callIndex, captureCurrentIterationExceptions, isStreaming, cancellationToken); + + results.Add(functionResult); + + // If any function requested termination, we should stop right away. + if (functionResult.Terminate) + { + shouldTerminate = true; + break; + } + } + } + + IList addedMessages = CreateResponseMessages(results.ToArray()); + ThrowIfNoFunctionResultsAdded(addedMessages); + UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); + messages.AddRange(addedMessages); + + return (shouldTerminate, consecutiveErrorCount, addedMessages); + } + } + +#pragma warning disable CA1851 // Possible multiple enumerations of 'IEnumerable' collection + /// + /// Updates the consecutive error count, and throws an exception if the count exceeds the maximum. + /// + /// Added messages. + /// Consecutive error count. + /// Thrown if the maximum consecutive error count is exceeded. + private void UpdateConsecutiveErrorCountOrThrow(IList added, ref int consecutiveErrorCount) + { + var allExceptions = added.SelectMany(m => m.Contents.OfType()) + .Select(frc => frc.Exception!) + .Where(e => e is not null); + + if (allExceptions.Any()) + { + consecutiveErrorCount++; + if (consecutiveErrorCount > _maximumConsecutiveErrorsPerRequest) + { + var allExceptionsArray = allExceptions.ToArray(); + if (allExceptionsArray.Length == 1) + { + ExceptionDispatchInfo.Capture(allExceptionsArray[0]).Throw(); + } + else + { + throw new AggregateException(allExceptionsArray); + } + } + } + else + { + consecutiveErrorCount = 0; + } + } +#pragma warning restore CA1851 + + /// + /// Throws an exception if doesn't create any messages. + /// + private void ThrowIfNoFunctionResultsAdded(IList? messages) + { + if (messages is null || messages.Count == 0) + { + Throw.InvalidOperationException($"{GetType().Name}.{nameof(CreateResponseMessages)} returned null or an empty collection of messages."); + } + } + + /// Processes the function call described in []. + /// The current chat contents, inclusive of the function call contents being processed. + /// The options used for the response being processed. + /// The function call contents representing all the functions being invoked. + /// The iteration number of how many roundtrips have been made to the inner client. + /// The 0-based index of the function being called out of . + /// If true, handles function-invocation exceptions by returning a value with . Otherwise, rethrows. + /// Whether the function calls are being processed in a streaming context. + /// The to monitor for cancellation requests. + /// A value indicating how the caller should proceed. + private async Task ProcessFunctionCallAsync( + List messages, ChatOptions? options, List callContents, + int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, CancellationToken cancellationToken) + { + var callContent = callContents[functionCallIndex]; + + // Look up the AIFunction for the function call. If the requested function isn't available, send back an error. + AIFunction? aiFunction = FindAIFunction(options?.Tools, callContent.Name) ?? FindAIFunction(AdditionalTools, callContent.Name); + if (aiFunction is null) + { + return new(terminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null); + } + + FunctionInvocationContext context = new() + { + Function = aiFunction, + Arguments = new(callContent.Arguments) { Services = FunctionInvocationServices }, + Messages = messages, + Options = options, + CallContent = callContent, + Iteration = iteration, + FunctionCallIndex = functionCallIndex, + FunctionCount = callContents.Count, + IsStreaming = isStreaming + }; + + object? result; + try + { + result = await InstrumentedInvokeFunctionAsync(context, cancellationToken); + } + catch (Exception e) when (!cancellationToken.IsCancellationRequested) + { + if (!captureExceptions) + { + throw; + } + + return new( + terminate: false, + FunctionInvocationStatus.Exception, + callContent, + result: null, + exception: e); + } + + return new( + terminate: context.Terminate, + FunctionInvocationStatus.RanToCompletion, + callContent, + result, + exception: null); + + static AIFunction? FindAIFunction(IList? tools, string functionName) + { + if (tools is not null) + { + int count = tools.Count; + for (int i = 0; i < count; i++) + { + if (tools[i] is AIFunction function && function.Name == functionName) + { + return function; + } + } + } + + return null; + } + } + + /// Creates one or more response messages for function invocation results. + /// Information about the function call invocations and results. + /// A list of all chat messages created from . + protected virtual IList CreateResponseMessages( + ReadOnlySpan results) + { + var contents = new List(results.Length); + for (int i = 0; i < results.Length; i++) + { + contents.Add(CreateFunctionResultContent(results[i])); + } + + return [new(ChatRole.Tool, contents)]; + + FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult result) + { + _ = Throw.IfNull(result); + + object? functionResult; + if (result.Status == FunctionInvocationStatus.RanToCompletion) + { + functionResult = result.Result ?? "Success: Function completed."; + } + else + { + string message = result.Status switch + { + FunctionInvocationStatus.NotFound => $"Error: Requested function \"{result.CallContent.Name}\" not found.", + FunctionInvocationStatus.Exception => "Error: Function failed.", + _ => "Error: Unknown error.", + }; + + if (IncludeDetailedErrors && result.Exception is not null) + { + message = $"{message} Exception: {result.Exception.Message}"; + } + + functionResult = message; + } + + return new FunctionResultContent(result.CallContent.CallId, functionResult) { Exception = result.Exception }; + } + } + + /// Invokes the function asynchronously. + /// + /// The function invocation context detailing the function to be invoked and its arguments along with additional request information. + /// + /// The to monitor for cancellation requests. The default is . + /// The result of the function invocation, or if the function invocation returned . + /// is . + private async Task InstrumentedInvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + { + _ = Throw.IfNull(context); + + using Activity? activity = _activitySource?.StartActivity( + $"{OpenTelemetryConsts.GenAI.ExecuteTool} {context.Function.Name}", + ActivityKind.Internal, + default(ActivityContext), + [ + new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"), + new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId), + new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name), + new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description), + ]); + + long startingTimestamp = 0; + if (_logger.IsEnabled(LogLevel.Debug)) + { + startingTimestamp = Stopwatch.GetTimestamp(); + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogInvokingSensitive(context.Function.Name, LoggingHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions)); + } + else + { + LogInvoking(context.Function.Name); + } + } + + object? result = null; + try + { + CurrentContext = context; // doesn't need to be explicitly reset after, as that's handled automatically at async method exit + result = await InvokeFunctionAsync(context, cancellationToken); + } + catch (Exception e) + { + if (activity is not null) + { + _ = activity.SetTag("error.type", e.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, e.Message); + } + + if (e is OperationCanceledException) + { + LogInvocationCanceled(context.Function.Name); + } + else + { + LogInvocationFailed(context.Function.Name, e); + } + + throw; + } + finally + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + TimeSpan elapsed = GetElapsedTime(startingTimestamp); + + if (result is not null && _logger.IsEnabled(LogLevel.Trace)) + { + LogInvocationCompletedSensitive(context.Function.Name, elapsed, LoggingHelpers.AsJson(result, context.Function.JsonSerializerOptions)); + } + else + { + LogInvocationCompleted(context.Function.Name, elapsed); + } + } + } + + return result; + } + + /// This method will invoke the function within the try block. + /// The function invocation context. + /// Cancellation token. + /// The function result. + protected virtual ValueTask InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + { + _ = Throw.IfNull(context); + + return FunctionInvoker is { } invoker ? + invoker(context, cancellationToken) : + context.Function.InvokeAsync(context.Arguments, cancellationToken); + } + + private static TimeSpan GetElapsedTime(long startingTimestamp) => +#if NET + Stopwatch.GetElapsedTime(startingTimestamp); +#else + new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * ((double)TimeSpan.TicksPerSecond / Stopwatch.Frequency))); +#endif + + [LoggerMessage(LogLevel.Debug, "Invoking {MethodName}.", SkipEnabledCheck = true)] + private partial void LogInvoking(string methodName); + + [LoggerMessage(LogLevel.Trace, "Invoking {MethodName}({Arguments}).", SkipEnabledCheck = true)] + private partial void LogInvokingSensitive(string methodName, string arguments); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invocation completed. Duration: {Duration}", SkipEnabledCheck = true)] + private partial void LogInvocationCompleted(string methodName, TimeSpan duration); + + [LoggerMessage(LogLevel.Trace, "{MethodName} invocation completed. Duration: {Duration}. Result: {Result}", SkipEnabledCheck = true)] + private partial void LogInvocationCompletedSensitive(string methodName, TimeSpan duration, string result); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invocation canceled.")] + private partial void LogInvocationCanceled(string methodName); + + [LoggerMessage(LogLevel.Error, "{MethodName} invocation failed.")] + private partial void LogInvocationFailed(string methodName, Exception error); + + /// Provides information about the invocation of a function call. + public sealed class FunctionInvocationResult + { + /// + /// Initializes a new instance of the class. + /// + /// Indicates whether the caller should terminate the processing loop. + /// Indicates the status of the function invocation. + /// Contains information about the function call. + /// The result of the function call. + /// The exception thrown by the function call, if any. + internal FunctionInvocationResult(bool terminate, FunctionInvocationStatus status, FunctionCallContent callContent, object? result, Exception? exception) + { + Terminate = terminate; + Status = status; + CallContent = callContent; + Result = result; + Exception = exception; + } + + /// Gets status about how the function invocation completed. + public FunctionInvocationStatus Status { get; } + + /// Gets the function call content information associated with this invocation. + public FunctionCallContent CallContent { get; } + + /// Gets the result of the function call. + public object? Result { get; } + + /// Gets any exception the function call threw. + public Exception? Exception { get; } + + /// Gets a value indicating whether the caller should terminate the processing loop. + public bool Terminate { get; } + } + + /// Provides error codes for when errors occur as part of the function calling loop. + public enum FunctionInvocationStatus + { + /// The operation completed successfully. + RanToCompletion, + + /// The requested function could not be found. + NotFound, + + /// The function call failed with an exception. + Exception, + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs similarity index 95% rename from dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalGeneratingChatClient.cs rename to dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs index 27abd56768..f3898298ad 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalGeneratingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs @@ -12,22 +12,22 @@ namespace Microsoft.Extensions.AI.Agents; /// -/// Represents a chat client that seeks user approval for function calls. +/// Represents a chat client that seeks user approval for function calls and sits behind the . /// -public class ApprovalGeneratingChatClient : DelegatingChatClient +public class PostFICCApprovalGeneratingChatClient : DelegatingChatClient { /// The logger to use for logging information about function approval. private readonly ILogger _logger; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The underlying , or the next instance in a chain of clients. /// An to use for logging information about function invocation. - public ApprovalGeneratingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null) + public PostFICCApprovalGeneratingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null) : base(innerClient) { - this._logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + this._logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; } /// diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs new file mode 100644 index 0000000000..5954f183f8 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI.Agents.MEAI; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI.Agents; + +/// +/// Represents a chat client that seeks user approval for function calls and sits before the . +/// +public class PreFICCApprovalGeneratingChatClient : DelegatingChatClient +{ + /// The logger to use for logging information about function approval. + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying , or the next instance in a chain of clients. + /// An to use for logging information about function invocation. + public PreFICCApprovalGeneratingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null) + : base(innerClient) + { + this._logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + } + + /// + public override async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + var messagesList = messages as IList ?? messages.ToList(); + + // If we got any FunctionApprovalResponseContent, we can remove the FunctionApprovalRequestContent for those responses, since the FunctionApprovalResponseContent + // will be turned into FunctionCallContent and FunctionResultContent later, but the FunctionApprovalRequestContent is now unecessary. + // If we got any approval request/responses, and we also already have FunctionResultContent for those, we can filter out those too requests/responses + // since they are already handled. + // This is since the downstream service, may not know what to do with the FunctionApprovalRequestContent/FunctionApprovalResponseContent. + RemoveExecutedApprovedApprovalRequests(messagesList); + + // Get all the remaining approval responses. + var approvalResponses = messagesList.SelectMany(x => x.Contents).OfType().ToList(); + + // If we have any functions in options, we should clone them and mark any that do not yet have an approval as not invocable. + options = MakeFunctionsNonInvocable(options, approvalResponses); + + // For rejections we need to replace them with function call content plus rejected function result content. + // For approvals we need to replace them just with function call content, since the inner client will invoke them. + ReplaceApprovalResponses(messagesList); + + var response = await base.GetResponseAsync(messagesList, options, cancellationToken).ConfigureAwait(false); + if (response is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); + } + + // Replace any FunctionCallContent in the response with FunctionApprovalRequestContent. + ReplaceFunctionCallsWithApprovalRequests(response.Messages); + + return response; + } + + /// + public override IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + return base.GetStreamingResponseAsync(messages, options, cancellationToken); + } + + private static void RemoveExecutedApprovedApprovalRequests(IList messages) + { + var functionResultCallIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.CallId).ToHashSet(); + var approvalResponsetIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.ApprovalId).ToHashSet(); + + int messageCount = messages.Count; + for (int i = 0; i < messageCount; i++) + { + // Get any content that is not a FunctionApprovalRequestContent/FunctionApprovalResponseContent or is a FunctionApprovalRequestContent/FunctionApprovalResponseContent that has not been executed. + var content = messages[i].Contents.Where(x => + (x is not FunctionApprovalRequestContent && x is not FunctionApprovalResponseContent) || + (x is FunctionApprovalRequestContent request && !approvalResponsetIds.Contains(request.ApprovalId) && !functionResultCallIds.Contains(request.FunctionCall.CallId)) || + (x is FunctionApprovalResponseContent approval && !functionResultCallIds.Contains(approval.FunctionCall.CallId))).ToList(); + + // Remove the entire message if there is no content left after filtering. + if (content.Count == 0) + { + messages.RemoveAt(i); + i--; // Adjust index since we removed an item. + messageCount--; // Adjust count since we removed an item. + continue; + } + + // Replace the message contents with the filtered content. + messages[i].Contents = content; + } + } + + private static void ReplaceApprovalResponses(IList messages) + { + List approvedFunctionCallContent = []; + + List rejectedFunctionCallContent = []; + List rejectedFunctionResultContent = []; + + int messageCount = messages.Count; + for (int i = 0; i < messageCount; i++) + { + var content = messages[i].Contents; + + int contentCount = content.Count; + + // ApprovalResponses are submitted as part of a user messages, but FunctionCallContent should be in an assistant message and + // FunctionResultContent should be in a tool message, so we need to remove them from the user messages, and add them to the appropriate + // mesages types later. + for (int j = 0; j < contentCount; j++) + { + // Find all responses that were approved, and add the FunctionCallContent for them to the list to add back later. + if (content[j] is FunctionApprovalResponseContent approval && approval.Approved) + { + content.RemoveAt(j); + j--; // Adjust index since we removed an item. + contentCount--; // Adjust count since we removed an item. + + approvedFunctionCallContent.Add(approval.FunctionCall); + continue; + } + + // Find all responses that were rejected, and add their FunctionCallContent and a FunctionResultContent indicating the rejection, to the lists to add back later. + if (content[j] is FunctionApprovalResponseContent rejection && !rejection.Approved) + { + content.RemoveAt(j); + j--; // Adjust index since we removed an item. + contentCount--; // Adjust count since we removed an item. + + var rejectedFunctionCall = new FunctionResultContent(rejection.FunctionCall.CallId, "Error: Function invocation approval was not granted."); + rejectedFunctionCallContent.Add(rejection.FunctionCall); + rejectedFunctionResultContent.Add(rejectedFunctionCall); + } + } + + // If we have no content left in the message after replacing, we can remove the message from the list. + if (content.Count == 0) + { + messages.RemoveAt(i); + i--; // Adjust index since we removed an item. + messageCount--; // Adjust count since we removed an item. + } + } + + if (rejectedFunctionCallContent.Count > 0) + { + messages.Add(new ChatMessage(ChatRole.Assistant, rejectedFunctionCallContent)); + messages.Add(new ChatMessage(ChatRole.Tool, rejectedFunctionResultContent)); + } + + if (approvedFunctionCallContent.Count != 0) + { + messages.Add(new ChatMessage(ChatRole.Assistant, approvedFunctionCallContent)); + } + } + + /// Replaces any from with . + private static void ReplaceFunctionCallsWithApprovalRequests(IList messages) + { + int count = messages.Count; + for (int i = 0; i < count; i++) + { + ReplaceFunctionCallsWithApprovalRequests(messages[i].Contents); + } + } + + /// Copies any from with . + private static void ReplaceFunctionCallsWithApprovalRequests(IList content) + { + int count = content.Count; + for (int i = 0; i < count; i++) + { + if (content[i] is FunctionCallContent functionCall) + { + content[i] = new FunctionApprovalRequestContent + { + FunctionCall = functionCall, + ApprovalId = functionCall.CallId + }; + } + } + } + + private static ChatOptions? MakeFunctionsNonInvocable(ChatOptions? options, List approvals) + { + if (options?.Tools?.Count is > 0) + { + options = options.Clone(); + options.Tools = options.Tools!.Select(x => + { + if (x is AIFunction function && !approvals.Any(y => y.FunctionCall.Name == function.Name)) + { + var f = new NonInvocableAIFunction(function); + return f; + } + return x; + }).ToList(); + } + + return options; + } +} From c8e42de59c3ae822dc3a9ce657deff3586546da1 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 6 Aug 2025 18:29:48 +0100 Subject: [PATCH 16/53] Fix namespaces --- .../MEAI/NonInvocableAIFunction.cs | 2 +- .../MEAI/NonInvocableAwareFunctionInvokingChatClient.cs | 2 -- .../MEAI/PostFICCApprovalGeneratingChatClient.cs | 2 +- .../MEAI/PreFICCApprovalGeneratingChatClient.cs | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs index 8173a699c5..c75748bf0a 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -namespace Microsoft.Extensions.AI.Agents.MEAI; +namespace Microsoft.Extensions.AI; /// /// Marks an existing with additional metadata to indicate that it is not invocable. diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs index 4abc195c3c..e086060b51 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs @@ -7,10 +7,8 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; -using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Agents.MEAI; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Shared.Diagnostics; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs index f3898298ad..c0cc744077 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs @@ -9,7 +9,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Shared.Diagnostics; -namespace Microsoft.Extensions.AI.Agents; +namespace Microsoft.Extensions.AI; /// /// Represents a chat client that seeks user approval for function calls and sits behind the . diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs index 5954f183f8..08173f7496 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Agents.MEAI; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Shared.Diagnostics; From 97352a00069b0a0373a227ca1352f84e1402c3f9 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Thu, 7 Aug 2025 19:11:12 +0100 Subject: [PATCH 17/53] Add combined FICC with approvals --- ...ep02_ChatClientAgent_UsingFunctionTools.cs | 1 + .../ChatCompletion/ChatClientExtensions.cs | 15 +- ...nInvokingChatClientWithBuiltInApprovals.cs | 1184 +++++++++++++++++ .../PreFICCApprovalGeneratingChatClient.cs | 2 +- 4 files changed, 1188 insertions(+), 14 deletions(-) create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs diff --git a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs index cd23b419da..07ec1f3ef7 100644 --- a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs +++ b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs @@ -149,6 +149,7 @@ public async Task ApprovalsWithTools(ChatClientProviders provider) // Respond to user input, invoking functions where appropriate. await RunAgentAsync("What is the special soup and its price?"); + await RunAgentAsync("What is the special drink?"); async Task RunAgentAsync(string input) { diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs index 56bf6724c6..2c2ed4fb0b 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/ChatCompletion/ChatClientExtensions.cs @@ -17,24 +17,13 @@ internal static IChatClient AsAgentInvokingChatClient(this IChatClient chatClien chatBuilder.UseAgentInvocation(); } - if (chatClient.GetService() is null) + if (chatClient.GetService() is null) { chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => { var loggerFactory = services.GetService(); - PreFICCApprovalGeneratingChatClient approvalGeneratingChatClient = new(innerClient, loggerFactory); - return approvalGeneratingChatClient; - }); - } - - if (chatClient.GetService() is null) - { - chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => - { - var loggerFactory = services.GetService(); - - return new NonInvocableAwareFunctionInvokingChatClient(innerClient, loggerFactory, services); + return new FunctionInvokingChatClientWithBuiltInApprovals(innerClient, loggerFactory, services); }); } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs new file mode 100644 index 0000000000..ff521532ed --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -0,0 +1,1184 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; + +#pragma warning disable CA2213 // Disposable fields should be disposed +#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test +#pragma warning disable SA1202 // 'protected' members should come before 'private' members +#pragma warning disable S107 // Methods should not have too many parameters + +// AF repo suppressions for code copied from MEAI. +#pragma warning disable IDE1006 // Naming Styles +#pragma warning disable IDE0009 // Member access should be qualified. +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task +#pragma warning disable VSTHRD111 // Use ConfigureAwait(bool) + +namespace Microsoft.Extensions.AI; + +/// +/// A delegating chat client that invokes functions defined on . +/// Include this in a chat pipeline to resolve function calls automatically. +/// +/// +/// +/// When this client receives a in a chat response, it responds +/// by calling the corresponding defined in , +/// producing a that it sends back to the inner client. This loop +/// is repeated until there are no more function calls to make, or until another stop condition is met, +/// such as hitting . +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// instances employed as part of the supplied are also safe. +/// The property can be used to control whether multiple function invocation +/// requests as part of the same request are invocable concurrently, but even with that set to +/// (the default), multiple concurrent requests to this same instance and using the same tools could result in those +/// tools being used concurrently (one per request). For example, a function that accesses the HttpContext of a specific +/// ASP.NET web request should only be used as part of a single at a time, and only with +/// set to , in case the inner client decided to issue multiple +/// invocation requests to that same function. +/// +/// +public partial class FunctionInvokingChatClientWithBuiltInApprovals : DelegatingChatClient +{ + /// The for the current function invocation. + private static readonly AsyncLocal _currentContext = new(); + + /// Gets the specified when constructing the , if any. + protected IServiceProvider? FunctionInvocationServices { get; } + + /// The logger to use for logging information about function invocation. + private readonly ILogger _logger; + + /// The to use for telemetry. + /// This component does not own the instance and should not dispose it. + private readonly ActivitySource? _activitySource; + + /// Maximum number of roundtrips allowed to the inner client. + private int _maximumIterationsPerRequest = 40; // arbitrary default to prevent runaway execution + + /// Maximum number of consecutive iterations that are allowed contain at least one exception result. If the limit is exceeded, we rethrow the exception instead of continuing. + private int _maximumConsecutiveErrorsPerRequest = 3; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying , or the next instance in a chain of clients. + /// An to use for logging information about function invocation. + /// An optional to use for resolving services required by the instances being invoked. + public FunctionInvokingChatClientWithBuiltInApprovals(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) + : base(innerClient) + { + _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + _activitySource = innerClient.GetService(); + FunctionInvocationServices = functionInvocationServices; + } + + /// + /// Gets or sets the for the current function invocation. + /// + /// + /// This value flows across async calls. + /// + public static FunctionInvocationContext? CurrentContext + { + get => _currentContext.Value; + protected set => _currentContext.Value = value; + } + + /// + /// Gets or sets a value indicating whether detailed exception information should be included + /// in the chat history when calling the underlying . + /// + /// + /// if the full exception message is added to the chat history + /// when calling the underlying . + /// if a generic error message is included in the chat history. + /// The default value is . + /// + /// + /// + /// Setting the value to prevents the underlying language model from disclosing + /// raw exception details to the end user, since it doesn't receive that information. Even in this + /// case, the raw object is available to application code by inspecting + /// the property. + /// + /// + /// Setting the value to can help the underlying bypass problems on + /// its own, for example by retrying the function call with different arguments. However it might + /// result in disclosing the raw exception information to external users, which can be a security + /// concern depending on the application scenario. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to whether detailed errors are provided during an in-flight request. + /// + /// + public bool IncludeDetailedErrors { get; set; } + + /// + /// Gets or sets a value indicating whether to allow concurrent invocation of functions. + /// + /// + /// if multiple function calls can execute in parallel. + /// if function calls are processed serially. + /// The default value is . + /// + /// + /// An individual response from the inner client might contain multiple function call requests. + /// By default, such function calls are processed serially. Set to + /// to enable concurrent invocation such that multiple function calls can execute in parallel. + /// + public bool AllowConcurrentInvocation { get; set; } + + /// + /// Gets or sets the maximum number of iterations per request. + /// + /// + /// The maximum number of iterations per request. + /// The default value is 40. + /// + /// + /// + /// Each request to this might end up making + /// multiple requests to the inner client. Each time the inner client responds with + /// a function call request, this client might perform that invocation and send the results + /// back to the inner client in a new request. This property limits the number of times + /// such a roundtrip is performed. The value must be at least one, as it includes the initial request. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to how many iterations are allowed for an in-flight request. + /// + /// + public int MaximumIterationsPerRequest + { + get => _maximumIterationsPerRequest; + set + { + if (value < 1) + { + Throw.ArgumentOutOfRangeException(nameof(value)); + } + + _maximumIterationsPerRequest = value; + } + } + + /// + /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error. + /// + /// + /// The maximum number of consecutive iterations that are allowed to fail with an error. + /// The default value is 3. + /// + /// + /// + /// When function invocations fail with an exception, the + /// continues to make requests to the inner client, optionally supplying exception information (as + /// controlled by ). This allows the to + /// recover from errors by trying other function parameters that may succeed. + /// + /// + /// However, in case function invocations continue to produce exceptions, this property can be used to + /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be + /// rethrown to the caller. + /// + /// + /// If the value is set to zero, all function calling exceptions immediately terminate the function + /// invocation loop and the exception will be rethrown to the caller. + /// + /// + /// Changing the value of this property while the client is in use might result in inconsistencies + /// as to how many iterations are allowed for an in-flight request. + /// + /// + public int MaximumConsecutiveErrorsPerRequest + { + get => _maximumConsecutiveErrorsPerRequest; + set => _maximumConsecutiveErrorsPerRequest = Throw.IfLessThan(value, 0); + } + + /// Gets or sets a collection of additional tools the client is able to invoke. + /// + /// These will not impact the requests sent by the , which will pass through the + /// unmodified. However, if the inner client requests the invocation of a tool + /// that was not in , this collection will also be consulted + /// to look for a corresponding tool to invoke. This is useful when the service may have been pre-configured to be aware + /// of certain tools that aren't also sent on each individual request. + /// + public IList? AdditionalTools { get; set; } + + /// Gets or sets a delegate used to invoke instances. + /// + /// By default, the protected method is called for each to be invoked, + /// invoking the instance and returning its result. If this delegate is set to a non- value, + /// will replace its normal invocation with a call to this delegate, enabling + /// this delegate to assume all invocation handling of the function. + /// + public Func>? FunctionInvoker { get; set; } + + /// + public override async Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + // A single request into this GetResponseAsync may result in multiple requests to the inner client. + // Create an activity to group them together for better observability. + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}"); + + // Copy the original messages in order to avoid enumerating the original messages multiple times. + // The IEnumerable can represent an arbitrary amount of work. + List originalMessages = [.. messages]; + messages = originalMessages; + + List? augmentedHistory = null; // the actual history of messages sent on turns other than the first + ChatResponse? response = null; // the response from the inner client, which is possibly modified and then eventually returned + List? responseMessages = null; // tracked list of messages, across multiple turns, to be used for the final response + UsageDetails? totalUsage = null; // tracked usage across all turns, to be used for the final response + List? functionCallContents = null; // function call contents that need responding to in the current turn + bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set + int consecutiveErrorCount = 0; + int iteration = 0; + + // ** Approvals additions on top of FICC - start **// + + // Remove any approval requests and approval request/response pairs that have already been executed. + var notExecutedResponses = ProcessApprovalRequestsAndResponses(originalMessages); + + // Generate failed function result contents for any rejected requests. + List rejectedFunctionCallMessages = []; + if (notExecutedResponses.rejections is { Count: > 0 }) + { + foreach (var rejectedCall in notExecutedResponses.rejections) + { + // Create a FunctionResultContent for the rejected function call. + var functionResult = new FunctionResultContent(rejectedCall.CallId, "Error: Function invocation approval was not granted."); + + rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Assistant, [rejectedCall])); + rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult])); + } + + originalMessages.AddRange(rejectedFunctionCallMessages); + } + + var rejectedFunctionCalls = notExecutedResponses.rejections; + var rejectedFunctionResults = rejectedFunctionCalls?.Select(x => new FunctionResultContent(x.CallId, "Error: Function invocation approval was not granted.")); + + // Check if there are any function calls to do from any approved functions and execute them. + if (notExecutedResponses.approvals is { Count: > 0 }) + { + originalMessages.AddRange(notExecutedResponses.approvals.Select(x => new ChatMessage(ChatRole.Assistant, [x]))); + + // Add the responses from the function calls into the augmented history and also into the tracked + // list of response messages. + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); + responseMessages = [.. modeAndMessages.MessagesAdded, ..rejectedFunctionCallMessages]; + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + if (modeAndMessages.ShouldTerminate) + { + return new ChatResponse(responseMessages); + } + } + + // ** Approvals additions on top of FICC - end **// + + for (; ; iteration++) + { + functionCallContents?.Clear(); + + // Make the call to the inner client. + response = await base.GetResponseAsync(messages, options, cancellationToken); + if (response is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); + } + + // ** Approvals additions on top of FICC - start **// + + // Before we do any function execution, make sure that any functions that require approval, have been turned into approval requests + // so that they don't get executed here. + ReplaceFunctionCallsWithApprovalRequests(response.Messages); + + // ** Approvals additions on top of FICC - end **// + + // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. + bool requiresFunctionInvocation = + (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && + iteration < MaximumIterationsPerRequest && + CopyFunctionCalls(response.Messages, ref functionCallContents); + + // In a common case where we make a request and there's no function calling work required, + // fast path out by just returning the original response. + if (iteration == 0 && !requiresFunctionInvocation) + { + return response; + } + + // Track aggregate details from the response, including all of the response messages and usage details. + (responseMessages ??= []).AddRange(response.Messages); + if (response.Usage is not null) + { + if (totalUsage is not null) + { + totalUsage.Add(response.Usage); + } + else + { + totalUsage = response.Usage; + } + } + + // If there are no tools to call, or for any other reason we should stop, we're done. + // Break out of the loop and allow the handling at the end to configure the response + // with aggregated data from previous requests. + if (!requiresFunctionInvocation) + { + break; + } + + // Prepare the history for the next iteration. + FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); + + // Add the responses from the function calls into the augmented history and also into the tracked + // list of response messages. + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); + responseMessages.AddRange(modeAndMessages.MessagesAdded); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + if (modeAndMessages.ShouldTerminate) + { + break; + } + + UpdateOptionsForNextIteration(ref options, response.ConversationId); + } + + Debug.Assert(responseMessages is not null, "Expected to only be here if we have response messages."); + response.Messages = responseMessages!; + response.Usage = totalUsage; + + AddUsageTags(activity, totalUsage); + + return response; + } + + /// + public override async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client. + // Create an activity to group them together for better observability. + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}"); + UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes + + // Copy the original messages in order to avoid enumerating the original messages multiple times. + // The IEnumerable can represent an arbitrary amount of work. + List originalMessages = [.. messages]; + messages = originalMessages; + + List? augmentedHistory = null; // the actual history of messages sent on turns other than the first + List? functionCallContents = null; // function call contents that need responding to in the current turn + List? responseMessages = null; // tracked list of messages, across multiple turns, to be used in fallback cases to reconstitute history + bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set + List updates = []; // updates from the current response + int consecutiveErrorCount = 0; + + for (int iteration = 0; ; iteration++) + { + updates.Clear(); + functionCallContents?.Clear(); + + await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) + { + if (update is null) + { + Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}."); + } + + updates.Add(update); + + _ = CopyFunctionCalls(update.Contents, ref functionCallContents); + + if (totalUsage is not null) + { + IList contents = update.Contents; + int contentsCount = contents.Count; + for (int i = 0; i < contentsCount; i++) + { + if (contents[i] is UsageContent uc) + { + totalUsage.Add(uc.Details); + } + } + } + + yield return update; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + // If there are no tools to call, or for any other reason we should stop, return the response. + if (functionCallContents is not { Count: > 0 } || + (options?.Tools is not { Count: > 0 } && AdditionalTools is not { Count: > 0 }) || + iteration >= _maximumIterationsPerRequest) + { + break; + } + + // Reconstitute a response from the response updates. + var response = updates.ToChatResponse(); + (responseMessages ??= []).AddRange(response.Messages); + + // Prepare the history for the next iteration. + FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); + + // Process all of the functions, adding their results into the history. + var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken); + responseMessages.AddRange(modeAndMessages.MessagesAdded); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + // This is a synthetic ID since we're generating the tool messages instead of getting them from + // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to + // use the same message ID for all of them within a given iteration, as this is a single logical + // message with multiple content items. We could also use different message IDs per tool content, + // but there's no benefit to doing so. + string toolResponseId = Guid.NewGuid().ToString("N"); + + // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages + // includes all activities, including generated function results. + foreach (var message in modeAndMessages.MessagesAdded) + { + var toolResultUpdate = new ChatResponseUpdate + { + AdditionalProperties = message.AdditionalProperties, + AuthorName = message.AuthorName, + ConversationId = response.ConversationId, + CreatedAt = DateTimeOffset.UtcNow, + Contents = message.Contents, + RawRepresentation = message.RawRepresentation, + ResponseId = toolResponseId, + MessageId = toolResponseId, // See above for why this can be the same as ResponseId + Role = message.Role, + }; + + yield return toolResultUpdate; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + + if (modeAndMessages.ShouldTerminate) + { + break; + } + + UpdateOptionsForNextIteration(ref options, response.ConversationId); + } + + AddUsageTags(activity, totalUsage); + } + + /// Adds tags to for usage details in . + private static void AddUsageTags(Activity? activity, UsageDetails? usage) + { + if (usage is not null && activity is { IsAllDataRequested: true }) + { + if (usage.InputTokenCount is long inputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, (int)inputTokens); + } + + if (usage.OutputTokenCount is long outputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.OutputTokens, (int)outputTokens); + } + } + } + + /// Prepares the various chat message lists after a response from the inner client and before invoking functions. + /// The original messages provided by the caller. + /// The messages reference passed to the inner client. + /// The augmented history containing all the messages to be sent. + /// The most recent response being handled. + /// A list of all response messages received up until this point. + /// Whether the previous iteration's response had a conversation ID. + private static void FixupHistories( + IEnumerable originalMessages, + ref IEnumerable messages, + [NotNull] ref List? augmentedHistory, + ChatResponse response, + List allTurnsResponseMessages, + ref bool lastIterationHadConversationId) + { + // We're now going to need to augment the history with function result contents. + // That means we need a separate list to store the augmented history. + if (response.ConversationId is not null) + { + // The response indicates the inner client is tracking the history, so we don't want to send + // anything we've already sent or received. + if (augmentedHistory is not null) + { + augmentedHistory.Clear(); + } + else + { + augmentedHistory = []; + } + + lastIterationHadConversationId = true; + } + else if (lastIterationHadConversationId) + { + // In the very rare case where the inner client returned a response with a conversation ID but then + // returned a subsequent response without one, we want to reconstitute the full history. To do that, + // we can populate the history with the original chat messages and then all of the response + // messages up until this point, which includes the most recent ones. + augmentedHistory ??= []; + augmentedHistory.Clear(); + augmentedHistory.AddRange(originalMessages); + augmentedHistory.AddRange(allTurnsResponseMessages); + + lastIterationHadConversationId = false; + } + else + { + // If augmentedHistory is already non-null, then we've already populated it with everything up + // until this point (except for the most recent response). If it's null, we need to seed it with + // the chat history provided by the caller. + augmentedHistory ??= originalMessages.ToList(); + + // Now add the most recent response messages. + augmentedHistory.AddMessages(response); + + lastIterationHadConversationId = false; + } + + // Use the augmented history as the new set of messages to send. + messages = augmentedHistory; + } + + /// Copies any from to . + private static bool CopyFunctionCalls( + IList messages, [NotNullWhen(true)] ref List? functionCalls) + { + bool any = false; + int count = messages.Count; + for (int i = 0; i < count; i++) + { + any |= CopyFunctionCalls(messages[i].Contents, ref functionCalls); + } + + return any; + } + + /// Copies any from to . + private static bool CopyFunctionCalls( + IList content, [NotNullWhen(true)] ref List? functionCalls) + { + bool any = false; + int count = content.Count; + for (int i = 0; i < count; i++) + { + if (content[i] is FunctionCallContent functionCall) + { + (functionCalls ??= []).Add(functionCall); + any = true; + } + } + + return any; + } + + private static void UpdateOptionsForNextIteration(ref ChatOptions? options, string? conversationId) + { + if (options is null) + { + if (conversationId is not null) + { + options = new() { ConversationId = conversationId }; + } + } + else if (options.ToolMode is RequiredChatToolMode) + { + // We have to reset the tool mode to be non-required after the first iteration, + // as otherwise we'll be in an infinite loop. + options = options.Clone(); + options.ToolMode = null; + options.ConversationId = conversationId; + } + else if (options.ConversationId != conversationId) + { + // As with the other modes, ensure we've propagated the chat conversation ID to the options. + // We only need to clone the options if we're actually mutating it. + options = options.Clone(); + options.ConversationId = conversationId; + } + } + + /// + /// Processes the function calls in the list. + /// + /// The current chat contents, inclusive of the function call contents being processed. + /// The options used for the response being processed. + /// The function call contents representing the functions to be invoked. + /// The iteration number of how many roundtrips have been made to the inner client. + /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors. + /// Whether the function calls are being processed in a streaming context. + /// The to monitor for cancellation requests. + /// A value indicating how the caller should proceed. + private async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync( + List messages, ChatOptions? options, List functionCallContents, int iteration, int consecutiveErrorCount, + bool isStreaming, CancellationToken cancellationToken) + { + // We must add a response for every tool call, regardless of whether we successfully executed it or not. + // If we successfully execute it, we'll add the result. If we don't, we'll add an error. + + Debug.Assert(functionCallContents.Count > 0, "Expected at least one function call."); + var shouldTerminate = false; + var captureCurrentIterationExceptions = consecutiveErrorCount < _maximumConsecutiveErrorsPerRequest; + + // Process all functions. If there's more than one and concurrent invocation is enabled, do so in parallel. + if (functionCallContents.Count == 1) + { + FunctionInvocationResult result = await ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, 0, captureCurrentIterationExceptions, isStreaming, cancellationToken); + + IList addedMessages = CreateResponseMessages([result]); + ThrowIfNoFunctionResultsAdded(addedMessages); + UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); + messages.AddRange(addedMessages); + + return (result.Terminate, consecutiveErrorCount, addedMessages); + } + else + { + List results = []; + + if (AllowConcurrentInvocation) + { + // Rather than awaiting each function before invoking the next, invoke all of them + // and then await all of them. We avoid forcibly introducing parallelism via Task.Run, + // but if a function invocation completes asynchronously, its processing can overlap + // with the processing of other the other invocation invocations. + results.AddRange(await Task.WhenAll( + from callIndex in Enumerable.Range(0, functionCallContents.Count) + select ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, callIndex, captureExceptions: true, isStreaming, cancellationToken))); + + shouldTerminate = results.Any(r => r.Terminate); + } + else + { + // Invoke each function serially. + for (int callIndex = 0; callIndex < functionCallContents.Count; callIndex++) + { + var functionResult = await ProcessFunctionCallAsync( + messages, options, functionCallContents, + iteration, callIndex, captureCurrentIterationExceptions, isStreaming, cancellationToken); + + results.Add(functionResult); + + // If any function requested termination, we should stop right away. + if (functionResult.Terminate) + { + shouldTerminate = true; + break; + } + } + } + + IList addedMessages = CreateResponseMessages(results.ToArray()); + ThrowIfNoFunctionResultsAdded(addedMessages); + UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); + messages.AddRange(addedMessages); + + return (shouldTerminate, consecutiveErrorCount, addedMessages); + } + } + +#pragma warning disable CA1851 // Possible multiple enumerations of 'IEnumerable' collection + /// + /// Updates the consecutive error count, and throws an exception if the count exceeds the maximum. + /// + /// Added messages. + /// Consecutive error count. + /// Thrown if the maximum consecutive error count is exceeded. + private void UpdateConsecutiveErrorCountOrThrow(IList added, ref int consecutiveErrorCount) + { + var allExceptions = added.SelectMany(m => m.Contents.OfType()) + .Select(frc => frc.Exception!) + .Where(e => e is not null); + + if (allExceptions.Any()) + { + consecutiveErrorCount++; + if (consecutiveErrorCount > _maximumConsecutiveErrorsPerRequest) + { + var allExceptionsArray = allExceptions.ToArray(); + if (allExceptionsArray.Length == 1) + { + ExceptionDispatchInfo.Capture(allExceptionsArray[0]).Throw(); + } + else + { + throw new AggregateException(allExceptionsArray); + } + } + } + else + { + consecutiveErrorCount = 0; + } + } +#pragma warning restore CA1851 + + /// + /// Throws an exception if doesn't create any messages. + /// + private void ThrowIfNoFunctionResultsAdded(IList? messages) + { + if (messages is null || messages.Count == 0) + { + Throw.InvalidOperationException($"{GetType().Name}.{nameof(CreateResponseMessages)} returned null or an empty collection of messages."); + } + } + + /// Processes the function call described in []. + /// The current chat contents, inclusive of the function call contents being processed. + /// The options used for the response being processed. + /// The function call contents representing all the functions being invoked. + /// The iteration number of how many roundtrips have been made to the inner client. + /// The 0-based index of the function being called out of . + /// If true, handles function-invocation exceptions by returning a value with . Otherwise, rethrows. + /// Whether the function calls are being processed in a streaming context. + /// The to monitor for cancellation requests. + /// A value indicating how the caller should proceed. + private async Task ProcessFunctionCallAsync( + List messages, ChatOptions? options, List callContents, + int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, CancellationToken cancellationToken) + { + var callContent = callContents[functionCallIndex]; + + // Look up the AIFunction for the function call. If the requested function isn't available, send back an error. + AIFunction? aiFunction = FindAIFunction(options?.Tools, callContent.Name) ?? FindAIFunction(AdditionalTools, callContent.Name); + if (aiFunction is null) + { + return new(terminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null); + } + + FunctionInvocationContext context = new() + { + Function = aiFunction, + Arguments = new(callContent.Arguments) { Services = FunctionInvocationServices }, + Messages = messages, + Options = options, + CallContent = callContent, + Iteration = iteration, + FunctionCallIndex = functionCallIndex, + FunctionCount = callContents.Count, + IsStreaming = isStreaming + }; + + object? result; + try + { + result = await InstrumentedInvokeFunctionAsync(context, cancellationToken); + } + catch (Exception e) when (!cancellationToken.IsCancellationRequested) + { + if (!captureExceptions) + { + throw; + } + + return new( + terminate: false, + FunctionInvocationStatus.Exception, + callContent, + result: null, + exception: e); + } + + return new( + terminate: context.Terminate, + FunctionInvocationStatus.RanToCompletion, + callContent, + result, + exception: null); + + static AIFunction? FindAIFunction(IList? tools, string functionName) + { + if (tools is not null) + { + int count = tools.Count; + for (int i = 0; i < count; i++) + { + if (tools[i] is AIFunction function && function.Name == functionName) + { + return function; + } + } + } + + return null; + } + } + + /// Creates one or more response messages for function invocation results. + /// Information about the function call invocations and results. + /// A list of all chat messages created from . + protected virtual IList CreateResponseMessages( + ReadOnlySpan results) + { + var contents = new List(results.Length); + for (int i = 0; i < results.Length; i++) + { + contents.Add(CreateFunctionResultContent(results[i])); + } + + return [new(ChatRole.Tool, contents)]; + + FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult result) + { + _ = Throw.IfNull(result); + + object? functionResult; + if (result.Status == FunctionInvocationStatus.RanToCompletion) + { + functionResult = result.Result ?? "Success: Function completed."; + } + else + { + string message = result.Status switch + { + FunctionInvocationStatus.NotFound => $"Error: Requested function \"{result.CallContent.Name}\" not found.", + FunctionInvocationStatus.Exception => "Error: Function failed.", + _ => "Error: Unknown error.", + }; + + if (IncludeDetailedErrors && result.Exception is not null) + { + message = $"{message} Exception: {result.Exception.Message}"; + } + + functionResult = message; + } + + return new FunctionResultContent(result.CallContent.CallId, functionResult) { Exception = result.Exception }; + } + } + + /// Invokes the function asynchronously. + /// + /// The function invocation context detailing the function to be invoked and its arguments along with additional request information. + /// + /// The to monitor for cancellation requests. The default is . + /// The result of the function invocation, or if the function invocation returned . + /// is . + private async Task InstrumentedInvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + { + _ = Throw.IfNull(context); + + using Activity? activity = _activitySource?.StartActivity( + $"{OpenTelemetryConsts.GenAI.ExecuteTool} {context.Function.Name}", + ActivityKind.Internal, + default(ActivityContext), + [ + new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"), + new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId), + new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name), + new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description), + ]); + + long startingTimestamp = 0; + if (_logger.IsEnabled(LogLevel.Debug)) + { + startingTimestamp = Stopwatch.GetTimestamp(); + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogInvokingSensitive(context.Function.Name, LoggingHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions)); + } + else + { + LogInvoking(context.Function.Name); + } + } + + object? result = null; + try + { + CurrentContext = context; // doesn't need to be explicitly reset after, as that's handled automatically at async method exit + result = await InvokeFunctionAsync(context, cancellationToken); + } + catch (Exception e) + { + if (activity is not null) + { + _ = activity.SetTag("error.type", e.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, e.Message); + } + + if (e is OperationCanceledException) + { + LogInvocationCanceled(context.Function.Name); + } + else + { + LogInvocationFailed(context.Function.Name, e); + } + + throw; + } + finally + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + TimeSpan elapsed = GetElapsedTime(startingTimestamp); + + if (result is not null && _logger.IsEnabled(LogLevel.Trace)) + { + LogInvocationCompletedSensitive(context.Function.Name, elapsed, LoggingHelpers.AsJson(result, context.Function.JsonSerializerOptions)); + } + else + { + LogInvocationCompleted(context.Function.Name, elapsed); + } + } + } + + return result; + } + + /// This method will invoke the function within the try block. + /// The function invocation context. + /// Cancellation token. + /// The function result. + protected virtual ValueTask InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) + { + _ = Throw.IfNull(context); + + return FunctionInvoker is { } invoker ? + invoker(context, cancellationToken) : + context.Function.InvokeAsync(context.Arguments, cancellationToken); + } + + /// + /// We want to get rid of any and that have already been executed. + /// We want to throw an exception for any that has no response, since it is an error state. + /// We want to return the from any that has no matching and for execution. + /// + private static (List? approvals, List? rejections) ProcessApprovalRequestsAndResponses(List messages) + { + // Get the list of function call ids that are already executed. + var functionResultCallIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.CallId).ToHashSet(); + List? notExecutedApprovedFunctionCalls = null; + List? notExecutedRejectedFunctionCalls = null; + HashSet? requestCallIds = null; + + for (int i = 0; i < messages.Count; i++) + { + var message = messages[i]; + + List? keptContents = null; + + // Find contents we want to keep. + for (int j = 0; j < message.Contents.Count; j++) + { + var content = message.Contents[j]; + + // Save response that are not yet executed, so that we can execute them later. + if (content is FunctionApprovalResponseContent response && !functionResultCallIds.Contains(response.FunctionCall.CallId)) + { + if (response.Approved) + { + notExecutedApprovedFunctionCalls ??= []; + notExecutedApprovedFunctionCalls.Add(response.FunctionCall); + } + else + { + notExecutedRejectedFunctionCalls ??= []; + notExecutedRejectedFunctionCalls.Add(response.FunctionCall); + } + } + + // Capture each call id for each approval request. + if (content is FunctionApprovalRequestContent request_) + { + requestCallIds ??= []; + requestCallIds.Add(request_.FunctionCall.CallId); + } + + // Remove the call id for each approval response. + if (content is FunctionApprovalResponseContent response_) + { + if (requestCallIds is null) + { + Throw.InvalidOperationException("FunctionApprovalResponseContent found without a matching FunctionApprovalRequestContent."); + } + + if (!requestCallIds.Contains(response_.FunctionCall.CallId)) + { + Throw.InvalidOperationException($"FunctionApprovalResponseContent found with a FunctionCall.CallId '{response_.FunctionCall.CallId}' that does not match any FunctionApprovalRequestContent."); + } + + requestCallIds.Remove(response_.FunctionCall.CallId); + } + + // Requests/responses that are already executed. + if (content is FunctionApprovalRequestContent request__ && functionResultCallIds.Contains(request__.FunctionCall.CallId) || + content is FunctionApprovalResponseContent response__ && functionResultCallIds.Contains(response__.FunctionCall.CallId)) + { + continue; + } + + // If we get to here, we should have just the contents that we want to keep. + keptContents ??= []; + keptContents.Add(content); + } + + if (message.Contents.Count > 0 && keptContents?.Count != message.Contents.Count) + { + if (keptContents is null || keptContents.Count == 0) + { + // If we have no contents left after filtering, we can remove the message. + messages.RemoveAt(i); + i--; // Adjust index since we removed an item. + continue; + } + + // If we have any contents left after filtering, we can keep the message. + messages[i] = new ChatMessage(message.Role, keptContents) + { + AuthorName = message.AuthorName, + AdditionalProperties = message.AdditionalProperties, + RawRepresentation = message.RawRepresentation, + MessageId = message.MessageId, + }; + } + } + + // If we got an approval for each request, we should have no call ids left. + if (requestCallIds?.Count is > 0) + { + Throw.InvalidOperationException($"FunctionApprovalRequestContent found with FunctionCall.CallId(s) '{string.Join(", ", requestCallIds)}' that have no matching FunctionApprovalResponseContent."); + } + + return (notExecutedApprovedFunctionCalls, notExecutedRejectedFunctionCalls); + } + + /// Replaces any from with . + private static void ReplaceFunctionCallsWithApprovalRequests(IList messages) + { + int count = messages.Count; + for (int i = 0; i < count; i++) + { + ReplaceFunctionCallsWithApprovalRequests(messages[i].Contents); + } + } + + /// Copies any from with . + private static void ReplaceFunctionCallsWithApprovalRequests(IList content) + { + int count = content.Count; + for (int i = 0; i < count; i++) + { + if (content[i] is FunctionCallContent functionCall) + { + content[i] = new FunctionApprovalRequestContent + { + FunctionCall = functionCall, + ApprovalId = functionCall.CallId + }; + } + } + } + + private static TimeSpan GetElapsedTime(long startingTimestamp) => +#if NET + Stopwatch.GetElapsedTime(startingTimestamp); +#else + new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * ((double)TimeSpan.TicksPerSecond / Stopwatch.Frequency))); +#endif + + [LoggerMessage(LogLevel.Debug, "Invoking {MethodName}.", SkipEnabledCheck = true)] + private partial void LogInvoking(string methodName); + + [LoggerMessage(LogLevel.Trace, "Invoking {MethodName}({Arguments}).", SkipEnabledCheck = true)] + private partial void LogInvokingSensitive(string methodName, string arguments); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invocation completed. Duration: {Duration}", SkipEnabledCheck = true)] + private partial void LogInvocationCompleted(string methodName, TimeSpan duration); + + [LoggerMessage(LogLevel.Trace, "{MethodName} invocation completed. Duration: {Duration}. Result: {Result}", SkipEnabledCheck = true)] + private partial void LogInvocationCompletedSensitive(string methodName, TimeSpan duration, string result); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invocation canceled.")] + private partial void LogInvocationCanceled(string methodName); + + [LoggerMessage(LogLevel.Error, "{MethodName} invocation failed.")] + private partial void LogInvocationFailed(string methodName, Exception error); + + /// Provides information about the invocation of a function call. + public sealed class FunctionInvocationResult + { + /// + /// Initializes a new instance of the class. + /// + /// Indicates whether the caller should terminate the processing loop. + /// Indicates the status of the function invocation. + /// Contains information about the function call. + /// The result of the function call. + /// The exception thrown by the function call, if any. + internal FunctionInvocationResult(bool terminate, FunctionInvocationStatus status, FunctionCallContent callContent, object? result, Exception? exception) + { + Terminate = terminate; + Status = status; + CallContent = callContent; + Result = result; + Exception = exception; + } + + /// Gets status about how the function invocation completed. + public FunctionInvocationStatus Status { get; } + + /// Gets the function call content information associated with this invocation. + public FunctionCallContent CallContent { get; } + + /// Gets the result of the function call. + public object? Result { get; } + + /// Gets any exception the function call threw. + public Exception? Exception { get; } + + /// Gets a value indicating whether the caller should terminate the processing loop. + public bool Terminate { get; } + } + + /// Provides error codes for when errors occur as part of the function calling loop. + public enum FunctionInvocationStatus + { + /// The operation completed successfully. + RanToCompletion, + + /// The requested function could not be found. + NotFound, + + /// The function call failed with an exception. + Exception, + } +} \ No newline at end of file diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs index 08173f7496..e2c07e79cb 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs @@ -38,7 +38,7 @@ public override async Task GetResponseAsync(IEnumerable Date: Thu, 7 Aug 2025 19:18:56 +0100 Subject: [PATCH 18/53] Bug fix. --- ...nInvokingChatClientWithBuiltInApprovals.cs | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index ff521532ed..87958c070f 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -1026,17 +1026,21 @@ private static (List? approvals, List? // Remove the call id for each approval response. if (content is FunctionApprovalResponseContent response_) { - if (requestCallIds is null) + // TODO: We cannot check for a matching request here, since the request is not availble for service managed threads, so consider if this still makes snese. + //if (requestCallIds is null) + //{ + // Throw.InvalidOperationException("FunctionApprovalResponseContent found without a matching FunctionApprovalRequestContent."); + //} + + //if (!requestCallIds.Contains(response_.FunctionCall.CallId)) + //{ + // Throw.InvalidOperationException($"FunctionApprovalResponseContent found with a FunctionCall.CallId '{response_.FunctionCall.CallId}' that does not match any FunctionApprovalRequestContent."); + //} + + if (requestCallIds is not null) { - Throw.InvalidOperationException("FunctionApprovalResponseContent found without a matching FunctionApprovalRequestContent."); + requestCallIds.Remove(response_.FunctionCall.CallId); } - - if (!requestCallIds.Contains(response_.FunctionCall.CallId)) - { - Throw.InvalidOperationException($"FunctionApprovalResponseContent found with a FunctionCall.CallId '{response_.FunctionCall.CallId}' that does not match any FunctionApprovalRequestContent."); - } - - requestCallIds.Remove(response_.FunctionCall.CallId); } // Requests/responses that are already executed. From 52935681aab87e47c8895a9f0242f12374065d36 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 8 Aug 2025 18:33:08 +0100 Subject: [PATCH 19/53] Add MCP and ApprovalRequiredAIFunction POC --- dotnet/Directory.Packages.props | 2 + dotnet/agent-framework-dotnet.slnx | 21 ++-- .../GettingStarted/GettingStarted.csproj | 1 + ...ep02_ChatClientAgent_UsingFunctionTools.cs | 35 +++++- .../MEAI/HostedMcpServerTool.cs | 61 ++++++++++ ...dMcpServerToolAlwaysRequireApprovalMode.cs | 14 +++ .../MEAI/HostedMcpServerToolApprovalMode.cs | 40 +++++++ ...dMcpServerToolNeverRequireApprovalMode .cs | 14 +++ ...pServerToolRequireSpecificApprovalMode .cs | 32 +++++ .../MEAI/ApprovalRequiredAIFunction.cs | 17 +++ ...nInvokingChatClientWithBuiltInApprovals.cs | 70 ++++++++--- .../HostedMCPChatClient.cs | 109 ++++++++++++++++++ ....Extensions.AI.ModelContextProtocol.csproj | 31 +++++ 13 files changed, 418 insertions(+), 29 deletions(-) create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerTool.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolAlwaysRequireApprovalMode.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolApprovalMode.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolNeverRequireApprovalMode .cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolRequireSpecificApprovalMode .cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/HostedMCPChatClient.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/Microsoft.Extensions.AI.ModelContextProtocol.csproj diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 19c7cb6f84..0233f83fb2 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -61,6 +61,8 @@ + + diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 7e976a0493..91c5bf7227 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -152,8 +152,20 @@ + + + + + + + + + + + + @@ -166,14 +178,5 @@ - - - - - - - - - diff --git a/dotnet/samples/GettingStarted/GettingStarted.csproj b/dotnet/samples/GettingStarted/GettingStarted.csproj index 3722c8294a..33191e0740 100644 --- a/dotnet/samples/GettingStarted/GettingStarted.csproj +++ b/dotnet/samples/GettingStarted/GettingStarted.csproj @@ -45,6 +45,7 @@ + diff --git a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs index 07ec1f3ef7..c94278d556 100644 --- a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs +++ b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs @@ -3,6 +3,8 @@ using System.ComponentModel; using Microsoft.Extensions.AI; using Microsoft.Extensions.AI.Agents; +using Microsoft.Extensions.AI.ModelContextProtocol; +using Microsoft.Extensions.DependencyInjection; namespace Steps; @@ -131,8 +133,13 @@ public async Task ApprovalsWithTools(ChatClientProviders provider) instructions: "Answer questions about the menu", tools: [ AIFunctionFactory.Create(menuTools.GetMenu), - AIFunctionFactory.Create(menuTools.GetSpecials), - AIFunctionFactory.Create(menuTools.GetItemPrice) + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), + AIFunctionFactory.Create(menuTools.GetItemPrice), + new HostedMcpServerTool("MyService", new Uri("https://mcp-server.example.com")) + { + AllowedTools = ["add"], + ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, + } ]); // Create the server-side agent Id when applicable (depending on the provider). @@ -141,8 +148,25 @@ public async Task ApprovalsWithTools(ChatClientProviders provider) // Get the chat client to use for the agent. using var chatClient = base.GetChatClient(provider, agentOptions); + var chatBuilder = chatClient.AsBuilder(); + if (chatClient.GetService() is null) + { + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + return new HostedMCPChatClient(innerClient); + }); + } + if (chatClient.GetService() is null) + { + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + return new FunctionInvokingChatClientWithBuiltInApprovals(innerClient, null, services); + }); + } + using var chatClientWithMCPAndApprovals = chatBuilder.Build(); + // Define the agent - var agent = new ChatClientAgent(chatClient, agentOptions); + var agent = new ChatClientAgent(chatClientWithMCPAndApprovals, agentOptions); // Create the chat history thread to capture the agent interaction. var thread = agent.GetNewThread(); @@ -150,6 +174,7 @@ public async Task ApprovalsWithTools(ChatClientProviders provider) // Respond to user input, invoking functions where appropriate. await RunAgentAsync("What is the special soup and its price?"); await RunAgentAsync("What is the special drink?"); + await RunAgentAsync("What is 2 + 2?"); async Task RunAgentAsync(string input) { @@ -164,8 +189,8 @@ async Task RunAgentAsync(string input) { List nextIterationMessages = []; - var approvedRequests = userInputRequests.OfType().Where(x => x.FunctionCall.Name == "GetSpecials").ToList(); - var rejectedRequests = userInputRequests.OfType().Where(x => x.FunctionCall.Name != "GetSpecials").ToList(); + var approvedRequests = userInputRequests.OfType().Where(x => x.FunctionCall.Name == "GetSpecials" || x.FunctionCall.Name == "add").ToList(); + var rejectedRequests = userInputRequests.OfType().Where(x => x.FunctionCall.Name != "GetSpecials" && x.FunctionCall.Name != "add").ToList(); approvedRequests.ForEach(x => Console.WriteLine($"Approving the {x.FunctionCall.Name} function call.")); rejectedRequests.ForEach(x => Console.WriteLine($"Rejecting the {x.FunctionCall.Name} function call.")); diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerTool.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerTool.cs new file mode 100644 index 0000000000..f5af2a85e6 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerTool.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a hosted MCP server tool that can be specified to an AI service. +/// +public class HostedMcpServerTool : AITool +{ + /// + /// Initializes a new instance of the class. + /// + /// The name of the remote MCP server. + /// The URL of the remote MCP server. + public HostedMcpServerTool(string serverName, Uri url) + { + ServerName = Throw.IfNullOrWhitespace(serverName); + Url = Throw.IfNull(url); + } + + /// + /// Gets the name of the remote MCP server that is used to identify it. + /// + public string ServerName { get; } + + /// + /// Gets the URL of the remote MCP server. + /// + public Uri Url { get; } + + /// + /// Gets or sets the description of the remote MCP server, used to provide more context to the AI service. + /// + public string? ServerDescription { get; set; } + + /// + /// Gets or sets the list of tools allowed to be used by the AI service. + /// + public IList? AllowedTools { get; set; } + + /// + /// Gets or sets the approval mode that indicates when the AI service should require user approval for tool calls to the remote MCP server. + /// + /// + /// You can set this property to to require approval for all tool calls, + /// or to to never require approval. + /// + public HostedMcpServerToolApprovalMode? ApprovalMode { get; set; } + + /// + /// Gets or sets the HTTP headers that the AI service should use when making tool calls to the remote MCP server. + /// + /// + /// This property is useful for specifying the authentication header or other headers required by the MCP server. + /// + public IDictionary? Headers { get; set; } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolAlwaysRequireApprovalMode.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolAlwaysRequireApprovalMode.cs new file mode 100644 index 0000000000..9a4103c882 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolAlwaysRequireApprovalMode.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Indicates that approval is always required for tool calls to a hosted MCP server. +/// +/// +/// Use to get an instance of . +/// +[DebuggerDisplay(nameof(AlwaysRequire))] +public sealed class HostedMcpServerToolAlwaysRequireApprovalMode : HostedMcpServerToolApprovalMode; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolApprovalMode.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolApprovalMode.cs new file mode 100644 index 0000000000..b9cd2624ca --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolApprovalMode.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; + +namespace Microsoft.Extensions.AI; + +/// +/// Describes how approval is required for tool calls to a hosted MCP server. +/// +/// +/// The predefined values , and are provided to specify handling for all tools. +/// To specify approval behavior for individual tool names, use . +/// +#pragma warning disable CA1052 // Static holder types should be Static or NotInheritable +public class HostedMcpServerToolApprovalMode +#pragma warning restore CA1052 +{ + /// + /// Gets a predefined indicating that all tool calls to a hosted MCP server always require approval. + /// + public static HostedMcpServerToolAlwaysRequireApprovalMode AlwaysRequire { get; } = new(); + + /// + /// Gets a predefined indicating that all tool calls to a hosted MCP server never require approval. + /// + public static HostedMcpServerToolNeverRequireApprovalMode NeverRequire { get; } = new(); + + private protected HostedMcpServerToolApprovalMode() + { + } + + /// + /// Instantiates a that specifies approval behavior for individual tool names. + /// + /// The list of tools names that always require approval. + /// The list of tools names that never require approval. + /// An instance of for the specified tool names. + public static HostedMcpServerToolRequireSpecificApprovalMode RequireSpecific(IList? alwaysRequireApprovalToolNames, IList? neverRequireApprovalToolNames) + => new(alwaysRequireApprovalToolNames, neverRequireApprovalToolNames); +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolNeverRequireApprovalMode .cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolNeverRequireApprovalMode .cs new file mode 100644 index 0000000000..4eed447217 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolNeverRequireApprovalMode .cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Indicates that approval is never required for tool calls to a hosted MCP server. +/// +/// +/// Use to get an instance of . +/// +[DebuggerDisplay(nameof(NeverRequire))] +public sealed class HostedMcpServerToolNeverRequireApprovalMode : HostedMcpServerToolApprovalMode; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolRequireSpecificApprovalMode .cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolRequireSpecificApprovalMode .cs new file mode 100644 index 0000000000..0979449887 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI/HostedMcpServerToolRequireSpecificApprovalMode .cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents a mode where approval behavior is specified for individual tool names. +/// +public sealed class HostedMcpServerToolRequireSpecificApprovalMode : HostedMcpServerToolApprovalMode +{ + /// + /// Initializes a new instance of the class that specifies approval behavior for individual tool names. + /// + /// The list of tools names that always require approval. + /// The list of tools names that never require approval. + public HostedMcpServerToolRequireSpecificApprovalMode(IList? alwaysRequireApprovalToolNames, IList? neverRequireApprovalToolNames) + { + AlwaysRequireApprovalToolNames = alwaysRequireApprovalToolNames; + NeverRequireApprovalToolNames = neverRequireApprovalToolNames; + } + + /// + /// Gets or sets the list of tool names that always require approval. + /// + public IList? AlwaysRequireApprovalToolNames { get; set; } + + /// + /// Gets or sets the list of tool names that never require approval. + /// + public IList? NeverRequireApprovalToolNames { get; set; } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs new file mode 100644 index 0000000000..e7b4d2bec8 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Extensions.AI; + +/// +/// Marks an existing with additional metadata to indicate that it requires approval. +/// +/// The that requires approval. +public sealed class ApprovalRequiredAIFunction(AIFunction function) : DelegatingAIFunction(function) +{ + /// + /// An optional callback that can be used to determine if the function call requires approval, instead of the default behavior, which is to always require approval. + /// + public Func RequiresApprovalCallback { get; set; } = delegate { return true; }; +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 87958c070f..f9fa994659 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -255,6 +255,10 @@ public override async Task GetResponseAsync( // ** Approvals additions on top of FICC - start **// + List? augmentedPreInvocationHistory = null; + List allTools = [.. options?.Tools ?? [], .. AdditionalTools ?? []]; + Dictionary approvalRequiredFunctionMap = allTools.OfType().ToDictionary(x => x.Name) ?? new(); + // Remove any approval requests and approval request/response pairs that have already been executed. var notExecutedResponses = ProcessApprovalRequestsAndResponses(originalMessages); @@ -271,7 +275,8 @@ public override async Task GetResponseAsync( rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult])); } - originalMessages.AddRange(rejectedFunctionCallMessages); + augmentedPreInvocationHistory ??= []; + augmentedPreInvocationHistory.AddRange(rejectedFunctionCallMessages); } var rejectedFunctionCalls = notExecutedResponses.rejections; @@ -280,18 +285,23 @@ public override async Task GetResponseAsync( // Check if there are any function calls to do from any approved functions and execute them. if (notExecutedResponses.approvals is { Count: > 0 }) { - originalMessages.AddRange(notExecutedResponses.approvals.Select(x => new ChatMessage(ChatRole.Assistant, [x]))); + var approvalFunctionCalls = notExecutedResponses.approvals.Select(x => new ChatMessage(ChatRole.Assistant, [x])).ToList(); + originalMessages.AddRange(approvalFunctionCalls); + augmentedPreInvocationHistory ??= []; + augmentedPreInvocationHistory.AddRange(approvalFunctionCalls); // Add the responses from the function calls into the augmented history and also into the tracked // list of response messages. var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); - responseMessages = [.. modeAndMessages.MessagesAdded, ..rejectedFunctionCallMessages]; + responseMessages = [.. modeAndMessages.MessagesAdded, .. rejectedFunctionCallMessages]; consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; if (modeAndMessages.ShouldTerminate) { return new ChatResponse(responseMessages); } + + augmentedPreInvocationHistory.AddRange(modeAndMessages.MessagesAdded); } // ** Approvals additions on top of FICC - end **// @@ -311,7 +321,7 @@ public override async Task GetResponseAsync( // Before we do any function execution, make sure that any functions that require approval, have been turned into approval requests // so that they don't get executed here. - ReplaceFunctionCallsWithApprovalRequests(response.Messages); + ReplaceFunctionCallsWithApprovalRequests(response.Messages, approvalRequiredFunctionMap); // ** Approvals additions on top of FICC - end **// @@ -321,6 +331,23 @@ public override async Task GetResponseAsync( iteration < MaximumIterationsPerRequest && CopyFunctionCalls(response.Messages, ref functionCallContents); + // ** Approvals additions on top of FICC - start **// + + // TODO: Ensure that this works correctly in all cases, and doesn't have any sideaffects. + // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, + // so they are processed as normal. + if (augmentedPreInvocationHistory?.Count > 0) + { + for (int i = augmentedPreInvocationHistory.Count - 1; i >= 0; i--) + { + response.Messages.Insert(0, augmentedPreInvocationHistory[i]); + } + + augmentedPreInvocationHistory = null; + } + + // ** Approvals additions on top of FICC - end **// + // In a common case where we make a request and there's no function calling work required, // fast path out by just returning the original response. if (iteration == 0 && !requiresFunctionInvocation) @@ -1086,24 +1113,37 @@ private static (List? approvals, List? } /// Replaces any from with . - private static void ReplaceFunctionCallsWithApprovalRequests(IList messages) + private static void ReplaceFunctionCallsWithApprovalRequests(IList messages, Dictionary approvalRequiredAIFunctionMap) { + bool anyApprovalRequired = false; + List<(int, int)>? functionsToReplace = null; + int count = messages.Count; for (int i = 0; i < count; i++) { - ReplaceFunctionCallsWithApprovalRequests(messages[i].Contents); + var content = messages[i].Contents; + int contentCount = content.Count; + + for (int j = 0; j < contentCount; j++) + { + if (content[j] is FunctionCallContent functionCall) + { + functionsToReplace ??= []; + functionsToReplace.Add((i, j)); + + anyApprovalRequired |= approvalRequiredAIFunctionMap.TryGetValue(functionCall.Name, out var approvalFunction) && approvalFunction.RequiresApprovalCallback(functionCall); + } + } } - } - /// Copies any from with . - private static void ReplaceFunctionCallsWithApprovalRequests(IList content) - { - int count = content.Count; - for (int i = 0; i < count; i++) + // If any function calls were found, and any of them required approval, we should replace all of them with approval requests. + // This is because we do not have a way to deal with cases where some function calls require approval and others do not, so we just replace all of them. + if (functionsToReplace is not null && anyApprovalRequired) { - if (content[i] is FunctionCallContent functionCall) + foreach (var (messageIndex, contentIndex) in functionsToReplace) { - content[i] = new FunctionApprovalRequestContent + var functionCall = (FunctionCallContent)messages[messageIndex].Contents[contentIndex]; + messages[messageIndex].Contents[contentIndex] = new FunctionApprovalRequestContent { FunctionCall = functionCall, ApprovalId = functionCall.CallId @@ -1185,4 +1225,4 @@ public enum FunctionInvocationStatus /// The function call failed with an exception. Exception, } -} \ No newline at end of file +} diff --git a/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/HostedMCPChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/HostedMCPChatClient.cs new file mode 100644 index 0000000000..27815973b1 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/HostedMCPChatClient.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using ModelContextProtocol.Client; + +namespace Microsoft.Extensions.AI.ModelContextProtocol; + +/// +/// Adds support for enabling MCP function invocation. +/// +public class HostedMCPChatClient : DelegatingChatClient +{ + /// The logger to use for logging information about function invocation. + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying , or the next instance in a chain of clients. + /// An to use for logging information about function invocation. + public HostedMCPChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null) + : base(innerClient) + { + this._logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + } + + /// + public override async Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + if (options?.Tools is not { Count: > 0 }) + { + // If there are no tools, just call the inner client. + return await base.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); + } + + List downstreamTools = []; + foreach (var tool in options.Tools ?? []) + { + if (tool is HostedMcpServerTool mcpTool) + { + // List all MCP functions from the specified MCP server. + // This will need some caching in a real-world scenario to avoid repeated calls. + var mcpClient = await CreateMcpClientAsync(mcpTool.Url).ConfigureAwait(false); + var mcpFunctions = await mcpClient.ListToolsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + + // Add the listed functions to our list of tools we'll pass to the inner client. + foreach (var mcpFunction in mcpFunctions) + { + if (mcpTool.AllowedTools is not null && !mcpTool.AllowedTools.Contains(mcpFunction.Name)) + { + this._logger.LogInformation("MCP function '{FunctionName}' is not allowed by the tool configuration.", mcpFunction.Name); + continue; + } + + switch (mcpTool.ApprovalMode) + { + case HostedMcpServerToolAlwaysRequireApprovalMode alwaysRequireApproval: + downstreamTools.Add(new ApprovalRequiredAIFunction(mcpFunction)); + break; + case HostedMcpServerToolNeverRequireApprovalMode neverRequireApproval: + downstreamTools.Add(mcpFunction); + break; + case HostedMcpServerToolRequireSpecificApprovalMode specificApprovalMode when specificApprovalMode.AlwaysRequireApprovalToolNames?.Contains(mcpFunction.Name) is true: + downstreamTools.Add(new ApprovalRequiredAIFunction(mcpFunction)); + break; + case HostedMcpServerToolRequireSpecificApprovalMode specificApprovalMode when specificApprovalMode.NeverRequireApprovalToolNames?.Contains(mcpFunction.Name) is true: + downstreamTools.Add(mcpFunction); + break; + default: + // Default to always require approval if no specific mode is set. + downstreamTools.Add(new ApprovalRequiredAIFunction(mcpFunction)); + break; + } + } + + // Skip adding the MCP tool itself, as we only want to add the functions it provides. + continue; + } + + // For other tools, we want to keep them in the list of tools. + downstreamTools.Add(tool); + } + + options = options.Clone(); + options.Tools = downstreamTools; + + // Make the call to the inner client. + return await base.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); + } + + private static async Task CreateMcpClientAsync(Uri mcpService) + { + // Create mock MCP client for demonstration purposes. + var clientTransport = new StdioClientTransport(new StdioClientTransportOptions + { + Name = "Everything", + Command = "npx", + Arguments = ["-y", "@modelcontextprotocol/server-everything"], + }); + + return await McpClientFactory.CreateAsync(clientTransport).ConfigureAwait(false); + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/Microsoft.Extensions.AI.ModelContextProtocol.csproj b/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/Microsoft.Extensions.AI.ModelContextProtocol.csproj new file mode 100644 index 0000000000..1b66862dda --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/Microsoft.Extensions.AI.ModelContextProtocol.csproj @@ -0,0 +1,31 @@ + + + + $(ProjectsTargetFrameworks) + $(ProjectsDebugTargetFrameworks) + alpha + + + + true + true + true + + + + + + + Microsoft Extensions AI Model Context Protocol extensions + Contains ChatClient that helps call MCP tooling automatically as part of a ChatClient stack. + + + + + + + + + + + From c5bedd0ff354a944500ae35ce9851e94e08bf8b3 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 11 Aug 2025 13:57:09 +0100 Subject: [PATCH 20/53] Bug fixes --- ...nInvokingChatClientWithBuiltInApprovals.cs | 77 ++++++++++--------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index f9fa994659..458354db99 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -277,23 +277,21 @@ public override async Task GetResponseAsync( augmentedPreInvocationHistory ??= []; augmentedPreInvocationHistory.AddRange(rejectedFunctionCallMessages); + originalMessages.AddRange(rejectedFunctionCallMessages); } - var rejectedFunctionCalls = notExecutedResponses.rejections; - var rejectedFunctionResults = rejectedFunctionCalls?.Select(x => new FunctionResultContent(x.CallId, "Error: Function invocation approval was not granted.")); - // Check if there are any function calls to do from any approved functions and execute them. if (notExecutedResponses.approvals is { Count: > 0 }) { - var approvalFunctionCalls = notExecutedResponses.approvals.Select(x => new ChatMessage(ChatRole.Assistant, [x])).ToList(); - originalMessages.AddRange(approvalFunctionCalls); + var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => new ChatMessage(ChatRole.Assistant, [x])).ToList(); + originalMessages.AddRange(approvedFunctionCalls); augmentedPreInvocationHistory ??= []; - augmentedPreInvocationHistory.AddRange(approvalFunctionCalls); + augmentedPreInvocationHistory.AddRange(approvedFunctionCalls); // Add the responses from the function calls into the augmented history and also into the tracked // list of response messages. var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); - responseMessages = [.. modeAndMessages.MessagesAdded, .. rejectedFunctionCallMessages]; + responseMessages = [.. rejectedFunctionCallMessages, .. approvedFunctionCalls, .. modeAndMessages.MessagesAdded]; consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; if (modeAndMessages.ShouldTerminate) @@ -331,27 +329,27 @@ public override async Task GetResponseAsync( iteration < MaximumIterationsPerRequest && CopyFunctionCalls(response.Messages, ref functionCallContents); - // ** Approvals additions on top of FICC - start **// - - // TODO: Ensure that this works correctly in all cases, and doesn't have any sideaffects. - // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, - // so they are processed as normal. - if (augmentedPreInvocationHistory?.Count > 0) + // In a common case where we make a request and there's no function calling work required, + // fast path out by just returning the original response. + if (iteration == 0 && !requiresFunctionInvocation) { - for (int i = augmentedPreInvocationHistory.Count - 1; i >= 0; i--) + // ** Approvals additions on top of FICC - start **// + + // TODO: Ensure that this works correctly in all cases, and doesn't have any sideaffects. + // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, + // so they are processed as normal. + if (augmentedPreInvocationHistory?.Count > 0) { - response.Messages.Insert(0, augmentedPreInvocationHistory[i]); - } + for (int i = augmentedPreInvocationHistory.Count - 1; i >= 0; i--) + { + response.Messages.Insert(0, augmentedPreInvocationHistory[i]); + } - augmentedPreInvocationHistory = null; - } + augmentedPreInvocationHistory = null; + } - // ** Approvals additions on top of FICC - end **// + // ** Approvals additions on top of FICC - end **// - // In a common case where we make a request and there's no function calling work required, - // fast path out by just returning the original response. - if (iteration == 0 && !requiresFunctionInvocation) - { return response; } @@ -1028,21 +1026,6 @@ private static (List? approvals, List? { var content = message.Contents[j]; - // Save response that are not yet executed, so that we can execute them later. - if (content is FunctionApprovalResponseContent response && !functionResultCallIds.Contains(response.FunctionCall.CallId)) - { - if (response.Approved) - { - notExecutedApprovedFunctionCalls ??= []; - notExecutedApprovedFunctionCalls.Add(response.FunctionCall); - } - else - { - notExecutedRejectedFunctionCalls ??= []; - notExecutedRejectedFunctionCalls.Add(response.FunctionCall); - } - } - // Capture each call id for each approval request. if (content is FunctionApprovalRequestContent request_) { @@ -1077,6 +1060,24 @@ private static (List? approvals, List? continue; } + // Build a list of response that are not yet executed, so that we can execute them before we invoke the LLM. + // We can also remove them from the list of messages, since they will be turned back into FunctionCallContent and FunctionResultContent. + if (content is FunctionApprovalResponseContent response && !functionResultCallIds.Contains(response.FunctionCall.CallId)) + { + if (response.Approved) + { + notExecutedApprovedFunctionCalls ??= []; + notExecutedApprovedFunctionCalls.Add(response.FunctionCall); + } + else + { + notExecutedRejectedFunctionCalls ??= []; + notExecutedRejectedFunctionCalls.Add(response.FunctionCall); + } + + continue; + } + // If we get to here, we should have just the contents that we want to keep. keptContents ??= []; keptContents.Add(content); From edcad347bb79266a52988d90b1c0bd717d142ae7 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 11 Aug 2025 15:00:55 +0100 Subject: [PATCH 21/53] Address PR Feedback. --- .../FunctionApprovalRequestContent.cs | 35 +++++++++---------- .../FunctionApprovalResponseContent.cs | 16 ++++----- .../MEAI/ApprovalRequiredAIFunction.cs | 3 +- ...nInvokingChatClientWithBuiltInApprovals.cs | 12 +++---- .../PostFICCApprovalGeneratingChatClient.cs | 6 +--- .../PreFICCApprovalGeneratingChatClient.cs | 6 +--- 6 files changed, 31 insertions(+), 47 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs index 0b86acf32b..d98c3d6d8c 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using Microsoft.Shared.Diagnostics; + namespace Microsoft.Extensions.AI; /// @@ -7,10 +9,21 @@ namespace Microsoft.Extensions.AI; /// public class FunctionApprovalRequestContent : UserInputRequestContent { + /// + /// Initializes a new instance of the class. + /// + /// The ID to uniquely identify the user input request/response pair. + /// The function call that requires user approval. + public FunctionApprovalRequestContent(string approvalId, FunctionCallContent functionCall) + { + this.ApprovalId = Throw.IfNullOrWhitespace(approvalId); + this.FunctionCall = Throw.IfNull(functionCall); + } + /// /// Gets or sets the function call that pre-invoke approval is required for. /// - public FunctionCallContent FunctionCall { get; set; } = default!; + public FunctionCallContent FunctionCall { get; } /// /// Creates a representing an approval response. @@ -18,15 +31,7 @@ public class FunctionApprovalRequestContent : UserInputRequestContent /// The representing the approval response. public ChatMessage Approve() { - return new ChatMessage(ChatRole.User, - [ - new FunctionApprovalResponseContent - { - ApprovalId = this.ApprovalId, - Approved = true, - FunctionCall = this.FunctionCall - } - ]); + return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.ApprovalId, true, this.FunctionCall)]); } /// @@ -35,14 +40,6 @@ public ChatMessage Approve() /// The representing the rejection response. public ChatMessage Reject() { - return new ChatMessage(ChatRole.User, - [ - new FunctionApprovalResponseContent - { - ApprovalId = this.ApprovalId, - Approved = false, - FunctionCall = this.FunctionCall - } - ]); + return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.ApprovalId, false, this.FunctionCall)]); } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs index 41ba57a978..0f7d5d183c 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using Microsoft.Shared.Diagnostics; + namespace Microsoft.Extensions.AI; /// @@ -10,17 +12,13 @@ public class FunctionApprovalResponseContent : UserInputResponseContent /// /// Initializes a new instance of the class. /// - public FunctionApprovalResponseContent() - { - } - - /// - /// Initializes a new instance of the class with the specified approval status. - /// + /// The ID to uniquely identify the user input request/response pair. /// Indicates whether the request was approved. - public FunctionApprovalResponseContent(bool approved) + /// The function call that requires user approval. + public FunctionApprovalResponseContent(string approvalId, bool approved, FunctionCallContent functionCall) { this.Approved = approved; + this.FunctionCall = Throw.IfNull(functionCall); } /// @@ -31,5 +29,5 @@ public FunctionApprovalResponseContent(bool approved) /// /// Gets or sets the function call that pre-invoke approval is required for. /// - public FunctionCallContent FunctionCall { get; set; } = default!; + public FunctionCallContent FunctionCall { get; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs index e7b4d2bec8..f84c7b1ec8 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Threading.Tasks; namespace Microsoft.Extensions.AI; @@ -13,5 +14,5 @@ public sealed class ApprovalRequiredAIFunction(AIFunction function) : Delegating /// /// An optional callback that can be used to determine if the function call requires approval, instead of the default behavior, which is to always require approval. /// - public Func RequiresApprovalCallback { get; set; } = delegate { return true; }; + public Func> RequiresApprovalCallback { get; set; } = delegate { return new ValueTask(true); }; } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 458354db99..7b0c4fb670 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -319,7 +319,7 @@ public override async Task GetResponseAsync( // Before we do any function execution, make sure that any functions that require approval, have been turned into approval requests // so that they don't get executed here. - ReplaceFunctionCallsWithApprovalRequests(response.Messages, approvalRequiredFunctionMap); + await ReplaceFunctionCallsWithApprovalRequests(response.Messages, approvalRequiredFunctionMap); // ** Approvals additions on top of FICC - end **// @@ -1114,7 +1114,7 @@ private static (List? approvals, List? } /// Replaces any from with . - private static void ReplaceFunctionCallsWithApprovalRequests(IList messages, Dictionary approvalRequiredAIFunctionMap) + private static async Task ReplaceFunctionCallsWithApprovalRequests(IList messages, Dictionary approvalRequiredAIFunctionMap) { bool anyApprovalRequired = false; List<(int, int)>? functionsToReplace = null; @@ -1132,7 +1132,7 @@ private static void ReplaceFunctionCallsWithApprovalRequests(IList functionsToReplace ??= []; functionsToReplace.Add((i, j)); - anyApprovalRequired |= approvalRequiredAIFunctionMap.TryGetValue(functionCall.Name, out var approvalFunction) && approvalFunction.RequiresApprovalCallback(functionCall); + anyApprovalRequired |= approvalRequiredAIFunctionMap.TryGetValue(functionCall.Name, out var approvalFunction) && await approvalFunction.RequiresApprovalCallback(functionCall); } } } @@ -1144,11 +1144,7 @@ private static void ReplaceFunctionCallsWithApprovalRequests(IList foreach (var (messageIndex, contentIndex) in functionsToReplace) { var functionCall = (FunctionCallContent)messages[messageIndex].Contents[contentIndex]; - messages[messageIndex].Contents[contentIndex] = new FunctionApprovalRequestContent - { - FunctionCall = functionCall, - ApprovalId = functionCall.CallId - }; + messages[messageIndex].Contents[contentIndex] = new FunctionApprovalRequestContent(functionCall.CallId, functionCall); } } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs index c0cc744077..f7e9dc428c 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs @@ -202,11 +202,7 @@ private static void ReplaceFunctionCallsWithApprovalRequests(IList co { if (content[i] is FunctionCallContent functionCall) { - content[i] = new FunctionApprovalRequestContent - { - FunctionCall = functionCall, - ApprovalId = functionCall.CallId - }; + content[i] = new FunctionApprovalRequestContent(functionCall.CallId, functionCall); } } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs index e2c07e79cb..788e3c8739 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs @@ -181,11 +181,7 @@ private static void ReplaceFunctionCallsWithApprovalRequests(IList co { if (content[i] is FunctionCallContent functionCall) { - content[i] = new FunctionApprovalRequestContent - { - FunctionCall = functionCall, - ApprovalId = functionCall.CallId - }; + content[i] = new FunctionApprovalRequestContent(functionCall.CallId, functionCall); } } } From 0a61294215fd7451e3efa014cb37eae49849f31d Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 11 Aug 2025 16:57:52 +0100 Subject: [PATCH 22/53] Add AIFunctionApprovalContext with some small efficiency improvements. --- .../MEAI/AIFunctionApprovalContext.cs | 26 +++++++++++++++++++ .../MEAI/ApprovalRequiredAIFunction.cs | 2 +- ...nInvokingChatClientWithBuiltInApprovals.cs | 15 ++++++----- 3 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/AIFunctionApprovalContext.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/AIFunctionApprovalContext.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/AIFunctionApprovalContext.cs new file mode 100644 index 0000000000..e735c1aa55 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/AIFunctionApprovalContext.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Extensions.AI; + +/// +/// Context object that provides information about the function call that requires approval. +/// +public class AIFunctionApprovalContext +{ + /// + /// Initializes a new instance of the class. + /// + /// The containing the details of the invocation. + /// + public AIFunctionApprovalContext(FunctionCallContent functionCall) + { + this.FunctionCall = functionCall ?? throw new ArgumentNullException(nameof(functionCall)); + } + + /// + /// Gets the containing the details of the invocation that will be made if approval is granted. + /// + public FunctionCallContent FunctionCall { get; } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs index f84c7b1ec8..8dfa63d7ae 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs @@ -14,5 +14,5 @@ public sealed class ApprovalRequiredAIFunction(AIFunction function) : Delegating /// /// An optional callback that can be used to determine if the function call requires approval, instead of the default behavior, which is to always require approval. /// - public Func> RequiresApprovalCallback { get; set; } = delegate { return new ValueTask(true); }; + public Func> RequiresApprovalCallback { get; set; } = delegate { return new ValueTask(true); }; } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 7b0c4fb670..2b30da9933 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -256,16 +256,19 @@ public override async Task GetResponseAsync( // ** Approvals additions on top of FICC - start **// List? augmentedPreInvocationHistory = null; - List allTools = [.. options?.Tools ?? [], .. AdditionalTools ?? []]; - Dictionary approvalRequiredFunctionMap = allTools.OfType().ToDictionary(x => x.Name) ?? new(); + Dictionary approvalRequiredFunctionMap = + (options?.Tools ?? []).Concat(AdditionalTools ?? []) + .OfType() + .ToDictionary(x => x.Name); // Remove any approval requests and approval request/response pairs that have already been executed. var notExecutedResponses = ProcessApprovalRequestsAndResponses(originalMessages); // Generate failed function result contents for any rejected requests. - List rejectedFunctionCallMessages = []; + List? rejectedFunctionCallMessages = null; if (notExecutedResponses.rejections is { Count: > 0 }) { + rejectedFunctionCallMessages = []; foreach (var rejectedCall in notExecutedResponses.rejections) { // Create a FunctionResultContent for the rejected function call. @@ -291,7 +294,7 @@ public override async Task GetResponseAsync( // Add the responses from the function calls into the augmented history and also into the tracked // list of response messages. var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); - responseMessages = [.. rejectedFunctionCallMessages, .. approvedFunctionCalls, .. modeAndMessages.MessagesAdded]; + responseMessages = [.. rejectedFunctionCallMessages ?? [], .. approvedFunctionCalls, .. modeAndMessages.MessagesAdded]; consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; if (modeAndMessages.ShouldTerminate) @@ -1036,7 +1039,7 @@ private static (List? approvals, List? // Remove the call id for each approval response. if (content is FunctionApprovalResponseContent response_) { - // TODO: We cannot check for a matching request here, since the request is not availble for service managed threads, so consider if this still makes snese. + // TODO: We cannot check for a matching request here, since the request is not availble for service managed threads, so consider if this still makes sense. //if (requestCallIds is null) //{ // Throw.InvalidOperationException("FunctionApprovalResponseContent found without a matching FunctionApprovalRequestContent."); @@ -1132,7 +1135,7 @@ private static async Task ReplaceFunctionCallsWithApprovalRequests(IList Date: Mon, 11 Aug 2025 17:04:47 +0100 Subject: [PATCH 23/53] Remove unecessary helpers that are only used in one place. --- .../AIContentExtensions.cs | 6 ------ .../AgentRunResponse.cs | 4 +++- .../AgentRunResponseUpdate.cs | 3 ++- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIContentExtensions.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIContentExtensions.cs index 73e416e764..22e0193e8b 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIContentExtensions.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AIContentExtensions.cs @@ -115,10 +115,4 @@ public static string ConcatText(this IList messages) #endif } } - - public static IEnumerable EnumerateUserInputRequests(this IList messages) - => messages.SelectMany(x => x.Contents).OfType(); - - public static IEnumerable EnumerateUserInputRequests(this IList contents) - => contents.OfType(); } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs index 240a1599c3..822cc188d1 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs @@ -6,6 +6,8 @@ #endif using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; + #if NET9_0_OR_GREATER using System.Text; #endif @@ -87,7 +89,7 @@ public IList Messages /// This property concatenates all instances in the response. /// [JsonIgnore] - public IEnumerable UserInputRequests => this._messages?.EnumerateUserInputRequests() ?? Array.Empty(); + public IEnumerable UserInputRequests => this._messages?.SelectMany(x => x.Contents).OfType() ?? Array.Empty(); /// Gets or sets the ID of the agent that produced the response. public string? AgentId { get; set; } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs index bd23c69eeb..fecb4194b3 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Text.Json.Serialization; using Microsoft.Shared.Diagnostics; @@ -97,7 +98,7 @@ public string? AuthorName /// This property concatenates all instances in the response. /// [JsonIgnore] - public IEnumerable UserInputRequests => this._contents?.EnumerateUserInputRequests() ?? Array.Empty(); + public IEnumerable UserInputRequests => this._contents?.OfType() ?? Array.Empty(); /// Gets or sets the agent run response update content items. [AllowNull] From 0d5fcbf3b6541c1c4866c4c873e5756c9d37d688 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 11 Aug 2025 17:32:27 +0100 Subject: [PATCH 24/53] Replace dictionary with array and lazy initialize. --- ...tionInvokingChatClientWithBuiltInApprovals.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 2b30da9933..3a7b50d2e9 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -256,10 +256,6 @@ public override async Task GetResponseAsync( // ** Approvals additions on top of FICC - start **// List? augmentedPreInvocationHistory = null; - Dictionary approvalRequiredFunctionMap = - (options?.Tools ?? []).Concat(AdditionalTools ?? []) - .OfType() - .ToDictionary(x => x.Name); // Remove any approval requests and approval request/response pairs that have already been executed. var notExecutedResponses = ProcessApprovalRequestsAndResponses(originalMessages); @@ -322,7 +318,7 @@ public override async Task GetResponseAsync( // Before we do any function execution, make sure that any functions that require approval, have been turned into approval requests // so that they don't get executed here. - await ReplaceFunctionCallsWithApprovalRequests(response.Messages, approvalRequiredFunctionMap); + await ReplaceFunctionCallsWithApprovalRequests(response.Messages, options?.Tools, AdditionalTools); // ** Approvals additions on top of FICC - end **// @@ -1117,11 +1113,13 @@ private static (List? approvals, List? } /// Replaces any from with . - private static async Task ReplaceFunctionCallsWithApprovalRequests(IList messages, Dictionary approvalRequiredAIFunctionMap) + private static async Task ReplaceFunctionCallsWithApprovalRequests(IList messages, IList? requestOptionsTools, IList? additionalTools) { bool anyApprovalRequired = false; List<(int, int)>? functionsToReplace = null; + ApprovalRequiredAIFunction[]? approvalRequiredFunctions = null; + int count = messages.Count; for (int i = 0; i < count; i++) { @@ -1135,7 +1133,11 @@ private static async Task ReplaceFunctionCallsWithApprovalRequests(IList() + .ToArray(); + + anyApprovalRequired |= approvalRequiredFunctions.FirstOrDefault(x => x.Name == functionCall.Name) is { } approvalFunction && await approvalFunction.RequiresApprovalCallback(new(functionCall)); } } } From 8beb0f6a554321f4c7847d44ed44737e0750dc2c Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 11 Aug 2025 18:46:11 +0100 Subject: [PATCH 25/53] Improve sample code --- .../Step02_ChatClientAgent_UsingFunctionTools.cs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs index c94278d556..fbe19f1feb 100644 --- a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs +++ b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs @@ -187,16 +187,14 @@ async Task RunAgentAsync(string input) // Loop until all user input requests are handled. while (userInputRequests.Count > 0) { - List nextIterationMessages = []; - - var approvedRequests = userInputRequests.OfType().Where(x => x.FunctionCall.Name == "GetSpecials" || x.FunctionCall.Name == "add").ToList(); - var rejectedRequests = userInputRequests.OfType().Where(x => x.FunctionCall.Name != "GetSpecials" && x.FunctionCall.Name != "add").ToList(); - - approvedRequests.ForEach(x => Console.WriteLine($"Approving the {x.FunctionCall.Name} function call.")); - rejectedRequests.ForEach(x => Console.WriteLine($"Rejecting the {x.FunctionCall.Name} function call.")); + List nextIterationMessages = userInputRequests?.Select((request) => request switch + { + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(), + FunctionApprovalRequestContent functionApprovalRequest => functionApprovalRequest.Reject(), + _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") + })?.ToList() ?? []; - nextIterationMessages.AddRange(approvedRequests.Select(x => x.Approve())); - nextIterationMessages.AddRange(rejectedRequests.Select(x => x.Reject())); + nextIterationMessages.ForEach(x => Console.WriteLine($"Approval for the {(x.Contents[0] as FunctionApprovalResponseContent)?.FunctionCall.Name} function call is set to {(x.Contents[0] as FunctionApprovalResponseContent)?.Approved}.")); response = await agent.RunAsync(nextIterationMessages, thread); this.WriteResponseOutput(response); From 9fbd0a7561b89cbb4cf50d5714cf0966c487b05a Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:05:24 +0100 Subject: [PATCH 26/53] Make user input id required and rename it to be more general. --- .../FunctionApprovalRequestContent.cs | 6 +++--- .../FunctionApprovalResponseContent.cs | 1 + .../MEAI.Contents/UserInputRequestContent.cs | 15 +++++++++++++-- .../MEAI.Contents/UserInputResponseContent.cs | 15 +++++++++++++-- .../MEAI/PreFICCApprovalGeneratingChatClient.cs | 4 ++-- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs index d98c3d6d8c..6391efcee3 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs @@ -15,8 +15,8 @@ public class FunctionApprovalRequestContent : UserInputRequestContent /// The ID to uniquely identify the user input request/response pair. /// The function call that requires user approval. public FunctionApprovalRequestContent(string approvalId, FunctionCallContent functionCall) + : base(approvalId) { - this.ApprovalId = Throw.IfNullOrWhitespace(approvalId); this.FunctionCall = Throw.IfNull(functionCall); } @@ -31,7 +31,7 @@ public FunctionApprovalRequestContent(string approvalId, FunctionCallContent fun /// The representing the approval response. public ChatMessage Approve() { - return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.ApprovalId, true, this.FunctionCall)]); + return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall)]); } /// @@ -40,6 +40,6 @@ public ChatMessage Approve() /// The representing the rejection response. public ChatMessage Reject() { - return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.ApprovalId, false, this.FunctionCall)]); + return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall)]); } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs index 0f7d5d183c..3fcce35b71 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs @@ -16,6 +16,7 @@ public class FunctionApprovalResponseContent : UserInputResponseContent /// Indicates whether the request was approved. /// The function call that requires user approval. public FunctionApprovalResponseContent(string approvalId, bool approved, FunctionCallContent functionCall) + : base(approvalId) { this.Approved = approved; this.FunctionCall = Throw.IfNull(functionCall); diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs index 29e163bef0..ba327b6717 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using Microsoft.Shared.Diagnostics; + namespace Microsoft.Extensions.AI; /// @@ -8,7 +10,16 @@ namespace Microsoft.Extensions.AI; public abstract class UserInputRequestContent : AIContent { /// - /// Gets or sets the ID to uniquely identify the user input request/response pair. + /// Initializes a new instance of the class. + /// + /// The ID to uniquely identify the user input request/response pair. + protected UserInputRequestContent(string id) + { + Id = Throw.IfNullOrWhitespace(id); + } + + /// + /// Gets the ID to uniquely identify the user input request/response pair. /// - public string ApprovalId { get; set; } = default!; + public string Id { get; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs index 140aa8a895..9c15927b7d 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using Microsoft.Shared.Diagnostics; + namespace Microsoft.Extensions.AI; /// @@ -8,7 +10,16 @@ namespace Microsoft.Extensions.AI; public abstract class UserInputResponseContent : AIContent { /// - /// Gets or sets the ID to uniquely identify the user input request/response pair. + /// Initializes a new instance of the class. + /// + /// The ID to uniquely identify the user input request/response pair. + protected UserInputResponseContent(string id) + { + Id = Throw.IfNullOrWhitespace(id); + } + + /// + /// Gets the ID to uniquely identify the user input request/response pair. /// - public string ApprovalId { get; set; } = default!; + public string Id { get; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs index 788e3c8739..5634c50792 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs @@ -74,7 +74,7 @@ public override IAsyncEnumerable GetStreamingResponseAsync(I private static void RemoveExecutedApprovedApprovalRequests(IList messages) { var functionResultCallIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.CallId).ToHashSet(); - var approvalResponsetIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.ApprovalId).ToHashSet(); + var approvalResponsetIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.Id).ToHashSet(); int messageCount = messages.Count; for (int i = 0; i < messageCount; i++) @@ -82,7 +82,7 @@ private static void RemoveExecutedApprovedApprovalRequests(IList me // Get any content that is not a FunctionApprovalRequestContent/FunctionApprovalResponseContent or is a FunctionApprovalRequestContent/FunctionApprovalResponseContent that has not been executed. var content = messages[i].Contents.Where(x => (x is not FunctionApprovalRequestContent && x is not FunctionApprovalResponseContent) || - (x is FunctionApprovalRequestContent request && !approvalResponsetIds.Contains(request.ApprovalId) && !functionResultCallIds.Contains(request.FunctionCall.CallId)) || + (x is FunctionApprovalRequestContent request && !approvalResponsetIds.Contains(request.Id) && !functionResultCallIds.Contains(request.FunctionCall.CallId)) || (x is FunctionApprovalResponseContent approval && !functionResultCallIds.Contains(approval.FunctionCall.CallId))).ToList(); // Remove the entire message if there is no content left after filtering. From 8ad466ba0b459b0d02a09c93ddd2adeb458f5f0e Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:31:14 +0100 Subject: [PATCH 27/53] Upgrade packages to resolve conflicts. --- dotnet/Directory.Packages.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index b903ec7382..fbbca52293 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -31,7 +31,7 @@ - + @@ -43,7 +43,7 @@ - + From 049c9d331cca14d75ee1ffe19879833ccab499f3 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:54:09 +0100 Subject: [PATCH 28/53] Move some code into methods to simply main GetResponse. --- ...nInvokingChatClientWithBuiltInApprovals.cs | 78 ++++++++++++------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 3a7b50d2e9..8f8982b2d9 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -255,6 +255,8 @@ public override async Task GetResponseAsync( // ** Approvals additions on top of FICC - start **// + // We should maintain a list of chat messages that need to be returned to the caller + // as part of the next response that were generated before the first iteration. List? augmentedPreInvocationHistory = null; // Remove any approval requests and approval request/response pairs that have already been executed. @@ -262,22 +264,11 @@ public override async Task GetResponseAsync( // Generate failed function result contents for any rejected requests. List? rejectedFunctionCallMessages = null; - if (notExecutedResponses.rejections is { Count: > 0 }) - { - rejectedFunctionCallMessages = []; - foreach (var rejectedCall in notExecutedResponses.rejections) - { - // Create a FunctionResultContent for the rejected function call. - var functionResult = new FunctionResultContent(rejectedCall.CallId, "Error: Function invocation approval was not granted."); + GenerateRejectedFunctionResults(notExecutedResponses.rejections, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); - rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Assistant, [rejectedCall])); - rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult])); - } - - augmentedPreInvocationHistory ??= []; - augmentedPreInvocationHistory.AddRange(rejectedFunctionCallMessages); - originalMessages.AddRange(rejectedFunctionCallMessages); - } + // We need to add FCC and FRC that were generated from the rejected approval requests into the original messages, + // so that the LLM can see them during the first iteration. + originalMessages.AddRange(rejectedFunctionCallMessages ?? []); // Check if there are any function calls to do from any approved functions and execute them. if (notExecutedResponses.approvals is { Count: > 0 }) @@ -334,18 +325,10 @@ public override async Task GetResponseAsync( { // ** Approvals additions on top of FICC - start **// - // TODO: Ensure that this works correctly in all cases, and doesn't have any sideaffects. // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, - // so they are processed as normal. - if (augmentedPreInvocationHistory?.Count > 0) - { - for (int i = augmentedPreInvocationHistory.Count - 1; i >= 0; i--) - { - response.Messages.Insert(0, augmentedPreInvocationHistory[i]); - } - - augmentedPreInvocationHistory = null; - } + // so they are returned to the caller. + UpdateResponseWithPreInvocationHistory(response, augmentedPreInvocationHistory); + augmentedPreInvocationHistory = null; // ** Approvals additions on top of FICC - end **// @@ -1001,6 +984,49 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul context.Function.InvokeAsync(context.Arguments, cancellationToken); } + /// + /// If we have any rejected approval responses, we need to generate failed function results for them. + /// + /// Any rejected approval responses. + /// The function call and result content for the rejected calls. + /// The list of messages generated before the first invocation, that need to be added to the user response. + private static void GenerateRejectedFunctionResults( + List? rejections, + ref List? rejectedFunctionCallMessages, + ref List? augmentedPreInvocationHistory) + { + if (rejections is { Count: > 0 }) + { + rejectedFunctionCallMessages = []; + foreach (var rejectedCall in rejections) + { + // Create a FunctionResultContent for the rejected function call. + var functionResult = new FunctionResultContent(rejectedCall.CallId, "Error: Function invocation approval was not granted."); + + rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Assistant, [rejectedCall])); + rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult])); + } + + augmentedPreInvocationHistory ??= []; + augmentedPreInvocationHistory.AddRange(rejectedFunctionCallMessages); + } + } + + /// + /// Insert the given at the start of the 's messages. + /// + private static void UpdateResponseWithPreInvocationHistory(ChatResponse response, List? augmentedPreInvocationHistory) + { + if (augmentedPreInvocationHistory?.Count > 0) + { + // Since these messages are pre-invocation, we want to insert them at the start of the response messages. + for (int i = augmentedPreInvocationHistory.Count - 1; i >= 0; i--) + { + response.Messages.Insert(0, augmentedPreInvocationHistory[i]); + } + } + } + /// /// We want to get rid of any and that have already been executed. /// We want to throw an exception for any that has no response, since it is an error state. From 39ef6bdf2fcdd7bf808a2c41cfeb526fbad7f674 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:28:52 +0100 Subject: [PATCH 29/53] Adding approvals support to streaming --- ...ep02_ChatClientAgent_UsingFunctionTools.cs | 92 ------- ...ntAgent_UsingFunctionToolsWithApprovals.cs | 231 ++++++++++++++++ .../FunctionApprovalRequestContent.cs | 6 +- .../FunctionApprovalResponseContent.cs | 2 +- .../UserInputOriginalMetadata.cs | 31 +++ .../MEAI.Contents/UserInputRequestContent.cs | 5 + .../MEAI.Contents/UserInputResponseContent.cs | 5 + ...nInvokingChatClientWithBuiltInApprovals.cs | 249 ++++++++++++++++-- 8 files changed, 506 insertions(+), 115 deletions(-) create mode 100644 dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs create mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputOriginalMetadata.cs diff --git a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs index fbe19f1feb..be945edfbf 100644 --- a/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs +++ b/dotnet/samples/GettingStarted/Steps/Step02_ChatClientAgent_UsingFunctionTools.cs @@ -3,8 +3,6 @@ using System.ComponentModel; using Microsoft.Extensions.AI; using Microsoft.Extensions.AI.Agents; -using Microsoft.Extensions.AI.ModelContextProtocol; -using Microsoft.Extensions.DependencyInjection; namespace Steps; @@ -116,96 +114,6 @@ async Task RunAgentAsync(string input) await base.AgentCleanUpAsync(provider, agent, thread); } - [Theory] - [InlineData(ChatClientProviders.AzureOpenAI)] - [InlineData(ChatClientProviders.AzureAIAgentsPersistent)] - [InlineData(ChatClientProviders.OpenAIAssistant)] - [InlineData(ChatClientProviders.OpenAIChatCompletion)] - [InlineData(ChatClientProviders.OpenAIResponses)] - public async Task ApprovalsWithTools(ChatClientProviders provider) - { - // Creating a MenuTools instance to be used by the agent. - var menuTools = new MenuTools(); - - // Define the options for the chat client agent. - var agentOptions = new ChatClientAgentOptions( - name: "Host", - instructions: "Answer questions about the menu", - tools: [ - AIFunctionFactory.Create(menuTools.GetMenu), - new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), - AIFunctionFactory.Create(menuTools.GetItemPrice), - new HostedMcpServerTool("MyService", new Uri("https://mcp-server.example.com")) - { - AllowedTools = ["add"], - ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, - } - ]); - - // Create the server-side agent Id when applicable (depending on the provider). - agentOptions.Id = await base.AgentCreateAsync(provider, agentOptions); - - // Get the chat client to use for the agent. - using var chatClient = base.GetChatClient(provider, agentOptions); - - var chatBuilder = chatClient.AsBuilder(); - if (chatClient.GetService() is null) - { - chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => - { - return new HostedMCPChatClient(innerClient); - }); - } - if (chatClient.GetService() is null) - { - chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => - { - return new FunctionInvokingChatClientWithBuiltInApprovals(innerClient, null, services); - }); - } - using var chatClientWithMCPAndApprovals = chatBuilder.Build(); - - // Define the agent - var agent = new ChatClientAgent(chatClientWithMCPAndApprovals, agentOptions); - - // Create the chat history thread to capture the agent interaction. - var thread = agent.GetNewThread(); - - // Respond to user input, invoking functions where appropriate. - await RunAgentAsync("What is the special soup and its price?"); - await RunAgentAsync("What is the special drink?"); - await RunAgentAsync("What is 2 + 2?"); - - async Task RunAgentAsync(string input) - { - this.WriteUserMessage(input); - var response = await agent.RunAsync(input, thread); - this.WriteResponseOutput(response); - - var userInputRequests = response.UserInputRequests.ToList(); - - // Loop until all user input requests are handled. - while (userInputRequests.Count > 0) - { - List nextIterationMessages = userInputRequests?.Select((request) => request switch - { - FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(), - FunctionApprovalRequestContent functionApprovalRequest => functionApprovalRequest.Reject(), - _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") - })?.ToList() ?? []; - - nextIterationMessages.ForEach(x => Console.WriteLine($"Approval for the {(x.Contents[0] as FunctionApprovalResponseContent)?.FunctionCall.Name} function call is set to {(x.Contents[0] as FunctionApprovalResponseContent)?.Approved}.")); - - response = await agent.RunAsync(nextIterationMessages, thread); - this.WriteResponseOutput(response); - userInputRequests = response.UserInputRequests.ToList(); - } - } - - // Clean up the server-side agent after use when applicable (depending on the provider). - await base.AgentCleanUpAsync(provider, agent, thread); - } - private sealed class MenuTools { [Description("Get the full menu items.")] diff --git a/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs new file mode 100644 index 0000000000..93550405e5 --- /dev/null +++ b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.AI.Agents; +using Microsoft.Extensions.AI.ModelContextProtocol; +using Microsoft.Extensions.DependencyInjection; + +namespace Steps; + +public sealed class Step10_ChatClientAgent_UsingFunctionToolsWithApprovals(ITestOutputHelper output) : AgentSample(output) +{ + [Theory] + [InlineData(ChatClientProviders.AzureOpenAI)] + [InlineData(ChatClientProviders.AzureAIAgentsPersistent)] + [InlineData(ChatClientProviders.OpenAIAssistant)] + [InlineData(ChatClientProviders.OpenAIChatCompletion)] + [InlineData(ChatClientProviders.OpenAIResponses)] + public async Task ApprovalsWithTools(ChatClientProviders provider) + { + // Creating a MenuTools instance to be used by the agent. + var menuTools = new MenuTools(); + + // Define the options for the chat client agent. + var agentOptions = new ChatClientAgentOptions( + name: "Host", + instructions: "Answer questions about the menu", + tools: [ + AIFunctionFactory.Create(menuTools.GetMenu), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), + AIFunctionFactory.Create(menuTools.GetItemPrice), + new HostedMcpServerTool("MyService", new Uri("https://mcp-server.example.com")) + { + AllowedTools = ["add"], + ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, + } + ]); + + // Create the server-side agent Id when applicable (depending on the provider). + agentOptions.Id = await base.AgentCreateAsync(provider, agentOptions); + + // Get the chat client to use for the agent. + using var chatClient = base.GetChatClient(provider, agentOptions); + + // Modify the chat client to include MCP and built-in approvals if not already present. + var chatBuilder = chatClient.AsBuilder(); + if (chatClient.GetService() is null) + { + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + return new HostedMCPChatClient(innerClient); + }); + } + if (chatClient.GetService() is null) + { + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + return new FunctionInvokingChatClientWithBuiltInApprovals(innerClient, null, services); + }); + } + using var chatClientWithMCPAndApprovals = chatBuilder.Build(); + + // Define the agent + var agent = new ChatClientAgent(chatClientWithMCPAndApprovals, agentOptions); + + // Create the chat history thread to capture the agent interaction. + var thread = agent.GetNewThread(); + + // Respond to user input, invoking functions where appropriate. + await RunAgentAsync("What is the special soup and its price?"); + await RunAgentAsync("What is the special drink?"); + await RunAgentAsync("What is 2 + 2?"); + + async Task RunAgentAsync(string input) + { + this.WriteUserMessage(input); + var response = await agent.RunAsync(input, thread); + this.WriteResponseOutput(response); + + var userInputRequests = response.UserInputRequests.ToList(); + + // Loop until all user input requests are handled. + while (userInputRequests.Count > 0) + { + List nextIterationMessages = userInputRequests?.Select((request) => request switch + { + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(), + FunctionApprovalRequestContent functionApprovalRequest => functionApprovalRequest.Reject(), + _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") + })?.ToList() ?? []; + + nextIterationMessages.ForEach(x => Console.WriteLine($"Approval for the {(x.Contents[0] as FunctionApprovalResponseContent)?.FunctionCall.Name} function call is set to {(x.Contents[0] as FunctionApprovalResponseContent)?.Approved}.")); + + response = await agent.RunAsync(nextIterationMessages, thread); + this.WriteResponseOutput(response); + userInputRequests = response.UserInputRequests.ToList(); + } + } + + // Clean up the server-side agent after use when applicable (depending on the provider). + await base.AgentCleanUpAsync(provider, agent, thread); + } + + [Theory] + [InlineData(ChatClientProviders.AzureOpenAI)] + [InlineData(ChatClientProviders.AzureAIAgentsPersistent)] + [InlineData(ChatClientProviders.OpenAIAssistant)] + [InlineData(ChatClientProviders.OpenAIChatCompletion)] + [InlineData(ChatClientProviders.OpenAIResponses)] + public async Task ApprovalsWithToolsStreaming(ChatClientProviders provider) + { + // Creating a MenuTools instance to be used by the agent. + var menuTools = new MenuTools(); + + // Define the options for the chat client agent. + var agentOptions = new ChatClientAgentOptions( + name: "Host", + instructions: "Answer questions about the menu", + tools: [ + AIFunctionFactory.Create(menuTools.GetMenu), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), + AIFunctionFactory.Create(menuTools.GetItemPrice), + new HostedMcpServerTool("MyService", new Uri("https://mcp-server.example.com")) + { + AllowedTools = ["add"], + ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, + } + ]); + + // Create the server-side agent Id when applicable (depending on the provider). + agentOptions.Id = await base.AgentCreateAsync(provider, agentOptions); + + // Get the chat client to use for the agent. + using var chatClient = base.GetChatClient(provider, agentOptions); + + // Modify the chat client to include MCP and built-in approvals if not already present. + var chatBuilder = chatClient.AsBuilder(); + if (chatClient.GetService() is null) + { + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + return new HostedMCPChatClient(innerClient); + }); + } + if (chatClient.GetService() is null) + { + chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => + { + return new FunctionInvokingChatClientWithBuiltInApprovals(innerClient, null, services); + }); + } + using var chatClientWithMCPAndApprovals = chatBuilder.Build(); + + // Define the agent + var agent = new ChatClientAgent(chatClientWithMCPAndApprovals, agentOptions); + + // Create the chat history thread to capture the agent interaction. + var thread = agent.GetNewThread(); + + // Respond to user input, invoking functions where appropriate. + await RunAgentAsync("What is the special soup and its price?"); + await RunAgentAsync("What is the special drink?"); + await RunAgentAsync("What is 2 + 2?"); + + async Task RunAgentAsync(string input) + { + this.WriteUserMessage(input); + var updates = await agent.RunStreamingAsync(input, thread).ToListAsync(); + this.WriteResponseOutput(updates.ToAgentRunResponse()); + var userInputRequests = updates.SelectMany(x => x.UserInputRequests).ToList(); + + // Loop until all user input requests are handled. + while (userInputRequests.Count > 0) + { + List nextIterationMessages = userInputRequests?.Select((request) => request switch + { + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(), + FunctionApprovalRequestContent functionApprovalRequest => functionApprovalRequest.Reject(), + _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") + })?.ToList() ?? []; + + nextIterationMessages.ForEach(x => Console.WriteLine($"Approval for the {(x.Contents[0] as FunctionApprovalResponseContent)?.FunctionCall.Name} function call is set to {(x.Contents[0] as FunctionApprovalResponseContent)?.Approved}.")); + + updates = await agent.RunStreamingAsync(nextIterationMessages, thread).ToListAsync(); + this.WriteResponseOutput(updates.ToAgentRunResponse()); + userInputRequests = updates.SelectMany(x => x.UserInputRequests).ToList(); + } + } + + // Clean up the server-side agent after use when applicable (depending on the provider). + await base.AgentCleanUpAsync(provider, agent, thread); + } + + private sealed class MenuTools + { + [Description("Get the full menu items.")] + public MenuItem[] GetMenu() + { + return s_menuItems; + } + + [Description("Get the specials from the menu.")] + public IEnumerable GetSpecials() + { + return s_menuItems.Where(i => i.IsSpecial); + } + + [Description("Get the price of a menu item.")] + public float? GetItemPrice([Description("The name of the menu item.")] string menuItem) + { + return s_menuItems.FirstOrDefault(i => i.Name.Equals(menuItem, StringComparison.OrdinalIgnoreCase))?.Price; + } + + private static readonly MenuItem[] s_menuItems = [ + new() { Category = "Soup", Name = "Clam Chowder", Price = 4.95f, IsSpecial = true }, + new() { Category = "Soup", Name = "Tomato Soup", Price = 4.95f, IsSpecial = false }, + new() { Category = "Salad", Name = "Cobb Salad", Price = 9.99f }, + new() { Category = "Salad", Name = "House Salad", Price = 4.95f }, + new() { Category = "Drink", Name = "Chai Tea", Price = 2.95f, IsSpecial = true }, + new() { Category = "Drink", Name = "Soda", Price = 1.95f }, + ]; + + public sealed class MenuItem + { + public string Category { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public float Price { get; set; } + public bool IsSpecial { get; set; } + } + } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs index 6391efcee3..063a8de13f 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs @@ -21,7 +21,7 @@ public FunctionApprovalRequestContent(string approvalId, FunctionCallContent fun } /// - /// Gets or sets the function call that pre-invoke approval is required for. + /// Gets the function call that pre-invoke approval is required for. /// public FunctionCallContent FunctionCall { get; } @@ -31,7 +31,7 @@ public FunctionApprovalRequestContent(string approvalId, FunctionCallContent fun /// The representing the approval response. public ChatMessage Approve() { - return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall)]); + return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall) { OriginalMessageMetadata = this.OriginalMessageMetadata }]); } /// @@ -40,6 +40,6 @@ public ChatMessage Approve() /// The representing the rejection response. public ChatMessage Reject() { - return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall)]); + return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall) { OriginalMessageMetadata = this.OriginalMessageMetadata }]); } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs index 3fcce35b71..577042598b 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs @@ -28,7 +28,7 @@ public FunctionApprovalResponseContent(string approvalId, bool approved, Functio public bool Approved { get; set; } /// - /// Gets or sets the function call that pre-invoke approval is required for. + /// Gets the function call that pre-invoke approval is required for. /// public FunctionCallContent FunctionCall { get; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputOriginalMetadata.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputOriginalMetadata.cs new file mode 100644 index 0000000000..380d87b661 --- /dev/null +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputOriginalMetadata.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents metadata for the original message or update that initiated a user input request. +/// +public class UserInputOriginalMetadata +{ + /// Gets or sets the ID of the chat message or update. + public string? MessageId { get; set; } + + /// Gets or sets the name of the author of the message or update. + public string? AuthorName { get; set; } + + /// Gets or sets any additional properties associated with the message or update. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } + + /// Gets or sets a timestamp for the response update. + public DateTimeOffset? CreatedAt { get; set; } + + /// Gets or sets the ID of the response of which the update is a part. + public string? ResponseId { get; set; } + + /// Gets or sets the raw representation of the chat message or update from an underlying implementation. + [JsonIgnore] + public object? RawRepresentation { get; set; } +} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs index ba327b6717..59a5ee487f 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs @@ -22,4 +22,9 @@ protected UserInputRequestContent(string id) /// Gets the ID to uniquely identify the user input request/response pair. /// public string Id { get; } + + /// + /// Gets or sets any metadata from the original message that this content is associated with. + /// + public UserInputOriginalMetadata? OriginalMessageMetadata { get; set; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs index 9c15927b7d..fafac5eb93 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs @@ -22,4 +22,9 @@ protected UserInputResponseContent(string id) /// Gets the ID to uniquely identify the user input request/response pair. /// public string Id { get; } + + /// + /// Gets or sets any metadata from the original message that this content is associated with. + /// + public UserInputOriginalMetadata? OriginalMessageMetadata { get; set; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 8f8982b2d9..3233f2bab9 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -251,7 +251,6 @@ public override async Task GetResponseAsync( List? functionCallContents = null; // function call contents that need responding to in the current turn bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set int consecutiveErrorCount = 0; - int iteration = 0; // ** Approvals additions on top of FICC - start **// @@ -264,7 +263,7 @@ public override async Task GetResponseAsync( // Generate failed function result contents for any rejected requests. List? rejectedFunctionCallMessages = null; - GenerateRejectedFunctionResults(notExecutedResponses.rejections, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); + GenerateRejectedFunctionResults(notExecutedResponses.rejections, null, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); // We need to add FCC and FRC that were generated from the rejected approval requests into the original messages, // so that the LLM can see them during the first iteration. @@ -273,14 +272,14 @@ public override async Task GetResponseAsync( // Check if there are any function calls to do from any approved functions and execute them. if (notExecutedResponses.approvals is { Count: > 0 }) { - var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => new ChatMessage(ChatRole.Assistant, [x])).ToList(); + var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => ConvertToFunctionCallContentMessage(x)).ToList(); originalMessages.AddRange(approvedFunctionCalls); augmentedPreInvocationHistory ??= []; augmentedPreInvocationHistory.AddRange(approvedFunctionCalls); // Add the responses from the function calls into the augmented history and also into the tracked // list of response messages. - var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: false, cancellationToken); responseMessages = [.. rejectedFunctionCallMessages ?? [], .. approvedFunctionCalls, .. modeAndMessages.MessagesAdded]; consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; @@ -294,7 +293,7 @@ public override async Task GetResponseAsync( // ** Approvals additions on top of FICC - end **// - for (; ; iteration++) + for (int iteration = 0; ; iteration++) { functionCallContents?.Clear(); @@ -406,11 +405,103 @@ public override async IAsyncEnumerable GetStreamingResponseA List updates = []; // updates from the current response int consecutiveErrorCount = 0; + // ** Approvals additions on top of FICC - start **// + + // This is a synthetic ID since we're generating the tool messages instead of getting them from + // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to + // use the same message ID for all of them within a given iteration, as this is a single logical + // message with multiple content items. We could also use different message IDs per tool content, + // but there's no benefit to doing so. + string toolResponseId = Guid.NewGuid().ToString("N"); + + ApprovalRequiredAIFunction[]? approvalRequiredFunctions = (options?.Tools ?? []).Concat(AdditionalTools ?? []).OfType().ToArray(); + bool hasApprovalRequiringFunctions = (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && approvalRequiredFunctions.Length > 0; + + var shouldTerminateFromPreInvocationFunctionCalls = false; + + // We should maintain a list of chat messages that need to be returned to the caller + // as part of the next response that were generated before the first iteration. + List? augmentedPreInvocationHistory = null; + + // Remove any approval requests and approval request/response pairs that have already been executed. + var notExecutedResponses = ProcessApprovalRequestsAndResponses(originalMessages); + + // Generate failed function result contents for any rejected requests. + List? rejectedFunctionCallMessages = null; + GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); + + // We need to add FCC and FRC that were generated from the rejected approval requests into the original messages, + // so that the LLM can see them during the first iteration. + originalMessages.AddRange(rejectedFunctionCallMessages ?? []); + + // Check if there are any function calls to do from any approved functions and execute them. + if (notExecutedResponses.approvals is { Count: > 0 }) + { + var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => ConvertToFunctionCallContentMessage(x)).ToList(); + originalMessages.AddRange(approvedFunctionCalls); + augmentedPreInvocationHistory ??= []; + augmentedPreInvocationHistory.AddRange(approvedFunctionCalls); + + // Add the responses from the function calls into the augmented history and also into the tracked + // list of response messages. + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: true, cancellationToken); + foreach (var message in modeAndMessages.MessagesAdded) + { + message.MessageId = toolResponseId; + } + + responseMessages = [.. rejectedFunctionCallMessages ?? [], .. approvedFunctionCalls, .. modeAndMessages.MessagesAdded]; + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + augmentedPreInvocationHistory.AddRange(modeAndMessages.MessagesAdded); + shouldTerminateFromPreInvocationFunctionCalls |= modeAndMessages.ShouldTerminate; + } + + // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages + // includes all activities, including generated function results. + if (augmentedPreInvocationHistory is { Count: > 0 }) + { + foreach (var message in augmentedPreInvocationHistory ?? []) + { + var toolResultUpdate = new ChatResponseUpdate + { + AdditionalProperties = message.AdditionalProperties, + AuthorName = message.AuthorName, + ConversationId = options?.ConversationId, + CreatedAt = DateTimeOffset.UtcNow, + Contents = message.Contents, + RawRepresentation = message.RawRepresentation, + ResponseId = message.MessageId, + MessageId = message.MessageId, + Role = message.Role, + }; + + yield return toolResultUpdate; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + + if (shouldTerminateFromPreInvocationFunctionCalls) + { + yield break; + } + + // ** Approvals additions on top of FICC - end **// + for (int iteration = 0; ; iteration++) { updates.Clear(); functionCallContents?.Clear(); + // ** Approvals additions on top of FICC - start **// + + bool hasApprovalRequiringFcc = false; + int lastApprovalCheckedFCCIndex = 0; + int lastYieldedUpdateIndex = 0; + string? currentMessageId = null; + + // ** Approvals additions on top of FICC - end **// + await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) { if (update is null) @@ -418,6 +509,11 @@ public override async IAsyncEnumerable GetStreamingResponseA Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}."); } + if (update.MessageId is { Length: > 0 }) + { + currentMessageId = update.MessageId; + } + updates.Add(update); _ = CopyFunctionCalls(update.Contents, ref functionCallContents); @@ -435,12 +531,66 @@ public override async IAsyncEnumerable GetStreamingResponseA } } - yield return update; + // ** Approvals modifications - start **// + if (functionCallContents?.Count is not > 0 || !hasApprovalRequiringFunctions) + { + // If there are no function calls to make yet, or if none of the functions require approval at all, + // we can yield the update as-is. + lastYieldedUpdateIndex++; + yield return update; + } + else + { + if (!hasApprovalRequiringFcc) + { + // Check if any of the function call contents in this update requires approval. + // We only do this until we find the first one that requires approval. + (hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex) = await CheckForApprovalRequiringFCCAsync( + functionCallContents, approvalRequiredFunctions, hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex); + } + + // We've encountered a function call content that requires approval (either in this update or ealier) + // so we need to ask for approval for all functions, since we cannot mix and match. + if (hasApprovalRequiringFcc) + { + // Convert all function call contents into approval requests from the last yielded update index + // and yield all those updates. + for (; lastYieldedUpdateIndex < updates.Count; lastYieldedUpdateIndex++) + { + var updateToYield = updates[lastYieldedUpdateIndex]; + if (updateToYield.Contents is { Count: > 0 }) + { + for (int i = 0; i < updateToYield.Contents.Count; i++) + { + if (updateToYield.Contents[i] is FunctionCallContent fcc) + { + var approvalRequest = ConvertToApprovalRequestContent(fcc, updateToYield); + if (approvalRequest.OriginalMessageMetadata!.MessageId is not { Length: > 0 }) + { + approvalRequest.OriginalMessageMetadata!.MessageId = currentMessageId; + } + updateToYield.Contents[i] = approvalRequest; + } + } + } + yield return updateToYield; + } + } + else + { + // We don't have any appoval requiring function calls yet, but we may receive some in future + // so we cannot yield the updates yet. We'll just keep them in the updates list + // for later. + } + } + // ** Approvals modifications - end **// + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } // If there are no tools to call, or for any other reason we should stop, return the response. if (functionCallContents is not { Count: > 0 } || + hasApprovalRequiringFcc || (options?.Tools is not { Count: > 0 } && AdditionalTools is not { Count: > 0 }) || iteration >= _maximumIterationsPerRequest) { @@ -459,12 +609,16 @@ public override async IAsyncEnumerable GetStreamingResponseA responseMessages.AddRange(modeAndMessages.MessagesAdded); consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + // ** Approvals removal - start **// + // This is a synthetic ID since we're generating the tool messages instead of getting them from // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to // use the same message ID for all of them within a given iteration, as this is a single logical // message with multiple content items. We could also use different message IDs per tool content, // but there's no benefit to doing so. - string toolResponseId = Guid.NewGuid().ToString("N"); + //string toolResponseId = Guid.NewGuid().ToString("N"); + + // ** Approvals removal - end **// // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages // includes all activities, including generated function results. @@ -609,6 +763,25 @@ private static bool CopyFunctionCalls( return any; } + private static async Task<(bool hasApprovalRequiringFcc, int lastApprovalCheckedFCCIndex)> CheckForApprovalRequiringFCCAsync( + List? functionCallContents, + ApprovalRequiredAIFunction[] approvalRequiredFunctions, + bool hasApprovalRequiringFcc, + int lastApprovalCheckedFCCIndex) + { + for (; lastApprovalCheckedFCCIndex < (functionCallContents?.Count ?? 0); lastApprovalCheckedFCCIndex++) + { + var fcc = functionCallContents![lastApprovalCheckedFCCIndex]; + if (approvalRequiredFunctions.FirstOrDefault(y => y.Name == fcc.Name) is ApprovalRequiredAIFunction approvalFunction && + await approvalFunction.RequiresApprovalCallback(new(fcc))) + { + hasApprovalRequiringFcc |= true; + } + } + + return (hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex); + } + private static void UpdateOptionsForNextIteration(ref ChatOptions? options, string? conversationId) { if (options is null) @@ -988,10 +1161,12 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul /// If we have any rejected approval responses, we need to generate failed function results for them. /// /// Any rejected approval responses. + /// The message id to use for the tool response. /// The function call and result content for the rejected calls. /// The list of messages generated before the first invocation, that need to be added to the user response. private static void GenerateRejectedFunctionResults( - List? rejections, + List? rejections, + string? toolResponseId, ref List? rejectedFunctionCallMessages, ref List? augmentedPreInvocationHistory) { @@ -1001,10 +1176,10 @@ private static void GenerateRejectedFunctionResults( foreach (var rejectedCall in rejections) { // Create a FunctionResultContent for the rejected function call. - var functionResult = new FunctionResultContent(rejectedCall.CallId, "Error: Function invocation approval was not granted."); + var functionResult = new FunctionResultContent(rejectedCall.FunctionCall.CallId, "Error: Function invocation approval was not granted."); - rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Assistant, [rejectedCall])); - rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult])); + rejectedFunctionCallMessages.Add(ConvertToFunctionCallContentMessage(rejectedCall)); + rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult]) { MessageId = toolResponseId }); } augmentedPreInvocationHistory ??= []; @@ -1032,12 +1207,12 @@ private static void UpdateResponseWithPreInvocationHistory(ChatResponse response /// We want to throw an exception for any that has no response, since it is an error state. /// We want to return the from any that has no matching and for execution. /// - private static (List? approvals, List? rejections) ProcessApprovalRequestsAndResponses(List messages) + private static (List? approvals, List? rejections) ProcessApprovalRequestsAndResponses(List messages) { // Get the list of function call ids that are already executed. var functionResultCallIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.CallId).ToHashSet(); - List? notExecutedApprovedFunctionCalls = null; - List? notExecutedRejectedFunctionCalls = null; + List? notExecutedApprovedFunctionCalls = null; + List? notExecutedRejectedFunctionCalls = null; HashSet? requestCallIds = null; for (int i = 0; i < messages.Count; i++) @@ -1092,12 +1267,12 @@ private static (List? approvals, List? if (response.Approved) { notExecutedApprovedFunctionCalls ??= []; - notExecutedApprovedFunctionCalls.Add(response.FunctionCall); + notExecutedApprovedFunctionCalls.Add(response); } else { notExecutedRejectedFunctionCalls ??= []; - notExecutedRejectedFunctionCalls.Add(response.FunctionCall); + notExecutedRejectedFunctionCalls.Add(response); } continue; @@ -1125,7 +1300,7 @@ private static (List? approvals, List? AdditionalProperties = message.AdditionalProperties, RawRepresentation = message.RawRepresentation, MessageId = message.MessageId, - }; + }; } } @@ -1174,12 +1349,48 @@ private static async Task ReplaceFunctionCallsWithApprovalRequests(IList new(functionCall.CallId, functionCall) + { + OriginalMessageMetadata = new() + { + MessageId = message.MessageId, + AuthorName = message.AuthorName, + AdditionalProperties = message.AdditionalProperties, + RawRepresentation = message.RawRepresentation + } + }; + + private static FunctionApprovalRequestContent ConvertToApprovalRequestContent(FunctionCallContent functionCall, ChatResponseUpdate update) + => new(functionCall.CallId, functionCall) + { + OriginalMessageMetadata = new() + { + MessageId = update.MessageId, + AuthorName = update.AuthorName, + AdditionalProperties = update.AdditionalProperties, + RawRepresentation = update.RawRepresentation, + CreatedAt = update.CreatedAt, + ResponseId = update.ResponseId + } + }; + + private static ChatMessage ConvertToFunctionCallContentMessage(FunctionApprovalResponseContent response) + => new(ChatRole.Assistant, [response.FunctionCall]) + { + MessageId = response.OriginalMessageMetadata?.MessageId, + AuthorName = response.OriginalMessageMetadata?.AuthorName, + AdditionalProperties = response.OriginalMessageMetadata?.AdditionalProperties, + RawRepresentation = response.OriginalMessageMetadata?.RawRepresentation, + }; + private static TimeSpan GetElapsedTime(long startingTimestamp) => #if NET Stopwatch.GetElapsedTime(startingTimestamp); From 14d477324833d45c3fc099c24bd5cd56c573abf6 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:47:56 +0100 Subject: [PATCH 30/53] Address PR comments. --- .../MEAI.Contents/FunctionApprovalRequestContent.cs | 6 +++--- .../MEAI.Contents/FunctionApprovalResponseContent.cs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs index 063a8de13f..f009c5bd3d 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs @@ -12,10 +12,10 @@ public class FunctionApprovalRequestContent : UserInputRequestContent /// /// Initializes a new instance of the class. /// - /// The ID to uniquely identify the user input request/response pair. + /// The ID to uniquely identify the function approval request/response pair. /// The function call that requires user approval. - public FunctionApprovalRequestContent(string approvalId, FunctionCallContent functionCall) - : base(approvalId) + public FunctionApprovalRequestContent(string id, FunctionCallContent functionCall) + : base(id) { this.FunctionCall = Throw.IfNull(functionCall); } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs index 577042598b..c12328837c 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs @@ -12,11 +12,11 @@ public class FunctionApprovalResponseContent : UserInputResponseContent /// /// Initializes a new instance of the class. /// - /// The ID to uniquely identify the user input request/response pair. + /// The ID to uniquely identify the function approval request/response pair. /// Indicates whether the request was approved. /// The function call that requires user approval. - public FunctionApprovalResponseContent(string approvalId, bool approved, FunctionCallContent functionCall) - : base(approvalId) + public FunctionApprovalResponseContent(string id, bool approved, FunctionCallContent functionCall) + : base(id) { this.Approved = approved; this.FunctionCall = Throw.IfNull(functionCall); From b288fed18fb95fdd43ce488af7cfdb30bbdffb3e Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 13 Aug 2025 14:52:14 +0100 Subject: [PATCH 31/53] Change approval response type and naming --- ...lientAgent_UsingFunctionToolsWithApprovals.cs | 12 ++++++++---- .../FunctionApprovalRequestContent.cs | 16 ++++++++-------- ...tionInvokingChatClientWithBuiltInApprovals.cs | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs index 93550405e5..6a8c455818 100644 --- a/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs +++ b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs @@ -84,8 +84,10 @@ async Task RunAgentAsync(string input) { List nextIterationMessages = userInputRequests?.Select((request) => request switch { - FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(), - FunctionApprovalRequestContent functionApprovalRequest => functionApprovalRequest.Reject(), + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateApproval()]), + FunctionApprovalRequestContent functionApprovalRequest => + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateRejection()]), _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") })?.ToList() ?? []; @@ -174,8 +176,10 @@ async Task RunAgentAsync(string input) { List nextIterationMessages = userInputRequests?.Select((request) => request switch { - FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => functionApprovalRequest.Approve(), - FunctionApprovalRequestContent functionApprovalRequest => functionApprovalRequest.Reject(), + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateApproval()]), + FunctionApprovalRequestContent functionApprovalRequest => + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateRejection()]), _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") })?.ToList() ?? []; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs index f009c5bd3d..4042fe7795 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs @@ -26,20 +26,20 @@ public FunctionApprovalRequestContent(string id, FunctionCallContent functionCal public FunctionCallContent FunctionCall { get; } /// - /// Creates a representing an approval response. + /// Creates a representing an approval response. /// - /// The representing the approval response. - public ChatMessage Approve() + /// The representing the approval response. + public FunctionApprovalResponseContent CreateApproval() { - return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall) { OriginalMessageMetadata = this.OriginalMessageMetadata }]); + return new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall) { OriginalMessageMetadata = this.OriginalMessageMetadata }; } /// - /// Creates a representing a rejection response. + /// Creates a representing a rejection response. /// - /// The representing the rejection response. - public ChatMessage Reject() + /// The representing the rejection response. + public FunctionApprovalResponseContent CreateRejection() { - return new ChatMessage(ChatRole.User, [new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall) { OriginalMessageMetadata = this.OriginalMessageMetadata }]); + return new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall) { OriginalMessageMetadata = this.OriginalMessageMetadata }; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 3233f2bab9..81fe2f881f 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -1300,7 +1300,7 @@ private static (List? approvals, List Date: Wed, 13 Aug 2025 17:04:33 +0100 Subject: [PATCH 32/53] Remove commented out validation checks that won't work for server side threads. --- ...nInvokingChatClientWithBuiltInApprovals.cs | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 81fe2f881f..99ff2fa411 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -1226,31 +1226,17 @@ private static (List? approvals, List? approvals, List Date: Thu, 14 Aug 2025 11:21:39 +0100 Subject: [PATCH 33/53] Remove original metadata and read metadata from approval request message instead --- .../FunctionApprovalRequestContent.cs | 4 +- .../MEAI.Contents/UserInputRequestContent.cs | 5 - .../MEAI.Contents/UserInputResponseContent.cs | 5 - ...nInvokingChatClientWithBuiltInApprovals.cs | 207 ++++++++++-------- 4 files changed, 112 insertions(+), 109 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs index 4042fe7795..408c3ffa7d 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs @@ -31,7 +31,7 @@ public FunctionApprovalRequestContent(string id, FunctionCallContent functionCal /// The representing the approval response. public FunctionApprovalResponseContent CreateApproval() { - return new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall) { OriginalMessageMetadata = this.OriginalMessageMetadata }; + return new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall); } /// @@ -40,6 +40,6 @@ public FunctionApprovalResponseContent CreateApproval() /// The representing the rejection response. public FunctionApprovalResponseContent CreateRejection() { - return new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall) { OriginalMessageMetadata = this.OriginalMessageMetadata }; + return new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall); } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs index 59a5ee487f..ba327b6717 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs @@ -22,9 +22,4 @@ protected UserInputRequestContent(string id) /// Gets the ID to uniquely identify the user input request/response pair. /// public string Id { get; } - - /// - /// Gets or sets any metadata from the original message that this content is associated with. - /// - public UserInputOriginalMetadata? OriginalMessageMetadata { get; set; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs index fafac5eb93..9c15927b7d 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs @@ -22,9 +22,4 @@ protected UserInputResponseContent(string id) /// Gets the ID to uniquely identify the user input request/response pair. /// public string Id { get; } - - /// - /// Gets or sets any metadata from the original message that this content is associated with. - /// - public UserInputOriginalMetadata? OriginalMessageMetadata { get; set; } } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 99ff2fa411..1dd5186b6e 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -263,7 +263,7 @@ public override async Task GetResponseAsync( // Generate failed function result contents for any rejected requests. List? rejectedFunctionCallMessages = null; - GenerateRejectedFunctionResults(notExecutedResponses.rejections, null, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); + GenerateRejectedFunctionResults(notExecutedResponses.rejections, null, null, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); // We need to add FCC and FRC that were generated from the rejected approval requests into the original messages, // so that the LLM can see them during the first iteration. @@ -272,14 +272,14 @@ public override async Task GetResponseAsync( // Check if there are any function calls to do from any approved functions and execute them. if (notExecutedResponses.approvals is { Count: > 0 }) { - var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => ConvertToFunctionCallContentMessage(x)).ToList(); + var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => ConvertToFunctionCallContentMessage(x, null)).ToList(); originalMessages.AddRange(approvedFunctionCalls); augmentedPreInvocationHistory ??= []; augmentedPreInvocationHistory.AddRange(approvedFunctionCalls); // Add the responses from the function calls into the augmented history and also into the tracked // list of response messages. - var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: false, cancellationToken); + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: false, cancellationToken); responseMessages = [.. rejectedFunctionCallMessages ?? [], .. approvedFunctionCalls, .. modeAndMessages.MessagesAdded]; consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; @@ -413,6 +413,7 @@ public override async IAsyncEnumerable GetStreamingResponseA // message with multiple content items. We could also use different message IDs per tool content, // but there's no benefit to doing so. string toolResponseId = Guid.NewGuid().ToString("N"); + string functionCallContentFallbackMessageId = Guid.NewGuid().ToString("N"); ApprovalRequiredAIFunction[]? approvalRequiredFunctions = (options?.Tools ?? []).Concat(AdditionalTools ?? []).OfType().ToArray(); bool hasApprovalRequiringFunctions = (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && approvalRequiredFunctions.Length > 0; @@ -428,7 +429,7 @@ public override async IAsyncEnumerable GetStreamingResponseA // Generate failed function result contents for any rejected requests. List? rejectedFunctionCallMessages = null; - GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); + GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId, functionCallContentFallbackMessageId, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); // We need to add FCC and FRC that were generated from the rejected approval requests into the original messages, // so that the LLM can see them during the first iteration. @@ -437,14 +438,14 @@ public override async IAsyncEnumerable GetStreamingResponseA // Check if there are any function calls to do from any approved functions and execute them. if (notExecutedResponses.approvals is { Count: > 0 }) { - var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => ConvertToFunctionCallContentMessage(x)).ToList(); + var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => ConvertToFunctionCallContentMessage(x, functionCallContentFallbackMessageId)).ToList(); originalMessages.AddRange(approvedFunctionCalls); augmentedPreInvocationHistory ??= []; augmentedPreInvocationHistory.AddRange(approvedFunctionCalls); // Add the responses from the function calls into the augmented history and also into the tracked // list of response messages. - var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: true, cancellationToken); + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: true, cancellationToken); foreach (var message in modeAndMessages.MessagesAdded) { message.MessageId = toolResponseId; @@ -498,7 +499,6 @@ public override async IAsyncEnumerable GetStreamingResponseA bool hasApprovalRequiringFcc = false; int lastApprovalCheckedFCCIndex = 0; int lastYieldedUpdateIndex = 0; - string? currentMessageId = null; // ** Approvals additions on top of FICC - end **// @@ -509,11 +509,6 @@ public override async IAsyncEnumerable GetStreamingResponseA Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}."); } - if (update.MessageId is { Length: > 0 }) - { - currentMessageId = update.MessageId; - } - updates.Add(update); _ = CopyFunctionCalls(update.Contents, ref functionCallContents); @@ -564,11 +559,7 @@ public override async IAsyncEnumerable GetStreamingResponseA { if (updateToYield.Contents[i] is FunctionCallContent fcc) { - var approvalRequest = ConvertToApprovalRequestContent(fcc, updateToYield); - if (approvalRequest.OriginalMessageMetadata!.MessageId is not { Length: > 0 }) - { - approvalRequest.OriginalMessageMetadata!.MessageId = currentMessageId; - } + var approvalRequest = new FunctionApprovalRequestContent(fcc.CallId, fcc); updateToYield.Contents[i] = approvalRequest; } } @@ -1162,11 +1153,13 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul /// /// Any rejected approval responses. /// The message id to use for the tool response. + /// Optional message id to use if no original approval request message is availble to copy from. /// The function call and result content for the rejected calls. /// The list of messages generated before the first invocation, that need to be added to the user response. private static void GenerateRejectedFunctionResults( - List? rejections, + List? rejections, string? toolResponseId, + string? fallbackMessageId, ref List? rejectedFunctionCallMessages, ref List? augmentedPreInvocationHistory) { @@ -1176,9 +1169,9 @@ private static void GenerateRejectedFunctionResults( foreach (var rejectedCall in rejections) { // Create a FunctionResultContent for the rejected function call. - var functionResult = new FunctionResultContent(rejectedCall.FunctionCall.CallId, "Error: Function invocation approval was not granted."); + var functionResult = new FunctionResultContent(rejectedCall.Response.FunctionCall.CallId, "Error: Function invocation approval was not granted."); - rejectedFunctionCallMessages.Add(ConvertToFunctionCallContentMessage(rejectedCall)); + rejectedFunctionCallMessages.Add(ConvertToFunctionCallContentMessage(rejectedCall, fallbackMessageId)); rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult]) { MessageId = toolResponseId }); } @@ -1187,6 +1180,19 @@ private static void GenerateRejectedFunctionResults( } } + private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWithRequestMessage resultWithRequestMessage, string? fallbackMessageId) + { + if (resultWithRequestMessage.RequestMessage is not null) + { + var functionCallMessage = resultWithRequestMessage.RequestMessage.Clone(); + functionCallMessage.Contents = [resultWithRequestMessage.Response.FunctionCall]; + functionCallMessage.MessageId ??= fallbackMessageId; + return functionCallMessage; + } + + return new ChatMessage(ChatRole.Assistant, [resultWithRequestMessage.Response.FunctionCall]) { MessageId = fallbackMessageId }; + } + /// /// Insert the given at the start of the 's messages. /// @@ -1203,17 +1209,32 @@ private static void UpdateResponseWithPreInvocationHistory(ChatResponse response } /// - /// We want to get rid of any and that have already been executed. - /// We want to throw an exception for any that has no response, since it is an error state. - /// We want to return the from any that has no matching and for execution. + /// This method extracts the approval requests and responses from the provided list of messages, validates them, filters them to ones that require execution and splits them into approved and rejected. /// - private static (List? approvals, List? rejections) ProcessApprovalRequestsAndResponses(List messages) + /// + /// 1st iteration: over all messages and content + /// ===== + /// Build a list of all function call ids that are already executed. + /// Build a list of all function approval requests and responses. + /// Build a list of the content we want to keep (everything except approval requests and responses) and create a new list of messages for those. + /// Validate that we have an approval response for each approval request. + /// + /// 2nd iteration: over all approval responses + /// ===== + /// Filter out any approval responses that already have a matching function result (i.e. already executed). + /// Find the matching function approval request for any response (where available). + /// Split the approval responses into two lists: approved and rejected, with their request messages (where available). + /// + /// We return the messages containing the approval requests since these are the same messages that originally contained the FunctionCallContent from the downstream service. + /// We can then use the metadata from these messages when we re-create the FunctionCallContent messages/updates to return to the caller. This way, when we finally do return + /// the FuncionCallContent to users it's part of a message/update that contains the same metadata as originally returned to the downstream service. + /// + private static (List? approvals, List? rejections) ProcessApprovalRequestsAndResponses(List messages) { - // Get the list of function call ids that are already executed. - var functionResultCallIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.CallId).ToHashSet(); - List? notExecutedApprovedFunctionCalls = null; - List? notExecutedRejectedFunctionCalls = null; - HashSet? requestCallIds = null; + Dictionary? allApprovalRequestsMessages = null; + List? allApprovalResponses = null; + HashSet? approvalRequestCallIds = null; + HashSet? functionResultCallIds = null; for (int i = 0; i < messages.Count; i++) { @@ -1226,41 +1247,39 @@ private static (List? approvals, List(); + allApprovalRequestsMessages.Add(approvalRequest.Id, message); continue; } - // Build a list of responses that are not yet executed, so that we can execute them before we invoke the LLM. - // We can also remove them from the list of messages, since they will be turned back into FunctionCallContent and FunctionResultContent. - if (content is FunctionApprovalResponseContent response && !functionResultCallIds.Contains(response.FunctionCall.CallId)) + if (content is FunctionApprovalResponseContent approvalResponse) { - if (response.Approved) - { - notExecutedApprovedFunctionCalls ??= []; - notExecutedApprovedFunctionCalls.Add(response); - } - else - { - notExecutedRejectedFunctionCalls ??= []; - notExecutedRejectedFunctionCalls.Add(response); - } - + allApprovalResponses ??= []; + allApprovalResponses.Add(approvalResponse); continue; } @@ -1279,21 +1298,44 @@ private static (List? approvals, List 0) + // Validation: If we got an approval for each request, we should have no call ids left. + if (approvalRequestCallIds?.Count is > 0) { - Throw.InvalidOperationException($"FunctionApprovalRequestContent found with FunctionCall.CallId(s) '{string.Join(", ", requestCallIds)}' that have no matching FunctionApprovalResponseContent."); + Throw.InvalidOperationException($"FunctionApprovalRequestContent found with FunctionCall.CallId(s) '{string.Join(", ", approvalRequestCallIds)}' that have no matching FunctionApprovalResponseContent."); + } + + List? notExecutedApprovedFunctionCalls = null; + List? notExecutedRejectedFunctionCalls = null; + + for (int i = 0; i < (allApprovalResponses?.Count ?? 0); i++) + { + var approvalResponse = allApprovalResponses![i]; + + // Skip any approval responses that have already been executed. + if (functionResultCallIds?.Contains(approvalResponse.FunctionCall.CallId) is not true) + { + ChatMessage? requestMessage = null; + allApprovalRequestsMessages?.TryGetValue(approvalResponse.FunctionCall.CallId, out requestMessage); + + // Split the responses into approved and rejected. + if (approvalResponse.Approved) + { + notExecutedApprovedFunctionCalls ??= []; + notExecutedApprovedFunctionCalls.Add(new ApprovalResultWithRequestMessage { Response = approvalResponse, RequestMessage = requestMessage }); + } + else + { + notExecutedRejectedFunctionCalls ??= []; + notExecutedRejectedFunctionCalls.Add(new ApprovalResultWithRequestMessage { Response = approvalResponse, RequestMessage = requestMessage }); + } + } } return (notExecutedApprovedFunctionCalls, notExecutedRejectedFunctionCalls); @@ -1337,46 +1379,11 @@ private static async Task ReplaceFunctionCallsWithApprovalRequests(IList new(functionCall.CallId, functionCall) - { - OriginalMessageMetadata = new() - { - MessageId = message.MessageId, - AuthorName = message.AuthorName, - AdditionalProperties = message.AdditionalProperties, - RawRepresentation = message.RawRepresentation - } - }; - - private static FunctionApprovalRequestContent ConvertToApprovalRequestContent(FunctionCallContent functionCall, ChatResponseUpdate update) - => new(functionCall.CallId, functionCall) - { - OriginalMessageMetadata = new() - { - MessageId = update.MessageId, - AuthorName = update.AuthorName, - AdditionalProperties = update.AdditionalProperties, - RawRepresentation = update.RawRepresentation, - CreatedAt = update.CreatedAt, - ResponseId = update.ResponseId - } - }; - - private static ChatMessage ConvertToFunctionCallContentMessage(FunctionApprovalResponseContent response) - => new(ChatRole.Assistant, [response.FunctionCall]) - { - MessageId = response.OriginalMessageMetadata?.MessageId, - AuthorName = response.OriginalMessageMetadata?.AuthorName, - AdditionalProperties = response.OriginalMessageMetadata?.AdditionalProperties, - RawRepresentation = response.OriginalMessageMetadata?.RawRepresentation, - }; - private static TimeSpan GetElapsedTime(long startingTimestamp) => #if NET Stopwatch.GetElapsedTime(startingTimestamp); @@ -1450,4 +1457,10 @@ public enum FunctionInvocationStatus /// The function call failed with an exception. Exception, } + + private struct ApprovalResultWithRequestMessage + { + public FunctionApprovalResponseContent Response { get; set; } + public ChatMessage? RequestMessage { get; set; } + } } From 6d8b20e1c439989e35e24cba2337b26fa09b21c2 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:02:59 +0100 Subject: [PATCH 34/53] Small code refactoring and clarifying comments. --- ...nInvokingChatClientWithBuiltInApprovals.cs | 211 +++++++++--------- 1 file changed, 106 insertions(+), 105 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs index 1dd5186b6e..efd41d8b5d 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/FunctionInvokingChatClientWithBuiltInApprovals.cs @@ -413,18 +413,21 @@ public override async IAsyncEnumerable GetStreamingResponseA // message with multiple content items. We could also use different message IDs per tool content, // but there's no benefit to doing so. string toolResponseId = Guid.NewGuid().ToString("N"); + + // We also need a synthetic ID for the function call content for approved function calls + // where we don't know what the original message id of the function call was. string functionCallContentFallbackMessageId = Guid.NewGuid().ToString("N"); ApprovalRequiredAIFunction[]? approvalRequiredFunctions = (options?.Tools ?? []).Concat(AdditionalTools ?? []).OfType().ToArray(); - bool hasApprovalRequiringFunctions = (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && approvalRequiredFunctions.Length > 0; + bool hasApprovalRequiringFunctions = approvalRequiredFunctions.Length > 0; var shouldTerminateFromPreInvocationFunctionCalls = false; - // We should maintain a list of chat messages that need to be returned to the caller - // as part of the next response that were generated before the first iteration. + // We should maintain a list of chat messages that were generated before the first iteration + // and that need to be returned to the caller as part of the next response. List? augmentedPreInvocationHistory = null; - // Remove any approval requests and approval request/response pairs that have already been executed. + // Extract and remove any approval requests and approval request/response pairs that have already been executed. var notExecutedResponses = ProcessApprovalRequestsAndResponses(originalMessages); // Generate failed function result contents for any rejected requests. @@ -464,20 +467,7 @@ public override async IAsyncEnumerable GetStreamingResponseA { foreach (var message in augmentedPreInvocationHistory ?? []) { - var toolResultUpdate = new ChatResponseUpdate - { - AdditionalProperties = message.AdditionalProperties, - AuthorName = message.AuthorName, - ConversationId = options?.ConversationId, - CreatedAt = DateTimeOffset.UtcNow, - Contents = message.Contents, - RawRepresentation = message.RawRepresentation, - ResponseId = message.MessageId, - MessageId = message.MessageId, - Role = message.Role, - }; - - yield return toolResultUpdate; + yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } } @@ -533,6 +523,7 @@ public override async IAsyncEnumerable GetStreamingResponseA // we can yield the update as-is. lastYieldedUpdateIndex++; yield return update; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } else { @@ -565,6 +556,7 @@ public override async IAsyncEnumerable GetStreamingResponseA } } yield return updateToYield; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } } else @@ -572,6 +564,8 @@ public override async IAsyncEnumerable GetStreamingResponseA // We don't have any appoval requiring function calls yet, but we may receive some in future // so we cannot yield the updates yet. We'll just keep them in the updates list // for later. + // We will yield the updates as soon as we receive a function call content that requires approval or + // when we reach the end of the updates stream. } } // ** Approvals modifications - end **// @@ -615,20 +609,7 @@ public override async IAsyncEnumerable GetStreamingResponseA // includes all activities, including generated function results. foreach (var message in modeAndMessages.MessagesAdded) { - var toolResultUpdate = new ChatResponseUpdate - { - AdditionalProperties = message.AdditionalProperties, - AuthorName = message.AuthorName, - ConversationId = response.ConversationId, - CreatedAt = DateTimeOffset.UtcNow, - Contents = message.Contents, - RawRepresentation = message.RawRepresentation, - ResponseId = toolResponseId, - MessageId = toolResponseId, // See above for why this can be the same as ResponseId - Role = message.Role, - }; - - yield return toolResultUpdate; + yield return ConvertToolResultMessageToUpdate(message, response.ConversationId, toolResponseId); Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } @@ -1148,66 +1129,6 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul context.Function.InvokeAsync(context.Arguments, cancellationToken); } - /// - /// If we have any rejected approval responses, we need to generate failed function results for them. - /// - /// Any rejected approval responses. - /// The message id to use for the tool response. - /// Optional message id to use if no original approval request message is availble to copy from. - /// The function call and result content for the rejected calls. - /// The list of messages generated before the first invocation, that need to be added to the user response. - private static void GenerateRejectedFunctionResults( - List? rejections, - string? toolResponseId, - string? fallbackMessageId, - ref List? rejectedFunctionCallMessages, - ref List? augmentedPreInvocationHistory) - { - if (rejections is { Count: > 0 }) - { - rejectedFunctionCallMessages = []; - foreach (var rejectedCall in rejections) - { - // Create a FunctionResultContent for the rejected function call. - var functionResult = new FunctionResultContent(rejectedCall.Response.FunctionCall.CallId, "Error: Function invocation approval was not granted."); - - rejectedFunctionCallMessages.Add(ConvertToFunctionCallContentMessage(rejectedCall, fallbackMessageId)); - rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult]) { MessageId = toolResponseId }); - } - - augmentedPreInvocationHistory ??= []; - augmentedPreInvocationHistory.AddRange(rejectedFunctionCallMessages); - } - } - - private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWithRequestMessage resultWithRequestMessage, string? fallbackMessageId) - { - if (resultWithRequestMessage.RequestMessage is not null) - { - var functionCallMessage = resultWithRequestMessage.RequestMessage.Clone(); - functionCallMessage.Contents = [resultWithRequestMessage.Response.FunctionCall]; - functionCallMessage.MessageId ??= fallbackMessageId; - return functionCallMessage; - } - - return new ChatMessage(ChatRole.Assistant, [resultWithRequestMessage.Response.FunctionCall]) { MessageId = fallbackMessageId }; - } - - /// - /// Insert the given at the start of the 's messages. - /// - private static void UpdateResponseWithPreInvocationHistory(ChatResponse response, List? augmentedPreInvocationHistory) - { - if (augmentedPreInvocationHistory?.Count > 0) - { - // Since these messages are pre-invocation, we want to insert them at the start of the response messages. - for (int i = augmentedPreInvocationHistory.Count - 1; i >= 0; i--) - { - response.Messages.Insert(0, augmentedPreInvocationHistory[i]); - } - } - } - /// /// This method extracts the approval requests and responses from the provided list of messages, validates them, filters them to ones that require execution and splits them into approved and rejected. /// @@ -1341,26 +1262,75 @@ private static (List? approvals, ListReplaces any from with . - private static async Task ReplaceFunctionCallsWithApprovalRequests(IList messages, IList? requestOptionsTools, IList? additionalTools) + /// + /// If we have any rejected approval responses, we need to generate failed function results for them. + /// + /// Any rejected approval responses. + /// The message id to use for the tool response. + /// Optional message id to use if no original approval request message is availble to copy from. + /// The function call and result content for the rejected calls. + /// The list of messages generated before the first invocation, that need to be added to the user response. + private static void GenerateRejectedFunctionResults( + List? rejections, + string? toolResponseId, + string? fallbackMessageId, + ref List? rejectedFunctionCallMessages, + ref List? augmentedPreInvocationHistory) { - bool anyApprovalRequired = false; - List<(int, int)>? functionsToReplace = null; + if (rejections is { Count: > 0 }) + { + rejectedFunctionCallMessages = []; + foreach (var rejectedCall in rejections) + { + // We want to add the original FunctionCallContent that was replaced with an ApprovalRequest back into the chat history as well + // otherwise we would have an ApprovalRequest, ApprovalResponse and FunctionResultContent, but no FunctionCallContent matching the FunctionResultContent. + rejectedFunctionCallMessages.Add(ConvertToFunctionCallContentMessage(rejectedCall, fallbackMessageId)); + + // Create a FunctionResultContent for the rejected function call. + var functionResult = new FunctionResultContent(rejectedCall.Response.FunctionCall.CallId, "Error: Function invocation approval was not granted."); + rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult]) { MessageId = toolResponseId }); + } + + augmentedPreInvocationHistory ??= []; + augmentedPreInvocationHistory.AddRange(rejectedFunctionCallMessages); + } + } + + private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWithRequestMessage resultWithRequestMessage, string? fallbackMessageId) + { + if (resultWithRequestMessage.RequestMessage is not null) + { + var functionCallMessage = resultWithRequestMessage.RequestMessage.Clone(); + functionCallMessage.Contents = [resultWithRequestMessage.Response.FunctionCall]; + functionCallMessage.MessageId ??= fallbackMessageId; + return functionCallMessage; + } + return new ChatMessage(ChatRole.Assistant, [resultWithRequestMessage.Response.FunctionCall]) { MessageId = fallbackMessageId }; + } + + /// + /// Replaces all from with + /// if any one of them requires approval. + /// + private static async Task ReplaceFunctionCallsWithApprovalRequests(IList messages, IList? requestOptionsTools, IList? additionalTools) + { ApprovalRequiredAIFunction[]? approvalRequiredFunctions = null; - int count = messages.Count; - for (int i = 0; i < count; i++) + bool anyApprovalRequired = false; + List<(int, int)>? allFunctionCallContentIndices = null; + + // Build a list of the indices of all FunctionCallContent items. + // Also check if any of them require approval. + for (int i = 0; i < messages.Count; i++) { var content = messages[i].Contents; - int contentCount = content.Count; - - for (int j = 0; j < contentCount; j++) + for (int j = 0; j < content.Count; j++) { if (content[j] is FunctionCallContent functionCall) { - functionsToReplace ??= []; - functionsToReplace.Add((i, j)); + allFunctionCallContentIndices ??= []; + allFunctionCallContentIndices.Add((i, j)); approvalRequiredFunctions ??= (requestOptionsTools ?? []).Concat(additionalTools ?? []) .OfType() @@ -1373,9 +1343,9 @@ private static async Task ReplaceFunctionCallsWithApprovalRequests(IList + /// Insert the given at the start of the 's messages. + /// + private static void UpdateResponseWithPreInvocationHistory(ChatResponse response, List? augmentedPreInvocationHistory) + { + if (augmentedPreInvocationHistory?.Count > 0) + { + // Since these messages are pre-invocation, we want to insert them at the start of the response messages. + for (int i = augmentedPreInvocationHistory.Count - 1; i >= 0; i--) + { + response.Messages.Insert(0, augmentedPreInvocationHistory[i]); + } + } + } + + private static ChatResponseUpdate ConvertToolResultMessageToUpdate(ChatMessage message, string? converationId, string? messageId) + { + return new() + { + AdditionalProperties = message.AdditionalProperties, + AuthorName = message.AuthorName, + ConversationId = converationId, + CreatedAt = DateTimeOffset.UtcNow, + Contents = message.Contents, + RawRepresentation = message.RawRepresentation, + ResponseId = messageId, + MessageId = messageId, + Role = message.Role, + }; + } + private static TimeSpan GetElapsedTime(long startingTimestamp) => #if NET Stopwatch.GetElapsedTime(startingTimestamp); From 2bc90af62a72c13afb77dc1ee3388e8ffbfa0dea Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:19:49 +0100 Subject: [PATCH 35/53] Remove failed POCs --- .../UserInputOriginalMetadata.cs | 31 - ...ApprovalAwareFunctionInvokingChatClient.cs | 1010 ---------------- .../MEAI/NonInvocableAIFunction.cs | 8 - ...nvocableAwareFunctionInvokingChatClient.cs | 1027 ----------------- .../PostFICCApprovalGeneratingChatClient.cs | 209 ---- .../PreFICCApprovalGeneratingChatClient.cs | 207 ---- 6 files changed, 2492 deletions(-) delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputOriginalMetadata.cs delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalAwareFunctionInvokingChatClient.cs delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputOriginalMetadata.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputOriginalMetadata.cs deleted file mode 100644 index 380d87b661..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputOriginalMetadata.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents metadata for the original message or update that initiated a user input request. -/// -public class UserInputOriginalMetadata -{ - /// Gets or sets the ID of the chat message or update. - public string? MessageId { get; set; } - - /// Gets or sets the name of the author of the message or update. - public string? AuthorName { get; set; } - - /// Gets or sets any additional properties associated with the message or update. - public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } - - /// Gets or sets a timestamp for the response update. - public DateTimeOffset? CreatedAt { get; set; } - - /// Gets or sets the ID of the response of which the update is a part. - public string? ResponseId { get; set; } - - /// Gets or sets the raw representation of the chat message or update from an underlying implementation. - [JsonIgnore] - public object? RawRepresentation { get; set; } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalAwareFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalAwareFunctionInvokingChatClient.cs deleted file mode 100644 index 0e9596d509..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalAwareFunctionInvokingChatClient.cs +++ /dev/null @@ -1,1010 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Runtime.ExceptionServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Shared.Diagnostics; - -#pragma warning disable CA2213 // Disposable fields should be disposed -#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test -#pragma warning disable SA1202 // 'protected' members should come before 'private' members -#pragma warning disable S107 // Methods should not have too many parameters - -#pragma warning disable IDE0009 // Member access should be qualified. -#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task -#pragma warning disable VSTHRD111 // Use ConfigureAwait(bool) - -namespace Microsoft.Extensions.AI; - -/// -/// A delegating chat client that invokes functions defined on . -/// Include this in a chat pipeline to resolve function calls automatically. -/// -/// -/// -/// When this client receives a in a chat response, it responds -/// by calling the corresponding defined in , -/// producing a that it sends back to the inner client. This loop -/// is repeated until there are no more function calls to make, or until another stop condition is met, -/// such as hitting . -/// -/// -/// The provided implementation of is thread-safe for concurrent use so long as the -/// instances employed as part of the supplied are also safe. -/// The property can be used to control whether multiple function invocation -/// requests as part of the same request are invocable concurrently, but even with that set to -/// (the default), multiple concurrent requests to this same instance and using the same tools could result in those -/// tools being used concurrently (one per request). For example, a function that accesses the HttpContext of a specific -/// ASP.NET web request should only be used as part of a single at a time, and only with -/// set to , in case the inner client decided to issue multiple -/// invocation requests to that same function. -/// -/// -public partial class ApprovalAwareFunctionInvokingChatClient : DelegatingChatClient -{ - /// The for the current function invocation. - private static readonly AsyncLocal s_currentContext = new(); - - /// Gets the specified when constructing the , if any. - protected IServiceProvider? FunctionInvocationServices { get; } - - /// The logger to use for logging information about function invocation. - private readonly ILogger _logger; - - /// The to use for telemetry. - /// This component does not own the instance and should not dispose it. - private readonly ActivitySource? _activitySource; - - /// Maximum number of roundtrips allowed to the inner client. - private int _maximumIterationsPerRequest = 40; // arbitrary default to prevent runaway execution - - /// Maximum number of consecutive iterations that are allowed contain at least one exception result. If the limit is exceeded, we rethrow the exception instead of continuing. - private int _maximumConsecutiveErrorsPerRequest = 3; - - /// - /// Initializes a new instance of the class. - /// - /// The underlying , or the next instance in a chain of clients. - /// An to use for logging information about function invocation. - /// An optional to use for resolving services required by the instances being invoked. - public ApprovalAwareFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) - : base(innerClient) - { - _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; - _activitySource = innerClient.GetService(); - FunctionInvocationServices = functionInvocationServices; - } - - /// - /// Gets or sets the for the current function invocation. - /// - /// - /// This value flows across async calls. - /// - public static FunctionInvocationContext? CurrentContext - { - get => s_currentContext.Value; - protected set => s_currentContext.Value = value; - } - - /// - /// Gets or sets a value indicating whether detailed exception information should be included - /// in the chat history when calling the underlying . - /// - /// - /// if the full exception message is added to the chat history - /// when calling the underlying . - /// if a generic error message is included in the chat history. - /// The default value is . - /// - /// - /// - /// Setting the value to prevents the underlying language model from disclosing - /// raw exception details to the end user, since it doesn't receive that information. Even in this - /// case, the raw object is available to application code by inspecting - /// the property. - /// - /// - /// Setting the value to can help the underlying bypass problems on - /// its own, for example by retrying the function call with different arguments. However it might - /// result in disclosing the raw exception information to external users, which can be a security - /// concern depending on the application scenario. - /// - /// - /// Changing the value of this property while the client is in use might result in inconsistencies - /// as to whether detailed errors are provided during an in-flight request. - /// - /// - public bool IncludeDetailedErrors { get; set; } - - /// - /// Gets or sets a value indicating whether to allow concurrent invocation of functions. - /// - /// - /// if multiple function calls can execute in parallel. - /// if function calls are processed serially. - /// The default value is . - /// - /// - /// An individual response from the inner client might contain multiple function call requests. - /// By default, such function calls are processed serially. Set to - /// to enable concurrent invocation such that multiple function calls can execute in parallel. - /// - public bool AllowConcurrentInvocation { get; set; } - - /// - /// Gets or sets the maximum number of iterations per request. - /// - /// - /// The maximum number of iterations per request. - /// The default value is 40. - /// - /// - /// - /// Each request to this might end up making - /// multiple requests to the inner client. Each time the inner client responds with - /// a function call request, this client might perform that invocation and send the results - /// back to the inner client in a new request. This property limits the number of times - /// such a roundtrip is performed. The value must be at least one, as it includes the initial request. - /// - /// - /// Changing the value of this property while the client is in use might result in inconsistencies - /// as to how many iterations are allowed for an in-flight request. - /// - /// - public int MaximumIterationsPerRequest - { - get => _maximumIterationsPerRequest; - set - { - if (value < 1) - { - Throw.ArgumentOutOfRangeException(nameof(value)); - } - - _maximumIterationsPerRequest = value; - } - } - - /// - /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error. - /// - /// - /// The maximum number of consecutive iterations that are allowed to fail with an error. - /// The default value is 3. - /// - /// - /// - /// When function invocations fail with an exception, the - /// continues to make requests to the inner client, optionally supplying exception information (as - /// controlled by ). This allows the to - /// recover from errors by trying other function parameters that may succeed. - /// - /// - /// However, in case function invocations continue to produce exceptions, this property can be used to - /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be - /// rethrown to the caller. - /// - /// - /// If the value is set to zero, all function calling exceptions immediately terminate the function - /// invocation loop and the exception will be rethrown to the caller. - /// - /// - /// Changing the value of this property while the client is in use might result in inconsistencies - /// as to how many iterations are allowed for an in-flight request. - /// - /// - public int MaximumConsecutiveErrorsPerRequest - { - get => _maximumConsecutiveErrorsPerRequest; - set => _maximumConsecutiveErrorsPerRequest = Throw.IfLessThan(value, 0); - } - - /// Gets or sets a collection of additional tools the client is able to invoke. - /// - /// These will not impact the requests sent by the , which will pass through the - /// unmodified. However, if the inner client requests the invocation of a tool - /// that was not in , this collection will also be consulted - /// to look for a corresponding tool to invoke. This is useful when the service may have been pre-configured to be aware - /// of certain tools that aren't also sent on each individual request. - /// - public IList? AdditionalTools { get; set; } - - /// Gets or sets a delegate used to invoke instances. - /// - /// By default, the protected method is called for each to be invoked, - /// invoking the instance and returning its result. If this delegate is set to a non- value, - /// will replace its normal invocation with a call to this delegate, enabling - /// this delegate to assume all invocation handling of the function. - /// - public Func>? FunctionInvoker { get; set; } - - /// - public override async Task GetResponseAsync( - IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(messages); - - // A single request into this GetResponseAsync may result in multiple requests to the inner client. - // Create an activity to group them together for better observability. - using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}"); - - // Copy the original messages in order to avoid enumerating the original messages multiple times. - // The IEnumerable can represent an arbitrary amount of work. - List originalMessages = [.. messages]; - messages = originalMessages; - - List? augmentedHistory = null; // the actual history of messages sent on turns other than the first - ChatResponse? response = null; // the response from the inner client, which is possibly modified and then eventually returned - List? responseMessages = null; // tracked list of messages, across multiple turns, to be used for the final response - UsageDetails? totalUsage = null; // tracked usage across all turns, to be used for the final response - List? functionCallContents = null; // function call contents that need responding to in the current turn - bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set - int consecutiveErrorCount = 0; - - for (int iteration = 0; ; iteration++) - { - functionCallContents?.Clear(); - - // Make the call to the inner client. - response = await base.GetResponseAsync(messages, options, cancellationToken); - if (response is null) - { - Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); - } - - // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. - bool requiresFunctionInvocation = - (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && - iteration < MaximumIterationsPerRequest && - CopyFunctionCalls(response.Messages, ref functionCallContents); - - // In a common case where we make a request and there's no function calling work required, - // fast path out by just returning the original response. - if (iteration == 0 && !requiresFunctionInvocation) - { - return response; - } - - // Track aggregate details from the response, including all of the response messages and usage details. - (responseMessages ??= []).AddRange(response.Messages); - if (response.Usage is not null) - { - if (totalUsage is not null) - { - totalUsage.Add(response.Usage); - } - else - { - totalUsage = response.Usage; - } - } - - // If there are no tools to call, or for any other reason we should stop, we're done. - // Break out of the loop and allow the handling at the end to configure the response - // with aggregated data from previous requests. - if (!requiresFunctionInvocation) - { - break; - } - - // Prepare the history for the next iteration. - FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); - - // Add the responses from the function calls into the augmented history and also into the tracked - // list of response messages. - var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); - responseMessages.AddRange(modeAndMessages.MessagesAdded); - consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; - - if (modeAndMessages.ShouldTerminate) - { - break; - } - - UpdateOptionsForNextIteration(ref options, response.ConversationId); - } - - Debug.Assert(responseMessages is not null, "Expected to only be here if we have response messages."); - response.Messages = responseMessages!; - response.Usage = totalUsage; - - AddUsageTags(activity, totalUsage); - - return response; - } - - /// - public override async IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(messages); - - // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client. - // Create an activity to group them together for better observability. - using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}"); - UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes - - // Copy the original messages in order to avoid enumerating the original messages multiple times. - // The IEnumerable can represent an arbitrary amount of work. - List originalMessages = [.. messages]; - messages = originalMessages; - - List? augmentedHistory = null; // the actual history of messages sent on turns other than the first - List? functionCallContents = null; // function call contents that need responding to in the current turn - List? responseMessages = null; // tracked list of messages, across multiple turns, to be used in fallback cases to reconstitute history - bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set - List updates = []; // updates from the current response - int consecutiveErrorCount = 0; - - for (int iteration = 0; ; iteration++) - { - updates.Clear(); - functionCallContents?.Clear(); - - await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) - { - if (update is null) - { - Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}."); - } - - updates.Add(update); - - _ = CopyFunctionCalls(update.Contents, ref functionCallContents); - - if (totalUsage is not null) - { - IList contents = update.Contents; - int contentsCount = contents.Count; - for (int i = 0; i < contentsCount; i++) - { - if (contents[i] is UsageContent uc) - { - totalUsage.Add(uc.Details); - } - } - } - - yield return update; - Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 - } - - // If there are no tools to call, or for any other reason we should stop, return the response. - if (functionCallContents is not { Count: > 0 } || - (options?.Tools is not { Count: > 0 } && AdditionalTools is not { Count: > 0 }) || - iteration >= _maximumIterationsPerRequest) - { - break; - } - - // Reconstitute a response from the response updates. - var response = updates.ToChatResponse(); - (responseMessages ??= []).AddRange(response.Messages); - - // Prepare the history for the next iteration. - FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); - - // Process all of the functions, adding their results into the history. - var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken); - responseMessages.AddRange(modeAndMessages.MessagesAdded); - consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; - - // This is a synthetic ID since we're generating the tool messages instead of getting them from - // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to - // use the same message ID for all of them within a given iteration, as this is a single logical - // message with multiple content items. We could also use different message IDs per tool content, - // but there's no benefit to doing so. - string toolResponseId = Guid.NewGuid().ToString("N"); - - // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages - // includes all activities, including generated function results. - foreach (var message in modeAndMessages.MessagesAdded) - { - var toolResultUpdate = new ChatResponseUpdate - { - AdditionalProperties = message.AdditionalProperties, - AuthorName = message.AuthorName, - ConversationId = response.ConversationId, - CreatedAt = DateTimeOffset.UtcNow, - Contents = message.Contents, - RawRepresentation = message.RawRepresentation, - ResponseId = toolResponseId, - MessageId = toolResponseId, // See above for why this can be the same as ResponseId - Role = message.Role, - }; - - yield return toolResultUpdate; - Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 - } - - if (modeAndMessages.ShouldTerminate) - { - break; - } - - UpdateOptionsForNextIteration(ref options, response.ConversationId); - } - - AddUsageTags(activity, totalUsage); - } - - /// Adds tags to for usage details in . - private static void AddUsageTags(Activity? activity, UsageDetails? usage) - { - if (usage is not null && activity is { IsAllDataRequested: true }) - { - if (usage.InputTokenCount is long inputTokens) - { - _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, (int)inputTokens); - } - - if (usage.OutputTokenCount is long outputTokens) - { - _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.OutputTokens, (int)outputTokens); - } - } - } - - /// Prepares the various chat message lists after a response from the inner client and before invoking functions. - /// The original messages provided by the caller. - /// The messages reference passed to the inner client. - /// The augmented history containing all the messages to be sent. - /// The most recent response being handled. - /// A list of all response messages received up until this point. - /// Whether the previous iteration's response had a conversation ID. - private static void FixupHistories( - IEnumerable originalMessages, - ref IEnumerable messages, - [NotNull] ref List? augmentedHistory, - ChatResponse response, - List allTurnsResponseMessages, - ref bool lastIterationHadConversationId) - { - // We're now going to need to augment the history with function result contents. - // That means we need a separate list to store the augmented history. - if (response.ConversationId is not null) - { - // The response indicates the inner client is tracking the history, so we don't want to send - // anything we've already sent or received. - if (augmentedHistory is not null) - { - augmentedHistory.Clear(); - } - else - { - augmentedHistory = []; - - // This is needed to allow the ApprovalGeneratingChatClient to work. - // It adds all FunctionApprovalResponseContent that were provided by the caller back into - // the downstream request messages so that they can be processed by the inner client. - // The ones that have matching FunctionCallContent are filtered out, and the ones without matching - // FunctionCallContent are converted to rejected FunctionCallContent. - var functionApprovals = originalMessages.SelectMany(x => x.Contents).OfType().Cast().ToList(); - if (functionApprovals.Count > 0) - { - augmentedHistory.Add(new ChatMessage(ChatRole.User, functionApprovals)); - } - } - - lastIterationHadConversationId = true; - } - else if (lastIterationHadConversationId) - { - // In the very rare case where the inner client returned a response with a conversation ID but then - // returned a subsequent response without one, we want to reconstitute the full history. To do that, - // we can populate the history with the original chat messages and then all of the response - // messages up until this point, which includes the most recent ones. - augmentedHistory ??= []; - augmentedHistory.Clear(); - augmentedHistory.AddRange(originalMessages); - augmentedHistory.AddRange(allTurnsResponseMessages); - - lastIterationHadConversationId = false; - } - else - { - // If augmentedHistory is already non-null, then we've already populated it with everything up - // until this point (except for the most recent response). If it's null, we need to seed it with - // the chat history provided by the caller. - augmentedHistory ??= originalMessages.ToList(); - - // Now add the most recent response messages. - augmentedHistory.AddMessages(response); - - lastIterationHadConversationId = false; - } - - // Use the augmented history as the new set of messages to send. - messages = augmentedHistory; - } - - /// Copies any from to . - private static bool CopyFunctionCalls( - IList messages, [NotNullWhen(true)] ref List? functionCalls) - { - bool any = false; - int count = messages.Count; - for (int i = 0; i < count; i++) - { - any |= CopyFunctionCalls(messages[i].Contents, ref functionCalls); - } - - return any; - } - - /// Copies any from to . - private static bool CopyFunctionCalls( - IList content, [NotNullWhen(true)] ref List? functionCalls) - { - bool any = false; - int count = content.Count; - for (int i = 0; i < count; i++) - { - if (content[i] is FunctionCallContent functionCall) - { - (functionCalls ??= []).Add(functionCall); - any = true; - } - } - - return any; - } - - private static void UpdateOptionsForNextIteration(ref ChatOptions? options, string? conversationId) - { - if (options is null) - { - if (conversationId is not null) - { - options = new() { ConversationId = conversationId }; - } - } - else if (options.ToolMode is RequiredChatToolMode) - { - // We have to reset the tool mode to be non-required after the first iteration, - // as otherwise we'll be in an infinite loop. - options = options.Clone(); - options.ToolMode = null; - options.ConversationId = conversationId; - } - else if (options.ConversationId != conversationId) - { - // As with the other modes, ensure we've propagated the chat conversation ID to the options. - // We only need to clone the options if we're actually mutating it. - options = options.Clone(); - options.ConversationId = conversationId; - } - } - - /// - /// Processes the function calls in the list. - /// - /// The current chat contents, inclusive of the function call contents being processed. - /// The options used for the response being processed. - /// The function call contents representing the functions to be invoked. - /// The iteration number of how many roundtrips have been made to the inner client. - /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors. - /// Whether the function calls are being processed in a streaming context. - /// The to monitor for cancellation requests. - /// A value indicating how the caller should proceed. - private async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync( - List messages, ChatOptions? options, List functionCallContents, int iteration, int consecutiveErrorCount, - bool isStreaming, CancellationToken cancellationToken) - { - // We must add a response for every tool call, regardless of whether we successfully executed it or not. - // If we successfully execute it, we'll add the result. If we don't, we'll add an error. - - Debug.Assert(functionCallContents.Count > 0, "Expected at least one function call."); - var shouldTerminate = false; - var captureCurrentIterationExceptions = consecutiveErrorCount < _maximumConsecutiveErrorsPerRequest; - - // Process all functions. If there's more than one and concurrent invocation is enabled, do so in parallel. - if (functionCallContents.Count == 1) - { - FunctionInvocationResult result = await ProcessFunctionCallAsync( - messages, options, functionCallContents, - iteration, 0, captureCurrentIterationExceptions, isStreaming, cancellationToken); - - IList addedMessages = CreateResponseMessages([result]); - ThrowIfNoFunctionResultsAdded(addedMessages); - UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); - messages.AddRange(addedMessages); - - return (result.Terminate, consecutiveErrorCount, addedMessages); - } - else - { - List results = []; - - if (AllowConcurrentInvocation) - { - // Rather than awaiting each function before invoking the next, invoke all of them - // and then await all of them. We avoid forcibly introducing parallelism via Task.Run, - // but if a function invocation completes asynchronously, its processing can overlap - // with the processing of other the other invocation invocations. - results.AddRange(await Task.WhenAll( - from callIndex in Enumerable.Range(0, functionCallContents.Count) - select ProcessFunctionCallAsync( - messages, options, functionCallContents, - iteration, callIndex, captureExceptions: true, isStreaming, cancellationToken))); - - shouldTerminate = results.Any(r => r.Terminate); - } - else - { - // Invoke each function serially. - for (int callIndex = 0; callIndex < functionCallContents.Count; callIndex++) - { - var functionResult = await ProcessFunctionCallAsync( - messages, options, functionCallContents, - iteration, callIndex, captureCurrentIterationExceptions, isStreaming, cancellationToken); - - results.Add(functionResult); - - // If any function requested termination, we should stop right away. - if (functionResult.Terminate) - { - shouldTerminate = true; - break; - } - } - } - - IList addedMessages = CreateResponseMessages(results.ToArray()); - ThrowIfNoFunctionResultsAdded(addedMessages); - UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); - messages.AddRange(addedMessages); - - return (shouldTerminate, consecutiveErrorCount, addedMessages); - } - } - -#pragma warning disable CA1851 // Possible multiple enumerations of 'IEnumerable' collection - /// - /// Updates the consecutive error count, and throws an exception if the count exceeds the maximum. - /// - /// Added messages. - /// Consecutive error count. - /// Thrown if the maximum consecutive error count is exceeded. - private void UpdateConsecutiveErrorCountOrThrow(IList added, ref int consecutiveErrorCount) - { - var allExceptions = added.SelectMany(m => m.Contents.OfType()) - .Select(frc => frc.Exception!) - .Where(e => e is not null); - - if (allExceptions.Any()) - { - consecutiveErrorCount++; - if (consecutiveErrorCount > _maximumConsecutiveErrorsPerRequest) - { - var allExceptionsArray = allExceptions.ToArray(); - if (allExceptionsArray.Length == 1) - { - ExceptionDispatchInfo.Capture(allExceptionsArray[0]).Throw(); - } - else - { - throw new AggregateException(allExceptionsArray); - } - } - } - else - { - consecutiveErrorCount = 0; - } - } -#pragma warning restore CA1851 - - /// - /// Throws an exception if doesn't create any messages. - /// - private void ThrowIfNoFunctionResultsAdded(IList? messages) - { - if (messages is null || messages.Count == 0) - { - Throw.InvalidOperationException($"{GetType().Name}.{nameof(CreateResponseMessages)} returned null or an empty collection of messages."); - } - } - - /// Processes the function call described in []. - /// The current chat contents, inclusive of the function call contents being processed. - /// The options used for the response being processed. - /// The function call contents representing all the functions being invoked. - /// The iteration number of how many roundtrips have been made to the inner client. - /// The 0-based index of the function being called out of . - /// If true, handles function-invocation exceptions by returning a value with . Otherwise, rethrows. - /// Whether the function calls are being processed in a streaming context. - /// The to monitor for cancellation requests. - /// A value indicating how the caller should proceed. - private async Task ProcessFunctionCallAsync( - List messages, ChatOptions? options, List callContents, - int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, CancellationToken cancellationToken) - { - var callContent = callContents[functionCallIndex]; - - // Look up the AIFunction for the function call. If the requested function isn't available, send back an error. - AIFunction? aiFunction = FindAIFunction(options?.Tools, callContent.Name) ?? FindAIFunction(AdditionalTools, callContent.Name); - if (aiFunction is null) - { - return new(terminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null); - } - - FunctionInvocationContext context = new() - { - Function = aiFunction, - Arguments = new(callContent.Arguments) { Services = FunctionInvocationServices }, - Messages = messages, - Options = options, - CallContent = callContent, - Iteration = iteration, - FunctionCallIndex = functionCallIndex, - FunctionCount = callContents.Count, - IsStreaming = isStreaming - }; - - object? result; - try - { - result = await InstrumentedInvokeFunctionAsync(context, cancellationToken); - } - catch (Exception e) when (!cancellationToken.IsCancellationRequested) - { - if (!captureExceptions) - { - throw; - } - - return new( - terminate: false, - FunctionInvocationStatus.Exception, - callContent, - result: null, - exception: e); - } - - return new( - terminate: context.Terminate, - FunctionInvocationStatus.RanToCompletion, - callContent, - result, - exception: null); - - static AIFunction? FindAIFunction(IList? tools, string functionName) - { - if (tools is not null) - { - int count = tools.Count; - for (int i = 0; i < count; i++) - { - if (tools[i] is AIFunction function && function.Name == functionName) - { - return function; - } - } - } - - return null; - } - } - - /// Creates one or more response messages for function invocation results. - /// Information about the function call invocations and results. - /// A list of all chat messages created from . - protected virtual IList CreateResponseMessages( - ReadOnlySpan results) - { - var contents = new List(results.Length); - for (int i = 0; i < results.Length; i++) - { - contents.Add(CreateFunctionResultContent(results[i])); - } - - return [new(ChatRole.Tool, contents)]; - - FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult result) - { - _ = Throw.IfNull(result); - - object? functionResult; - if (result.Status == FunctionInvocationStatus.RanToCompletion) - { - functionResult = result.Result ?? "Success: Function completed."; - } - else - { - string message = result.Status switch - { - FunctionInvocationStatus.NotFound => $"Error: Requested function \"{result.CallContent.Name}\" not found.", - FunctionInvocationStatus.Exception => "Error: Function failed.", - _ => "Error: Unknown error.", - }; - - if (IncludeDetailedErrors && result.Exception is not null) - { - message = $"{message} Exception: {result.Exception.Message}"; - } - - functionResult = message; - } - - return new FunctionResultContent(result.CallContent.CallId, functionResult) { Exception = result.Exception }; - } - } - - /// Invokes the function asynchronously. - /// - /// The function invocation context detailing the function to be invoked and its arguments along with additional request information. - /// - /// The to monitor for cancellation requests. The default is . - /// The result of the function invocation, or if the function invocation returned . - /// is . - private async Task InstrumentedInvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) - { - _ = Throw.IfNull(context); - - using Activity? activity = _activitySource?.StartActivity( - $"{OpenTelemetryConsts.GenAI.ExecuteTool} {context.Function.Name}", - ActivityKind.Internal, - default(ActivityContext), - [ - new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"), - new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId), - new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name), - new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description), - ]); - - long startingTimestamp = 0; - if (_logger.IsEnabled(LogLevel.Debug)) - { - startingTimestamp = Stopwatch.GetTimestamp(); - if (_logger.IsEnabled(LogLevel.Trace)) - { - LogInvokingSensitive(context.Function.Name, LoggingHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions)); - } - else - { - LogInvoking(context.Function.Name); - } - } - - object? result = null; - try - { - CurrentContext = context; // doesn't need to be explicitly reset after, as that's handled automatically at async method exit - result = await InvokeFunctionAsync(context, cancellationToken); - } - catch (Exception e) - { - if (activity is not null) - { - _ = activity.SetTag("error.type", e.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, e.Message); - } - - if (e is OperationCanceledException) - { - LogInvocationCanceled(context.Function.Name); - } - else - { - LogInvocationFailed(context.Function.Name, e); - } - - throw; - } - finally - { - if (_logger.IsEnabled(LogLevel.Debug)) - { - TimeSpan elapsed = GetElapsedTime(startingTimestamp); - - if (result is not null && _logger.IsEnabled(LogLevel.Trace)) - { - LogInvocationCompletedSensitive(context.Function.Name, elapsed, LoggingHelpers.AsJson(result, context.Function.JsonSerializerOptions)); - } - else - { - LogInvocationCompleted(context.Function.Name, elapsed); - } - } - } - - return result; - } - - /// This method will invoke the function within the try block. - /// The function invocation context. - /// Cancellation token. - /// The function result. - protected virtual ValueTask InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) - { - _ = Throw.IfNull(context); - - return FunctionInvoker is { } invoker ? - invoker(context, cancellationToken) : - context.Function.InvokeAsync(context.Arguments, cancellationToken); - } - - private static TimeSpan GetElapsedTime(long startingTimestamp) => -#if NET - Stopwatch.GetElapsedTime(startingTimestamp); -#else - new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * ((double)TimeSpan.TicksPerSecond / Stopwatch.Frequency))); -#endif - - [LoggerMessage(LogLevel.Debug, "Invoking {MethodName}.", SkipEnabledCheck = true)] - private partial void LogInvoking(string methodName); - - [LoggerMessage(LogLevel.Trace, "Invoking {MethodName}({Arguments}).", SkipEnabledCheck = true)] - private partial void LogInvokingSensitive(string methodName, string arguments); - - [LoggerMessage(LogLevel.Debug, "{MethodName} invocation completed. Duration: {Duration}", SkipEnabledCheck = true)] - private partial void LogInvocationCompleted(string methodName, TimeSpan duration); - - [LoggerMessage(LogLevel.Trace, "{MethodName} invocation completed. Duration: {Duration}. Result: {Result}", SkipEnabledCheck = true)] - private partial void LogInvocationCompletedSensitive(string methodName, TimeSpan duration, string result); - - [LoggerMessage(LogLevel.Debug, "{MethodName} invocation canceled.")] - private partial void LogInvocationCanceled(string methodName); - - [LoggerMessage(LogLevel.Error, "{MethodName} invocation failed.")] - private partial void LogInvocationFailed(string methodName, Exception error); - - /// Provides information about the invocation of a function call. - public sealed class FunctionInvocationResult - { - /// - /// Initializes a new instance of the class. - /// - /// Indicates whether the caller should terminate the processing loop. - /// Indicates the status of the function invocation. - /// Contains information about the function call. - /// The result of the function call. - /// The exception thrown by the function call, if any. - internal FunctionInvocationResult(bool terminate, FunctionInvocationStatus status, FunctionCallContent callContent, object? result, Exception? exception) - { - Terminate = terminate; - Status = status; - CallContent = callContent; - Result = result; - Exception = exception; - } - - /// Gets status about how the function invocation completed. - public FunctionInvocationStatus Status { get; } - - /// Gets the function call content information associated with this invocation. - public FunctionCallContent CallContent { get; } - - /// Gets the result of the function call. - public object? Result { get; } - - /// Gets any exception the function call threw. - public Exception? Exception { get; } - - /// Gets a value indicating whether the caller should terminate the processing loop. - public bool Terminate { get; } - } - - /// Provides error codes for when errors occur as part of the function calling loop. - public enum FunctionInvocationStatus - { - /// The operation completed successfully. - RanToCompletion, - - /// The requested function could not be found. - NotFound, - - /// The function call failed with an exception. - Exception, - } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs deleted file mode 100644 index c75748bf0a..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAIFunction.cs +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.AI; - -/// -/// Marks an existing with additional metadata to indicate that it is not invocable. -/// -internal class NonInvocableAIFunction(AIFunction innerFunction) : DelegatingAIFunction(innerFunction); diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs deleted file mode 100644 index e086060b51..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NonInvocableAwareFunctionInvokingChatClient.cs +++ /dev/null @@ -1,1027 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Runtime.ExceptionServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Shared.Diagnostics; - -#pragma warning disable CA2213 // Disposable fields should be disposed -#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test -#pragma warning disable SA1202 // 'protected' members should come before 'private' members -#pragma warning disable S107 // Methods should not have too many parameters - -#pragma warning disable IDE0009 // Member access should be qualified. -#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task -#pragma warning disable VSTHRD111 // Use ConfigureAwait(bool) - -namespace Microsoft.Extensions.AI; - -/// -/// A delegating chat client that invokes functions defined on . -/// Include this in a chat pipeline to resolve function calls automatically. -/// -/// -/// -/// When this client receives a in a chat response, it responds -/// by calling the corresponding defined in , -/// producing a that it sends back to the inner client. This loop -/// is repeated until there are no more function calls to make, or until another stop condition is met, -/// such as hitting . -/// -/// -/// The provided implementation of is thread-safe for concurrent use so long as the -/// instances employed as part of the supplied are also safe. -/// The property can be used to control whether multiple function invocation -/// requests as part of the same request are invocable concurrently, but even with that set to -/// (the default), multiple concurrent requests to this same instance and using the same tools could result in those -/// tools being used concurrently (one per request). For example, a function that accesses the HttpContext of a specific -/// ASP.NET web request should only be used as part of a single at a time, and only with -/// set to , in case the inner client decided to issue multiple -/// invocation requests to that same function. -/// -/// -public partial class NonInvocableAwareFunctionInvokingChatClient : DelegatingChatClient -{ - /// The for the current function invocation. - private static readonly AsyncLocal s_currentContext = new(); - - /// Gets the specified when constructing the , if any. - protected IServiceProvider? FunctionInvocationServices { get; } - - /// The logger to use for logging information about function invocation. - private readonly ILogger _logger; - - /// The to use for telemetry. - /// This component does not own the instance and should not dispose it. - private readonly ActivitySource? _activitySource; - - /// Maximum number of roundtrips allowed to the inner client. - private int _maximumIterationsPerRequest = 40; // arbitrary default to prevent runaway execution - - /// Maximum number of consecutive iterations that are allowed contain at least one exception result. If the limit is exceeded, we rethrow the exception instead of continuing. - private int _maximumConsecutiveErrorsPerRequest = 3; - - /// - /// Initializes a new instance of the class. - /// - /// The underlying , or the next instance in a chain of clients. - /// An to use for logging information about function invocation. - /// An optional to use for resolving services required by the instances being invoked. - public NonInvocableAwareFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null) - : base(innerClient) - { - _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; - _activitySource = innerClient.GetService(); - FunctionInvocationServices = functionInvocationServices; - } - - /// - /// Gets or sets the for the current function invocation. - /// - /// - /// This value flows across async calls. - /// - public static FunctionInvocationContext? CurrentContext - { - get => s_currentContext.Value; - protected set => s_currentContext.Value = value; - } - - /// - /// Gets or sets a value indicating whether detailed exception information should be included - /// in the chat history when calling the underlying . - /// - /// - /// if the full exception message is added to the chat history - /// when calling the underlying . - /// if a generic error message is included in the chat history. - /// The default value is . - /// - /// - /// - /// Setting the value to prevents the underlying language model from disclosing - /// raw exception details to the end user, since it doesn't receive that information. Even in this - /// case, the raw object is available to application code by inspecting - /// the property. - /// - /// - /// Setting the value to can help the underlying bypass problems on - /// its own, for example by retrying the function call with different arguments. However it might - /// result in disclosing the raw exception information to external users, which can be a security - /// concern depending on the application scenario. - /// - /// - /// Changing the value of this property while the client is in use might result in inconsistencies - /// as to whether detailed errors are provided during an in-flight request. - /// - /// - public bool IncludeDetailedErrors { get; set; } - - /// - /// Gets or sets a value indicating whether to allow concurrent invocation of functions. - /// - /// - /// if multiple function calls can execute in parallel. - /// if function calls are processed serially. - /// The default value is . - /// - /// - /// An individual response from the inner client might contain multiple function call requests. - /// By default, such function calls are processed serially. Set to - /// to enable concurrent invocation such that multiple function calls can execute in parallel. - /// - public bool AllowConcurrentInvocation { get; set; } - - /// - /// Gets or sets the maximum number of iterations per request. - /// - /// - /// The maximum number of iterations per request. - /// The default value is 40. - /// - /// - /// - /// Each request to this might end up making - /// multiple requests to the inner client. Each time the inner client responds with - /// a function call request, this client might perform that invocation and send the results - /// back to the inner client in a new request. This property limits the number of times - /// such a roundtrip is performed. The value must be at least one, as it includes the initial request. - /// - /// - /// Changing the value of this property while the client is in use might result in inconsistencies - /// as to how many iterations are allowed for an in-flight request. - /// - /// - public int MaximumIterationsPerRequest - { - get => _maximumIterationsPerRequest; - set - { - if (value < 1) - { - Throw.ArgumentOutOfRangeException(nameof(value)); - } - - _maximumIterationsPerRequest = value; - } - } - - /// - /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error. - /// - /// - /// The maximum number of consecutive iterations that are allowed to fail with an error. - /// The default value is 3. - /// - /// - /// - /// When function invocations fail with an exception, the - /// continues to make requests to the inner client, optionally supplying exception information (as - /// controlled by ). This allows the to - /// recover from errors by trying other function parameters that may succeed. - /// - /// - /// However, in case function invocations continue to produce exceptions, this property can be used to - /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be - /// rethrown to the caller. - /// - /// - /// If the value is set to zero, all function calling exceptions immediately terminate the function - /// invocation loop and the exception will be rethrown to the caller. - /// - /// - /// Changing the value of this property while the client is in use might result in inconsistencies - /// as to how many iterations are allowed for an in-flight request. - /// - /// - public int MaximumConsecutiveErrorsPerRequest - { - get => _maximumConsecutiveErrorsPerRequest; - set => _maximumConsecutiveErrorsPerRequest = Throw.IfLessThan(value, 0); - } - - /// Gets or sets a collection of additional tools the client is able to invoke. - /// - /// These will not impact the requests sent by the , which will pass through the - /// unmodified. However, if the inner client requests the invocation of a tool - /// that was not in , this collection will also be consulted - /// to look for a corresponding tool to invoke. This is useful when the service may have been pre-configured to be aware - /// of certain tools that aren't also sent on each individual request. - /// - public IList? AdditionalTools { get; set; } - - /// Gets or sets a delegate used to invoke instances. - /// - /// By default, the protected method is called for each to be invoked, - /// invoking the instance and returning its result. If this delegate is set to a non- value, - /// will replace its normal invocation with a call to this delegate, enabling - /// this delegate to assume all invocation handling of the function. - /// - public Func>? FunctionInvoker { get; set; } - - /// - public override async Task GetResponseAsync( - IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(messages); - - // A single request into this GetResponseAsync may result in multiple requests to the inner client. - // Create an activity to group them together for better observability. - using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}"); - - // Copy the original messages in order to avoid enumerating the original messages multiple times. - // The IEnumerable can represent an arbitrary amount of work. - List originalMessages = [.. messages]; - messages = originalMessages; - - List? augmentedHistory = null; // the actual history of messages sent on turns other than the first - ChatResponse? response = null; // the response from the inner client, which is possibly modified and then eventually returned - List? responseMessages = null; // tracked list of messages, across multiple turns, to be used for the final response - UsageDetails? totalUsage = null; // tracked usage across all turns, to be used for the final response - List? functionCallContents = null; // function call contents that need responding to in the current turn - bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set - int consecutiveErrorCount = 0; - int iteration = 0; - - NonInvocableAIFunction[] nonInvocableAIFunctions = [.. options?.Tools?.OfType() ?? [], .. AdditionalTools?.OfType() ?? []]; - HashSet nonInvocableAIFunctionNames = new(nonInvocableAIFunctions.Select(f => f.Name)); - HashSet alreadyInvokedFunctionCalls = new(originalMessages.SelectMany(m => m.Contents).OfType().Select(frc => frc.CallId)); - - // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. - bool requiresPreGetResponseFunctionInvocation = - (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && - CopyFunctionCalls(originalMessages, nonInvocableAIFunctionNames, alreadyInvokedFunctionCalls, ref functionCallContents); - - if (requiresPreGetResponseFunctionInvocation) - { - // Add the responses from the function calls into the augmented history and also into the tracked - // list of response messages. - var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, functionCallContents!, iteration++, consecutiveErrorCount, isStreaming: false, cancellationToken); - responseMessages = [.. modeAndMessages.MessagesAdded]; - consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; - - if (modeAndMessages.ShouldTerminate) - { - return new ChatResponse(responseMessages); - } - } - - for (; ; iteration++) - { - // Do pre-invocation function calls, that may be passed in. - - functionCallContents?.Clear(); - - // Make the call to the inner client. - response = await base.GetResponseAsync(messages, options, cancellationToken); - if (response is null) - { - Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); - } - - // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. - bool requiresFunctionInvocation = - (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && - iteration < MaximumIterationsPerRequest && - CopyFunctionCalls(response.Messages, nonInvocableAIFunctionNames, [], ref functionCallContents); - - // In a common case where we make a request and there's no function calling work required, - // fast path out by just returning the original response. - if (iteration == 0 && !requiresFunctionInvocation) - { - return response; - } - - // Track aggregate details from the response, including all of the response messages and usage details. - (responseMessages ??= []).AddRange(response.Messages); - if (response.Usage is not null) - { - if (totalUsage is not null) - { - totalUsage.Add(response.Usage); - } - else - { - totalUsage = response.Usage; - } - } - - // If there are no tools to call, or for any other reason we should stop, we're done. - // Break out of the loop and allow the handling at the end to configure the response - // with aggregated data from previous requests. - if (!requiresFunctionInvocation) - { - break; - } - - // Prepare the history for the next iteration. - FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); - - // Add the responses from the function calls into the augmented history and also into the tracked - // list of response messages. - var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken); - responseMessages.AddRange(modeAndMessages.MessagesAdded); - consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; - - if (modeAndMessages.ShouldTerminate) - { - break; - } - - UpdateOptionsForNextIteration(ref options, response.ConversationId); - } - - Debug.Assert(responseMessages is not null, "Expected to only be here if we have response messages."); - response.Messages = responseMessages!; - response.Usage = totalUsage; - - AddUsageTags(activity, totalUsage); - - return response; - } - - /// - public override async IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(messages); - - // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client. - // Create an activity to group them together for better observability. - using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}"); - UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes - - // Copy the original messages in order to avoid enumerating the original messages multiple times. - // The IEnumerable can represent an arbitrary amount of work. - List originalMessages = [.. messages]; - messages = originalMessages; - - List? augmentedHistory = null; // the actual history of messages sent on turns other than the first - List? functionCallContents = null; // function call contents that need responding to in the current turn - List? responseMessages = null; // tracked list of messages, across multiple turns, to be used in fallback cases to reconstitute history - bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set - List updates = []; // updates from the current response - int consecutiveErrorCount = 0; - NonInvocableAIFunction[] nonInvocableAIFunctions = [.. options?.Tools?.OfType() ?? [], .. AdditionalTools?.OfType() ?? []]; - HashSet nonInvocableAIFunctionNames = new(nonInvocableAIFunctions.Select(f => f.Name)); - - for (int iteration = 0; ; iteration++) - { - updates.Clear(); - functionCallContents?.Clear(); - - await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) - { - if (update is null) - { - Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}."); - } - - updates.Add(update); - - _ = CopyFunctionCalls(update.Contents, nonInvocableAIFunctionNames, [], ref functionCallContents); - - if (totalUsage is not null) - { - IList contents = update.Contents; - int contentsCount = contents.Count; - for (int i = 0; i < contentsCount; i++) - { - if (contents[i] is UsageContent uc) - { - totalUsage.Add(uc.Details); - } - } - } - - yield return update; - Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 - } - - // If there are no tools to call, or for any other reason we should stop, return the response. - if (functionCallContents is not { Count: > 0 } || - (options?.Tools is not { Count: > 0 } && AdditionalTools is not { Count: > 0 }) || - iteration >= _maximumIterationsPerRequest) - { - break; - } - - // Reconstitute a response from the response updates. - var response = updates.ToChatResponse(); - (responseMessages ??= []).AddRange(response.Messages); - - // Prepare the history for the next iteration. - FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadConversationId); - - // Process all of the functions, adding their results into the history. - var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken); - responseMessages.AddRange(modeAndMessages.MessagesAdded); - consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; - - // This is a synthetic ID since we're generating the tool messages instead of getting them from - // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to - // use the same message ID for all of them within a given iteration, as this is a single logical - // message with multiple content items. We could also use different message IDs per tool content, - // but there's no benefit to doing so. - string toolResponseId = Guid.NewGuid().ToString("N"); - - // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages - // includes all activities, including generated function results. - foreach (var message in modeAndMessages.MessagesAdded) - { - var toolResultUpdate = new ChatResponseUpdate - { - AdditionalProperties = message.AdditionalProperties, - AuthorName = message.AuthorName, - ConversationId = response.ConversationId, - CreatedAt = DateTimeOffset.UtcNow, - Contents = message.Contents, - RawRepresentation = message.RawRepresentation, - ResponseId = toolResponseId, - MessageId = toolResponseId, // See above for why this can be the same as ResponseId - Role = message.Role, - }; - - yield return toolResultUpdate; - Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 - } - - if (modeAndMessages.ShouldTerminate) - { - break; - } - - UpdateOptionsForNextIteration(ref options, response.ConversationId); - } - - AddUsageTags(activity, totalUsage); - } - - /// Adds tags to for usage details in . - private static void AddUsageTags(Activity? activity, UsageDetails? usage) - { - if (usage is not null && activity is { IsAllDataRequested: true }) - { - if (usage.InputTokenCount is long inputTokens) - { - _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.InputTokens, (int)inputTokens); - } - - if (usage.OutputTokenCount is long outputTokens) - { - _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.OutputTokens, (int)outputTokens); - } - } - } - - /// Prepares the various chat message lists after a response from the inner client and before invoking functions. - /// The original messages provided by the caller. - /// The messages reference passed to the inner client. - /// The augmented history containing all the messages to be sent. - /// The most recent response being handled. - /// A list of all response messages received up until this point. - /// Whether the previous iteration's response had a conversation ID. - private static void FixupHistories( - IEnumerable originalMessages, - ref IEnumerable messages, - [NotNull] ref List? augmentedHistory, - ChatResponse response, - List allTurnsResponseMessages, - ref bool lastIterationHadConversationId) - { - // We're now going to need to augment the history with function result contents. - // That means we need a separate list to store the augmented history. - if (response.ConversationId is not null) - { - // The response indicates the inner client is tracking the history, so we don't want to send - // anything we've already sent or received. - if (augmentedHistory is not null) - { - augmentedHistory.Clear(); - } - else - { - augmentedHistory = []; - } - - lastIterationHadConversationId = true; - } - else if (lastIterationHadConversationId) - { - // In the very rare case where the inner client returned a response with a conversation ID but then - // returned a subsequent response without one, we want to reconstitute the full history. To do that, - // we can populate the history with the original chat messages and then all of the response - // messages up until this point, which includes the most recent ones. - augmentedHistory ??= []; - augmentedHistory.Clear(); - augmentedHistory.AddRange(originalMessages); - augmentedHistory.AddRange(allTurnsResponseMessages); - - lastIterationHadConversationId = false; - } - else - { - // If augmentedHistory is already non-null, then we've already populated it with everything up - // until this point (except for the most recent response). If it's null, we need to seed it with - // the chat history provided by the caller. - augmentedHistory ??= originalMessages.ToList(); - - // Now add the most recent response messages. - augmentedHistory.AddMessages(response); - - lastIterationHadConversationId = false; - } - - // Use the augmented history as the new set of messages to send. - messages = augmentedHistory; - } - - /// Copies any from to . - private static bool CopyFunctionCalls( - IList messages, HashSet nonInvocableAIFunctionNames, HashSet alreadyInvokedFunctionCalls, [NotNullWhen(true)] ref List? functionCalls) - { - bool any = false; - int count = messages.Count; - for (int i = 0; i < count; i++) - { - any |= CopyFunctionCalls(messages[i].Contents, nonInvocableAIFunctionNames, alreadyInvokedFunctionCalls, ref functionCalls); - } - - return any; - } - - /// Copies any from to . - private static bool CopyFunctionCalls( - IList content, HashSet nonInvocableAIFunctionNames, HashSet alreadyInvokedFunctionCalls, [NotNullWhen(true)] ref List? functionCalls) - { - bool any = false; - int count = content.Count; - for (int i = 0; i < count; i++) - { - if (content[i] is FunctionCallContent functionCall && !nonInvocableAIFunctionNames.Contains(functionCall.Name) && !alreadyInvokedFunctionCalls.Contains(functionCall.CallId)) - { - (functionCalls ??= []).Add(functionCall); - any = true; - } - } - - return any; - } - - private static void UpdateOptionsForNextIteration(ref ChatOptions? options, string? conversationId) - { - if (options is null) - { - if (conversationId is not null) - { - options = new() { ConversationId = conversationId }; - } - } - else if (options.ToolMode is RequiredChatToolMode) - { - // We have to reset the tool mode to be non-required after the first iteration, - // as otherwise we'll be in an infinite loop. - options = options.Clone(); - options.ToolMode = null; - options.ConversationId = conversationId; - } - else if (options.ConversationId != conversationId) - { - // As with the other modes, ensure we've propagated the chat conversation ID to the options. - // We only need to clone the options if we're actually mutating it. - options = options.Clone(); - options.ConversationId = conversationId; - } - } - - /// - /// Processes the function calls in the list. - /// - /// The current chat contents, inclusive of the function call contents being processed. - /// The options used for the response being processed. - /// The function call contents representing the functions to be invoked. - /// The iteration number of how many roundtrips have been made to the inner client. - /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors. - /// Whether the function calls are being processed in a streaming context. - /// The to monitor for cancellation requests. - /// A value indicating how the caller should proceed. - private async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync( - List messages, ChatOptions? options, List functionCallContents, int iteration, int consecutiveErrorCount, - bool isStreaming, CancellationToken cancellationToken) - { - // We must add a response for every tool call, regardless of whether we successfully executed it or not. - // If we successfully execute it, we'll add the result. If we don't, we'll add an error. - - Debug.Assert(functionCallContents.Count > 0, "Expected at least one function call."); - var shouldTerminate = false; - var captureCurrentIterationExceptions = consecutiveErrorCount < _maximumConsecutiveErrorsPerRequest; - - // Process all functions. If there's more than one and concurrent invocation is enabled, do so in parallel. - if (functionCallContents.Count == 1) - { - FunctionInvocationResult result = await ProcessFunctionCallAsync( - messages, options, functionCallContents, - iteration, 0, captureCurrentIterationExceptions, isStreaming, cancellationToken); - - IList addedMessages = CreateResponseMessages([result]); - ThrowIfNoFunctionResultsAdded(addedMessages); - UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); - messages.AddRange(addedMessages); - - return (result.Terminate, consecutiveErrorCount, addedMessages); - } - else - { - List results = []; - - if (AllowConcurrentInvocation) - { - // Rather than awaiting each function before invoking the next, invoke all of them - // and then await all of them. We avoid forcibly introducing parallelism via Task.Run, - // but if a function invocation completes asynchronously, its processing can overlap - // with the processing of other the other invocation invocations. - results.AddRange(await Task.WhenAll( - from callIndex in Enumerable.Range(0, functionCallContents.Count) - select ProcessFunctionCallAsync( - messages, options, functionCallContents, - iteration, callIndex, captureExceptions: true, isStreaming, cancellationToken))); - - shouldTerminate = results.Any(r => r.Terminate); - } - else - { - // Invoke each function serially. - for (int callIndex = 0; callIndex < functionCallContents.Count; callIndex++) - { - var functionResult = await ProcessFunctionCallAsync( - messages, options, functionCallContents, - iteration, callIndex, captureCurrentIterationExceptions, isStreaming, cancellationToken); - - results.Add(functionResult); - - // If any function requested termination, we should stop right away. - if (functionResult.Terminate) - { - shouldTerminate = true; - break; - } - } - } - - IList addedMessages = CreateResponseMessages(results.ToArray()); - ThrowIfNoFunctionResultsAdded(addedMessages); - UpdateConsecutiveErrorCountOrThrow(addedMessages, ref consecutiveErrorCount); - messages.AddRange(addedMessages); - - return (shouldTerminate, consecutiveErrorCount, addedMessages); - } - } - -#pragma warning disable CA1851 // Possible multiple enumerations of 'IEnumerable' collection - /// - /// Updates the consecutive error count, and throws an exception if the count exceeds the maximum. - /// - /// Added messages. - /// Consecutive error count. - /// Thrown if the maximum consecutive error count is exceeded. - private void UpdateConsecutiveErrorCountOrThrow(IList added, ref int consecutiveErrorCount) - { - var allExceptions = added.SelectMany(m => m.Contents.OfType()) - .Select(frc => frc.Exception!) - .Where(e => e is not null); - - if (allExceptions.Any()) - { - consecutiveErrorCount++; - if (consecutiveErrorCount > _maximumConsecutiveErrorsPerRequest) - { - var allExceptionsArray = allExceptions.ToArray(); - if (allExceptionsArray.Length == 1) - { - ExceptionDispatchInfo.Capture(allExceptionsArray[0]).Throw(); - } - else - { - throw new AggregateException(allExceptionsArray); - } - } - } - else - { - consecutiveErrorCount = 0; - } - } -#pragma warning restore CA1851 - - /// - /// Throws an exception if doesn't create any messages. - /// - private void ThrowIfNoFunctionResultsAdded(IList? messages) - { - if (messages is null || messages.Count == 0) - { - Throw.InvalidOperationException($"{GetType().Name}.{nameof(CreateResponseMessages)} returned null or an empty collection of messages."); - } - } - - /// Processes the function call described in []. - /// The current chat contents, inclusive of the function call contents being processed. - /// The options used for the response being processed. - /// The function call contents representing all the functions being invoked. - /// The iteration number of how many roundtrips have been made to the inner client. - /// The 0-based index of the function being called out of . - /// If true, handles function-invocation exceptions by returning a value with . Otherwise, rethrows. - /// Whether the function calls are being processed in a streaming context. - /// The to monitor for cancellation requests. - /// A value indicating how the caller should proceed. - private async Task ProcessFunctionCallAsync( - List messages, ChatOptions? options, List callContents, - int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, CancellationToken cancellationToken) - { - var callContent = callContents[functionCallIndex]; - - // Look up the AIFunction for the function call. If the requested function isn't available, send back an error. - AIFunction? aiFunction = FindAIFunction(options?.Tools, callContent.Name) ?? FindAIFunction(AdditionalTools, callContent.Name); - if (aiFunction is null) - { - return new(terminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null); - } - - FunctionInvocationContext context = new() - { - Function = aiFunction, - Arguments = new(callContent.Arguments) { Services = FunctionInvocationServices }, - Messages = messages, - Options = options, - CallContent = callContent, - Iteration = iteration, - FunctionCallIndex = functionCallIndex, - FunctionCount = callContents.Count, - IsStreaming = isStreaming - }; - - object? result; - try - { - result = await InstrumentedInvokeFunctionAsync(context, cancellationToken); - } - catch (Exception e) when (!cancellationToken.IsCancellationRequested) - { - if (!captureExceptions) - { - throw; - } - - return new( - terminate: false, - FunctionInvocationStatus.Exception, - callContent, - result: null, - exception: e); - } - - return new( - terminate: context.Terminate, - FunctionInvocationStatus.RanToCompletion, - callContent, - result, - exception: null); - - static AIFunction? FindAIFunction(IList? tools, string functionName) - { - if (tools is not null) - { - int count = tools.Count; - for (int i = 0; i < count; i++) - { - if (tools[i] is AIFunction function && function.Name == functionName) - { - return function; - } - } - } - - return null; - } - } - - /// Creates one or more response messages for function invocation results. - /// Information about the function call invocations and results. - /// A list of all chat messages created from . - protected virtual IList CreateResponseMessages( - ReadOnlySpan results) - { - var contents = new List(results.Length); - for (int i = 0; i < results.Length; i++) - { - contents.Add(CreateFunctionResultContent(results[i])); - } - - return [new(ChatRole.Tool, contents)]; - - FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult result) - { - _ = Throw.IfNull(result); - - object? functionResult; - if (result.Status == FunctionInvocationStatus.RanToCompletion) - { - functionResult = result.Result ?? "Success: Function completed."; - } - else - { - string message = result.Status switch - { - FunctionInvocationStatus.NotFound => $"Error: Requested function \"{result.CallContent.Name}\" not found.", - FunctionInvocationStatus.Exception => "Error: Function failed.", - _ => "Error: Unknown error.", - }; - - if (IncludeDetailedErrors && result.Exception is not null) - { - message = $"{message} Exception: {result.Exception.Message}"; - } - - functionResult = message; - } - - return new FunctionResultContent(result.CallContent.CallId, functionResult) { Exception = result.Exception }; - } - } - - /// Invokes the function asynchronously. - /// - /// The function invocation context detailing the function to be invoked and its arguments along with additional request information. - /// - /// The to monitor for cancellation requests. The default is . - /// The result of the function invocation, or if the function invocation returned . - /// is . - private async Task InstrumentedInvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) - { - _ = Throw.IfNull(context); - - using Activity? activity = _activitySource?.StartActivity( - $"{OpenTelemetryConsts.GenAI.ExecuteTool} {context.Function.Name}", - ActivityKind.Internal, - default(ActivityContext), - [ - new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"), - new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId), - new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name), - new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description), - ]); - - long startingTimestamp = 0; - if (_logger.IsEnabled(LogLevel.Debug)) - { - startingTimestamp = Stopwatch.GetTimestamp(); - if (_logger.IsEnabled(LogLevel.Trace)) - { - LogInvokingSensitive(context.Function.Name, LoggingHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions)); - } - else - { - LogInvoking(context.Function.Name); - } - } - - object? result = null; - try - { - CurrentContext = context; // doesn't need to be explicitly reset after, as that's handled automatically at async method exit - result = await InvokeFunctionAsync(context, cancellationToken); - } - catch (Exception e) - { - if (activity is not null) - { - _ = activity.SetTag("error.type", e.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, e.Message); - } - - if (e is OperationCanceledException) - { - LogInvocationCanceled(context.Function.Name); - } - else - { - LogInvocationFailed(context.Function.Name, e); - } - - throw; - } - finally - { - if (_logger.IsEnabled(LogLevel.Debug)) - { - TimeSpan elapsed = GetElapsedTime(startingTimestamp); - - if (result is not null && _logger.IsEnabled(LogLevel.Trace)) - { - LogInvocationCompletedSensitive(context.Function.Name, elapsed, LoggingHelpers.AsJson(result, context.Function.JsonSerializerOptions)); - } - else - { - LogInvocationCompleted(context.Function.Name, elapsed); - } - } - } - - return result; - } - - /// This method will invoke the function within the try block. - /// The function invocation context. - /// Cancellation token. - /// The function result. - protected virtual ValueTask InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken) - { - _ = Throw.IfNull(context); - - return FunctionInvoker is { } invoker ? - invoker(context, cancellationToken) : - context.Function.InvokeAsync(context.Arguments, cancellationToken); - } - - private static TimeSpan GetElapsedTime(long startingTimestamp) => -#if NET - Stopwatch.GetElapsedTime(startingTimestamp); -#else - new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * ((double)TimeSpan.TicksPerSecond / Stopwatch.Frequency))); -#endif - - [LoggerMessage(LogLevel.Debug, "Invoking {MethodName}.", SkipEnabledCheck = true)] - private partial void LogInvoking(string methodName); - - [LoggerMessage(LogLevel.Trace, "Invoking {MethodName}({Arguments}).", SkipEnabledCheck = true)] - private partial void LogInvokingSensitive(string methodName, string arguments); - - [LoggerMessage(LogLevel.Debug, "{MethodName} invocation completed. Duration: {Duration}", SkipEnabledCheck = true)] - private partial void LogInvocationCompleted(string methodName, TimeSpan duration); - - [LoggerMessage(LogLevel.Trace, "{MethodName} invocation completed. Duration: {Duration}. Result: {Result}", SkipEnabledCheck = true)] - private partial void LogInvocationCompletedSensitive(string methodName, TimeSpan duration, string result); - - [LoggerMessage(LogLevel.Debug, "{MethodName} invocation canceled.")] - private partial void LogInvocationCanceled(string methodName); - - [LoggerMessage(LogLevel.Error, "{MethodName} invocation failed.")] - private partial void LogInvocationFailed(string methodName, Exception error); - - /// Provides information about the invocation of a function call. - public sealed class FunctionInvocationResult - { - /// - /// Initializes a new instance of the class. - /// - /// Indicates whether the caller should terminate the processing loop. - /// Indicates the status of the function invocation. - /// Contains information about the function call. - /// The result of the function call. - /// The exception thrown by the function call, if any. - internal FunctionInvocationResult(bool terminate, FunctionInvocationStatus status, FunctionCallContent callContent, object? result, Exception? exception) - { - Terminate = terminate; - Status = status; - CallContent = callContent; - Result = result; - Exception = exception; - } - - /// Gets status about how the function invocation completed. - public FunctionInvocationStatus Status { get; } - - /// Gets the function call content information associated with this invocation. - public FunctionCallContent CallContent { get; } - - /// Gets the result of the function call. - public object? Result { get; } - - /// Gets any exception the function call threw. - public Exception? Exception { get; } - - /// Gets a value indicating whether the caller should terminate the processing loop. - public bool Terminate { get; } - } - - /// Provides error codes for when errors occur as part of the function calling loop. - public enum FunctionInvocationStatus - { - /// The operation completed successfully. - RanToCompletion, - - /// The requested function could not be found. - NotFound, - - /// The function call failed with an exception. - Exception, - } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs deleted file mode 100644 index f7e9dc428c..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PostFICCApprovalGeneratingChatClient.cs +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -/// -/// Represents a chat client that seeks user approval for function calls and sits behind the . -/// -public class PostFICCApprovalGeneratingChatClient : DelegatingChatClient -{ - /// The logger to use for logging information about function approval. - private readonly ILogger _logger; - - /// - /// Initializes a new instance of the class. - /// - /// The underlying , or the next instance in a chain of clients. - /// An to use for logging information about function invocation. - public PostFICCApprovalGeneratingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null) - : base(innerClient) - { - this._logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; - } - - /// - public override async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(messages); - - var messagesList = messages as IList ?? messages.ToList(); - - // If we got any approval responses, and we also got FunctionResultContent for those approvals, we can filter out those approval responses - // since they are already handled. - RemoveExecutedApprovedApprovalRequests(messagesList); - - // Get all the remaining approval responses. - var approvalResponses = messagesList.SelectMany(x => x.Contents).OfType().ToList(); - - if (approvalResponses.Count == 0) - { - // We have no approval responses, so we can just call the inner client. - var response = await base.GetResponseAsync(messagesList, options, cancellationToken).ConfigureAwait(false); - if (response is null) - { - Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); - } - - // Replace any FunctionCallContent in the response with FunctionApprovalRequestContent. - ReplaceFunctionCallsWithApprovalRequests(response.Messages); - - return response; - } - - if (approvalResponses.All(x => !x.Approved)) - { - // If we only have rejections, we can call the inner client with rejected function calls. - // Replace all rejected FunctionApprovalResponseContent with rejected FunctionResultContent. - ReplaceRejectedFunctionCallRequests(messagesList); - - var response = await base.GetResponseAsync(messagesList, options, cancellationToken).ConfigureAwait(false); - if (response is null) - { - Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); - } - - // Replace any FunctionCallContent in the response with FunctionApprovalRequestContent. - ReplaceFunctionCallsWithApprovalRequests(response.Messages); - - return response; - } - - // We have a mix of approvals and rejections, so we need to return the approved function calls - // to the upper layer for invocation. - // We do nothing with the rejected ones. They must be supplied by the caller again - // on the next invocation, and then we will convert them to rejected FunctionResultContent. - var approvedToolCalls = approvalResponses.Where(x => x.Approved).Select(x => x.FunctionCall).Cast().ToList(); - return new ChatResponse - { - ConversationId = options?.ConversationId, - CreatedAt = DateTimeOffset.UtcNow, - FinishReason = ChatFinishReason.ToolCalls, - ResponseId = Guid.NewGuid().ToString(), - Messages = - [ - new ChatMessage(ChatRole.Assistant, approvedToolCalls) - ] - }; - } - - /// - public override IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) - { - return base.GetStreamingResponseAsync(messages, options, cancellationToken); - } - - private static void RemoveExecutedApprovedApprovalRequests(IList messages) - { - var functionResultCallIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.CallId).ToHashSet(); - - int messageCount = messages.Count; - for (int i = 0; i < messageCount; i++) - { - // Get any content that is not a FunctionApprovalResponseContent or is a FunctionApprovalResponseContent that has not been executed. - var content = messages[i].Contents.Where(x => x is not FunctionApprovalResponseContent || (x is FunctionApprovalResponseContent approval && !functionResultCallIds.Contains(approval.FunctionCall.CallId))).ToList(); - - // Remove the entire message if there is no content left after filtering. - if (content.Count == 0) - { - messages.RemoveAt(i); - i--; // Adjust index since we removed an item. - messageCount--; // Adjust count since we removed an item. - continue; - } - - // Replace the message contents with the filtered content. - messages[i].Contents = content; - } - } - - private static void ReplaceRejectedFunctionCallRequests(IList messages) - { - List newMessages = []; - - int messageCount = messages.Count; - for (int i = 0; i < messageCount; i++) - { - var content = messages[i].Contents; - - List replacedContent = []; - List toolCalls = []; - int contentCount = content.Count; - for (int j = 0; j < contentCount; j++) - { - // Find all responses that were rejected, and replace them with a FunctionResultContent indicating the rejection. - if (content[j] is FunctionApprovalResponseContent approval && !approval.Approved) - { - var rejectedFunctionCall = new FunctionResultContent(approval.FunctionCall.CallId, "Error: Function invocation approval was not granted."); - replacedContent.Add(rejectedFunctionCall); - content[j] = rejectedFunctionCall; - toolCalls.Add(approval.FunctionCall); - } - } - - // Since approvals are submitted as part of a user messages, we have to move the - // replaced function results to tool messages. - if (replacedContent.Count == contentCount) - { - // If all content was replaced, we can replace the entire message with a new tool message. - messages.RemoveAt(i); - i--; // Adjust index since we removed an item. - messageCount--; // Adjust count since we removed an item. - - newMessages.Add(new ChatMessage(ChatRole.Assistant, toolCalls)); - newMessages.Add(new ChatMessage(ChatRole.Tool, replacedContent)); - } - else if (replacedContent.Count > 0) - { - // If only some content was replaced, we move the updated content to a new tool message. - foreach (var replacedItem in replacedContent) - { - messages[i].Contents.Remove(replacedItem); - } - - newMessages.Add(new ChatMessage(ChatRole.Assistant, toolCalls)); - newMessages.Add(new ChatMessage(ChatRole.Tool, replacedContent)); - } - } - - if (newMessages.Count > 0) - { - // If we have new messages, we add them to the original messages. - foreach (var newMessage in newMessages) - { - messages.Add(newMessage); - } - } - } - - /// Replaces any from with . - private static void ReplaceFunctionCallsWithApprovalRequests(IList messages) - { - int count = messages.Count; - for (int i = 0; i < count; i++) - { - ReplaceFunctionCallsWithApprovalRequests(messages[i].Contents); - } - } - - /// Copies any from with . - private static void ReplaceFunctionCallsWithApprovalRequests(IList content) - { - int count = content.Count; - for (int i = 0; i < count; i++) - { - if (content[i] is FunctionCallContent functionCall) - { - content[i] = new FunctionApprovalRequestContent(functionCall.CallId, functionCall); - } - } - } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs deleted file mode 100644 index 5634c50792..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/PreFICCApprovalGeneratingChatClient.cs +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI.Agents; - -/// -/// Represents a chat client that seeks user approval for function calls and sits before the . -/// -public class PreFICCApprovalGeneratingChatClient : DelegatingChatClient -{ - /// The logger to use for logging information about function approval. - private readonly ILogger _logger; - - /// - /// Initializes a new instance of the class. - /// - /// The underlying , or the next instance in a chain of clients. - /// An to use for logging information about function invocation. - public PreFICCApprovalGeneratingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null) - : base(innerClient) - { - this._logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; - } - - /// - public override async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(messages); - - var messagesList = messages as IList ?? messages.ToList(); - - // If we got any FunctionApprovalResponseContent, we can remove the FunctionApprovalRequestContent for those responses, since the FunctionApprovalResponseContent - // will be turned into FunctionCallContent and FunctionResultContent later, but the FunctionApprovalRequestContent is now unecessary. - // If we got any approval request/responses, and we also already have FunctionResultContent for those, we can filter out those requests/responses too - // since they are already handled. - // This is since the downstream service, may not know what to do with the FunctionApprovalRequestContent/FunctionApprovalResponseContent. - RemoveExecutedApprovedApprovalRequests(messagesList); - - // Get all the remaining approval responses. - var approvalResponses = messagesList.SelectMany(x => x.Contents).OfType().ToList(); - - // If we have any functions in options, we should clone them and mark any that do not yet have an approval as not invocable. - options = MakeFunctionsNonInvocable(options, approvalResponses); - - // For rejections we need to replace them with function call content plus rejected function result content. - // For approvals we need to replace them just with function call content, since the inner client will invoke them. - ReplaceApprovalResponses(messagesList); - - var response = await base.GetResponseAsync(messagesList, options, cancellationToken).ConfigureAwait(false); - if (response is null) - { - Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); - } - - // Replace any FunctionCallContent in the response with FunctionApprovalRequestContent. - ReplaceFunctionCallsWithApprovalRequests(response.Messages); - - return response; - } - - /// - public override IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) - { - return base.GetStreamingResponseAsync(messages, options, cancellationToken); - } - - private static void RemoveExecutedApprovedApprovalRequests(IList messages) - { - var functionResultCallIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.CallId).ToHashSet(); - var approvalResponsetIds = messages.SelectMany(x => x.Contents).OfType().Select(x => x.Id).ToHashSet(); - - int messageCount = messages.Count; - for (int i = 0; i < messageCount; i++) - { - // Get any content that is not a FunctionApprovalRequestContent/FunctionApprovalResponseContent or is a FunctionApprovalRequestContent/FunctionApprovalResponseContent that has not been executed. - var content = messages[i].Contents.Where(x => - (x is not FunctionApprovalRequestContent && x is not FunctionApprovalResponseContent) || - (x is FunctionApprovalRequestContent request && !approvalResponsetIds.Contains(request.Id) && !functionResultCallIds.Contains(request.FunctionCall.CallId)) || - (x is FunctionApprovalResponseContent approval && !functionResultCallIds.Contains(approval.FunctionCall.CallId))).ToList(); - - // Remove the entire message if there is no content left after filtering. - if (content.Count == 0) - { - messages.RemoveAt(i); - i--; // Adjust index since we removed an item. - messageCount--; // Adjust count since we removed an item. - continue; - } - - // Replace the message contents with the filtered content. - messages[i].Contents = content; - } - } - - private static void ReplaceApprovalResponses(IList messages) - { - List approvedFunctionCallContent = []; - - List rejectedFunctionCallContent = []; - List rejectedFunctionResultContent = []; - - int messageCount = messages.Count; - for (int i = 0; i < messageCount; i++) - { - var content = messages[i].Contents; - - int contentCount = content.Count; - - // ApprovalResponses are submitted as part of a user messages, but FunctionCallContent should be in an assistant message and - // FunctionResultContent should be in a tool message, so we need to remove them from the user messages, and add them to the appropriate - // mesages types later. - for (int j = 0; j < contentCount; j++) - { - // Find all responses that were approved, and add the FunctionCallContent for them to the list to add back later. - if (content[j] is FunctionApprovalResponseContent approval && approval.Approved) - { - content.RemoveAt(j); - j--; // Adjust index since we removed an item. - contentCount--; // Adjust count since we removed an item. - - approvedFunctionCallContent.Add(approval.FunctionCall); - continue; - } - - // Find all responses that were rejected, and add their FunctionCallContent and a FunctionResultContent indicating the rejection, to the lists to add back later. - if (content[j] is FunctionApprovalResponseContent rejection && !rejection.Approved) - { - content.RemoveAt(j); - j--; // Adjust index since we removed an item. - contentCount--; // Adjust count since we removed an item. - - var rejectedFunctionCall = new FunctionResultContent(rejection.FunctionCall.CallId, "Error: Function invocation approval was not granted."); - rejectedFunctionCallContent.Add(rejection.FunctionCall); - rejectedFunctionResultContent.Add(rejectedFunctionCall); - } - } - - // If we have no content left in the message after replacing, we can remove the message from the list. - if (content.Count == 0) - { - messages.RemoveAt(i); - i--; // Adjust index since we removed an item. - messageCount--; // Adjust count since we removed an item. - } - } - - if (rejectedFunctionCallContent.Count > 0) - { - messages.Add(new ChatMessage(ChatRole.Assistant, rejectedFunctionCallContent)); - messages.Add(new ChatMessage(ChatRole.Tool, rejectedFunctionResultContent)); - } - - if (approvedFunctionCallContent.Count != 0) - { - messages.Add(new ChatMessage(ChatRole.Assistant, approvedFunctionCallContent)); - } - } - - /// Replaces any from with . - private static void ReplaceFunctionCallsWithApprovalRequests(IList messages) - { - int count = messages.Count; - for (int i = 0; i < count; i++) - { - ReplaceFunctionCallsWithApprovalRequests(messages[i].Contents); - } - } - - /// Copies any from with . - private static void ReplaceFunctionCallsWithApprovalRequests(IList content) - { - int count = content.Count; - for (int i = 0; i < count; i++) - { - if (content[i] is FunctionCallContent functionCall) - { - content[i] = new FunctionApprovalRequestContent(functionCall.CallId, functionCall); - } - } - } - - private static ChatOptions? MakeFunctionsNonInvocable(ChatOptions? options, List approvals) - { - if (options?.Tools?.Count is > 0) - { - options = options.Clone(); - options.Tools = options.Tools!.Select(x => - { - if (x is AIFunction function && !approvals.Any(y => y.FunctionCall.Name == function.Name)) - { - var f = new NonInvocableAIFunction(function); - return f; - } - return x; - }).ToList(); - } - - return options; - } -} From d12cdf89fbaf9d67b3d1ce71e967d453803bed9f Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Thu, 14 Aug 2025 13:55:35 +0100 Subject: [PATCH 36/53] Update ADR with latest changes --- .../00NN-userapproval-content-types.md | 303 ++++++++++-------- 1 file changed, 178 insertions(+), 125 deletions(-) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index f89d238517..38659484ad 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -19,11 +19,22 @@ Inference services are also increasingly supporting built-in tools or service si This document aims to provide options and capture the decision on how to model this user approval interaction with the agent caller. +See various features that would need to be supported via this type of mechanism, plus how various other frameworks support this: + +- Also see [dotnet issue 6492](https://github.com/dotnet/extensions/issues/6492), which discusses the need for a similar pattern in the context of MCP approvals. +- Also see [the openai RunToolApprovalItem](https://openai.github.io/openai-agents-js/openai/agents/classes/runtoolapprovalitem/). +- Also see [the openai human-in-the-loop guide](https://openai.github.io/openai-agents-js/guides/human-in-the-loop/#approval-requests). +- Also see [the openai MCP guide](https://openai.github.io/openai-agents-js/guides/mcp/#optional-approval-flow). +- Also see [MCP Approval Requests from OpenAI](https://platform.openai.com/docs/guides/tools-remote-mcp#approvals). +- Also see [Azure AI Foundry MCP Approvals](https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/model-context-protocol-samples?pivots=rest#submit-your-approval). +- Also see [MCP Elicitation requests](https://modelcontextprotocol.io/specification/draft/client/elicitation) + ## Decision Drivers - Agents should encapsulate their internal logic and not leak it to the caller. - We need to support approvals for local actions as well as remote actions. - We need to support approvals for service-side tool use, such as remote MCP tool invocations +- We should consider how other user input requests will be modeled, so that we can have a consistent approach for user input requests and approvals. ## Considered Options @@ -44,7 +55,7 @@ This approach allows a caller to provide a callback that the agent can invoke wh This approach is easy to use when the user and agent are in the same application context, such as a desktop application, where the application can show the approval request to the user and get their response from the callback before continuing the agent run. This approach does not work well for cases where the agent is hosted in a remote service, and where there is no user available to provide the approval in the same application context. -For cases like this, the agent needs to be suspended, and a network response must be sent to the client app. After the user provides their approval, the client app must call the service that hosts the agent again with the user's decision, and the agent needs to be resumed. However, with a callback, the agent is deep in the call stack and cannot be suspended or resumed like this. +For cases like this, the agent needs to be suspended, and a network response must be sent to the client app. After the user provides their approval, the client app must call the service that hosts the agent again, with the user's decision, and the agent needs to be resumed. However, with a callback, the agent is deep in the call stack and cannot be suspended or resumed like this. ```csharp class AgentRunOptions @@ -60,10 +71,9 @@ agent.RunAsync("Please book me a flight for Friday to Paris.", thread, new Agent // The user can then approve or reject the request. // The optional FunctionCallContent can be used to show the user what function the agent wants to call with the parameter set: // approvalRequest.FunctionCall?.Arguments. - // The Text property of the ApprovalRequestContent can also be used to show the user any additional textual context about the request. // If the user approves: - return approvalRequest.Approve(); + return true; } }); ``` @@ -81,9 +91,8 @@ It is up to the agent to decide when and if a user approval is required, and the `ApprovalRequestContent` and `ApprovalResponseContent` will not necessarily always map to a supported content type for the underlying service or agent thread storage. Specifically, when we are deciding in the IChatClient stack to ask for approval from the user, for a function call, this does not mean that the underlying ai service or -service side thread type (where applicable) supports the concept of a function call approval request. We therefore need the ability to temporarily store the approval request in the -AgentThread, without it becoming part of the thread history. This will serve as a temporary record of the fact that there is an outstanding approval request that the agent is waiting for to continue. -There will be no long term record of an approval request in the chat history, but if the server side thread doesn't support this, there is nothing we can do to change that. +service side thread type (where applicable) supports the concept of a function call approval request. While we can store the approval requests and response in local +threads, service managed threads won't necessarily support this. For service managed threads, there will therefore be no long term record of the approval request in the chat history. We should however log approvals so that there is a trace of this for debugging and auditing purposes. Suggested Types: @@ -92,7 +101,7 @@ Suggested Types: class ApprovalRequestContent : AIContent { // An ID to uniquely identify the approval request/response pair. - public string ApprovalId { get; set; } + public string Id { get; set; } // An optional user targeted message to explain what needs to be approved. public string? Text { get; set; } @@ -100,37 +109,31 @@ class ApprovalRequestContent : AIContent // Optional: If the approval is for a function call, this will contain the function call content. public FunctionCallContent? FunctionCall { get; set; } - public ChatMessage Approve() + public ApprovalResponseContent CreateApproval() { - return new ChatMessage(ChatRole.User, - [ - new ApprovalResponseContent - { - ApprovalId = this.ApprovalId, - Approved = true, - FunctionCall = this.FunctionCall - } - ]); + return new ApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = true, + FunctionCall = this.FunctionCall + }; } - public ChatMessage Reject() + public ApprovalResponseContent CreateRejection() { - return new ChatMessage(ChatRole.User, - [ - new ApprovalResponseContent - { - ApprovalId = this.ApprovalId, - Approved = false, - FunctionCall = this.FunctionCall - } - ]); + return new ApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = false, + FunctionCall = this.FunctionCall + }; } } class ApprovalResponseContent : AIContent { // An ID to uniquely identify the approval request/response pair. - public string ApprovalId { get; set; } + public string Id { get; set; } // Indicates whether the user approved the request. public bool Approved { get; set; } @@ -152,8 +155,7 @@ while (response.ApprovalRequests.Count > 0) // The Text property of the ApprovalRequestContent can also be used to show the user any additional textual context about the request. // If the user approves: - var approvalMessage = approvalRequest.Approve(); - messages.Add(approvalMessage); + messages.Add(new ChatMessage(ChatRole.User, [approvalRequest.CreateApproval()])); } // Get the next response from the agent. @@ -166,61 +168,12 @@ class AgentRunResponse // A new property on AgentRunResponse to aggregate the ApprovalRequestContent items from // the response messages (Similar to the Text property). - public IReadOnlyList ApprovalRequests { get; set; } - - ... -} - -class AgentThread -{ - ... - - // The thread state may need to store the approval requests and responses. - public List ActiveApprovalRequests { get; set; } + public IEnumerable ApprovalRequests { get; set; } ... } ``` -- Also see [dotnet issue 6492](https://github.com/dotnet/extensions/issues/6492), which discusses the need for a similar pattern in the context of MCP approvals. -- Also see [the openai RunToolApprovalItem](https://openai.github.io/openai-agents-js/openai/agents/classes/runtoolapprovalitem/). -- Also see [the openai human-in-the-loop guide](https://openai.github.io/openai-agents-js/guides/human-in-the-loop/#approval-requests). -- Also see [MCP Approval Requests from OpenAI](https://platform.openai.com/docs/guides/tools-remote-mcp#approvals). -- Also see [Azure AI Foundry MCP Approvals](https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/model-context-protocol-samples?pivots=rest#submit-your-approval). -- Also see [MCP Elicitation requests](https://modelcontextprotocol.io/specification/draft/client/elicitation) - -#### ChatClientAgent Approval Process Flow - -1. User asks agent to perform a task and request is added to the thread. -1. Agent calls model with registered functions. -1. Model responds with function calls to make. -1. ConfirmingFunctionInvokingChatClient decorator (new feature / enhancement to FunctionInvokingChatClient) identifies any function calls that require user approval and returns an ApprovalRequestContent. - ChatClient implementations should also convert any approval requests from the service into ApprovalRequestContent. -1. Agent updates the thread with the FunctionCallContent (or this may have already been done by a service threaded agent) if the approval request is for a function call. -1. Agent stores the ApprovalRequestContent in its AgentThread under ActiveApprovalRequests, so that it knows that there is an outstanding user request. -1. Agent returns the ApprovalRequestContent to the caller which shows it to the user in the appropriate format. -1. User (via caller) invokes the agent again with ApprovalResponseContent. -1. Agent removes the ApprovalRequestContent from its AgentThread ActiveApprovalRequests. -1. Agent invokes IChatClient with ApprovalResponseContent and the ConfirmingFunctionInvokingChatClient decorator identifies the response as an approval for the function call. - If it isn't an approval for a manual function call, it can be passed through to the underlying ChatClient to be converted to the appropriate Approval content type for the service. -1. ConfirmingFunctionInvokingChatClient decorator invokes the function call and invokes the underlying IChatClient with a FunctionResultContent. -1. Model responds with the result. -1. Agent responds to caller with result message and thread is updated with the result message. - -At construction time the set of functions that require user approval will need to be registered with the `ConfirmingFunctionInvokingChatClient` decorator -so that it can identify which function calls should be returned as an `ApprovalRequestContent`. - -#### CustomAgent Approval Process Flow - -1. User asks agent to perform a task and request is added to the thread. -1. Agent executes various steps. -1. Agent encounters a step for which it requires user approval to continue. -1. Agent responds with an ApprovalRequestContent. -1. Agent adds the ApprovalRequestContent to its AgentThread ActiveApprovalRequests. -1. User (via caller) invokes the agent again with ApprovalResponseContent. -1. Agent removes its ApprovalRequestContent from its AgentThread ActiveApprovalRequests. -1. Agent responds to caller with result message and thread is updated with the result message. - ### 4. Introduce new Container UserInputRequestContent and UserInputResponseContent types This approach is similar to the `ApprovalRequestContent` and `ApprovalResponseContent` types, but is more generic and can be used for any type of user input request, not just approvals. @@ -266,7 +219,7 @@ class UserInputResponseContent : AIContent } var response = await agent.RunAsync("Please book me a flight for Friday to Paris.", thread); -while (response.UserInputRequests.Count > 0) +while (response.UserInputRequests.Any()) { List messages = new List(); foreach (var userInputRequest in response.UserInputRequests) @@ -309,16 +262,6 @@ class AgentRunResponse ... } - -class AgentThread -{ - ... - - // The thread state may need to store the user input requests. - public List ActiveUserInputRequests { get; set; } - - ... -} ``` ### 5. Introduce new Base UserInputRequestContent and UserInputResponseContent types @@ -331,13 +274,13 @@ Suggested Types: class UserInputRequestContent : AIContent { // An ID to uniquely identify the approval request/response pair. - public string ApprovalId { get; set; } + public string Id { get; set; } } class UserInputResponseContent : AIContent { // An ID to uniquely identify the approval request/response pair. - public string ApprovalId { get; set; } + public string Id { get; set; } } // ----------------------------------- @@ -347,30 +290,24 @@ class FunctionApprovalRequestContent : UserInputRequestContent // Contains the function call that the agent wants to invoke. public FunctionCallContent FunctionCall { get; set; } - public ChatMessage Approve() + public ApprovalResponseContent CreateApproval() { - return new ChatMessage(ChatRole.User, - [ - new FunctionApprovalResponseContent - { - ApprovalId = this.ApprovalId, - Approved = true, - FunctionCall = this.FunctionCall - } - ]); + return new ApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = true, + FunctionCall = this.FunctionCall + }; } - public ChatMessage Reject() + public ApprovalResponseContent CreateRejection() { - return new ChatMessage(ChatRole.User, - [ - new FunctionApprovalResponseContent - { - ApprovalId = this.ApprovalId, - Approved = false, - FunctionCall = this.FunctionCall - } - ]); + return new ApprovalResponseContent + { + ApprovalId = this.ApprovalId, + Approved = false, + FunctionCall = this.FunctionCall + }; } } class FunctionApprovalResponseContent : UserInputResponseContent @@ -412,7 +349,7 @@ class StructuredDataInputResponseContent : UserInputResponseContent } var response = await agent.RunAsync("Please book me a flight for Friday to Paris.", thread); -while (response.UserInputRequests.Count > 0) +while (response.UserInputRequests.Any()) { List messages = new List(); foreach (var userInputRequest in response.UserInputRequests) @@ -422,8 +359,7 @@ while (response.UserInputRequests.Count > 0) // Here we need to show the user an approval request. // We can use the FunctionCall property to show e.g. the function call that the agent wants to invoke. // If the user approves: - var approvalMessage = approvalRequest.Approve(); - messages.Add(approvalMessage); + messages.Add(new ChatMessage(ChatRole.User, approvalRequest.CreateApproval())); } } @@ -441,18 +377,135 @@ class AgentRunResponse ... } +``` -class AgentThread -{ - ... +## Decision Outcome - // The thread state may need to store the user input requests. - public List ActiveUserInputRequests { get; set; } +Chosen option 5. + +## Appendices + +### ChatClientAgent Approval Process Flow + +1. User passes a User message to the agent with a request. +1. Agent calls IChatClient with any functions registered on the agent. + (IChatClient has FunctionInvokingChatClient) +1. Model responds with FunctionCallContent indicating function calls required. +1. FunctionInvokingChatClient decorator identifies any function calls that require user approval and returns an FunctionApprovalRequestContent. + (If there are multiple parallel function calls, all function calls will be returned as FunctionApprovalRequestContent even if only some require approval.) +1. Agent updates the thread with the FunctionApprovalRequestContent (or this may have already been done by a service threaded agent). +1. Agent returns the FunctionApprovalRequestContent to the caller which shows it to the user in the appropriate format. +1. User (via caller) invokes the agent again with FunctionApprovalResponseContent. +1. Agent adds the FunctionApprovalResponseContent to the thread. +1. Agent calls IChatClient with the provided FunctionApprovalResponseContent. +1. Agent invokes IChatClient with FunctionApprovalResponseContent and the FunctionInvokingChatClient decorator identifies the response as an approval for the function call. + Any rejected approvals are converted to FunctionResultContent with a message indicating that the function invocation was denied. + Any approved approvals are executed by the FunctionInvokingChatClient decorator. +1. FunctionInvokingChatClient decorator passes the FunctionCallContent and FunctionResultContent for the approved and rejected function calls to the model. +1. Model responds with the result. +1. FunctionInvokingChatClient returns the FunctionCallContent, FunctionResultContent, and the result message to the agent. +1. Agent responds to caller with the same messages and updates the thread with these as well. - ... -} +### CustomAgent Approval Process Flow + +1. User passes a User message to the agent with a request. +1. Agent adds this message to the thread. +1. Agent executes various steps. +1. Agent encounters a step for which it requires user input to continue. +1. Agent responds with an UserInputRequestContent and also adds it to its thread. +1. User (via caller) invokes the agent again with UserInputResponseContent. +1. Agent adds the UserInputResponseContent to the thread. +1. Agent responds to caller with result message and thread is updated with the result message. + +### Sequence Diagram: FunctionInvokingChatClient with built in Approval Generation + +This is a ChatClient Approval Stack option has been proven to work via a proof of concept implementation. + +```mermaid +--- +title: Multiple Functions with partial approval +--- + +sequenceDiagram + note right of Developer: Developer asks question with two functions. + Developer->>+FunctionInvokingChatClient: What is the special soup today?
[GetMenu, GetSpecials] + FunctionInvokingChatClient->>+ResponseChatClient: What is the special soup today?
[GetMenu, GetSpecials] + + ResponseChatClient-->>-FunctionInvokingChatClient: [FunctionCallContent(GetMenu)],
[FunctionCallContent(GetSpecials)] + note right of FunctionInvokingChatClient: FICC turns FunctionCallContent
into FunctionApprovalRequestContent + FunctionInvokingChatClient->>+Developer: [FunctionApprovalRequestContent(GetMenu)]
[FunctionApprovalRequestContent(GetSpecials)] + + note right of Developer:Developer asks user for approval + Developer->>+FunctionInvokingChatClient: [FunctionApprovalRequestContent(GetMenu, approved=false)]
[FunctionApprovalRequestContent(GetSpecials, approved=true)] + note right of FunctionInvokingChatClient:FunctionInvokingChatClient executes the approved
function and generates a failed FunctionResultContent
for the rejected one, before invoking the model again. + FunctionInvokingChatClient->>+ResponseChatClient: What is the special soup today?
[FunctionCallContent(GetMenu)],
[FunctionCallContent(GetSpecials)],
[FunctionResultContent(GetMenu, Function invocation denied")]
[FunctionResultContent(GetSpecials, "Special Soup: Clam Chouder...")] + + ResponseChatClient-->>-FunctionInvokingChatClient: [TextContent("The specials soup is...")] + FunctionInvokingChatClient->>+Developer: [FunctionCallContent(GetMenu)],
[FunctionCallContent(GetSpecials)],
[FunctionResultContent(GetMenu, Function invocation denied")]
[FunctionResultContent(GetSpecials, "Special Soup: Clam Chouder...")]
[TextContent("The specials soup is...")] ``` -## Decision Outcome +### Sequence Diagram: Post FunctionInvokingChatClient ApprovalGeneratingChatClient - Multiple function calls with partial approval + +This is a discarded ChatClient Approval Stack option, but is included here for reference. + +```mermaid +--- +title: Multiple Functions with partial approval +--- + +sequenceDiagram + note right of Developer: Developer asks question with two functions. + Developer->>+FunctionInvokingChatClient: What is the special soup today? [GetMenu, GetSpecials] + FunctionInvokingChatClient->>+ApprovalGeneratingChatClient: What is the special soup today? [GetMenu, GetSpecials] + ApprovalGeneratingChatClient->>+ResponseChatClient: What is the special soup today? [GetMenu, GetSpecials] + + ResponseChatClient-->>-ApprovalGeneratingChatClient: [FunctionCallContent(GetMenu)],
[FunctionCallContent(GetSpecials)] + ApprovalGeneratingChatClient-->>-FunctionInvokingChatClient: [FunctionApprovalRequestContent(GetMenu)],
[FunctionApprovalRequestContent(GetSpecials)] + FunctionInvokingChatClient-->>-Developer: [FunctionApprovalRequestContent(GetMenu)]
[FunctionApprovalRequestContent(GetSpecials)] -Approach TBD. + note right of Developer: Developer approves one function call and rejects the other. + Developer->>+FunctionInvokingChatClient: [FunctionApprovalResponseContent(GetMenu, approved=true)]
[FunctionApprovalResponseContent(GetSpecials, approved=false)] + FunctionInvokingChatClient->>+ApprovalGeneratingChatClient: [FunctionApprovalResponseContent(GetMenu, approved=true)]
[FunctionApprovalResponseContent(GetSpecials, approved=false)] + + note right of FunctionInvokingChatClient: ApprovalGeneratingChatClient only returns FunctionCallContent
for approved FunctionApprovalResponseContent. + ApprovalGeneratingChatClient-->>-FunctionInvokingChatClient: [FunctionCallContent(GetMenu)] + note right of FunctionInvokingChatClient: FunctionInvokingChatClient has to also include all
FunctionApprovalResponseContent in the new downstream request. + FunctionInvokingChatClient->>+ApprovalGeneratingChatClient: [FunctionResultContent(GetMenu, "mains.... deserts...")]
[FunctionApprovalResponseContent(GetMenu, approved=true)]
[FunctionApprovalResponseContent(GetSpecials, approved=false)] + + note right of ApprovalGeneratingChatClient: ApprovalGeneratingChatClient now throws away
approvals for executed functions, and creates
failed FunctionResultContent for denied function calls. + ApprovalGeneratingChatClient->>+ResponseChatClient: [FunctionResultContent(GetMenu, "mains.... deserts...")]
[FunctionResultContent(GetSpecials, "Function invocation denied")] +``` + +### Sequence Diagram: Pre FunctionInvokingChatClient ApprovalGeneratingChatClient - Multiple function calls with partial approval + +This is a discarded ChatClient Approval Stack option, but is included here for reference. + +```mermaid +--- +title: Multiple Functions with partial approval +--- + +sequenceDiagram + note right of Developer: Developer asks question with two functions. + Developer->>+ApprovalGeneratingChatClient: What is the special soup today? [GetMenu, GetSpecials] + note right of ApprovalGeneratingChatClient: AGCC marks functions as not-invocable + ApprovalGeneratingChatClient->>+FunctionInvokingChatClient: What is the special soup today?
[GetMenu(invocable=false)]
[GetSpecials(invocable=false)] + FunctionInvokingChatClient->>+ResponseChatClient: What is the special soup today?
[GetMenu(invocable=false)]
[GetSpecials(invocable=false)] + + ResponseChatClient-->>-FunctionInvokingChatClient: [FunctionCallContent(GetMenu)],
[FunctionCallContent(GetSpecials)] + note right of FunctionInvokingChatClient: FICC doesn't invoke functions since they are not invocable. + FunctionInvokingChatClient-->>-ApprovalGeneratingChatClient: [FunctionCallContent(GetMenu)],
[FunctionCallContent(GetSpecials)] + note right of ApprovalGeneratingChatClient: AGCC turns functions into approval requests + ApprovalGeneratingChatClient-->>-Developer: [FunctionApprovalRequestContent(GetMenu)]
[FunctionApprovalRequestContent(GetSpecials)] + + note right of Developer: Developer approves one function call and rejects the other. + Developer->>+ApprovalGeneratingChatClient: [FunctionApprovalResponseContent(GetMenu, approved=true)]
[FunctionApprovalResponseContent(GetSpecials, approved=false)] + note right of ApprovalGeneratingChatClient: AGCC turns turns approval requests
into FCC or failed function calls + ApprovalGeneratingChatClient->>+FunctionInvokingChatClient: [FunctionCallContent(GetMenu)]
[FunctionCallContent(GetSpecials)
[FunctionResultContent(GetSpecials, "Function invocation denied"))] + note right of FunctionInvokingChatClient: FICC invokes GetMenu since it's the only remaining one. + FunctionInvokingChatClient->>+ResponseChatClient: [FunctionCallContent(GetMenu)]
[FunctionResultContent(GetMenu, "mains.... deserts...")]
[FunctionCallContent(GetSpecials)
[FunctionResultContent(GetSpecials, "Function invocation denied"))] + + ResponseChatClient-->>-FunctionInvokingChatClient: [FunctionCallContent(GetMenu)]
[FunctionResultContent(GetMenu, "mains.... deserts...")]
[FunctionCallContent(GetSpecials)
[FunctionResultContent(GetSpecials, "Function invocation denied"))]
[TextContent("The specials soup is...")] + FunctionInvokingChatClient-->>-ApprovalGeneratingChatClient: [FunctionCallContent(GetMenu)]
[FunctionResultContent(GetMenu, "mains.... deserts...")]
[FunctionCallContent(GetSpecials)
[FunctionResultContent(GetSpecials, "Function invocation denied"))]
[TextContent("The specials soup is...")] + ApprovalGeneratingChatClient-->>-Developer: [FunctionCallContent(GetMenu)]
[FunctionResultContent(GetMenu, "mains.... deserts...")]
[FunctionCallContent(GetSpecials)
[FunctionResultContent(GetSpecials, "Function invocation denied"))]
[TextContent("The specials soup is...")] +``` From 3420337fec630489d09b7194e7a8b59f4368005f Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:02:00 +0100 Subject: [PATCH 37/53] Explain case when pre FICC AGCC fails --- docs/decisions/00NN-userapproval-content-types.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/decisions/00NN-userapproval-content-types.md b/docs/decisions/00NN-userapproval-content-types.md index 38659484ad..800f4755ed 100644 --- a/docs/decisions/00NN-userapproval-content-types.md +++ b/docs/decisions/00NN-userapproval-content-types.md @@ -480,6 +480,17 @@ sequenceDiagram This is a discarded ChatClient Approval Stack option, but is included here for reference. +It doesn't work for the scenario where we have multiple function calls for the same function in serial with different arguments. + +Flow: + +- AGCC turns AIFunctions into AIFunctionDefinitions (not invocable) and FICC ignores these. +- We get back a FunctionCall for one of these and it gets approved. +- We invoke the FICC again, this time with an AIFunction. +- We call the service with the FCC and FRC. +- We get back a new Function call for the same function again with different arguments. +- Since we were passed an AIFunction instead of an AIFunctionDefinition, we now incorrectly execute this FC without approval. + ```mermaid --- title: Multiple Functions with partial approval From 6a6f9ddd39c777119eedc776484095693d3d70e1 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Thu, 14 Aug 2025 18:03:02 +0100 Subject: [PATCH 38/53] Add support for mcp servers over http. --- dotnet/Directory.Packages.props | 1 + dotnet/agent-framework-dotnet.slnx | 4 +- .../GettingStarted/GettingStarted.csproj | 1 + ...ntAgent_UsingFunctionToolsWithApprovals.cs | 23 ++-- .../HostedMCPChatClient.cs | 114 +++++++++++++++--- 5 files changed, 114 insertions(+), 29 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index fbbca52293..2eafaf4af4 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -31,6 +31,7 @@ + diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 7717ca43fd..0cbcf9a9e5 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -125,9 +125,7 @@ - - - + diff --git a/dotnet/samples/GettingStarted/GettingStarted.csproj b/dotnet/samples/GettingStarted/GettingStarted.csproj index 33191e0740..8ae4c1be5e 100644 --- a/dotnet/samples/GettingStarted/GettingStarted.csproj +++ b/dotnet/samples/GettingStarted/GettingStarted.csproj @@ -38,6 +38,7 @@ + diff --git a/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs index 6a8c455818..a79cec5837 100644 --- a/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs +++ b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. using System.ComponentModel; +#if NETFRAMEWORK +using System.Net.Http; +#endif using Microsoft.Extensions.AI; using Microsoft.Extensions.AI.Agents; using Microsoft.Extensions.AI.ModelContextProtocol; @@ -29,9 +32,9 @@ public async Task ApprovalsWithTools(ChatClientProviders provider) AIFunctionFactory.Create(menuTools.GetMenu), new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), AIFunctionFactory.Create(menuTools.GetItemPrice), - new HostedMcpServerTool("MyService", new Uri("https://mcp-server.example.com")) + new HostedMcpServerTool("Tiktoken Documentation", new Uri("https://gitmcp.io/openai/tiktoken")) { - AllowedTools = ["add"], + AllowedTools = ["search_tiktoken_documentation", "fetch_tiktoken_documentation"], ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, } ]); @@ -48,7 +51,7 @@ public async Task ApprovalsWithTools(ChatClientProviders provider) { chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => { - return new HostedMCPChatClient(innerClient); + return new HostedMCPChatClient(innerClient, new HttpClient()); }); } if (chatClient.GetService() is null) @@ -69,7 +72,7 @@ public async Task ApprovalsWithTools(ChatClientProviders provider) // Respond to user input, invoking functions where appropriate. await RunAgentAsync("What is the special soup and its price?"); await RunAgentAsync("What is the special drink?"); - await RunAgentAsync("What is 2 + 2?"); + await RunAgentAsync("how does tiktoken work?"); async Task RunAgentAsync(string input) { @@ -84,7 +87,7 @@ async Task RunAgentAsync(string input) { List nextIterationMessages = userInputRequests?.Select((request) => request switch { - FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" || functionApprovalRequest.FunctionCall.Name == "search_tiktoken_documentation" => new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateApproval()]), FunctionApprovalRequestContent functionApprovalRequest => new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateRejection()]), @@ -122,9 +125,9 @@ public async Task ApprovalsWithToolsStreaming(ChatClientProviders provider) AIFunctionFactory.Create(menuTools.GetMenu), new ApprovalRequiredAIFunction(AIFunctionFactory.Create(menuTools.GetSpecials)), AIFunctionFactory.Create(menuTools.GetItemPrice), - new HostedMcpServerTool("MyService", new Uri("https://mcp-server.example.com")) + new HostedMcpServerTool("Tiktoken Documentation", new Uri("https://gitmcp.io/openai/tiktoken")) { - AllowedTools = ["add"], + AllowedTools = ["search_tiktoken_documentation", "fetch_tiktoken_documentation"], ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire, } ]); @@ -141,7 +144,7 @@ public async Task ApprovalsWithToolsStreaming(ChatClientProviders provider) { chatBuilder.Use((IChatClient innerClient, IServiceProvider services) => { - return new HostedMCPChatClient(innerClient); + return new HostedMCPChatClient(innerClient, new HttpClient()); }); } if (chatClient.GetService() is null) @@ -162,7 +165,7 @@ public async Task ApprovalsWithToolsStreaming(ChatClientProviders provider) // Respond to user input, invoking functions where appropriate. await RunAgentAsync("What is the special soup and its price?"); await RunAgentAsync("What is the special drink?"); - await RunAgentAsync("What is 2 + 2?"); + await RunAgentAsync("how does tiktoken work?"); async Task RunAgentAsync(string input) { @@ -176,7 +179,7 @@ async Task RunAgentAsync(string input) { List nextIterationMessages = userInputRequests?.Select((request) => request switch { - FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" => + FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" || functionApprovalRequest.FunctionCall.Name == "search_tiktoken_documentation" => new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateApproval()]), FunctionApprovalRequestContent functionApprovalRequest => new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateRejection()]), diff --git a/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/HostedMCPChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/HostedMCPChatClient.cs index 27815973b1..27023825c8 100644 --- a/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/HostedMCPChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.ModelContextProtocol/HostedMCPChatClient.cs @@ -1,11 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; using ModelContextProtocol.Client; namespace Microsoft.Extensions.AI.ModelContextProtocol; @@ -15,18 +19,29 @@ namespace Microsoft.Extensions.AI.ModelContextProtocol; /// public class HostedMCPChatClient : DelegatingChatClient { + private readonly ILoggerFactory? _loggerFactory; + /// The logger to use for logging information about function invocation. private readonly ILogger _logger; + /// The HTTP client to use when connecting to the remote MCP server. + private readonly HttpClient _httpClient; + + /// A dictionary of cached mcp clients, keyed by the MCP server URL. + private ConcurrentDictionary? _mcpClients = null; + /// /// Initializes a new instance of the class. /// /// The underlying , or the next instance in a chain of clients. + /// The to use when connecting to the remote MCP server. /// An to use for logging information about function invocation. - public HostedMCPChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null) + public HostedMCPChatClient(IChatClient innerClient, HttpClient httpClient, ILoggerFactory? loggerFactory = null) : base(innerClient) { + this._loggerFactory = loggerFactory; this._logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; + this._httpClient = Throw.IfNull(httpClient); } /// @@ -39,14 +54,47 @@ public override async Task GetResponseAsync( return await base.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); } - List downstreamTools = []; - foreach (var tool in options.Tools ?? []) + var downstreamTools = await this.BuildDownstreamAIToolsAsync(options.Tools, cancellationToken).ConfigureAwait(false); + options = options.Clone(); + options.Tools = downstreamTools; + + // Make the call to the inner client. + return await base.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); + } + + /// + public override async IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (options?.Tools is not { Count: > 0 }) + { + // If there are no tools, just call the inner client. + await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken).ConfigureAwait(false)) + { + yield return update; + } + } + + var downstreamTools = await this.BuildDownstreamAIToolsAsync(options!.Tools, cancellationToken).ConfigureAwait(false); + options = options.Clone(); + options.Tools = downstreamTools; + + // Make the call to the inner client. + await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken).ConfigureAwait(false)) + { + yield return update; + } + } + + private async Task?> BuildDownstreamAIToolsAsync(IList? inputTools, CancellationToken cancellationToken) + { + List? downstreamTools = null; + foreach (var tool in inputTools ?? []) { if (tool is HostedMcpServerTool mcpTool) { // List all MCP functions from the specified MCP server. // This will need some caching in a real-world scenario to avoid repeated calls. - var mcpClient = await CreateMcpClientAsync(mcpTool.Url).ConfigureAwait(false); + var mcpClient = await this.CreateMcpClientAsync(mcpTool.Url, mcpTool.ServerName).ConfigureAwait(false); var mcpFunctions = await mcpClient.ListToolsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); // Add the listed functions to our list of tools we'll pass to the inner client. @@ -58,6 +106,7 @@ public override async Task GetResponseAsync( continue; } + downstreamTools ??= new List(); switch (mcpTool.ApprovalMode) { case HostedMcpServerToolAlwaysRequireApprovalMode alwaysRequireApproval: @@ -84,26 +133,59 @@ public override async Task GetResponseAsync( } // For other tools, we want to keep them in the list of tools. + downstreamTools ??= new List(); downstreamTools.Add(tool); } - options = options.Clone(); - options.Tools = downstreamTools; + return downstreamTools; + } - // Make the call to the inner client. - return await base.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + // Dispose of the HTTP client if it was created by this client. + this._httpClient?.Dispose(); + + if (this._mcpClients is not null) + { + // Dispose of all cached MCP clients. + foreach (var client in this._mcpClients.Values) + { +#pragma warning disable CA2012 // Use ValueTasks correctly + _ = client.DisposeAsync(); +#pragma warning restore CA2012 // Use ValueTasks correctly + } + + this._mcpClients.Clear(); + } + } + + base.Dispose(disposing); } - private static async Task CreateMcpClientAsync(Uri mcpService) + private async Task CreateMcpClientAsync(Uri mcpServiceUri, string serverName) { - // Create mock MCP client for demonstration purposes. - var clientTransport = new StdioClientTransport(new StdioClientTransportOptions + if (this._mcpClients is null) + { + this._mcpClients = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + } + + if (this._mcpClients.TryGetValue(mcpServiceUri.ToString(), out var cachedClient)) + { + // Return the cached client if it exists. + return cachedClient; + } + +#pragma warning disable CA2000 // Dispose objects before losing scope - This should be disposed by the mcp client. + var transport = new SseClientTransport(new() { - Name = "Everything", - Command = "npx", - Arguments = ["-y", "@modelcontextprotocol/server-everything"], - }); + Endpoint = mcpServiceUri, + Name = serverName, + }, this._httpClient, this._loggerFactory); +#pragma warning restore CA2000 // Dispose objects before losing scope - return await McpClientFactory.CreateAsync(clientTransport).ConfigureAwait(false); + return await McpClientFactory.CreateAsync(transport).ConfigureAwait(false); } } From 31059411b58ee852922508cd63696c57c0bbed0d Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:52:30 +0100 Subject: [PATCH 39/53] Addressing PR comments. --- .../AgentRunResponse.cs | 2 +- .../FunctionApprovalRequestContent.cs | 2 +- .../FunctionApprovalResponseContent.cs | 4 +-- .../MEAI/AIFunctionApprovalContext.cs | 26 ------------------- .../MEAI/ApprovalRequiredAIFunction.cs | 23 +++++++++++++++- 5 files changed, 26 insertions(+), 31 deletions(-) delete mode 100644 dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/AIFunctionApprovalContext.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs index 822cc188d1..9279b9ae38 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs @@ -89,7 +89,7 @@ public IList Messages /// This property concatenates all instances in the response. /// [JsonIgnore] - public IEnumerable UserInputRequests => this._messages?.SelectMany(x => x.Contents).OfType() ?? Array.Empty(); + public IEnumerable UserInputRequests => this._messages?.SelectMany(x => x.Contents).OfType() ?? []; /// Gets or sets the ID of the agent that produced the response. public string? AgentId { get; set; } diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs index 408c3ffa7d..344bb4c901 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs @@ -7,7 +7,7 @@ namespace Microsoft.Extensions.AI; /// /// Represents a request for user approval of a function call. /// -public class FunctionApprovalRequestContent : UserInputRequestContent +public sealed class FunctionApprovalRequestContent : UserInputRequestContent { /// /// Initializes a new instance of the class. diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs index c12328837c..11bb01b5f6 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs @@ -7,7 +7,7 @@ namespace Microsoft.Extensions.AI; /// /// Represents a response to a function approval request. /// -public class FunctionApprovalResponseContent : UserInputResponseContent +public sealed class FunctionApprovalResponseContent : UserInputResponseContent { /// /// Initializes a new instance of the class. @@ -25,7 +25,7 @@ public FunctionApprovalResponseContent(string id, bool approved, FunctionCallCon /// /// Gets or sets a value indicating whether the user approved the request. /// - public bool Approved { get; set; } + public bool Approved { get; } /// /// Gets the function call that pre-invoke approval is required for. diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/AIFunctionApprovalContext.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/AIFunctionApprovalContext.cs deleted file mode 100644 index e735c1aa55..0000000000 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/AIFunctionApprovalContext.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.AI; - -/// -/// Context object that provides information about the function call that requires approval. -/// -public class AIFunctionApprovalContext -{ - /// - /// Initializes a new instance of the class. - /// - /// The containing the details of the invocation. - /// - public AIFunctionApprovalContext(FunctionCallContent functionCall) - { - this.FunctionCall = functionCall ?? throw new ArgumentNullException(nameof(functionCall)); - } - - /// - /// Gets the containing the details of the invocation that will be made if approval is granted. - /// - public FunctionCallContent FunctionCall { get; } -} diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs index 8dfa63d7ae..b85059b476 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs @@ -14,5 +14,26 @@ public sealed class ApprovalRequiredAIFunction(AIFunction function) : Delegating /// /// An optional callback that can be used to determine if the function call requires approval, instead of the default behavior, which is to always require approval. /// - public Func> RequiresApprovalCallback { get; set; } = delegate { return new ValueTask(true); }; + public Func> RequiresApprovalCallback { get; set; } = delegate { return new ValueTask(true); }; + + /// + /// Context object that provides information about the function call that requires approval. + /// + public sealed class ApprovalContext + { + /// + /// Initializes a new instance of the class. + /// + /// The containing the details of the invocation. + /// + public ApprovalContext(FunctionCallContent functionCall) + { + this.FunctionCall = functionCall ?? throw new ArgumentNullException(nameof(functionCall)); + } + + /// + /// Gets the containing the details of the invocation that will be made if approval is granted. + /// + public FunctionCallContent FunctionCall { get; } + } } From 26eb742c92ae869464251a2255b802e24509f418 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 15 Aug 2025 11:18:25 +0100 Subject: [PATCH 40/53] Address PR comments --- .../AgentRunResponseUpdate.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs index fecb4194b3..a40fac7c3c 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponseUpdate.cs @@ -98,7 +98,7 @@ public string? AuthorName /// This property concatenates all instances in the response. /// [JsonIgnore] - public IEnumerable UserInputRequests => this._contents?.OfType() ?? Array.Empty(); + public IEnumerable UserInputRequests => this._contents?.OfType() ?? []; /// Gets or sets the agent run response update content items. [AllowNull] From 5e52c1171b91fa14ab135275799eb9281c187e0b Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 19 Aug 2025 16:23:31 +0100 Subject: [PATCH 41/53] Add FICC unit tests and fix bugs. --- .../MEAI/NewFunctionInvokingChatClient.cs | 209 +++--- .../MEAI/AssertExtensions.cs | 86 +++ .../NewFunctionInvokingChatClientTests.cs | 689 ++++++++++++++++++ .../MEAI/TestChatClient.cs | 51 ++ 4 files changed, 950 insertions(+), 85 deletions(-) create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/AssertExtensions.cs create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs create mode 100644 dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/TestChatClient.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index 05b6553fc7..70bb436c18 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -56,7 +56,6 @@ namespace Microsoft.Extensions.AI; /// invocation requests to that same function. /// /// -[ExcludeFromCodeCoverage] public partial class NewFunctionInvokingChatClient : DelegatingChatClient { /// The for the current function invocation. @@ -261,41 +260,50 @@ public override async Task GetResponseAsync( // ** Approvals additions on top of FICC - start **// - // We should maintain a list of chat messages that need to be returned to the caller - // as part of the next response that were generated before the first iteration. - List? augmentedPreInvocationHistory = null; - - // Remove any approval requests and approval request/response pairs that have already been executed. - var notExecutedResponses = ProcessApprovalRequestsAndResponses(originalMessages); + // Extract any approval responses where we need to execute or reject the function calls. + // The original messages are also modified to remove all approval requests and responses. + var notExecutedResponses = ExtractAndRemoveApprovalRequestsAndResponses(originalMessages); - // Generate failed function result contents for any rejected requests. - List? rejectedFunctionCallMessages = null; - GenerateRejectedFunctionResults(notExecutedResponses.rejections, null, null, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); + // Wrap the function call content in message(s). + ICollection? allPreInvocationCallMessages = ConvertToFunctionCallContentMessages( + [.. notExecutedResponses.rejections ?? [], .. notExecutedResponses.approvals ?? []], null); - // We need to add FCC and FRC that were generated from the rejected approval requests into the original messages, - // so that the LLM can see them during the first iteration. - originalMessages.AddRange(rejectedFunctionCallMessages ?? []); + // Generate failed function result contents for any rejected requests and wrap it in a message. + List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notExecutedResponses.rejections, null, null); + ChatMessage? rejectedPreInvocationResultsMessage = rejectedFunctionCallResults != null ? + new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) : + null; - // Check if there are any function calls to do from any approved functions and execute them. - if (notExecutedResponses.approvals is { Count: > 0 }) + // Add all the FCC and FRC that we generated into the original messages list so that they are passed to the + // inner client and can be used to generate a result. Also add them to the augmented pre-invocation history + // so that they can be returned to the caller as part of the next response. + List? augmentedPreInvocationHistory = null; + if (allPreInvocationCallMessages is not null || rejectedPreInvocationResultsMessage is not null) { - var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => ConvertToFunctionCallContentMessage(x, null)).ToList(); - originalMessages.AddRange(approvedFunctionCalls); augmentedPreInvocationHistory ??= []; - augmentedPreInvocationHistory.AddRange(approvedFunctionCalls); + foreach (var message in (allPreInvocationCallMessages ?? []).Concat(rejectedPreInvocationResultsMessage == null ? [] : [rejectedPreInvocationResultsMessage])) + { + originalMessages.Add(message); + augmentedPreInvocationHistory.Add(message); + } + } - // Add the responses from the function calls into the augmented history and also into the tracked - // list of response messages. + // Check if there are any function calls to do for any approved functions and execute them. + if (notExecutedResponses.approvals is { Count: > 0 }) + { + // The FRC that is generated here is already added to originalMessages by ProcessFunctionCallsAsync. var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: false, cancellationToken); - responseMessages = [.. rejectedFunctionCallMessages ?? [], .. approvedFunctionCalls, .. modeAndMessages.MessagesAdded]; consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + // We need to add the generated FRC to the list we'll return to callers as part of the next response. + augmentedPreInvocationHistory ??= []; + augmentedPreInvocationHistory.AddRange(modeAndMessages.MessagesAdded); + if (modeAndMessages.ShouldTerminate) { + responseMessages = [.. allPreInvocationCallMessages ?? [], .. modeAndMessages.MessagesAdded]; return new ChatResponse(responseMessages); } - - augmentedPreInvocationHistory.AddRange(modeAndMessages.MessagesAdded); } // ** Approvals additions on top of FICC - end **// @@ -315,7 +323,7 @@ public override async Task GetResponseAsync( // Before we do any function execution, make sure that any functions that require approval, have been turned into approval requests // so that they don't get executed here. - await ReplaceFunctionCallsWithApprovalRequests(response.Messages, options?.Tools, AdditionalTools); + response.Messages = await ReplaceFunctionCallsWithApprovalRequests(response.Messages, options?.Tools, AdditionalTools); // ** Approvals additions on top of FICC - end **// @@ -333,7 +341,7 @@ public override async Task GetResponseAsync( // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, // so they are returned to the caller. - UpdateResponseWithPreInvocationHistory(response, augmentedPreInvocationHistory); + response.Messages = UpdateResponseMessagesWithPreInvocationHistory(response.Messages, augmentedPreInvocationHistory); augmentedPreInvocationHistory = null; // ** Approvals additions on top of FICC - end **// @@ -428,60 +436,55 @@ public override async IAsyncEnumerable GetStreamingResponseA ApprovalRequiredAIFunction[]? approvalRequiredFunctions = (options?.Tools ?? []).Concat(AdditionalTools ?? []).OfType().ToArray(); bool hasApprovalRequiringFunctions = approvalRequiredFunctions.Length > 0; - var shouldTerminateFromPreInvocationFunctionCalls = false; + // Extract any approval responses where we need to execute or reject the function calls. + // The original messages are also modified to remove all approval requests and responses. + var notExecutedResponses = ExtractAndRemoveApprovalRequestsAndResponses(originalMessages); - // We should maintain a list of chat messages that were generated before the first iteration - // and that need to be returned to the caller as part of the next response. - List? augmentedPreInvocationHistory = null; + // Wrap the function call content in message(s). + ICollection? allPreInvocationCallMessages = ConvertToFunctionCallContentMessages( + [.. notExecutedResponses.rejections ?? [], .. notExecutedResponses.approvals ?? []], null); - // Extract and remove any approval requests and approval request/response pairs that have already been executed. - var notExecutedResponses = ProcessApprovalRequestsAndResponses(originalMessages); + // Generate failed function results contents for any rejected requests and turn them into messages. + List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId, functionCallContentFallbackMessageId); + ChatMessage? rejectedPreInvocationResultsMessage = rejectedFunctionCallResults != null ? new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) { MessageId = toolResponseId } : null; - // Generate failed function result contents for any rejected requests. - List? rejectedFunctionCallMessages = null; - GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId, functionCallContentFallbackMessageId, ref rejectedFunctionCallMessages, ref augmentedPreInvocationHistory); + // Add all the FCC and FRC that we generated into the original messages list so that they are passed to the + // inner client and can be used to generate a result. Also add them to the augmented pre-invocation history + // so that they can be returned to the caller as part of the next response. + List? augmentedPreInvocationHistory = null; + if (allPreInvocationCallMessages is not null || rejectedPreInvocationResultsMessage is not null) + { + augmentedPreInvocationHistory ??= []; + foreach (var message in (allPreInvocationCallMessages ?? []).Concat(rejectedPreInvocationResultsMessage == null ? [] : [rejectedPreInvocationResultsMessage])) + { + originalMessages.Add(message); + augmentedPreInvocationHistory.Add(message); - // We need to add FCC and FRC that were generated from the rejected approval requests into the original messages, - // so that the LLM can see them during the first iteration. - originalMessages.AddRange(rejectedFunctionCallMessages ?? []); + yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } // Check if there are any function calls to do from any approved functions and execute them. if (notExecutedResponses.approvals is { Count: > 0 }) { - var approvedFunctionCalls = notExecutedResponses.approvals.Select(x => ConvertToFunctionCallContentMessage(x, functionCallContentFallbackMessageId)).ToList(); - originalMessages.AddRange(approvedFunctionCalls); - augmentedPreInvocationHistory ??= []; - augmentedPreInvocationHistory.AddRange(approvedFunctionCalls); - - // Add the responses from the function calls into the augmented history and also into the tracked - // list of response messages. var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: true, cancellationToken); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + augmentedPreInvocationHistory ??= []; foreach (var message in modeAndMessages.MessagesAdded) { message.MessageId = toolResponseId; - } - - responseMessages = [.. rejectedFunctionCallMessages ?? [], .. approvedFunctionCalls, .. modeAndMessages.MessagesAdded]; - consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + augmentedPreInvocationHistory.Add(message); - augmentedPreInvocationHistory.AddRange(modeAndMessages.MessagesAdded); - shouldTerminateFromPreInvocationFunctionCalls |= modeAndMessages.ShouldTerminate; - } - - // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages - // includes all activities, including generated function results. - if (augmentedPreInvocationHistory is { Count: > 0 }) - { - foreach (var message in augmentedPreInvocationHistory ?? []) - { yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } - } - if (shouldTerminateFromPreInvocationFunctionCalls) - { - yield break; + if (modeAndMessages.ShouldTerminate) + { + yield break; + } } // ** Approvals additions on top of FICC - end **// @@ -1157,7 +1160,7 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul /// We can then use the metadata from these messages when we re-create the FunctionCallContent messages/updates to return to the caller. This way, when we finally do return /// the FuncionCallContent to users it's part of a message/update that contains the same metadata as originally returned to the downstream service. /// - private static (List? approvals, List? rejections) ProcessApprovalRequestsAndResponses(List messages) + private static (List? approvals, List? rejections) ExtractAndRemoveApprovalRequestsAndResponses(List messages) { Dictionary? allApprovalRequestsMessages = null; List? allApprovalResponses = null; @@ -1275,32 +1278,56 @@ private static (List? approvals, ListAny rejected approval responses. /// The message id to use for the tool response. /// Optional message id to use if no original approval request message is availble to copy from. - /// The function call and result content for the rejected calls. - /// The list of messages generated before the first invocation, that need to be added to the user response. - private static void GenerateRejectedFunctionResults( + /// The for the rejected function calls. + private static List? GenerateRejectedFunctionResults( List? rejections, string? toolResponseId, - string? fallbackMessageId, - ref List? rejectedFunctionCallMessages, - ref List? augmentedPreInvocationHistory) + string? fallbackMessageId) { + List? functionResultContent = null; + if (rejections is { Count: > 0 }) { - rejectedFunctionCallMessages = []; + functionResultContent = []; + foreach (var rejectedCall in rejections) { - // We want to add the original FunctionCallContent that was replaced with an ApprovalRequest back into the chat history as well - // otherwise we would have an ApprovalRequest, ApprovalResponse and FunctionResultContent, but no FunctionCallContent matching the FunctionResultContent. - rejectedFunctionCallMessages.Add(ConvertToFunctionCallContentMessage(rejectedCall, fallbackMessageId)); - // Create a FunctionResultContent for the rejected function call. var functionResult = new FunctionResultContent(rejectedCall.Response.FunctionCall.CallId, "Error: Function invocation approval was not granted."); - rejectedFunctionCallMessages.Add(new ChatMessage(ChatRole.Tool, [functionResult]) { MessageId = toolResponseId }); + functionResultContent.Add(functionResult); } + } - augmentedPreInvocationHistory ??= []; - augmentedPreInvocationHistory.AddRange(rejectedFunctionCallMessages); + return functionResultContent; + } + +#pragma warning disable CA1859 // Use concrete types when possible for improved performance + private static ICollection? ConvertToFunctionCallContentMessages(IEnumerable? resultWithRequestMessages, string? fallbackMessageId) +#pragma warning restore CA1859 // Use concrete types when possible for improved performance + { + if (resultWithRequestMessages is not null) + { + Dictionary? messagesById = null; + + foreach (var resultWithRequestMessage in resultWithRequestMessages) + { + messagesById ??= new(); + + if (messagesById.TryGetValue(resultWithRequestMessage.RequestMessage?.MessageId ?? string.Empty, out var message)) + { + message.Contents.Add(resultWithRequestMessage.Response.FunctionCall); + } + else + { + var functionCallMessage = ConvertToFunctionCallContentMessage(resultWithRequestMessage, fallbackMessageId); + messagesById[functionCallMessage.MessageId ?? string.Empty] = functionCallMessage; + } + } + + return messagesById?.Values; } + + return null; } private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWithRequestMessage resultWithRequestMessage, string? fallbackMessageId) @@ -1320,8 +1347,9 @@ private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWit /// Replaces all from with /// if any one of them requires approval. /// - private static async Task ReplaceFunctionCallsWithApprovalRequests(IList messages, IList? requestOptionsTools, IList? additionalTools) + private static async Task> ReplaceFunctionCallsWithApprovalRequests(IList messages, IList? requestOptionsTools, IList? additionalTools) { + var outputMessages = messages; ApprovalRequiredAIFunction[]? approvalRequiredFunctions = null; bool anyApprovalRequired = false; @@ -1352,28 +1380,39 @@ private static async Task ReplaceFunctionCallsWithApprovalRequests(IList - /// Insert the given at the start of the 's messages. + /// Insert the given at the start of the . /// - private static void UpdateResponseWithPreInvocationHistory(ChatResponse response, List? augmentedPreInvocationHistory) + private static IList UpdateResponseMessagesWithPreInvocationHistory(IList responseMessages, List? augmentedPreInvocationHistory) { if (augmentedPreInvocationHistory?.Count > 0) { // Since these messages are pre-invocation, we want to insert them at the start of the response messages. - for (int i = augmentedPreInvocationHistory.Count - 1; i >= 0; i--) - { - response.Messages.Insert(0, augmentedPreInvocationHistory[i]); - } + return [.. augmentedPreInvocationHistory, .. responseMessages]; } + + return responseMessages; } private static ChatResponseUpdate ConvertToolResultMessageToUpdate(ChatMessage message, string? converationId, string? messageId) diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/AssertExtensions.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/AssertExtensions.cs new file mode 100644 index 0000000000..da95df83d0 --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/AssertExtensions.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft. All rights reserved. + +// WARNING: +// This class has been temporarily copied here from MEAI, to allow prototyping +// functionality that will be moved to MEAI in the future. +// This file is not intended to be modified. + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Xunit.Sdk; + +namespace Microsoft.Extensions.AI; + +internal static class AssertExtensions +{ + /// + /// Asserts that the two function call parameters are equal, up to JSON equivalence. + /// + public static void EqualFunctionCallParameters( + IDictionary? expected, + IDictionary? actual, + JsonSerializerOptions? options = null) + { + if (expected is null || actual is null) + { + Assert.Equal(expected, actual); + return; + } + + foreach (var expectedEntry in expected) + { + if (!actual.TryGetValue(expectedEntry.Key, out object? actualValue)) + { + throw new XunitException($"Expected parameter '{expectedEntry.Key}' not found in actual value."); + } + + AreJsonEquivalentValues(expectedEntry.Value, actualValue, options, propertyName: expectedEntry.Key); + } + + if (expected.Count != actual.Count) + { + var extraParameters = actual + .Where(e => !expected.ContainsKey(e.Key)) + .Select(e => $"'{e.Key}'") + .First(); + + throw new XunitException($"Actual value contains additional parameters {string.Join(", ", extraParameters)} not found in expected value."); + } + } + + /// + /// Asserts that the two function call results are equal, up to JSON equivalence. + /// + public static void EqualFunctionCallResults(object? expected, object? actual, JsonSerializerOptions? options = null) + => AreJsonEquivalentValues(expected, actual, options); + + /// + /// Asserts that the two JSON values are equal. + /// + public static void EqualJsonValues(JsonElement expectedJson, JsonElement actualJson, string? propertyName = null) + { + if (!JsonNode.DeepEquals( + JsonSerializer.SerializeToNode(expectedJson, AIJsonUtilities.DefaultOptions), + JsonSerializer.SerializeToNode(actualJson, AIJsonUtilities.DefaultOptions))) + { + string message = propertyName is null + ? $"JSON result does not match expected JSON.\r\nExpected: {expectedJson.GetRawText()}\r\nActual: {actualJson.GetRawText()}" + : $"Parameter '{propertyName}' does not match expected JSON.\r\nExpected: {expectedJson.GetRawText()}\r\nActual: {actualJson.GetRawText()}"; + + throw new XunitException(message); + } + } + + private static void AreJsonEquivalentValues(object? expected, object? actual, JsonSerializerOptions? options, string? propertyName = null) + { + options ??= AIJsonUtilities.DefaultOptions; + JsonElement expectedElement = NormalizeToElement(expected, options); + JsonElement actualElement = NormalizeToElement(actual, options); + EqualJsonValues(expectedElement, actualElement, propertyName); + + static JsonElement NormalizeToElement(object? value, JsonSerializerOptions options) + => value is JsonElement e ? e : JsonSerializer.SerializeToElement(value, options); + } +} diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs new file mode 100644 index 0000000000..381bb136ac --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs @@ -0,0 +1,689 @@ +// Copyright (c) Microsoft. All rights reserved. + +// AF repo suppressions for code copied from MEAI. +#pragma warning disable CA5394 // Do not use insecure randomness +#pragma warning disable IDE0009 // Member access should be qualified. +#pragma warning disable IDE0039 // Use local function + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +public class NewFunctionInvokingChatClientTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AllFunctionCallsReplacedWithApprovalsWhenAllRequireApprovalAsync(bool useAdditionalTools) + { + AITool[] tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2")) { RequiresApprovalCallback = async (context) => context.FunctionCall.Name == "Func2" }, + ]; + + var options = new ChatOptions + { + Tools = useAdditionalTools ? null : tools + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + ]; + + List expectedOutput = + [ + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, expectedOutput, additionalTools: useAdditionalTools ? tools : null); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, expectedOutput, additionalTools: useAdditionalTools ? tools : null); + } + + [Fact] + public async Task AllFunctionCallsReplacedWithApprovalsWhenAnyRequireApprovalAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + ]; + + List expectedOutput = + [ + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, expectedOutput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, expectedOutput); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AllFunctionCallsReplacedWithApprovalsWhenAnyRequestOrAdditionalRequireApprovalAsync(bool additionalToolsRequireApproval) + { + AIFunction func1 = AIFunctionFactory.Create(() => "Result 1", "Func1"); + AIFunction func2 = AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"); + AITool[] additionalTools = + [ + additionalToolsRequireApproval ? new ApprovalRequiredAIFunction(func1) : func1, + ]; + + var options = new ChatOptions + { + Tools = + [ + additionalToolsRequireApproval ? func2 : new ApprovalRequiredAIFunction(func2), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + ]; + + List expectedOutput = + [ + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, expectedOutput, additionalTools: additionalTools); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, expectedOutput, additionalTools: additionalTools); + } + + [Fact] + public async Task ApprovedApprovalResponsesAreExecutedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + [Fact] + public async Task RejectedApprovalResponsesAreFailedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", false, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", false, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted."), new FunctionResultContent("callId2", result: "Error: Function invocation approval was not granted.")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted."), new FunctionResultContent("callId2", result: "Error: Function invocation approval was not granted.")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + [Fact] + public async Task MixedApprovedAndRejectedApprovalResponsesAreExecutedAndFailedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", false, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted.")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List nonStreamingOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted.")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List streamingOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Error: Function invocation approval was not granted."), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, nonStreamingOutput, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, streamingOutput, expectedDownstreamClientInput); + } + + [Fact] + public async Task ApprovedInputsAreExecutedAndFunctionResultsAreConvertedAsync() + { + var options = new ChatOptions + { + Tools = + [ + AIFunctionFactory.Create(() => "Result 1", "Func1"), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2")), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 3 } })]), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, [new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 3 } }))]), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + + /// + /// Since we do not have a way of supporting both functions that require approval and those that do not + /// in one invocation, we always require all function calls to be approved if any require approval. + /// If we are therefore unsure as to whether we will encounter a function call that requires approval, + /// we have to wait until we find one before yielding any function call content. + /// If we don't have any function calls that require approval at all though, we can just yield all content normally + /// since this issue won't apply. + /// + [Fact] + public async Task FunctionCallContentIsYieldedImmediatelyIfNoApprovalRequiredWhenStreamingAsync() + { + var options = new ChatOptions + { + Tools = + [ + AIFunctionFactory.Create(() => "Result 1", "Func1"), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = [new ChatMessage(ChatRole.User, "hello")]; + + Func configurePipeline = b => b.Use(s => new NewFunctionInvokingChatClient(s)); + using CancellationTokenSource cts = new(); + + var updateYieldCount = 0; + + async IAsyncEnumerable YieldInnerClientUpdates(IEnumerable contents, ChatOptions? actualOptions, [EnumeratorCancellation] CancellationToken actualCancellationToken) + { + Assert.Equal(cts.Token, actualCancellationToken); + await Task.Yield(); + var messageId = Guid.NewGuid().ToString("N"); + + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]) { MessageId = messageId }; + } + + using var innerClient = new TestChatClient { GetStreamingResponseAsyncCallback = YieldInnerClientUpdates }; + IChatClient service = configurePipeline(innerClient.AsBuilder()).Build(); + + var updates = service.GetStreamingResponseAsync(new EnumeratedOnceEnumerable(input), options, cts.Token); + + var updateCount = 0; + await foreach (var update in updates) + { + if (updateCount < 2) + { + var functionCall = update.Contents.OfType().First(); + if (functionCall.CallId == "callId1") + { + Assert.Equal("Func1", functionCall.Name); + Assert.Equal(1, updateYieldCount); + } + else if (functionCall.CallId == "callId2") + { + Assert.Equal("Func2", functionCall.Name); + Assert.Equal(2, updateYieldCount); + } + } + + updateCount++; + } + } + + /// + /// Since we do not have a way of supporting both functions that require approval and those that do not + /// in one invocation, we always require all function calls to be approved if any require approval. + /// If we are therefore unsure as to whether we will encounter a function call that requires approval, + /// we have to wait until we find one before yielding any function call content. + /// We can however, yield any other content until we encounter the first function call. + /// + [Fact] + public async Task FunctionCalsAreBufferedUntilApprovalRequirementEncounteredWhenStreamingAsync() + { + var options = new ChatOptions + { + Tools = + [ + AIFunctionFactory.Create(() => "Result 1", "Func1"), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2")), + AIFunctionFactory.Create(() => "Result 3", "Func3"), + ] + }; + + List input = [new ChatMessage(ChatRole.User, "hello")]; + + Func configurePipeline = b => b.Use(s => new NewFunctionInvokingChatClient(s)); + using CancellationTokenSource cts = new(); + + var updateYieldCount = 0; + + async IAsyncEnumerable YieldInnerClientUpdates(IEnumerable contents, ChatOptions? actualOptions, [EnumeratorCancellation] CancellationToken actualCancellationToken) + { + Assert.Equal(cts.Token, actualCancellationToken); + await Task.Yield(); + var messageId = Guid.NewGuid().ToString("N"); + + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new TextContent("Text 1")]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new TextContent("Text 2")]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]) { MessageId = messageId }; + updateYieldCount++; + yield return new ChatResponseUpdate(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func3")]) { MessageId = messageId }; + } + + using var innerClient = new TestChatClient { GetStreamingResponseAsyncCallback = YieldInnerClientUpdates }; + IChatClient service = configurePipeline(innerClient.AsBuilder()).Build(); + + var updates = service.GetStreamingResponseAsync(new EnumeratedOnceEnumerable(input), options, cts.Token); + + var updateCount = 0; + await foreach (var update in updates) + { + switch (updateCount) + { + case 0: + Assert.Equal("Text 1", update.Contents.OfType().First().Text); + // First content should be yielded immedately, since we don't have any function calls yet. + Assert.Equal(1, updateYieldCount); + break; + case 1: + Assert.Equal("Text 2", update.Contents.OfType().First().Text); + // Second content should be yielded immedately, since we don't have any function calls yet. + Assert.Equal(2, updateYieldCount); + break; + case 2: + var approvalRequest1 = update.Contents.OfType().First(); + Assert.Equal("callId1", approvalRequest1.FunctionCall.CallId); + Assert.Equal("Func1", approvalRequest1.FunctionCall.Name); + // Third content should have been buffered, since we have not yet encountered a function call that requires approval. + Assert.Equal(4, updateYieldCount); + break; + case 3: + var approvalRequest2 = update.Contents.OfType().First(); + Assert.Equal("callId2", approvalRequest2.FunctionCall.CallId); + Assert.Equal("Func2", approvalRequest2.FunctionCall.Name); + // Fourth content can be yielded immediately, since it is the first function call that requires approval. + Assert.Equal(4, updateYieldCount); + break; + case 4: + var approvalRequest3 = update.Contents.OfType().First(); + Assert.Equal("callId1", approvalRequest3.FunctionCall.CallId); + Assert.Equal("Func3", approvalRequest3.FunctionCall.Name); + // Fifth content can be yielded immediately, since we previously encountered a function call that requires approval. + Assert.Equal(5, updateYieldCount); + break; + } + + updateCount++; + } + } + + private static async Task> InvokeAndAssertAsync( + ChatOptions? options, + List input, + List downstreamClientOutput, + List expectedOutput, + List? expectedDownstreamClientInput = null, + Func? configurePipeline = null, + AITool[]? additionalTools = null, + IServiceProvider? services = null) + { + Assert.NotEmpty(input); + + configurePipeline ??= b => b.Use(s => new NewFunctionInvokingChatClient(s) { AdditionalTools = additionalTools }); + + using CancellationTokenSource cts = new(); + long expectedTotalTokenCounts = 0; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = async (contents, actualOptions, actualCancellationToken) => + { + Assert.Equal(cts.Token, actualCancellationToken); + if (expectedDownstreamClientInput is not null) + { + CompareMessageLists(expectedDownstreamClientInput, contents.ToList()); + } + + await Task.Yield(); + + var usage = CreateRandomUsage(); + expectedTotalTokenCounts += usage.InputTokenCount!.Value; + + downstreamClientOutput.ForEach(m => m.MessageId = Guid.NewGuid().ToString("N")); + return new ChatResponse(downstreamClientOutput) { Usage = usage }; + } + }; + + IChatClient service = configurePipeline(innerClient.AsBuilder()).Build(services); + + var result = await service.GetResponseAsync(new EnumeratedOnceEnumerable(input), options, cts.Token); + Assert.NotNull(result); + + var actualOutput = result.Messages as List ?? result.Messages.ToList(); + CompareMessageLists(expectedOutput, actualOutput); + + // Usage should be aggregated over all responses, including AdditionalUsage + var actualUsage = result.Usage!; + Assert.Equal(expectedTotalTokenCounts, actualUsage.InputTokenCount); + Assert.Equal(expectedTotalTokenCounts, actualUsage.OutputTokenCount); + Assert.Equal(expectedTotalTokenCounts, actualUsage.TotalTokenCount); + Assert.Equal(2, actualUsage.AdditionalCounts!.Count); + Assert.Equal(expectedTotalTokenCounts, actualUsage.AdditionalCounts["firstValue"]); + Assert.Equal(expectedTotalTokenCounts, actualUsage.AdditionalCounts["secondValue"]); + + return actualOutput; + } + + private static UsageDetails CreateRandomUsage() + { + // We'll set the same random number on all the properties so that, when determining the + // correct sum in tests, we only have to total the values once + var value = new Random().Next(100); + return new UsageDetails + { + InputTokenCount = value, + OutputTokenCount = value, + TotalTokenCount = value, + AdditionalCounts = new() { ["firstValue"] = value, ["secondValue"] = value }, + }; + } + + private static async Task> InvokeAndAssertStreamingAsync( + ChatOptions? options, + List input, + List downstreamClientOutput, + List expectedOutput, + List? expectedDownstreamClientInput = null, + Func? configurePipeline = null, + AITool[]? additionalTools = null, + IServiceProvider? services = null) + { + Assert.NotEmpty(input); + + configurePipeline ??= b => b.Use(s => new NewFunctionInvokingChatClient(s) { AdditionalTools = additionalTools }); + + using CancellationTokenSource cts = new(); + + using var innerClient = new TestChatClient + { + GetStreamingResponseAsyncCallback = (contents, actualOptions, actualCancellationToken) => + { + Assert.Equal(cts.Token, actualCancellationToken); + if (expectedDownstreamClientInput is not null) + { + CompareMessageLists(expectedDownstreamClientInput, contents.ToList()); + } + + downstreamClientOutput.ForEach(m => m.MessageId = Guid.NewGuid().ToString("N")); + return YieldAsync(new ChatResponse(downstreamClientOutput).ToChatResponseUpdates()); + } + }; + + IChatClient service = configurePipeline(innerClient.AsBuilder()).Build(services); + + var result = await service.GetStreamingResponseAsync(new EnumeratedOnceEnumerable(input), options, cts.Token).ToChatResponseAsync(); + Assert.NotNull(result); + + var actualOutput = result.Messages as List ?? result.Messages.ToList(); + + expectedOutput ??= input; + CompareMessageLists(expectedOutput, actualOutput); + + return actualOutput; + } + + private static async IAsyncEnumerable YieldAsync(params T[] items) + { + await Task.Yield(); + foreach (var item in items) + { + yield return item; + } + } + + private static void CompareMessageLists(List expectedMessages, List actualMessages) + { + Assert.Equal(expectedMessages.Count, actualMessages.Count); + for (int i = 0; i < expectedMessages.Count; i++) + { + var expectedMessage = expectedMessages[i]; + var chatMessage = actualMessages[i]; + + Assert.Equal(expectedMessage.Role, chatMessage.Role); + Assert.Equal(expectedMessage.Text, chatMessage.Text); + Assert.Equal(expectedMessage.GetType(), chatMessage.GetType()); + + Assert.Equal(expectedMessage.Contents.Count, chatMessage.Contents.Count); + for (int j = 0; j < expectedMessage.Contents.Count; j++) + { + var expectedItem = expectedMessage.Contents[j]; + var chatItem = chatMessage.Contents[j]; + + Assert.Equal(expectedItem.GetType(), chatItem.GetType()); + Assert.Equal(expectedItem.ToString(), chatItem.ToString()); + if (expectedItem is FunctionCallContent expectedFunctionCall) + { + var chatFunctionCall = (FunctionCallContent)chatItem; + Assert.Equal(expectedFunctionCall.Name, chatFunctionCall.Name); + AssertExtensions.EqualFunctionCallParameters(expectedFunctionCall.Arguments, chatFunctionCall.Arguments); + } + else if (expectedItem is FunctionResultContent expectedFunctionResult) + { + var chatFunctionResult = (FunctionResultContent)chatItem; + AssertExtensions.EqualFunctionCallResults(expectedFunctionResult.Result, chatFunctionResult.Result); + } + } + } + } + + private sealed class EnumeratedOnceEnumerable(IEnumerable items) : IEnumerable + { + private int _iterated; + + public IEnumerator GetEnumerator() + { + if (Interlocked.Exchange(ref _iterated, 1) != 0) + { + throw new InvalidOperationException("This enumerable can only be enumerated once."); + } + + foreach (var item in items) + { + yield return item; + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/TestChatClient.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/TestChatClient.cs new file mode 100644 index 0000000000..436d4308ab --- /dev/null +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/TestChatClient.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. + +// WARNING: +// This class has been temporarily copied here from MEAI, to allow prototyping +// functionality that will be moved to MEAI in the future. +// This file is not intended to be modified. + +// AF repo suppressions for code copied from MEAI. +#pragma warning disable IDE0009 // Member access should be qualified. +#pragma warning disable CA1859 // Use concrete types when possible for improved performance +#pragma warning disable CA1063 // Implement IDisposable Correctly + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +public sealed class TestChatClient : IChatClient +{ + public TestChatClient() + { + GetServiceCallback = DefaultGetServiceCallback; + } + + public IServiceProvider? Services { get; set; } + + public Func, ChatOptions?, CancellationToken, Task>? GetResponseAsyncCallback { get; set; } + + public Func, ChatOptions?, CancellationToken, IAsyncEnumerable>? GetStreamingResponseAsyncCallback { get; set; } + + public Func GetServiceCallback { get; set; } + + private object? DefaultGetServiceCallback(Type serviceType, object? serviceKey) => + serviceType is not null && serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + => GetResponseAsyncCallback!.Invoke(messages, options, cancellationToken); + + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + => GetStreamingResponseAsyncCallback!.Invoke(messages, options, cancellationToken); + + public object? GetService(Type serviceType, object? serviceKey = null) + => GetServiceCallback(serviceType, serviceKey); + + void IDisposable.Dispose() + { + // No resources need disposing. + } +} From b1289e38516d4931faa40c42617f298bb55eab8b Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Tue, 19 Aug 2025 18:16:56 +0100 Subject: [PATCH 42/53] Add additional unit test. --- .../NewFunctionInvokingChatClientTests.cs | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs index 381bb136ac..bafb558424 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs @@ -350,6 +350,69 @@ public async Task ApprovedInputsAreExecutedAndFunctionResultsAreConvertedAsync() await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); } + [Fact] + public async Task AlreadyExecutedApprovalsAreIgnoredAsync() + { + var options = new ChatOptions + { + Tools = + [ + AIFunctionFactory.Create(() => "Result 1", "Func1"), + new ApprovalRequiredAIFunction(AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2")), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId3", new FunctionCallContent("callId3", "Func1")), + ]) { MessageId = "resp2" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId3", true, new FunctionCallContent("callId3", "Func1")), + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId3", "Func1")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId3", result: "Result 1")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "World"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId3", "Func1")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId3", result: "Result 1")]), + new ChatMessage(ChatRole.Assistant, "World"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + /// /// Since we do not have a way of supporting both functions that require approval and those that do not /// in one invocation, we always require all function calls to be approved if any require approval. From 0b4405e400858dd5b7f4ea3adb2adc3d0900d863 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 20 Aug 2025 11:29:24 +0100 Subject: [PATCH 43/53] Refactor ConvertToFunctionCallContentMessages to avoid unecessary allocations and add tests for it. --- .../MEAI/NewFunctionInvokingChatClient.cs | 30 +++++++--- .../NewFunctionInvokingChatClientTests.cs | 59 +++++++++++++++++++ 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index 70bb436c18..a8fd812638 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -301,7 +301,7 @@ public override async Task GetResponseAsync( if (modeAndMessages.ShouldTerminate) { - responseMessages = [.. allPreInvocationCallMessages ?? [], .. modeAndMessages.MessagesAdded]; + responseMessages = [.. augmentedPreInvocationHistory]; return new ChatResponse(responseMessages); } } @@ -1307,24 +1307,40 @@ private static (List? approvals, List? messagesById = null; foreach (var resultWithRequestMessage in resultWithRequestMessages) { - messagesById ??= new(); + if (currentMessage is not null && messagesById is null && currentMessage.MessageId != resultWithRequestMessage.RequestMessage?.MessageId) + { + // The majority of the time, all FCC would be part of a single message, so no need to create a dictionary for this case. + // If we are dealing with multiple messages though, we need to keep track of them by their message ID. + messagesById ??= new(); + messagesById[currentMessage.MessageId ?? string.Empty] = currentMessage; + } + + if (messagesById is not null) + { + messagesById.TryGetValue(resultWithRequestMessage.RequestMessage?.MessageId ?? string.Empty, out currentMessage); + } - if (messagesById.TryGetValue(resultWithRequestMessage.RequestMessage?.MessageId ?? string.Empty, out var message)) + if (currentMessage is null) { - message.Contents.Add(resultWithRequestMessage.Response.FunctionCall); + currentMessage = ConvertToFunctionCallContentMessage(resultWithRequestMessage, fallbackMessageId); } else { - var functionCallMessage = ConvertToFunctionCallContentMessage(resultWithRequestMessage, fallbackMessageId); - messagesById[functionCallMessage.MessageId ?? string.Empty] = functionCallMessage; + currentMessage.Contents.Add(resultWithRequestMessage.Response.FunctionCall); + } + + if (messagesById is not null) + { + messagesById[currentMessage.MessageId ?? string.Empty] = currentMessage; } } - return messagesById?.Values; + return messagesById?.Values as ICollection ?? (currentMessage != null ? [currentMessage!] : null); } return null; diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs index bafb558424..baf0eabddc 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs @@ -188,6 +188,65 @@ public async Task ApprovedApprovalResponsesAreExecutedAsync() await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); } + [Fact] + public async Task ApprovedApprovalResponsesFromSeparateFCCMessagesAreExecutedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + ]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId2", new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]) { MessageId = "resp2" }, + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + ]), + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]) { MessageId = "resp2" }, + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]) { MessageId = "resp1" }, + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]) { MessageId = "resp2" }, + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + [Fact] public async Task RejectedApprovalResponsesAreFailedAsync() { From 545c231685168e1367a97f14fdd161c893e3b3a8 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:22:56 +0100 Subject: [PATCH 44/53] Refactor preinvocation logic to share between streaming and non-streaming --- .../MEAI/NewFunctionInvokingChatClient.cs | 149 ++++++++++-------- 1 file changed, 80 insertions(+), 69 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index a8fd812638..f9f0378769 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -260,50 +260,23 @@ public override async Task GetResponseAsync( // ** Approvals additions on top of FICC - start **// - // Extract any approval responses where we need to execute or reject the function calls. - // The original messages are also modified to remove all approval requests and responses. - var notExecutedResponses = ExtractAndRemoveApprovalRequestsAndResponses(originalMessages); + // Process approval requests (remove from originalMessages) and rejected approval responses (re-create FCC and create failed FRC). + var (augmentedPreInvocationHistory, notExecutedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId: null, functionCallContentFallbackMessageId: null); - // Wrap the function call content in message(s). - ICollection? allPreInvocationCallMessages = ConvertToFunctionCallContentMessages( - [.. notExecutedResponses.rejections ?? [], .. notExecutedResponses.approvals ?? []], null); + // Execute approved approval responses, which generates some additional FRC. + (IList? approvedExecutedPreInvocationFRC, bool shouldTerminate, consecutiveErrorCount) = + await ExecuteApprovedPreInvocationFunctionApprovalResponses(notExecutedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: false, cancellationToken); - // Generate failed function result contents for any rejected requests and wrap it in a message. - List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notExecutedResponses.rejections, null, null); - ChatMessage? rejectedPreInvocationResultsMessage = rejectedFunctionCallResults != null ? - new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) : - null; - - // Add all the FCC and FRC that we generated into the original messages list so that they are passed to the - // inner client and can be used to generate a result. Also add them to the augmented pre-invocation history - // so that they can be returned to the caller as part of the next response. - List? augmentedPreInvocationHistory = null; - if (allPreInvocationCallMessages is not null || rejectedPreInvocationResultsMessage is not null) + if (approvedExecutedPreInvocationFRC is not null) { + // We need to add the generated FRC to the list we'll return to callers as part of the next response. augmentedPreInvocationHistory ??= []; - foreach (var message in (allPreInvocationCallMessages ?? []).Concat(rejectedPreInvocationResultsMessage == null ? [] : [rejectedPreInvocationResultsMessage])) - { - originalMessages.Add(message); - augmentedPreInvocationHistory.Add(message); - } + augmentedPreInvocationHistory.AddRange(approvedExecutedPreInvocationFRC); } - // Check if there are any function calls to do for any approved functions and execute them. - if (notExecutedResponses.approvals is { Count: > 0 }) + if (shouldTerminate) { - // The FRC that is generated here is already added to originalMessages by ProcessFunctionCallsAsync. - var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: false, cancellationToken); - consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; - - // We need to add the generated FRC to the list we'll return to callers as part of the next response. - augmentedPreInvocationHistory ??= []; - augmentedPreInvocationHistory.AddRange(modeAndMessages.MessagesAdded); - - if (modeAndMessages.ShouldTerminate) - { - responseMessages = [.. augmentedPreInvocationHistory]; - return new ChatResponse(responseMessages); - } + return new ChatResponse(augmentedPreInvocationHistory); } // ** Approvals additions on top of FICC - end **// @@ -436,52 +409,31 @@ public override async IAsyncEnumerable GetStreamingResponseA ApprovalRequiredAIFunction[]? approvalRequiredFunctions = (options?.Tools ?? []).Concat(AdditionalTools ?? []).OfType().ToArray(); bool hasApprovalRequiringFunctions = approvalRequiredFunctions.Length > 0; - // Extract any approval responses where we need to execute or reject the function calls. - // The original messages are also modified to remove all approval requests and responses. - var notExecutedResponses = ExtractAndRemoveApprovalRequestsAndResponses(originalMessages); - - // Wrap the function call content in message(s). - ICollection? allPreInvocationCallMessages = ConvertToFunctionCallContentMessages( - [.. notExecutedResponses.rejections ?? [], .. notExecutedResponses.approvals ?? []], null); - - // Generate failed function results contents for any rejected requests and turn them into messages. - List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId, functionCallContentFallbackMessageId); - ChatMessage? rejectedPreInvocationResultsMessage = rejectedFunctionCallResults != null ? new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) { MessageId = toolResponseId } : null; - - // Add all the FCC and FRC that we generated into the original messages list so that they are passed to the - // inner client and can be used to generate a result. Also add them to the augmented pre-invocation history - // so that they can be returned to the caller as part of the next response. - List? augmentedPreInvocationHistory = null; - if (allPreInvocationCallMessages is not null || rejectedPreInvocationResultsMessage is not null) + // Process approval requests (remove from original messages) and rejected approval responses (re-create FCC and create failed FRC). + var (preInvocationFCCWithRejectedFRC, notExecutedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId, functionCallContentFallbackMessageId); + if (preInvocationFCCWithRejectedFRC is not null) { - augmentedPreInvocationHistory ??= []; - foreach (var message in (allPreInvocationCallMessages ?? []).Concat(rejectedPreInvocationResultsMessage == null ? [] : [rejectedPreInvocationResultsMessage])) + foreach (var message in preInvocationFCCWithRejectedFRC) { - originalMessages.Add(message); - augmentedPreInvocationHistory.Add(message); - yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } } - // Check if there are any function calls to do from any approved functions and execute them. - if (notExecutedResponses.approvals is { Count: > 0 }) - { - var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedResponses.approvals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming: true, cancellationToken); - consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + // Execute approved approval responses, which generates some additional FRC. + (IList? approvedExecutedPreInvocationFRC, bool shouldTerminate, consecutiveErrorCount) = + await ExecuteApprovedPreInvocationFunctionApprovalResponses(notExecutedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: true, cancellationToken); - augmentedPreInvocationHistory ??= []; - foreach (var message in modeAndMessages.MessagesAdded) + if (approvedExecutedPreInvocationFRC is not null) + { + foreach (var message in approvedExecutedPreInvocationFRC) { message.MessageId = toolResponseId; - augmentedPreInvocationHistory.Add(message); - yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } - if (modeAndMessages.ShouldTerminate) + if (shouldTerminate) { yield break; } @@ -1139,6 +1091,65 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul context.Function.InvokeAsync(context.Arguments, cancellationToken); } + /// + /// 1. Remove all and from the . + /// 2. Recreate for any that haven't been executed yet. + /// 3. Genreate failed for any rejected . + /// 4. add all the new content items to and return them as the augmented pre-invocation history. + /// + private static (List? augmentedPreInvocationHistory, List? approvals) ProcessPreInvocationFunctionApprovalResponses( + List originalMessages, string? toolResponseId, string? functionCallContentFallbackMessageId) + { + // Extract any approval responses where we need to execute or reject the function calls. + // The original messages are also modified to remove all approval requests and responses. + var notExecutedResponses = ExtractAndRemoveApprovalRequestsAndResponses(originalMessages); + + // Wrap the function call content in message(s). + ICollection? allPreInvocationCallMessages = ConvertToFunctionCallContentMessages( + [.. notExecutedResponses.rejections ?? [], .. notExecutedResponses.approvals ?? []], null); + + // Generate failed function result contents for any rejected requests and wrap it in a message. + List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId, functionCallContentFallbackMessageId); + ChatMessage? rejectedPreInvocationResultsMessage = rejectedFunctionCallResults != null ? + new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) { MessageId = toolResponseId } : + null; + + // Add all the FCC and FRC that we generated into the original messages list so that they are passed to the + // inner client and can be used to generate a result. Also add them to the augmented pre-invocation history + // so that they can be returned to the caller as part of the next response. + List? augmentedPreInvocationHistory = null; + if (allPreInvocationCallMessages is not null || rejectedPreInvocationResultsMessage is not null) + { + augmentedPreInvocationHistory ??= []; + foreach (var message in (allPreInvocationCallMessages ?? []).Concat(rejectedPreInvocationResultsMessage == null ? [] : [rejectedPreInvocationResultsMessage])) + { + originalMessages.Add(message); + augmentedPreInvocationHistory.Add(message); + } + } + + return (augmentedPreInvocationHistory, notExecutedResponses.approvals); + } + + /// + /// Execute the provided and return the resulting . + /// + private async Task<(IList? functionResultContent, bool shouldTerminate, int consecutiveErrorCount)> ExecuteApprovedPreInvocationFunctionApprovalResponses( + List? notExecutedApprovals, List originalMessages, ChatOptions? options, int consecutiveErrorCount, bool isStreaming, CancellationToken cancellationToken) + { + // Check if there are any function calls to do for any approved functions and execute them. + if (notExecutedApprovals is { Count: > 0 }) + { + // The FRC that is generated here is already added to originalMessages by ProcessFunctionCallsAsync. + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedApprovals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming, cancellationToken); + consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; + + return (modeAndMessages.MessagesAdded, modeAndMessages.ShouldTerminate, consecutiveErrorCount); + } + + return (null, false, consecutiveErrorCount); + } + /// /// This method extracts the approval requests and responses from the provided list of messages, validates them, filters them to ones that require execution and splits them into approved and rejected. /// From e6aeebb6b1e0b4c145afdbad45121b1508911ae8 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:41:12 +0100 Subject: [PATCH 45/53] Move some approval generation code to a separate method to simplify main execution method. --- .../MEAI/NewFunctionInvokingChatClient.cs | 73 ++++++++++++------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index f9f0378769..bbabbe978e 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -506,16 +506,9 @@ public override async IAsyncEnumerable GetStreamingResponseA for (; lastYieldedUpdateIndex < updates.Count; lastYieldedUpdateIndex++) { var updateToYield = updates[lastYieldedUpdateIndex]; - if (updateToYield.Contents is { Count: > 0 }) + if (TryReplaceFunctionCallsWithApprovalRequests(updateToYield.Contents, out var updatedContents)) { - for (int i = 0; i < updateToYield.Contents.Count; i++) - { - if (updateToYield.Contents[i] is FunctionCallContent fcc) - { - var approvalRequest = new FunctionApprovalRequestContent(fcc.CallId, fcc); - updateToYield.Contents[i] = approvalRequest; - } - } + updateToYield.Contents = updatedContents; } yield return updateToYield; Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 @@ -697,25 +690,6 @@ private static bool CopyFunctionCalls( return any; } - private static async Task<(bool hasApprovalRequiringFcc, int lastApprovalCheckedFCCIndex)> CheckForApprovalRequiringFCCAsync( - List? functionCallContents, - ApprovalRequiredAIFunction[] approvalRequiredFunctions, - bool hasApprovalRequiringFcc, - int lastApprovalCheckedFCCIndex) - { - for (; lastApprovalCheckedFCCIndex < (functionCallContents?.Count ?? 0); lastApprovalCheckedFCCIndex++) - { - var fcc = functionCallContents![lastApprovalCheckedFCCIndex]; - if (approvalRequiredFunctions.FirstOrDefault(y => y.Name == fcc.Name) is ApprovalRequiredAIFunction approvalFunction && - await approvalFunction.RequiresApprovalCallback(new(fcc))) - { - hasApprovalRequiringFcc |= true; - } - } - - return (hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex); - } - private static void UpdateOptionsForNextIteration(ref ChatOptions? options, string? conversationId) { if (options is null) @@ -1370,6 +1344,49 @@ private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWit return new ChatMessage(ChatRole.Assistant, [resultWithRequestMessage.Response.FunctionCall]) { MessageId = fallbackMessageId }; } + private static async Task<(bool hasApprovalRequiringFcc, int lastApprovalCheckedFCCIndex)> CheckForApprovalRequiringFCCAsync( + List? functionCallContents, + ApprovalRequiredAIFunction[] approvalRequiredFunctions, + bool hasApprovalRequiringFcc, + int lastApprovalCheckedFCCIndex) + { + for (; lastApprovalCheckedFCCIndex < (functionCallContents?.Count ?? 0); lastApprovalCheckedFCCIndex++) + { + var fcc = functionCallContents![lastApprovalCheckedFCCIndex]; + if (approvalRequiredFunctions.FirstOrDefault(y => y.Name == fcc.Name) is ApprovalRequiredAIFunction approvalFunction && + await approvalFunction.RequiresApprovalCallback(new(fcc))) + { + hasApprovalRequiringFcc |= true; + } + } + + return (hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex); + } + + /// + /// Replaces all with and ouputs a new list if any of them were replaced. + /// + /// true if any was replaced, false otherwise. + private static bool TryReplaceFunctionCallsWithApprovalRequests(IList content, out IList? updatedContent) + { + updatedContent = null; + + if (content is { Count: > 0 }) + { + for (int i = 0; i < content.Count; i++) + { + if (content[i] is FunctionCallContent fcc) + { + updatedContent ??= [.. content]; // Clone the list if we haven't already + var approvalRequest = new FunctionApprovalRequestContent(fcc.CallId, fcc); + updatedContent[i] = approvalRequest; + } + } + } + + return updatedContent is not null; + } + /// /// Replaces all from with /// if any one of them requires approval. From a1b2973e95d1dca7a13e98428acddfd58bba05fc Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:40:25 +0100 Subject: [PATCH 46/53] More refactoring, additional unit tests and bug fixes. --- .../MEAI/NewFunctionInvokingChatClient.cs | 43 +++++++---- .../NewFunctionInvokingChatClientTests.cs | 76 +++++++++++++++++++ 2 files changed, 106 insertions(+), 13 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index bbabbe978e..cf6f3fa549 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -489,13 +489,10 @@ public override async IAsyncEnumerable GetStreamingResponseA } else { - if (!hasApprovalRequiringFcc) - { - // Check if any of the function call contents in this update requires approval. - // We only do this until we find the first one that requires approval. - (hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex) = await CheckForApprovalRequiringFCCAsync( - functionCallContents, approvalRequiredFunctions, hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex); - } + // Check if any of the function call contents in this update requires approval. + // Once we find the first one that requires approval, this method becomes a no-op. + (hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex) = await CheckForApprovalRequiringFCCAsync( + functionCallContents, approvalRequiredFunctions, hasApprovalRequiringFcc, lastApprovalCheckedFCCIndex); // We've encountered a function call content that requires approval (either in this update or ealier) // so we need to ask for approval for all functions, since we cannot mix and match. @@ -1080,10 +1077,10 @@ private static (List? augmentedPreInvocationHistory, List? allPreInvocationCallMessages = ConvertToFunctionCallContentMessages( - [.. notExecutedResponses.rejections ?? [], .. notExecutedResponses.approvals ?? []], null); + [.. notExecutedResponses.rejections ?? [], .. notExecutedResponses.approvals ?? []], functionCallContentFallbackMessageId); // Generate failed function result contents for any rejected requests and wrap it in a message. - List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId, functionCallContentFallbackMessageId); + List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId); ChatMessage? rejectedPreInvocationResultsMessage = rejectedFunctionCallResults != null ? new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) { MessageId = toolResponseId } : null; @@ -1262,12 +1259,10 @@ private static (List? approvals, List /// Any rejected approval responses. /// The message id to use for the tool response. - /// Optional message id to use if no original approval request message is availble to copy from. /// The for the rejected function calls. private static List? GenerateRejectedFunctionResults( List? rejections, - string? toolResponseId, - string? fallbackMessageId) + string? toolResponseId) { List? functionResultContent = null; @@ -1286,6 +1281,11 @@ private static (List? approvals, List + /// Extracts the from the provided to recreate the original function call messages. + /// The output messages tries to mimic the original messages that contained the , e.g. if the had been split into separate messages, + /// this method will recreate similarly split messages, each with their own . + /// #pragma warning disable CA1859 // Use concrete types when possible for improved performance private static ICollection? ConvertToFunctionCallContentMessages(IEnumerable? resultWithRequestMessages, string? fallbackMessageId) #pragma warning restore CA1859 // Use concrete types when possible for improved performance @@ -1297,7 +1297,9 @@ private static (List? approvals, List? approvals, List + /// Takes the from the and wraps it in a + /// using the same message id that the was originally returned with from the downstream . + /// private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWithRequestMessage resultWithRequestMessage, string? fallbackMessageId) { if (resultWithRequestMessage.RequestMessage is not null) @@ -1344,12 +1350,23 @@ private static ChatMessage ConvertToFunctionCallContentMessage(ApprovalResultWit return new ChatMessage(ChatRole.Assistant, [resultWithRequestMessage.Response.FunctionCall]) { MessageId = fallbackMessageId }; } + /// + /// Check if any of the provided require approval. + /// Supports checking from a provided index up to the end of the list, to allow efficient incremental checking + /// when streaming. + /// private static async Task<(bool hasApprovalRequiringFcc, int lastApprovalCheckedFCCIndex)> CheckForApprovalRequiringFCCAsync( List? functionCallContents, ApprovalRequiredAIFunction[] approvalRequiredFunctions, bool hasApprovalRequiringFcc, int lastApprovalCheckedFCCIndex) { + // If we already found an approval requiring FCC, we can skip checking the rest. + if (hasApprovalRequiringFcc) + { + return (true, functionCallContents?.Count ?? 0); + } + for (; lastApprovalCheckedFCCIndex < (functionCallContents?.Count ?? 0); lastApprovalCheckedFCCIndex++) { var fcc = functionCallContents![lastApprovalCheckedFCCIndex]; diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs index baf0eabddc..c06b7d7a54 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs @@ -472,6 +472,82 @@ public async Task AlreadyExecutedApprovalsAreIgnoredAsync() await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); } + [Fact] + public async Task ApprovalRequestWithoutApprovalResponseThrowsAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, + [ + new FunctionApprovalRequestContent("callId1", new FunctionCallContent("callId1", "Func1")), + ]) { MessageId = "resp1" }, + ]; + + var invokeException = await Assert.ThrowsAsync( + async () => await InvokeAndAssertAsync(options, input, [], [], [])); + Assert.Equal("FunctionApprovalRequestContent found with FunctionCall.CallId(s) 'callId1' that have no matching FunctionApprovalResponseContent.", invokeException.Message); + + var invokeStreamingException = await Assert.ThrowsAsync( + async () => await InvokeAndAssertStreamingAsync(options, input, [], [], [])); + Assert.Equal("FunctionApprovalRequestContent found with FunctionCall.CallId(s) 'callId1' that have no matching FunctionApprovalResponseContent.", invokeStreamingException.Message); + } + + [Fact] + public async Task ApprovedApprovalResponsesWithoutApprovalRequestAreExecutedAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ] + }; + + List input = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + //await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + /// /// Since we do not have a way of supporting both functions that require approval and those that do not /// in one invocation, we always require all function calls to be approved if any require approval. From 906ad2f87124f827ad6261e949a2cdfd29c76ba7 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:46:31 +0100 Subject: [PATCH 47/53] Remove comments delineating additions. --- .../MEAI/NewFunctionInvokingChatClient.cs | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index cf6f3fa549..907f5a857a 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -258,8 +258,6 @@ public override async Task GetResponseAsync( bool lastIterationHadConversationId = false; // whether the last iteration's response had a ConversationId set int consecutiveErrorCount = 0; - // ** Approvals additions on top of FICC - start **// - // Process approval requests (remove from originalMessages) and rejected approval responses (re-create FCC and create failed FRC). var (augmentedPreInvocationHistory, notExecutedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId: null, functionCallContentFallbackMessageId: null); @@ -279,8 +277,6 @@ public override async Task GetResponseAsync( return new ChatResponse(augmentedPreInvocationHistory); } - // ** Approvals additions on top of FICC - end **// - for (int iteration = 0; ; iteration++) { functionCallContents?.Clear(); @@ -292,14 +288,10 @@ public override async Task GetResponseAsync( Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}."); } - // ** Approvals additions on top of FICC - start **// - // Before we do any function execution, make sure that any functions that require approval, have been turned into approval requests // so that they don't get executed here. response.Messages = await ReplaceFunctionCallsWithApprovalRequests(response.Messages, options?.Tools, AdditionalTools); - // ** Approvals additions on top of FICC - end **// - // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents. bool requiresFunctionInvocation = (options?.Tools is { Count: > 0 } || AdditionalTools is { Count: > 0 }) && @@ -310,15 +302,11 @@ public override async Task GetResponseAsync( // fast path out by just returning the original response. if (iteration == 0 && !requiresFunctionInvocation) { - // ** Approvals additions on top of FICC - start **// - // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, // so they are returned to the caller. response.Messages = UpdateResponseMessagesWithPreInvocationHistory(response.Messages, augmentedPreInvocationHistory); augmentedPreInvocationHistory = null; - // ** Approvals additions on top of FICC - end **// - return response; } @@ -393,8 +381,6 @@ public override async IAsyncEnumerable GetStreamingResponseA List updates = []; // updates from the current response int consecutiveErrorCount = 0; - // ** Approvals additions on top of FICC - start **// - // This is a synthetic ID since we're generating the tool messages instead of getting them from // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to // use the same message ID for all of them within a given iteration, as this is a single logical @@ -439,21 +425,15 @@ public override async IAsyncEnumerable GetStreamingResponseA } } - // ** Approvals additions on top of FICC - end **// - for (int iteration = 0; ; iteration++) { updates.Clear(); functionCallContents?.Clear(); - // ** Approvals additions on top of FICC - start **// - bool hasApprovalRequiringFcc = false; int lastApprovalCheckedFCCIndex = 0; int lastYieldedUpdateIndex = 0; - // ** Approvals additions on top of FICC - end **// - await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken)) { if (update is null) @@ -478,7 +458,6 @@ public override async IAsyncEnumerable GetStreamingResponseA } } - // ** Approvals modifications - start **// if (functionCallContents?.Count is not > 0 || !hasApprovalRequiringFunctions) { // If there are no function calls to make yet, or if none of the functions require approval at all, @@ -520,7 +499,6 @@ public override async IAsyncEnumerable GetStreamingResponseA // when we reach the end of the updates stream. } } - // ** Approvals modifications - end **// Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } @@ -546,17 +524,6 @@ public override async IAsyncEnumerable GetStreamingResponseA responseMessages.AddRange(modeAndMessages.MessagesAdded); consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; - // ** Approvals removal - start **// - - // This is a synthetic ID since we're generating the tool messages instead of getting them from - // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to - // use the same message ID for all of them within a given iteration, as this is a single logical - // message with multiple content items. We could also use different message IDs per tool content, - // but there's no benefit to doing so. - //string toolResponseId = Guid.NewGuid().ToString("N"); - - // ** Approvals removal - end **// - // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages // includes all activities, including generated function results. foreach (var message in modeAndMessages.MessagesAdded) From 66b904e8401be3acdfd75e3e3eaf11228d2c748c Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:54:08 +0100 Subject: [PATCH 48/53] Fixing some typos. --- .../MEAI.Contents/UserInputResponseContent.cs | 2 +- .../MEAI/NewFunctionInvokingChatClient.cs | 4 ++-- .../MEAI/NewFunctionInvokingChatClientTests.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs index 9c15927b7d..78b138779e 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs @@ -10,7 +10,7 @@ namespace Microsoft.Extensions.AI; public abstract class UserInputResponseContent : AIContent { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The ID to uniquely identify the user input request/response pair. protected UserInputResponseContent(string id) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index 907f5a857a..40e195ad28 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -1443,13 +1443,13 @@ private static IList UpdateResponseMessagesWithPreInvocationHistory return responseMessages; } - private static ChatResponseUpdate ConvertToolResultMessageToUpdate(ChatMessage message, string? converationId, string? messageId) + private static ChatResponseUpdate ConvertToolResultMessageToUpdate(ChatMessage message, string? conversationId, string? messageId) { return new() { AdditionalProperties = message.AdditionalProperties, AuthorName = message.AuthorName, - ConversationId = converationId, + ConversationId = conversationId, CreatedAt = DateTimeOffset.UtcNow, Contents = message.Contents, RawRepresentation = message.RawRepresentation, diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs index c06b7d7a54..37ab4c3700 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs @@ -543,7 +543,7 @@ public async Task ApprovedApprovalResponsesWithoutApprovalRequestAreExecutedAsyn new ChatMessage(ChatRole.Assistant, "world"), ]; - //await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); } From 21680bebeeb1bff3c9320cdf20c0e0477bc50bd5 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 22 Aug 2025 12:06:12 +0100 Subject: [PATCH 49/53] Address PR comments. --- .../FunctionApprovalRequestContent.cs | 0 .../FunctionApprovalResponseContent.cs | 0 .../{MEAI.Contents => MEAI.A}/UserInputRequestContent.cs | 0 .../{MEAI.Contents => MEAI.A}/UserInputResponseContent.cs | 0 .../MEAI/ApprovalRequiredAIFunction.cs | 7 ++++--- 5 files changed, 4 insertions(+), 3 deletions(-) rename dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/{MEAI.Contents => MEAI.A}/FunctionApprovalRequestContent.cs (100%) rename dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/{MEAI.Contents => MEAI.A}/FunctionApprovalResponseContent.cs (100%) rename dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/{MEAI.Contents => MEAI.A}/UserInputRequestContent.cs (100%) rename dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/{MEAI.Contents => MEAI.A}/UserInputResponseContent.cs (100%) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs similarity index 100% rename from dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalRequestContent.cs rename to dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalResponseContent.cs similarity index 100% rename from dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/FunctionApprovalResponseContent.cs rename to dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalResponseContent.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputRequestContent.cs similarity index 100% rename from dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputRequestContent.cs rename to dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputRequestContent.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputResponseContent.cs similarity index 100% rename from dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.Contents/UserInputResponseContent.cs rename to dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/UserInputResponseContent.cs diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs index b85059b476..f1913efb0f 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/ApprovalRequiredAIFunction.cs @@ -2,6 +2,7 @@ using System; using System.Threading.Tasks; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -14,7 +15,7 @@ public sealed class ApprovalRequiredAIFunction(AIFunction function) : Delegating /// /// An optional callback that can be used to determine if the function call requires approval, instead of the default behavior, which is to always require approval. /// - public Func> RequiresApprovalCallback { get; set; } = delegate { return new ValueTask(true); }; + public Func> RequiresApprovalCallback { get; set; } = _ => new(true); /// /// Context object that provides information about the function call that requires approval. @@ -25,10 +26,10 @@ public sealed class ApprovalContext /// Initializes a new instance of the class. /// /// The containing the details of the invocation. - /// + /// is null. public ApprovalContext(FunctionCallContent functionCall) { - this.FunctionCall = functionCall ?? throw new ArgumentNullException(nameof(functionCall)); + this.FunctionCall = Throw.IfNull(functionCall); } /// From a712408da98d3dbf5f03e3ec3b010f8d47dd22a3 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 22 Aug 2025 12:39:48 +0100 Subject: [PATCH 50/53] Address PR comments. --- .../MEAI/NewFunctionInvokingChatClient.cs | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index 40e195ad28..58bf6a3c2d 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -259,22 +259,22 @@ public override async Task GetResponseAsync( int consecutiveErrorCount = 0; // Process approval requests (remove from originalMessages) and rejected approval responses (re-create FCC and create failed FRC). - var (augmentedPreInvocationHistory, notExecutedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId: null, functionCallContentFallbackMessageId: null); + var (preInvocationHistory, notInvokedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId: null, functionCallContentFallbackMessageId: null); // Execute approved approval responses, which generates some additional FRC. (IList? approvedExecutedPreInvocationFRC, bool shouldTerminate, consecutiveErrorCount) = - await ExecuteApprovedPreInvocationFunctionApprovalResponses(notExecutedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: false, cancellationToken); + await ExecuteApprovedPreInvocationFunctionApprovalResponses(notInvokedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: false, cancellationToken); if (approvedExecutedPreInvocationFRC is not null) { // We need to add the generated FRC to the list we'll return to callers as part of the next response. - augmentedPreInvocationHistory ??= []; - augmentedPreInvocationHistory.AddRange(approvedExecutedPreInvocationFRC); + preInvocationHistory ??= []; + preInvocationHistory.AddRange(approvedExecutedPreInvocationFRC); } if (shouldTerminate) { - return new ChatResponse(augmentedPreInvocationHistory); + return new ChatResponse(preInvocationHistory); } for (int iteration = 0; ; iteration++) @@ -304,8 +304,8 @@ public override async Task GetResponseAsync( { // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, // so they are returned to the caller. - response.Messages = UpdateResponseMessagesWithPreInvocationHistory(response.Messages, augmentedPreInvocationHistory); - augmentedPreInvocationHistory = null; + response.Messages = UpdateResponseMessagesWithPreInvocationHistory(response.Messages, preInvocationHistory); + preInvocationHistory = null; return response; } @@ -396,7 +396,7 @@ public override async IAsyncEnumerable GetStreamingResponseA bool hasApprovalRequiringFunctions = approvalRequiredFunctions.Length > 0; // Process approval requests (remove from original messages) and rejected approval responses (re-create FCC and create failed FRC). - var (preInvocationFCCWithRejectedFRC, notExecutedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId, functionCallContentFallbackMessageId); + var (preInvocationFCCWithRejectedFRC, notInvokedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId, functionCallContentFallbackMessageId); if (preInvocationFCCWithRejectedFRC is not null) { foreach (var message in preInvocationFCCWithRejectedFRC) @@ -408,7 +408,7 @@ public override async IAsyncEnumerable GetStreamingResponseA // Execute approved approval responses, which generates some additional FRC. (IList? approvedExecutedPreInvocationFRC, bool shouldTerminate, consecutiveErrorCount) = - await ExecuteApprovedPreInvocationFunctionApprovalResponses(notExecutedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: true, cancellationToken); + await ExecuteApprovedPreInvocationFunctionApprovalResponses(notInvokedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: true, cancellationToken); if (approvedExecutedPreInvocationFRC is not null) { @@ -1033,53 +1033,53 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul /// 1. Remove all and from the . /// 2. Recreate for any that haven't been executed yet. /// 3. Genreate failed for any rejected . - /// 4. add all the new content items to and return them as the augmented pre-invocation history. + /// 4. add all the new content items to and return them as the pre-invocation history. /// - private static (List? augmentedPreInvocationHistory, List? approvals) ProcessPreInvocationFunctionApprovalResponses( + private static (List? preInvocationHistory, List? approvals) ProcessPreInvocationFunctionApprovalResponses( List originalMessages, string? toolResponseId, string? functionCallContentFallbackMessageId) { // Extract any approval responses where we need to execute or reject the function calls. // The original messages are also modified to remove all approval requests and responses. - var notExecutedResponses = ExtractAndRemoveApprovalRequestsAndResponses(originalMessages); + var notInvokedResponses = ExtractAndRemoveApprovalRequestsAndResponses(originalMessages); // Wrap the function call content in message(s). ICollection? allPreInvocationCallMessages = ConvertToFunctionCallContentMessages( - [.. notExecutedResponses.rejections ?? [], .. notExecutedResponses.approvals ?? []], functionCallContentFallbackMessageId); + [.. notInvokedResponses.rejections ?? [], .. notInvokedResponses.approvals ?? []], functionCallContentFallbackMessageId); // Generate failed function result contents for any rejected requests and wrap it in a message. - List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notExecutedResponses.rejections, toolResponseId); + List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notInvokedResponses.rejections, toolResponseId); ChatMessage? rejectedPreInvocationResultsMessage = rejectedFunctionCallResults != null ? new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) { MessageId = toolResponseId } : null; // Add all the FCC and FRC that we generated into the original messages list so that they are passed to the - // inner client and can be used to generate a result. Also add them to the augmented pre-invocation history + // inner client and can be used to generate a result. Also add them to the pre-invocation history // so that they can be returned to the caller as part of the next response. - List? augmentedPreInvocationHistory = null; + List? preInvocationHistory = null; if (allPreInvocationCallMessages is not null || rejectedPreInvocationResultsMessage is not null) { - augmentedPreInvocationHistory ??= []; + preInvocationHistory ??= []; foreach (var message in (allPreInvocationCallMessages ?? []).Concat(rejectedPreInvocationResultsMessage == null ? [] : [rejectedPreInvocationResultsMessage])) { originalMessages.Add(message); - augmentedPreInvocationHistory.Add(message); + preInvocationHistory.Add(message); } } - return (augmentedPreInvocationHistory, notExecutedResponses.approvals); + return (preInvocationHistory, notInvokedResponses.approvals); } /// /// Execute the provided and return the resulting . /// - private async Task<(IList? functionResultContent, bool shouldTerminate, int consecutiveErrorCount)> ExecuteApprovedPreInvocationFunctionApprovalResponses( - List? notExecutedApprovals, List originalMessages, ChatOptions? options, int consecutiveErrorCount, bool isStreaming, CancellationToken cancellationToken) + private async Task<(IList? FunctionResultContent, bool ShouldTerminate, int ConsecutiveErrorCount)> ExecuteApprovedPreInvocationFunctionApprovalResponses( + List? notInvokedApprovals, List originalMessages, ChatOptions? options, int consecutiveErrorCount, bool isStreaming, CancellationToken cancellationToken) { // Check if there are any function calls to do for any approved functions and execute them. - if (notExecutedApprovals is { Count: > 0 }) + if (notInvokedApprovals is { Count: > 0 }) { // The FRC that is generated here is already added to originalMessages by ProcessFunctionCallsAsync. - var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notExecutedApprovals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming, cancellationToken); + var modeAndMessages = await ProcessFunctionCallsAsync(originalMessages, options, notInvokedApprovals.Select(x => x.Response.FunctionCall).ToList(), 0, consecutiveErrorCount, isStreaming, cancellationToken); consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount; return (modeAndMessages.MessagesAdded, modeAndMessages.ShouldTerminate, consecutiveErrorCount); @@ -1191,8 +1191,8 @@ private static (List? approvals, List? notExecutedApprovedFunctionCalls = null; - List? notExecutedRejectedFunctionCalls = null; + List? approvedFunctionCalls = null; + List? rejectedFunctionCalls = null; for (int i = 0; i < (allApprovalResponses?.Count ?? 0); i++) { @@ -1207,18 +1207,18 @@ private static (List? approvals, List @@ -1430,14 +1430,14 @@ private static async Task> ReplaceFunctionCallsWithApprovalRe } /// - /// Insert the given at the start of the . + /// Insert the given at the start of the . /// - private static IList UpdateResponseMessagesWithPreInvocationHistory(IList responseMessages, List? augmentedPreInvocationHistory) + private static IList UpdateResponseMessagesWithPreInvocationHistory(IList responseMessages, List? preInvocationHistory) { - if (augmentedPreInvocationHistory?.Count > 0) + if (preInvocationHistory?.Count > 0) { // Since these messages are pre-invocation, we want to insert them at the start of the response messages. - return [.. augmentedPreInvocationHistory, .. responseMessages]; + return [.. preInvocationHistory, .. responseMessages]; } return responseMessages; From c1da46b0fc2410e8a785eeb26e99a6f7bd8abd0a Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 22 Aug 2025 15:51:38 +0100 Subject: [PATCH 51/53] Address PR feedback. --- ...entAgent_UsingFunctionToolsWithApprovals.cs | 8 ++++---- .../MEAI.A/FunctionApprovalRequestContent.cs | 18 ++++-------------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs index 72c5c83449..40d4a45460 100644 --- a/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs +++ b/dotnet/samples/GettingStarted/Steps/Step10_ChatClientAgent_UsingFunctionToolsWithApprovals.cs @@ -88,9 +88,9 @@ async Task RunAgentAsync(string input) List nextIterationMessages = userInputRequests?.Select((request) => request switch { FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" || functionApprovalRequest.FunctionCall.Name == "search_tiktoken_documentation" => - new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateApproval()]), + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved: true)]), FunctionApprovalRequestContent functionApprovalRequest => - new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateRejection()]), + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved: false)]), _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") })?.ToList() ?? []; @@ -180,9 +180,9 @@ async Task RunAgentAsync(string input) List nextIterationMessages = userInputRequests?.Select((request) => request switch { FunctionApprovalRequestContent functionApprovalRequest when functionApprovalRequest.FunctionCall.Name == "GetSpecials" || functionApprovalRequest.FunctionCall.Name == "add" || functionApprovalRequest.FunctionCall.Name == "search_tiktoken_documentation" => - new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateApproval()]), + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved: true)]), FunctionApprovalRequestContent functionApprovalRequest => - new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateRejection()]), + new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved: false)]), _ => throw new NotSupportedException($"Unsupported request type: {request.GetType().Name}") })?.ToList() ?? []; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs index 344bb4c901..b6c561411f 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/MEAI.A/FunctionApprovalRequestContent.cs @@ -26,20 +26,10 @@ public FunctionApprovalRequestContent(string id, FunctionCallContent functionCal public FunctionCallContent FunctionCall { get; } /// - /// Creates a representing an approval response. + /// Creates a to indicate whether the function call is approved or rejected based on the value of . /// + /// if the function call is approved; otherwise, . /// The representing the approval response. - public FunctionApprovalResponseContent CreateApproval() - { - return new FunctionApprovalResponseContent(this.Id, true, this.FunctionCall); - } - - /// - /// Creates a representing a rejection response. - /// - /// The representing the rejection response. - public FunctionApprovalResponseContent CreateRejection() - { - return new FunctionApprovalResponseContent(this.Id, false, this.FunctionCall); - } + public FunctionApprovalResponseContent CreateResponse(bool approved) + => new(this.Id, approved, this.FunctionCall); } From 091f754e37fe03f4968e2cfb16c77c2c541c34c1 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 22 Aug 2025 15:52:02 +0100 Subject: [PATCH 52/53] Fix bug for server side threads. --- .../MEAI/NewFunctionInvokingChatClient.cs | 30 +++++++++---- .../NewFunctionInvokingChatClientTests.cs | 44 +++++++++++++++++++ 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index 58bf6a3c2d..8e0b249737 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -259,7 +259,7 @@ public override async Task GetResponseAsync( int consecutiveErrorCount = 0; // Process approval requests (remove from originalMessages) and rejected approval responses (re-create FCC and create failed FRC). - var (preInvocationHistory, notInvokedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId: null, functionCallContentFallbackMessageId: null); + var (preInvocationHistory, notInvokedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, !string.IsNullOrWhiteSpace(options?.ConversationId), toolResponseId: null, functionCallContentFallbackMessageId: null); // Execute approved approval responses, which generates some additional FRC. (IList? approvedExecutedPreInvocationFRC, bool shouldTerminate, consecutiveErrorCount) = @@ -396,7 +396,7 @@ public override async IAsyncEnumerable GetStreamingResponseA bool hasApprovalRequiringFunctions = approvalRequiredFunctions.Length > 0; // Process approval requests (remove from original messages) and rejected approval responses (re-create FCC and create failed FRC). - var (preInvocationFCCWithRejectedFRC, notInvokedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, toolResponseId, functionCallContentFallbackMessageId); + var (preInvocationFCCWithRejectedFRC, notInvokedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, !string.IsNullOrWhiteSpace(options?.ConversationId), toolResponseId, functionCallContentFallbackMessageId); if (preInvocationFCCWithRejectedFRC is not null) { foreach (var message in preInvocationFCCWithRejectedFRC) @@ -1036,7 +1036,7 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul /// 4. add all the new content items to and return them as the pre-invocation history. /// private static (List? preInvocationHistory, List? approvals) ProcessPreInvocationFunctionApprovalResponses( - List originalMessages, string? toolResponseId, string? functionCallContentFallbackMessageId) + List originalMessages, bool hasConversationId, string? toolResponseId, string? functionCallContentFallbackMessageId) { // Extract any approval responses where we need to execute or reject the function calls. // The original messages are also modified to remove all approval requests and responses. @@ -1052,20 +1052,32 @@ private static (List? preInvocationHistory, List? preInvocationHistory = null; - if (allPreInvocationCallMessages is not null || rejectedPreInvocationResultsMessage is not null) + if (allPreInvocationCallMessages is not null) { preInvocationHistory ??= []; - foreach (var message in (allPreInvocationCallMessages ?? []).Concat(rejectedPreInvocationResultsMessage == null ? [] : [rejectedPreInvocationResultsMessage])) + foreach (var message in allPreInvocationCallMessages) { - originalMessages.Add(message); preInvocationHistory.Add(message); + if (!hasConversationId) + { + originalMessages.Add(message); + } } } + // Add all the FRC that we generated to the pre-invocation history so that they can be returned to the caller as part of the next response. + // Also, add them into the original messages list so that they are passed to the inner client and can be used to generate a result. + if (rejectedPreInvocationResultsMessage is not null) + { + preInvocationHistory ??= []; + originalMessages.Add(rejectedPreInvocationResultsMessage); + preInvocationHistory.Add(rejectedPreInvocationResultsMessage); + } + return (preInvocationHistory, notInvokedResponses.approvals); } diff --git a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs index 37ab4c3700..b03ac5fdd3 100644 --- a/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs +++ b/dotnet/tests/Microsoft.Extensions.AI.Agents.UnitTests/MEAI/NewFunctionInvokingChatClientTests.cs @@ -548,6 +548,50 @@ public async Task ApprovedApprovalResponsesWithoutApprovalRequestAreExecutedAsyn await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); } + [Fact] + public async Task FunctionCallContentIsNotPassedToDownstreamServiceWithServiceThreadsAsync() + { + var options = new ChatOptions + { + Tools = + [ + new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Result 1", "Func1")), + AIFunctionFactory.Create((int i) => $"Result 2: {i}", "Func2"), + ], + ConversationId = "test-conversation", + }; + + List input = + [ + new ChatMessage(ChatRole.User, + [ + new FunctionApprovalResponseContent("callId1", true, new FunctionCallContent("callId1", "Func1")), + new FunctionApprovalResponseContent("callId2", true, new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })) + ]), + ]; + + List expectedDownstreamClientInput = + [ + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + ]; + + List downstreamClientOutput = + [ + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + List output = + [ + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1"), new FunctionCallContent("callId2", "Func2", arguments: new Dictionary { { "i", 42 } })]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1"), new FunctionResultContent("callId2", result: "Result 2: 42")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + await InvokeAndAssertAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + + await InvokeAndAssertStreamingAsync(options, input, downstreamClientOutput, output, expectedDownstreamClientInput); + } + /// /// Since we do not have a way of supporting both functions that require approval and those that do not /// in one invocation, we always require all function calls to be approved if any require approval. From 7518b707e50a7f14acf4dab5737367afa831f0e4 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 25 Aug 2025 11:15:06 +0100 Subject: [PATCH 53/53] Address PR comments --- .../AgentRunResponse.cs | 4 +- .../MEAI/NewFunctionInvokingChatClient.cs | 76 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs index 9279b9ae38..0f28bfbae9 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents.Abstractions/AgentRunResponse.cs @@ -1,14 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. using System; -#if NET9_0_OR_GREATER +#if NET8_0_OR_GREATER using System.Buffers; #endif using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; -#if NET9_0_OR_GREATER +#if NET8_0_OR_GREATER using System.Text; #endif using System.Text.Json; diff --git a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs index 8e0b249737..2b41af5671 100644 --- a/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs +++ b/dotnet/src/Microsoft.Extensions.AI.Agents/MEAI/NewFunctionInvokingChatClient.cs @@ -259,22 +259,22 @@ public override async Task GetResponseAsync( int consecutiveErrorCount = 0; // Process approval requests (remove from originalMessages) and rejected approval responses (re-create FCC and create failed FRC). - var (preInvocationHistory, notInvokedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, !string.IsNullOrWhiteSpace(options?.ConversationId), toolResponseId: null, functionCallContentFallbackMessageId: null); + var (preDownstreamCallHistory, notInvokedApprovals) = ProcessFunctionApprovalResponses(originalMessages, !string.IsNullOrWhiteSpace(options?.ConversationId), toolResponseId: null, functionCallContentFallbackMessageId: null); - // Execute approved approval responses, which generates some additional FRC. - (IList? approvedExecutedPreInvocationFRC, bool shouldTerminate, consecutiveErrorCount) = - await ExecuteApprovedPreInvocationFunctionApprovalResponses(notInvokedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: false, cancellationToken); + // Invoke approved approval responses, which generates some additional FRC wrapped in ChatMessage. + (IList? invokedApprovedFunctionApprovalResponses, bool shouldTerminate, consecutiveErrorCount) = + await InvokeApprovedFunctionApprovalResponses(notInvokedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: false, cancellationToken); - if (approvedExecutedPreInvocationFRC is not null) + if (invokedApprovedFunctionApprovalResponses is not null) { // We need to add the generated FRC to the list we'll return to callers as part of the next response. - preInvocationHistory ??= []; - preInvocationHistory.AddRange(approvedExecutedPreInvocationFRC); + preDownstreamCallHistory ??= []; + preDownstreamCallHistory.AddRange(invokedApprovedFunctionApprovalResponses); } if (shouldTerminate) { - return new ChatResponse(preInvocationHistory); + return new ChatResponse(preDownstreamCallHistory); } for (int iteration = 0; ; iteration++) @@ -304,8 +304,8 @@ public override async Task GetResponseAsync( { // Insert any pre-invocation FCC and FRC that were converted from approval responses into the response here, // so they are returned to the caller. - response.Messages = UpdateResponseMessagesWithPreInvocationHistory(response.Messages, preInvocationHistory); - preInvocationHistory = null; + response.Messages = UpdateResponseMessagesWithPreDownstreamCallHistory(response.Messages, preDownstreamCallHistory); + preDownstreamCallHistory = null; return response; } @@ -396,23 +396,23 @@ public override async IAsyncEnumerable GetStreamingResponseA bool hasApprovalRequiringFunctions = approvalRequiredFunctions.Length > 0; // Process approval requests (remove from original messages) and rejected approval responses (re-create FCC and create failed FRC). - var (preInvocationFCCWithRejectedFRC, notInvokedApprovals) = ProcessPreInvocationFunctionApprovalResponses(originalMessages, !string.IsNullOrWhiteSpace(options?.ConversationId), toolResponseId, functionCallContentFallbackMessageId); - if (preInvocationFCCWithRejectedFRC is not null) + var (preDownstreamCallHistory, notInvokedApprovals) = ProcessFunctionApprovalResponses(originalMessages, !string.IsNullOrWhiteSpace(options?.ConversationId), toolResponseId, functionCallContentFallbackMessageId); + if (preDownstreamCallHistory is not null) { - foreach (var message in preInvocationFCCWithRejectedFRC) + foreach (var message in preDownstreamCallHistory) { yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 } } - // Execute approved approval responses, which generates some additional FRC. - (IList? approvedExecutedPreInvocationFRC, bool shouldTerminate, consecutiveErrorCount) = - await ExecuteApprovedPreInvocationFunctionApprovalResponses(notInvokedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: true, cancellationToken); + // Invoke approved approval responses, which generates some additional FRC wrapped in ChatMessage. + (IList? invokedApprovedFunctionApprovalResponses, bool shouldTerminate, consecutiveErrorCount) = + await InvokeApprovedFunctionApprovalResponses(notInvokedApprovals, originalMessages, options, consecutiveErrorCount, isStreaming: true, cancellationToken); - if (approvedExecutedPreInvocationFRC is not null) + if (invokedApprovedFunctionApprovalResponses is not null) { - foreach (var message in approvedExecutedPreInvocationFRC) + foreach (var message in invokedApprovedFunctionApprovalResponses) { message.MessageId = toolResponseId; yield return ConvertToolResultMessageToUpdate(message, options?.ConversationId, message.MessageId); @@ -1035,7 +1035,7 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul /// 3. Genreate failed for any rejected . /// 4. add all the new content items to and return them as the pre-invocation history. /// - private static (List? preInvocationHistory, List? approvals) ProcessPreInvocationFunctionApprovalResponses( + private static (List? preDownstreamCallHistory, List? approvals) ProcessFunctionApprovalResponses( List originalMessages, bool hasConversationId, string? toolResponseId, string? functionCallContentFallbackMessageId) { // Extract any approval responses where we need to execute or reject the function calls. @@ -1043,25 +1043,25 @@ private static (List? preInvocationHistory, List? allPreInvocationCallMessages = ConvertToFunctionCallContentMessages( + ICollection? allPreDownstreamCallMessages = ConvertToFunctionCallContentMessages( [.. notInvokedResponses.rejections ?? [], .. notInvokedResponses.approvals ?? []], functionCallContentFallbackMessageId); // Generate failed function result contents for any rejected requests and wrap it in a message. List? rejectedFunctionCallResults = GenerateRejectedFunctionResults(notInvokedResponses.rejections, toolResponseId); - ChatMessage? rejectedPreInvocationResultsMessage = rejectedFunctionCallResults != null ? + ChatMessage? rejectedPreDownstreamCallResultsMessage = rejectedFunctionCallResults != null ? new ChatMessage(ChatRole.Tool, rejectedFunctionCallResults) { MessageId = toolResponseId } : null; - // Add all the FCC that we generated to the pre-invocation history so that they can be returned to the caller as part of the next response. + // Add all the FCC that we generated to the pre-downstream-call history so that they can be returned to the caller as part of the next response. // Also, if we are not dealing with a service thread (i.e. we don't have a conversation ID), add them // into the original messages list so that they are passed to the inner client and can be used to generate a result. - List? preInvocationHistory = null; - if (allPreInvocationCallMessages is not null) + List? preDownstreamCallHistory = null; + if (allPreDownstreamCallMessages is not null) { - preInvocationHistory ??= []; - foreach (var message in allPreInvocationCallMessages) + preDownstreamCallHistory ??= []; + foreach (var message in allPreDownstreamCallMessages) { - preInvocationHistory.Add(message); + preDownstreamCallHistory.Add(message); if (!hasConversationId) { originalMessages.Add(message); @@ -1069,22 +1069,22 @@ private static (List? preInvocationHistory, List /// Execute the provided and return the resulting . /// - private async Task<(IList? FunctionResultContent, bool ShouldTerminate, int ConsecutiveErrorCount)> ExecuteApprovedPreInvocationFunctionApprovalResponses( + private async Task<(IList? FunctionResultContent, bool ShouldTerminate, int ConsecutiveErrorCount)> InvokeApprovedFunctionApprovalResponses( List? notInvokedApprovals, List originalMessages, ChatOptions? options, int consecutiveErrorCount, bool isStreaming, CancellationToken cancellationToken) { // Check if there are any function calls to do for any approved functions and execute them. @@ -1442,14 +1442,14 @@ private static async Task> ReplaceFunctionCallsWithApprovalRe } /// - /// Insert the given at the start of the . + /// Insert the given at the start of the . /// - private static IList UpdateResponseMessagesWithPreInvocationHistory(IList responseMessages, List? preInvocationHistory) + private static IList UpdateResponseMessagesWithPreDownstreamCallHistory(IList responseMessages, List? preDownstreamCallHistory) { - if (preInvocationHistory?.Count > 0) + if (preDownstreamCallHistory?.Count > 0) { // Since these messages are pre-invocation, we want to insert them at the start of the response messages. - return [.. preInvocationHistory, .. responseMessages]; + return [.. preDownstreamCallHistory, .. responseMessages]; } return responseMessages;