possible fix for #133

Anthropic.SDK.Tests/NonAscii.cs

@@ -0,0 +1,50 @@
+using Anthropic.SDK.Constants;
+using Anthropic.SDK.Messaging;
+using Google.Api;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Extensions.AI;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Microsoft.SemanticKernel.Connectors.OpenAI;
+
+namespace Anthropic.SDK.Tests
+{
+    [TestClass]
+    public class NonAscii
+    {
+        [TestMethod]
+        public async Task TestNonAscii()
+        {
+            var client = new AnthropicClient();
+            var messages = new List<Message>();
+            messages.Add(new Message(RoleType.User, "Bonjour, je voudrais que tu me fournisses en réponse toutes les lettre de l'alphabet avec leurs variations d'accents (ex: e puis é, è, ê, ë).  Merci!"));
+            var parameters = new MessageParameters()
+            {
+                Messages = messages,
+                MaxTokens = 512,
+                Model = AnthropicModels.Claude4Sonnet,
+                Stream = false,
+                Temperature = 1.0m,
+            };
+            var res = await client.Messages.GetClaudeMessageAsync(parameters);
+            Assert.IsNotNull(res.Message.ToString());
+        }
+        [TestMethod]
+        public async Task TestNonAsciiSK()
+        {
+            var client = new AnthropicClient().Messages.AsChatCompletionService();
+            var res = await client.GetChatMessageContentsAsync(
+                "Bonjour, je voudrais que tu me fournisses en réponse toutes les lettre de l'alphabet avec leurs variations d'accents (ex: e puis é, è, ê, ë).  Merci!", new OpenAIPromptExecutionSettings()
+                {
+                    ModelId = AnthropicModels.Claude4Sonnet,
+                    MaxTokens = 1000
+                });
+
+            Assert.IsNotNull(res.First().ToString());
+        }
+    }
+}

Anthropic.SDK/AnthropicClient.cs

@@ -3,6 +3,8 @@
 using Anthropic.SDK.Messaging;
 using System.Text.Json.Serialization;
 using System.Text.Json;
+using System.Text.Encodings.Web;
+using System.Text.Unicode;
 using Anthropic.SDK.Batches;
 using Anthropic.SDK.Models;
 
@@ -72,6 +74,8 @@ public AnthropicClient(APIAuthentication apiKeys = null, HttpClient client = nul
             DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
             Converters = { new JsonStringEnumConverter() },
             ReferenceHandler = ReferenceHandler.IgnoreCycles,
+            // Ensure proper Unicode handling for all characters
+            Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
         };
 
 

Anthropic.SDK/BaseEndpoint.cs

@@ -8,6 +8,8 @@
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.Encodings.Web;
+using System.Text.Unicode;
 using System.Threading;
 using System.Threading.Tasks;
 using Anthropic.SDK.Messaging;
@@ -53,7 +55,9 @@ protected async Task<TResponse> HttpRequestMessages<TResponse>(string url = null
 
             var options = new JsonSerializerOptions
             {
-                Converters = { ContentConverter.Instance }
+                Converters = { ContentConverter.Instance },
+                // Ensure proper Unicode handling for all characters
+                Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)
             };
 
             using var ms = new MemoryStream(Encoding.UTF8.GetBytes(resultAsString));
@@ -127,7 +131,9 @@ protected async Task<HttpResponseMessage> HttpRequestRaw(string url = null, Http
                     var options = new JsonSerializerOptions
                     {
                         DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
-                        Converters = { ContentConverter.Instance }
+                        Converters = { ContentConverter.Instance },
+                        // Ensure proper Unicode handling for all characters
+                        Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)
                     };
                     string jsonContent = JsonSerializer.Serialize(postData, options);
                     req.Content = new StringContent(jsonContent, Encoding.UTF8, "application/json");

Anthropic.SDK/EndpointBase.cs

@@ -11,6 +11,8 @@
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.Encodings.Web;
+using System.Text.Unicode;
 using System.Threading;
 using System.Threading.Tasks;
 using Anthropic.SDK.Batches;
@@ -123,7 +125,9 @@
             {
                 var options = new JsonSerializerOptions
                 {
-                    Converters = { ContentConverter.Instance }
+                    Converters = { ContentConverter.Instance },
+                    // Ensure proper Unicode handling for all characters
+                    Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)
                 };
                 using var ms = new MemoryStream(Encoding.UTF8.GetBytes(line));
                 var res = await JsonSerializer.DeserializeAsync<BatchLine>(ms, options, cancellationToken: ctx).ConfigureAwait(false);
@@ -158,7 +162,7 @@
         /// <summary>
         /// Handle error responses from the API
         /// </summary>
         protected override async Task<Exception> HandleErrorResponseAsync(HttpResponseMessage response, string resultAsString, string url)
         {
 #if NET6_0_OR_GREATER
             if (response.StatusCode == HttpStatusCode.TooManyRequests)
@@ -241,15 +245,22 @@
                         using var ms = new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data));
                         var res = await JsonSerializer.DeserializeAsync<MessageResponse>(ms, new JsonSerializerOptions()
                         {
-                            Converters = { ContentConverter.Instance }
+                            Converters = { ContentConverter.Instance },
+                            // Ensure proper Unicode handling for all characters
+                            Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)
                         }, cancellationToken: ctx).ConfigureAwait(false);
                         res.RateLimits = GetRateLimits(response);
                         yield return res;
                     }
                     else if (currentEvent.EventType == "error")
                     {
                         using var ms = new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data));
-                        var res = await JsonSerializer.DeserializeAsync<ErrorResponse>(ms, cancellationToken: ctx).ConfigureAwait(false);
+                        var errorOptions = new JsonSerializerOptions
+                        {
+                            // Ensure proper Unicode handling for all characters
+                            Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)
+                        };
+                        var res = await JsonSerializer.DeserializeAsync<ErrorResponse>(ms, errorOptions, cancellationToken: ctx).ConfigureAwait(false);
                         throw new Exception(res.Error.Message);
                     }
                     currentEvent = new SseEvent();

Anthropic.SDK/VertexAIClient.cs

@@ -2,6 +2,8 @@
 using System.Net.Http;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.Encodings.Web;
+using System.Text.Unicode;
 using Anthropic.SDK.Messaging;
 
 namespace Anthropic.SDK
@@ -53,6 +55,8 @@ public VertexAIClient(VertexAIAuthentication auth = null, HttpClient client = nu
             DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
             Converters = { new JsonStringEnumConverter() },
             ReferenceHandler = ReferenceHandler.IgnoreCycles,
+            // Ensure proper Unicode handling for all characters
+            Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
         };
 
         private HttpClient SetupClient(HttpClient client)

Anthropic.SDK/VertexAIEndpointBase.cs

@@ -10,6 +10,8 @@
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.Encodings.Web;
+using System.Text.Unicode;
 using System.Threading;
 using System.Threading.Tasks;
 using Anthropic.SDK.Extensions;
@@ -138,7 +140,7 @@
         /// <summary>
         /// Handle error responses from the API
         /// </summary>
         protected override async Task<Exception> HandleErrorResponseAsync(HttpResponseMessage response, string resultAsString, string url)
         {
 #if NET6_0_OR_GREATER
             if (response.StatusCode == HttpStatusCode.TooManyRequests)
@@ -223,14 +225,25 @@
                         try
                         {
                             using var ms = new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data));
-                            result = await JsonSerializer.DeserializeAsync<MessageResponse>(ms, cancellationToken: ctx).ConfigureAwait(false);
+                            var options = new JsonSerializerOptions
+                            {
+                                Converters = { ContentConverter.Instance },
+                                // Ensure proper Unicode handling for all characters
+                                Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)
+                            };
+                            result = await JsonSerializer.DeserializeAsync<MessageResponse>(ms, options, cancellationToken: ctx).ConfigureAwait(false);
                         }
                         catch (JsonException)
                         {
                             // Try to parse as a Vertex AI response
                             try
                             {
-                                var vertexResponse = JsonSerializer.Deserialize<JsonElement>(currentEvent.Data);
+                                var vertexOptions = new JsonSerializerOptions
+                                {
+                                    // Ensure proper Unicode handling for all characters
+                                    Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)
+                                };
+                                var vertexResponse = JsonSerializer.Deserialize<JsonElement>(currentEvent.Data, vertexOptions);
 
                                 // Check if it has predictions
                                 if (vertexResponse.TryGetProperty("predictions", out var predictions) &&