From 064cc7498ed52a9d5d5f8e3b619df26c225938e8 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 2 May 2025 13:04:21 -0400 Subject: [PATCH 1/3] Add DataContent.Base64Data --- .../Contents/DataContent.cs | 99 +++++++++++-------- .../AIContentExtensions.cs | 16 --- .../ContentSafetyServicePayloadUtilities.cs | 14 +-- .../OllamaChatClient.cs | 7 +- .../Contents/DataContentTests.cs | 26 +++++ test/Shared/ImageDataUri/ImageDataUri.cs | 4 +- 6 files changed, 88 insertions(+), 78 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs index 6353586208f..18b46cc4df6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs @@ -2,8 +2,15 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +#if NET +using System.Buffers; +using System.Buffers.Text; +#endif using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +#if !NET +using System.Runtime.InteropServices; +#endif using System.Text.Json.Serialization; using Microsoft.Shared.Diagnostics; @@ -70,39 +77,35 @@ public DataContent(Uri uri, string? mediaType = null) [JsonConstructor] public DataContent([StringSyntax(StringSyntaxAttribute.Uri)] string uri, string? mediaType = null) { + // Store and validate the data URI. _uri = Throw.IfNullOrWhitespace(uri); - if (!uri.StartsWith(DataUriParser.Scheme, StringComparison.OrdinalIgnoreCase)) { Throw.ArgumentException(nameof(uri), "The provided URI is not a data URI."); } + // Parse the data URI to extract the data and media type. _dataUri = DataUriParser.Parse(uri.AsMemory()); + // Validate and store the media type. + mediaType ??= _dataUri.MediaType; if (mediaType is null) { - mediaType = _dataUri.MediaType; - if (mediaType is null) - { - Throw.ArgumentNullException(nameof(mediaType), $"{nameof(uri)} did not contain a media type, and {nameof(mediaType)} was not provided."); - } - } - else - { - if (mediaType != _dataUri.MediaType) - { - // If the data URI contains a media type that's different from a non-null media type - // explicitly provided, prefer the one explicitly provided as an override. - - // Extract the bytes from the data URI and null out the uri. - // Then we'll lazily recreate it later if needed based on the updated media type. - _data = _dataUri.ToByteArray(); - _dataUri = null; - _uri = null; - } + Throw.ArgumentNullException(nameof(mediaType), $"{nameof(uri)} did not contain a media type, and {nameof(mediaType)} was not provided."); } MediaType = DataUriParser.ThrowIfInvalidMediaType(mediaType); + + if (!_dataUri.IsBase64 || mediaType != _dataUri.MediaType) + { + // In rare cases, the data URI may contain non-base64 data, in which case we + // want to normalize it to base64. The supplied media type may also be different + // from the one in the data URI. In either case, we extract the bytes from the data URI + // and then throw away the uri; we'll recreate it lazily in the canonical form. + _data = _dataUri.ToByteArray(); + _dataUri = null; + _uri = null; + } } /// @@ -134,9 +137,8 @@ public DataContent(ReadOnlyMemory data, string mediaType) /// Gets the data URI for this . /// - /// The returned URI is always a valid URI string, even if the instance was constructed from a - /// or from a . In the case of a , this property returns a data URI containing - /// that data. + /// The returned URI is always a valid data URI string, even if the instance was constructed from a + /// or from a . /// [StringSyntax(StringSyntaxAttribute.Uri)] public string Uri @@ -145,27 +147,26 @@ public string Uri { if (_uri is null) { - if (_dataUri is null) - { - Debug.Assert(_data is not null, "Expected _data to be initialized."); - _uri = string.Concat("data:", MediaType, ";base64,", Convert.ToBase64String(_data.GetValueOrDefault() -#if NET - .Span)); -#else - .Span.ToArray())); -#endif - } - else - { - _uri = _dataUri.IsBase64 ? + Debug.Assert(_data is not null, "Expected _data to be initialized."); + ReadOnlyMemory data = _data.GetValueOrDefault(); + #if NET - $"data:{MediaType};base64,{_dataUri.Data.Span}" : - $"data:{MediaType};,{_dataUri.Data.Span}"; + char[] array = ArrayPool.Shared.Rent( + "data:".Length + MediaType.Length + ";base64,".Length + Base64.GetMaxEncodedToUtf8Length(data.Length)); + + bool wrote = array.AsSpan().TryWrite($"data:{MediaType};base64,", out int prefixLength); + wrote |= Convert.TryToBase64Chars(data.Span, array.AsSpan(prefixLength), out int dataLength); + Debug.Assert(wrote, "Expected to successfully write the data URI."); + _uri = array.AsSpan(0, prefixLength + dataLength).ToString(); + + ArrayPool.Shared.Return(array); #else - $"data:{MediaType};base64,{_dataUri.Data}" : - $"data:{MediaType};,{_dataUri.Data}"; + string base64 = MemoryMarshal.TryGetArray(data, out ArraySegment segment) ? + Convert.ToBase64String(segment.Array!, segment.Offset, segment.Count) : + Convert.ToBase64String(data.ToArray()); + + _uri = $"data:{MediaType};base64,{base64}"; #endif - } } return _uri; @@ -205,6 +206,22 @@ public ReadOnlyMemory Data } } + /// Gets the data represented by this instance as a Base64 character sequence. + /// The base64 representation of the data. + [JsonIgnore] + public ReadOnlyMemory Base64Data + { + get + { + const string Base64Separator = ";base64,"; + string uri = Uri; + int pos = uri.IndexOf(Base64Separator, StringComparison.OrdinalIgnoreCase); + Debug.Assert(pos >= 0, "Expected base64 to be present in the URI."); + pos += Base64Separator.Length; + return uri.AsMemory(pos); + } + } + /// Gets a string representing this instance to display in the debugger. [DebuggerBrowsable(DebuggerBrowsableState.Never)] private string DebuggerDisplay diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs index 4e33c64c305..6ec3793d0da 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs @@ -13,22 +13,6 @@ internal static bool IsImageWithSupportedFormat(this AIContent content) => (content is UriContent uriContent && IsSupportedImageFormat(uriContent.MediaType)) || (content is DataContent dataContent && IsSupportedImageFormat(dataContent.MediaType)); - internal static bool IsUriBase64Encoded(this DataContent dataContent) - { - ReadOnlyMemory uri = dataContent.Uri.AsMemory(); - - int commaIndex = uri.Span.IndexOf(','); - if (commaIndex == -1) - { - return false; - } - - ReadOnlyMemory metadata = uri.Slice(0, commaIndex); - - bool isBase64Encoded = metadata.Span.EndsWith(";base64".AsSpan(), StringComparison.OrdinalIgnoreCase); - return isBase64Encoded; - } - private static bool IsSupportedImageFormat(string mediaType) { // 'image/jpeg' is the official MIME type for JPEG. However, some systems recognize 'image/jpg' as well. diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs index a2694669106..feecec3be46 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs @@ -343,25 +343,13 @@ IEnumerable GetContents(ChatMessage message) } else if (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")) { - string url; - if (dataContent.IsUriBase64Encoded()) - { - url = dataContent.Uri; - } - else - { - BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data); - string base64ImageData = Convert.ToBase64String(imageBytes.ToArray()); - url = $"data:{dataContent.MediaType};base64,{base64ImageData}"; - } - yield return new JsonObject { ["type"] = "image_url", ["image_url"] = new JsonObject { - ["url"] = url + ["url"] = dataContent.Uri } }; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs index f42f1e1edfb..210d8f7c92d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs @@ -402,12 +402,7 @@ private IEnumerable ToOllamaChatRequestMessages(ChatMe if (item is DataContent dataContent && dataContent.HasTopLevelMediaType("image")) { IList images = currentTextMessage?.Images ?? []; - images.Add(Convert.ToBase64String(dataContent.Data -#if NET - .Span)); -#else - .ToArray())); -#endif + images.Add(dataContent.Base64Data.ToString()); if (currentTextMessage is not null) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs index 83f09c66889..0afa131899a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs @@ -66,21 +66,27 @@ public void Ctor_ValidMediaType_Roundtrips(string mediaType) { var content = new DataContent("data:image/png;base64,aGVsbG8=", mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("aGVsbG8=", content.Base64Data.ToString()); content = new DataContent("data:,", mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("", content.Base64Data.ToString()); content = new DataContent("data:text/plain,", mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("", content.Base64Data.ToString()); content = new DataContent(new Uri("data:text/plain,"), mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("", content.Base64Data.ToString()); content = new DataContent(new byte[] { 0, 1, 2 }, mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("AAEC", content.Base64Data.ToString()); content = new DataContent(content.Uri); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("AAEC", content.Base64Data.ToString()); } [Fact] @@ -91,10 +97,12 @@ public void Ctor_NoMediaType_Roundtrips() content = new DataContent("data:image/png;base64,aGVsbG8="); Assert.Equal("data:image/png;base64,aGVsbG8=", content.Uri); Assert.Equal("image/png", content.MediaType); + Assert.Equal("aGVsbG8=", content.Base64Data.ToString()); content = new DataContent(new Uri("data:image/png;base64,aGVsbG8=")); Assert.Equal("data:image/png;base64,aGVsbG8=", content.Uri); Assert.Equal("image/png", content.MediaType); + Assert.Equal("aGVsbG8=", content.Base64Data.ToString()); } [Fact] @@ -128,6 +136,7 @@ public void Deserialize_MatchesExpectedData() Assert.Equal("data:application/octet-stream;base64,AQIDBA==", content.Uri); Assert.Equal([0x01, 0x02, 0x03, 0x04], content.Data.ToArray()); + Assert.Equal("AQIDBA==", content.Base64Data.ToString()); Assert.Equal("application/octet-stream", content.MediaType); // Uri referenced content-only @@ -150,6 +159,7 @@ public void Deserialize_MatchesExpectedData() Assert.Equal("data:audio/wav;base64,AQIDBA==", content.Uri); Assert.Equal([0x01, 0x02, 0x03, 0x04], content.Data.ToArray()); + Assert.Equal("AQIDBA==", content.Base64Data.ToString()); Assert.Equal("audio/wav", content.MediaType); Assert.Equal("value", content.AdditionalProperties!["key"]!.ToString()); } @@ -224,4 +234,20 @@ public void HasMediaTypePrefix_ReturnsFalse(string mediaType, string prefix) var content = new DataContent("data:application/octet-stream;base64,AQIDBA==", mediaType); Assert.False(content.HasTopLevelMediaType(prefix)); } + + [Fact] + public void Data_Roundtrips() + { + Random rand = new(42); + for (int length = 0; length < 100; length++) + { + byte[] data = new byte[length]; + rand.NextBytes(data); + + var content = new DataContent(data, "application/octet-stream"); + Assert.Equal(data, content.Data.ToArray()); + Assert.Equal(Convert.ToBase64String(data), content.Base64Data.ToString()); + Assert.Equal($"data:application/octet-stream;base64,{Convert.ToBase64String(data)}", content.Uri); + } + } } diff --git a/test/Shared/ImageDataUri/ImageDataUri.cs b/test/Shared/ImageDataUri/ImageDataUri.cs index b2c91809305..582839a298a 100644 --- a/test/Shared/ImageDataUri/ImageDataUri.cs +++ b/test/Shared/ImageDataUri/ImageDataUri.cs @@ -19,7 +19,7 @@ internal static Uri GetImageDataUri() Assert.NotNull(s); MemoryStream ms = new(); s.CopyTo(ms); - return new Uri($"data:image/png;base64,{Convert.ToBase64String(ms.ToArray())}"); + return new Uri(new DataContent(ms.ToArray(), "image/png").Uri); } internal static Uri GetPdfDataUri() @@ -28,6 +28,6 @@ internal static Uri GetPdfDataUri() PdfPageBuilder page = builder.AddPage(PageSize.A4); PdfDocumentBuilder.AddedFont font = builder.AddStandard14Font(Standard14Font.Helvetica); page.AddText("Hello World!", 12, new PdfPoint(25, 700), font); - return new Uri($"data:application/pdf;base64,{Convert.ToBase64String(builder.Build())}"); + return new Uri(new DataContent(builder.Build(), "application/pdf").Uri); } } From 885ddcb195e2d1edcd2e6df5b4bf43d060a68da0 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 2 May 2025 13:42:32 -0400 Subject: [PATCH 2/3] Revert ImageDataUri test change --- test/Shared/ImageDataUri/ImageDataUri.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Shared/ImageDataUri/ImageDataUri.cs b/test/Shared/ImageDataUri/ImageDataUri.cs index 582839a298a..b2c91809305 100644 --- a/test/Shared/ImageDataUri/ImageDataUri.cs +++ b/test/Shared/ImageDataUri/ImageDataUri.cs @@ -19,7 +19,7 @@ internal static Uri GetImageDataUri() Assert.NotNull(s); MemoryStream ms = new(); s.CopyTo(ms); - return new Uri(new DataContent(ms.ToArray(), "image/png").Uri); + return new Uri($"data:image/png;base64,{Convert.ToBase64String(ms.ToArray())}"); } internal static Uri GetPdfDataUri() @@ -28,6 +28,6 @@ internal static Uri GetPdfDataUri() PdfPageBuilder page = builder.AddPage(PageSize.A4); PdfDocumentBuilder.AddedFont font = builder.AddStandard14Font(Standard14Font.Helvetica); page.AddText("Hello World!", 12, new PdfPoint(25, 700), font); - return new Uri(new DataContent(builder.Build(), "application/pdf").Uri); + return new Uri($"data:application/pdf;base64,{Convert.ToBase64String(builder.Build())}"); } } From 22e182423c6343cfc50ecfe4966ac07766220d4d Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 2 May 2025 21:36:56 -0400 Subject: [PATCH 3/3] Search only for comma --- .../Contents/DataContent.cs | 9 ++++----- .../Contents/DataContentTests.cs | 10 ++++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs index 18b46cc4df6..5bbde1e1444 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs @@ -17,6 +17,7 @@ #pragma warning disable S3996 // URI properties should not be strings #pragma warning disable CA1054 // URI-like parameters should not be strings #pragma warning disable CA1056 // URI-like properties should not be strings +#pragma warning disable CA1307 // Specify StringComparison for clarity namespace Microsoft.Extensions.AI; @@ -213,12 +214,10 @@ public ReadOnlyMemory Base64Data { get { - const string Base64Separator = ";base64,"; string uri = Uri; - int pos = uri.IndexOf(Base64Separator, StringComparison.OrdinalIgnoreCase); - Debug.Assert(pos >= 0, "Expected base64 to be present in the URI."); - pos += Base64Separator.Length; - return uri.AsMemory(pos); + int pos = uri.IndexOf(','); + Debug.Assert(pos >= 0, "Expected comma to be present in the URI."); + return uri.AsMemory(pos + 1); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs index 0afa131899a..0f5b6b22d92 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Text; using System.Text.Json; using Xunit; @@ -250,4 +251,13 @@ public void Data_Roundtrips() Assert.Equal($"data:application/octet-stream;base64,{Convert.ToBase64String(data)}", content.Uri); } } + + [Fact] + public void NonBase64Data_Normalized() + { + var content = new DataContent("data:text/plain,hello world"); + Assert.Equal("data:text/plain;base64,aGVsbG8gd29ybGQ=", content.Uri); + Assert.Equal("aGVsbG8gd29ybGQ=", content.Base64Data.ToString()); + Assert.Equal("hello world", Encoding.ASCII.GetString(content.Data.ToArray())); + } }