diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index ed8880c117feae..fa415f8df784fb 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -461,6 +461,9 @@ public void WritePropertyName(System.ReadOnlySpan utf8PropertyName) { } public void WritePropertyName(System.ReadOnlySpan propertyName) { } public void WritePropertyName(string propertyName) { } public void WritePropertyName(System.Text.Json.JsonEncodedText propertyName) { } + public void WriteRawValue(string json, bool skipInputValidation = false) { } + public void WriteRawValue(System.ReadOnlySpan utf8Json, bool skipInputValidation = false) { } + public void WriteRawValue(System.ReadOnlySpan json, bool skipInputValidation = false) { } public void WriteStartArray() { } public void WriteStartArray(System.ReadOnlySpan utf8PropertyName) { } public void WriteStartArray(System.ReadOnlySpan propertyName) { } diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 8a6315b1fe7c48..8cb1375c6d08e1 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -257,6 +257,7 @@ + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs index e82811377a6820..a2e91cd3b08fcc 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs @@ -64,6 +64,13 @@ internal static class JsonConstants // All other UTF-16 characters can be represented by either 1 or 2 UTF-8 bytes. public const int MaxExpansionFactorWhileTranscoding = 3; + // When transcoding from UTF8 -> UTF16, the byte count threshold where we rent from the array pool before performing a normal alloc. + public const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; + + // The maximum number of characters allowed when writing raw UTF-16 JSON. This is the maximum length that we can guarantee can + // be safely transcoded to UTF-8 and fit within an integer-length span, given the max expansion factor of a single character (3). + public const int MaxUtf16RawValueLength = int.MaxValue / MaxExpansionFactorWhileTranscoding; + public const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value. public const int MaxUnescapedTokenSize = MaxEscapedTokenSize / MaxExpansionFactorWhileEscaping; // 166_666_666 bytes public const int MaxBase64ValueTokenSize = (MaxEscapedTokenSize >> 2) * 3 / MaxExpansionFactorWhileEscaping; // 125_000_000 bytes diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs index 1cd7f8cc6eb11b..29e06ef3742f1b 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs @@ -355,12 +355,10 @@ public static partial class JsonSerializer private static TValue? ReadUsingMetadata(ReadOnlySpan json, JsonTypeInfo jsonTypeInfo) { - const long ArrayPoolMaxSizeBeforeUsingNormalAlloc = 1024 * 1024; - byte[]? tempArray = null; // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. - Span utf8 = json.Length <= (ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? + Span utf8 = json.Length <= (JsonConstants.ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? // Use a pooled alloc. tempArray = ArrayPool.Shared.Rent(json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) : // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs new file mode 100644 index 00000000000000..5ac9c8cebe315e --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Raw.cs @@ -0,0 +1,201 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Diagnostics; + +namespace System.Text.Json +{ + public sealed partial class Utf8JsonWriter + { + /// + /// Writes the input as JSON content. It is expected that the input content is a single complete JSON value. + /// + /// The raw JSON content to write. + /// Whether to validate if the input is an RFC 8259-compliant JSON payload. + /// Thrown if is . + /// Thrown if the length of the input is zero or greater than 715,827,882 ( / 3). + /// + /// Thrown if is , and the input + /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) + /// or the input JSON exceeds a recursive depth of 64. + /// + /// + /// When writing untrused JSON values, do not set to as this can result in invalid JSON + /// being written, and/or the overall payload being written to the writer instance being invalid. + /// + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled). + /// + /// The value for the writer instance is honored when using this method. + /// + /// The and values for the writer instance are not applied when using this method. + /// + public void WriteRawValue(string json, bool skipInputValidation = false) + { + if (!_options.SkipValidation) + { + ValidateWritingValue(); + } + + if (json == null) + { + throw new ArgumentNullException(nameof(json)); + } + + TranscodeAndWriteRawValue(json.AsSpan(), skipInputValidation); + } + + /// + /// Writes the input as JSON content. It is expected that the input content is a single complete JSON value. + /// + /// The raw JSON content to write. + /// Whether to validate if the input is an RFC 8259-compliant JSON payload. + /// Thrown if the length of the input is zero or greater than 715,827,882 ( / 3). + /// + /// Thrown if is , and the input + /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) + /// or the input JSON exceeds a recursive depth of 64. + /// + /// + /// When writing untrused JSON values, do not set to as this can result in invalid JSON + /// being written, and/or the overall payload being written to the writer instance being invalid. + /// + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled). + /// + /// The value for the writer instance is honored when using this method. + /// + /// The and values for the writer instance are not applied when using this method. + /// + public void WriteRawValue(ReadOnlySpan json, bool skipInputValidation = false) + { + if (!_options.SkipValidation) + { + ValidateWritingValue(); + } + + TranscodeAndWriteRawValue(json, skipInputValidation); + } + + /// + /// Writes the input as JSON content. It is expected that the input content is a single complete JSON value. + /// + /// The raw JSON content to write. + /// Whether to validate if the input is an RFC 8259-compliant JSON payload. + /// Thrown if the length of the input is zero or equal to . + /// + /// Thrown if is , and the input + /// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259) + /// or the input JSON exceeds a recursive depth of 64. + /// + /// + /// When writing untrused JSON values, do not set to as this can result in invalid JSON + /// being written, and/or the overall payload being written to the writer instance being invalid. + /// + /// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled). + /// + /// The value for the writer instance is honored when using this method. + /// + /// The and values for the writer instance are not applied when using this method. + /// + public void WriteRawValue(ReadOnlySpan utf8Json, bool skipInputValidation = false) + { + if (!_options.SkipValidation) + { + ValidateWritingValue(); + } + + if (utf8Json.Length == int.MaxValue) + { + ThrowHelper.ThrowArgumentException_ValueTooLarge(int.MaxValue); + } + + WriteRawValueCore(utf8Json, skipInputValidation); + } + + private void TranscodeAndWriteRawValue(ReadOnlySpan json, bool skipInputValidation) + { + if (json.Length > JsonConstants.MaxUtf16RawValueLength) + { + ThrowHelper.ThrowArgumentException_ValueTooLarge(json.Length); + } + + byte[]? tempArray = null; + + // For performance, avoid obtaining actual byte count unless memory usage is higher than the threshold. + Span utf8Json = json.Length <= (JsonConstants.ArrayPoolMaxSizeBeforeUsingNormalAlloc / JsonConstants.MaxExpansionFactorWhileTranscoding) ? + // Use a pooled alloc. + tempArray = ArrayPool.Shared.Rent(json.Length * JsonConstants.MaxExpansionFactorWhileTranscoding) : + // Use a normal alloc since the pool would create a normal alloc anyway based on the threshold (per current implementation) + // and by using a normal alloc we can avoid the Clear(). + new byte[JsonReaderHelper.GetUtf8ByteCount(json)]; + + try + { + int actualByteCount = JsonReaderHelper.GetUtf8FromText(json, utf8Json); + utf8Json = utf8Json.Slice(0, actualByteCount); + WriteRawValueCore(utf8Json, skipInputValidation); + } + finally + { + if (tempArray != null) + { + utf8Json.Clear(); + ArrayPool.Shared.Return(tempArray); + } + } + } + + private void WriteRawValueCore(ReadOnlySpan utf8Json, bool skipInputValidation) + { + int len = utf8Json.Length; + + if (len == 0) + { + ThrowHelper.ThrowArgumentException(SR.ExpectedJsonTokens); + } + + // In the UTF-16-based entry point methods above, we validate that the payload length <= int.MaxValue /3. + // The result of this division will be rounded down, so even if every input character needs to be transcoded + // (with expansion factor of 3), the resulting payload would be less than int.MaxValue, + // as (int.MaxValue/3) * 3 is less than int.MaxValue. + Debug.Assert(len < int.MaxValue); + + if (skipInputValidation) + { + // Treat all unvalidated raw JSON value writes as string. If the payload is valid, this approach does + // not affect structural validation since a string token is equivalent to a complete object, array, + // or other complete JSON tokens when considering structural validation on subsequent writer calls. + // If the payload is not valid, then we make no guarantees about the structural validation of the final payload. + _tokenType = JsonTokenType.String; + } + else + { + // Utilize reader validation. + Utf8JsonReader reader = new(utf8Json); + while (reader.Read()); + _tokenType = reader.TokenType; + } + + // TODO (https://github.com/dotnet/runtime/issues/29293): + // investigate writing this in chunks, rather than requesting one potentially long, contiguous buffer. + int maxRequired = len + 1; // Optionally, 1 list separator. We've guarded against integer overflow earlier in the call stack. + + if (_memory.Length - BytesPending < maxRequired) + { + Grow(maxRequired); + } + + Span output = _memory.Span; + + if (_currentDepth < 0) + { + output[BytesPending++] = JsonConstants.ListSeparator; + } + + utf8Json.CopyTo(output.Slice(BytesPending)); + BytesPending += len; + + SetFlagToAddListSeparatorBeforeNextItem(); + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj index ee45bdc5e97e9d..964f2c6d33a79f 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj @@ -186,6 +186,7 @@ + diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs new file mode 100644 index 00000000000000..eb59af3510cb9c --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.WriteRaw.cs @@ -0,0 +1,548 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit; + +namespace System.Text.Json.Tests +{ + public partial class Utf8JsonWriterTests + { + private const string TestGuidAsStr = "eb97fadd-3ebf-4781-8722-f4773989160e"; + private readonly static Guid s_guid = Guid.Parse(TestGuidAsStr); + + private static byte[] s_oneAsJson = new byte[] { (byte)'1' }; + + [Theory] + [MemberData(nameof(GetRootLevelPrimitives))] + [MemberData(nameof(GetArrays))] + public static void WriteRawValidJson(byte[] rawJson, Action verifyWithDeserialize) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + RunTests(skipInputValidation: true); + RunTests(skipInputValidation: false); + + void RunTests(bool skipInputValidation) + { + // ROS + writer.Reset(); + ms.SetLength(0); + writer.WriteRawValue(rawJson, skipInputValidation); + writer.Flush(); + verifyWithDeserialize(ms.ToArray()); + + // string + string rawJsonAsStr = Encoding.UTF8.GetString(rawJson); + writer.Reset(); + ms.SetLength(0); + writer.WriteRawValue(rawJsonAsStr, skipInputValidation); + writer.Flush(); + verifyWithDeserialize(ms.ToArray()); + + // ROS + writer.Reset(); + ms.SetLength(0); + writer.WriteRawValue(rawJsonAsStr.AsSpan(), skipInputValidation); + writer.Flush(); + verifyWithDeserialize(ms.ToArray()); + } + } + + public static IEnumerable GetRootLevelPrimitives() + { + Action validate; + + validate = (data) => Assert.Equal(123456789, JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes("123456789"), validate }; + + validate = (data) => Assert.Equal(1234.56789, JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes("1234.56789"), validate }; + + validate = (data) => Assert.Equal(1234.56789, JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes(" 1234.56789 "), validate }; + + validate = (data) => Assert.Equal(@"Hello", JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes(@"""Hello"""), validate }; + + validate = (data) => Assert.Equal(@"Hello", JsonSerializer.Deserialize(data)); + yield return new object[] { Encoding.UTF8.GetBytes(@" ""Hello"" "), validate }; + + validate = (data) => Assert.Equal(s_guid, JsonSerializer.Deserialize(data)); + byte[] guidAsJson = WrapInQuotes(Encoding.UTF8.GetBytes(TestGuidAsStr)); + yield return new object[] { guidAsJson, validate }; + } + + public static IEnumerable GetArrays() + { + Action validate; + + byte[] json = JsonSerializer.SerializeToUtf8Bytes(Enumerable.Repeat(1234.56789, 4)); + validate = (data) => + { + foreach (double d in JsonSerializer.Deserialize(data)) + { + Assert.Equal(1234.56789, d); + } + }; + yield return new object[] { json, validate }; + + json = JsonSerializer.SerializeToUtf8Bytes(Enumerable.Repeat("Hello", 4)); + validate = (data) => + { + foreach (string str in JsonSerializer.Deserialize(data)) + { + Assert.Equal("Hello", str); + } + }; + yield return new object[] { json, validate }; + + json = JsonSerializer.SerializeToUtf8Bytes(Enumerable.Repeat("Hello", 4)); + validate = (data) => + { + foreach (string str in JsonSerializer.Deserialize(data)) + { + Assert.Equal("Hello", str); + } + }; + yield return new object[] { json, validate }; + + json = Encoding.UTF8.GetBytes("[ 1, 1,1,1,1 ] "); + validate = (data) => + { + foreach (int val in JsonSerializer.Deserialize(data)) + { + Assert.Equal(1, val); + } + }; + yield return new object[] { json, validate }; + } + + public static IEnumerable GetObjects() + { + Action validate; + + byte[] json = Encoding.UTF8.GetBytes(@"{""Hello"":""World""}"); ; + validate = (data) => + { + KeyValuePair kvp = JsonSerializer.Deserialize>(data).Single(); + Assert.Equal("Hello", kvp.Key); + Assert.Equal("World", kvp.Value); + }; + yield return new object[] { json, validate }; + + json = Encoding.UTF8.GetBytes(@" { ""Hello"" :""World"" } "); ; + validate = (data) => + { + KeyValuePair kvp = JsonSerializer.Deserialize>(data).Single(); + Assert.Equal("Hello", kvp.Key); + Assert.Equal("World", kvp.Value); + }; + yield return new object[] { json, validate }; + } + + private static byte[] WrapInQuotes(ReadOnlySpan buffer) + { + byte[] quotedBuffer = new byte[buffer.Length + 2]; + quotedBuffer[0] = (byte)'"'; + buffer.CopyTo(quotedBuffer.AsSpan(1)); + quotedBuffer[buffer.Length + 1] = (byte)'"'; + return quotedBuffer; + } + + [Theory] + [InlineData(true, 0, "[]")] + [InlineData(false, 0, "[]")] + [InlineData(true, 1, "[1]")] + [InlineData(false, 1, "[1]")] + [InlineData(true, 5, "[1,1,1,1,1]")] + [InlineData(false, 5, "[1,1,1,1,1]")] + public static void WriteRawArrayElements(bool skipInputValidation, int numElements, string expectedJson) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + writer.WriteStartArray(); + + for (int i = 0; i < numElements; i++) + { + writer.WriteRawValue(s_oneAsJson, skipInputValidation); + } + + writer.WriteEndArray(); + + writer.Flush(); + Assert.Equal(expectedJson, Encoding.UTF8.GetString(ms.ToArray())); + } + + [Theory] + [InlineData(true, 0, "{}")] + [InlineData(false, 0, "{}")] + [InlineData(true, 1, @"{""int"":1}")] + [InlineData(false, 1, @"{""int"":1}")] + [InlineData(true, 3, @"{""int"":1,""int"":1,""int"":1}")] + [InlineData(false, 3, @"{""int"":1,""int"":1,""int"":1}")] + public static void WriteRawObjectProperty(bool skipInputValidation, int numElements, string expectedJson) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + writer.WriteStartObject(); + + for (int i = 0; i < numElements; i++) + { + writer.WritePropertyName("int"); + writer.WriteRawValue(s_oneAsJson, skipInputValidation); + } + + writer.WriteEndObject(); + + writer.Flush(); + Assert.Equal(expectedJson, Encoding.UTF8.GetString(ms.ToArray())); + } + + [Theory] + [InlineData("[")] + [InlineData("}")] + [InlineData("[}")] + [InlineData("xxx")] + [InlineData("{hello:")] + [InlineData("\\u007Bhello:")] + [InlineData(@"{""hello:""""")] + [InlineData(" ")] + [InlineData("// This is a single line comment")] + [InlineData("/* This is a multi-\nline comment*/")] + public static void WriteRawInvalidJson(string json) + { + RunTest(true); + RunTest(false); + + void RunTest(bool skipValidation) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + if (!skipValidation) + { + Assert.ThrowsAny(() => writer.WriteRawValue(json)); + } + else + { + writer.WriteRawValue(json, true); + writer.Flush(); + Assert.True(Encoding.UTF8.GetBytes(json).SequenceEqual(ms.ToArray())); + } + } + } + + [Fact] + public static void WriteRawNullOrEmptyTokenInvalid() + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + Assert.Throws(() => writer.WriteRawValue(json: default(string))); + Assert.Throws(() => writer.WriteRawValue(json: "")); + Assert.Throws(() => writer.WriteRawValue(json: default(ReadOnlySpan))); + Assert.Throws(() => writer.WriteRawValue(utf8Json: default)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public static void WriteRawHonorSkipValidation(bool skipValidation) + { + RunTest(true); + RunTest(false); + + void RunTest(bool skipInputValidation) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms, new JsonWriterOptions { SkipValidation = skipValidation }); + + writer.WriteStartObject(); + + if (skipValidation) + { + writer.WriteRawValue(@"{}", skipInputValidation); + writer.Flush(); + Assert.True(ms.ToArray().SequenceEqual(new byte[] { (byte)'{', (byte)'{', (byte)'}' })); + } + else + { + Assert.Throws(() => writer.WriteRawValue(@"{}", skipInputValidation)); + } + } + } + + [Fact] + public static void WriteRawDepthExceedsMaxOf64Fail() + { + + RunTest(GenerateJsonUsingDepth(1), false); + RunTest(GenerateJsonUsingDepth(64), false); + RunTest(GenerateJsonUsingDepth(65), true); + RunTest(GenerateJsonUsingDepth(65), false, true); + + void RunTest(string json, bool expectFail, bool skipInputValidation = false) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + if (expectFail) + { + Assert.ThrowsAny(() => writer.WriteRawValue(json, skipInputValidation)); + } + else + { + writer.WriteRawValue(json, skipInputValidation); + writer.Flush(); + + Assert.Equal(json, Encoding.UTF8.GetString(ms.ToArray())); + } + } + } + + private static string GenerateJsonUsingDepth(int depth) + { + Assert.True(depth > 0 && depth <= 65, "Test depth out of range"); + + StringBuilder sb = new(); + sb.Append("{"); + + for (int i = 0; i < depth - 1; i++) + { + sb.Append(@"""prop"":{"); + } + + for (int i = 0; i < depth - 1; i++) + { + sb.Append("}"); + } + + sb.Append("}"); + + return sb.ToString(); + } + + /// + /// This test is constrained to run on Windows and MacOSX because it causes + /// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + /// succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + /// time the memory is accessed which triggers the full memory allocation. + /// Also see + /// + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalFact(nameof(IsX64))] + [OuterLoop] + public void WriteRawLargeJsonToStreamWithoutFlushing() + { + var largeArray = new char[150_000_000]; + largeArray.AsSpan().Fill('a'); + + // Text size chosen so that after several doublings of the underlying buffer we reach ~2 GB (but don't go over) + JsonEncodedText text1 = JsonEncodedText.Encode(largeArray.AsSpan(0, 7_500)); + JsonEncodedText text2 = JsonEncodedText.Encode(largeArray.AsSpan(0, 5_000)); + JsonEncodedText text3 = JsonEncodedText.Encode(largeArray.AsSpan(0, 150_000_000)); + + using (var output = new MemoryStream()) + using (var writer = new Utf8JsonWriter(output)) + { + writer.WriteStartArray(); + writer.WriteRawValue(WrapInQuotes(text1.EncodedUtf8Bytes)); + Assert.Equal(7_503, writer.BytesPending); + + for (int i = 0; i < 30_000; i++) + { + writer.WriteRawValue(WrapInQuotes(text2.EncodedUtf8Bytes)); + } + Assert.Equal(150_097_503, writer.BytesPending); + + for (int i = 0; i < 13; i++) + { + writer.WriteRawValue(WrapInQuotes(text3.EncodedUtf8Bytes)); + } + Assert.Equal(2_100_097_542, writer.BytesPending); + + // Next write forces a grow beyond max array length + + Assert.Throws(() => writer.WriteRawValue(WrapInQuotes(text3.EncodedUtf8Bytes))); + + Assert.Equal(2_100_097_542, writer.BytesPending); + + var text4 = JsonEncodedText.Encode(largeArray.AsSpan(0, 1)); + for (int i = 0; i < 10_000_000; i++) + { + writer.WriteRawValue(WrapInQuotes(text4.EncodedUtf8Bytes)); + } + + Assert.Equal(2_100_097_542 + (4 * 10_000_000), writer.BytesPending); + } + } + + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalTheory(nameof(IsX64))] + [OuterLoop] + [InlineData(JsonTokenType.String)] + [InlineData(JsonTokenType.StartArray)] + [InlineData(JsonTokenType.StartObject)] + public static void WriteRawMaxUtf16InputLength(JsonTokenType tokenType) + { + // Max raw payload length supported by the writer. + int maxLength = int.MaxValue / 3; + + StringBuilder sb = new(); + sb.Append('"'); + + for (int i = 1; i < maxLength - 1; i++) + { + sb.Append('a'); + } + + sb.Append('"'); + + string payload = sb.ToString(); + + RunTest(OverloadParamType.ROSChar); + RunTest(OverloadParamType.String); + RunTest(OverloadParamType.ByteArray); + + void RunTest(OverloadParamType paramType) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + switch (tokenType) + { + case JsonTokenType.String: + WriteRawValueWithSetting(writer, payload, paramType); + writer.Flush(); + Assert.Equal(payload.Length, writer.BytesCommitted); + break; + case JsonTokenType.StartArray: + writer.WriteStartArray(); + WriteRawValueWithSetting(writer, payload, paramType); + WriteRawValueWithSetting(writer, payload, paramType); + writer.WriteEndArray(); + writer.Flush(); + // Start/EndArray + comma, 2 array elements + Assert.Equal(3 + (payload.Length * 2), writer.BytesCommitted); + break; + case JsonTokenType.StartObject: + writer.WriteStartObject(); + writer.WritePropertyName("1"); + WriteRawValueWithSetting(writer, payload, paramType); + writer.WritePropertyName("2"); + WriteRawValueWithSetting(writer, payload, paramType); + writer.WriteEndObject(); + writer.Flush(); + // Start/EndToken + comma, 2 property names, 2 property values + Assert.Equal(3 + (4 * 2) + (payload.Length * 2), writer.BytesCommitted); + break; + default: + Assert.True(false, "Unexpected test configuration"); + break; + } + } + } + + private enum OverloadParamType + { + ROSChar, + String, + ByteArray + } + + private static void WriteRawValueWithSetting(Utf8JsonWriter writer, string payload, OverloadParamType param) + { + switch (param) + { + case OverloadParamType.ROSChar: + writer.WriteRawValue(payload.AsSpan()); + break; + case OverloadParamType.String: + writer.WriteRawValue(payload); + break; + case OverloadParamType.ByteArray: + byte[] payloadAsBytes = Encoding.UTF8.GetBytes(payload); + writer.WriteRawValue(payloadAsBytes); + break; + } + } + + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalTheory(nameof(IsX64))] + [InlineData((int.MaxValue / 3) + 1)] + [InlineData(int.MaxValue / 3 + 2)] + [OuterLoop] + public static void WriteRawUtf16LengthGreaterThanMax(int len) + { + StringBuilder sb = new(); + sb.Append('"'); + + for (int i = 1; i < len - 1; i++) + { + sb.Append('a'); + } + + sb.Append('"'); + + string payload = sb.ToString(); + + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + // UTF-16 overloads not compatible with this length. + Assert.Throws(() => WriteRawValueWithSetting(writer, payload, OverloadParamType.ROSChar)); + Assert.Throws(() => WriteRawValueWithSetting(writer, payload, OverloadParamType.String)); + + // UTF-8 overload is okay. + WriteRawValueWithSetting(writer, payload, OverloadParamType.ByteArray); + writer.Flush(); + + Assert.Equal(payload.Length, Encoding.UTF8.GetString(ms.ToArray()).Length); + } + + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalFact(nameof(IsX64))] + [OuterLoop] + public static void WriteRawTranscodeFromUtf16ToUtf8TooLong() + { + // Max raw payload length supported by the writer. + int maxLength = int.MaxValue / 3; + + StringBuilder sb = new(); + sb.Append('"'); + + for (int i = 1; i < maxLength - 1; i++) + { + sb.Append('的'); // Non-UTF-8 character than will expand during transcoding + } + + sb.Append('"'); + + string payload = sb.ToString(); + + RunTest(OverloadParamType.ROSChar); + RunTest(OverloadParamType.String); + RunTest(OverloadParamType.ByteArray); + + void RunTest(OverloadParamType paramType) + { + using MemoryStream ms = new(); + using Utf8JsonWriter writer = new(ms); + + try + { + WriteRawValueWithSetting(writer, payload, paramType); + writer.Flush(); + + // All characters in the payload will be expanded during transcoding, except for the quotes. + int expectedLength = ((payload.Length - 2) * 3) + 2; + Assert.Equal(expectedLength, writer.BytesCommitted); + } + catch (OutOfMemoryException) { } // OutOfMemoryException is okay since the transcoding output is probably too large. + } + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs index f4796dcf60e40e..f18f23a64f3c6e 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs @@ -17,7 +17,7 @@ namespace System.Text.Json.Tests { - public class Utf8JsonWriterTests + public partial class Utf8JsonWriterTests { private const int MaxExpansionFactorWhileEscaping = 6; private const int MaxEscapedTokenSize = 1_000_000_000; // Max size for already escaped value. @@ -734,10 +734,13 @@ private static string GetExpectedLargeArrayOfStrings(int length) return stringBuilder.ToString(); } - // NOTE: WritingTooLargeProperty test is constrained to run on Windows and MacOSX because it causes - // problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can - // succeed even if there is not enough memory but then the test may get killed by the OOM killer at the - // time the memory is accessed which triggers the full memory allocation. + /// + /// This test is constrained to run on Windows and MacOSX because it causes + /// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + /// succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + /// time the memory is accessed which triggers the full memory allocation. + /// Also see + /// [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] [ConditionalFact(nameof(IsX64))] [OuterLoop]