diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 5132148ae58325..12e0ea00d2bc54 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -6,6 +6,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; using Internal.Runtime.CompilerServices; @@ -194,7 +195,7 @@ public static unsafe int IndexOf(ref byte searchSpace, byte value, int length) nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations nuint lengthToExamine = (nuint)(uint)length; - if (Avx2.IsSupported || Sse2.IsSupported) + if (Sse2.IsSupported || AdvSimd.Arm64.IsSupported) { // Avx2 branch also operates on Sse2 sizes, so check is combined. if (length >= Vector128.Count * 2) @@ -370,6 +371,40 @@ public static unsafe int IndexOf(ref byte searchSpace, byte value, int length) } } } + else if (AdvSimd.Arm64.IsSupported) + { + if (offset < (nuint)(uint)length) + { + lengthToExamine = GetByteVector128SpanLength(offset, length); + + // Mask to help find the first lane in compareResult that is set. + // MSB 0x10 corresponds to 1st lane, 0x01 corresponds to 0th lane and so forth. + Vector128 mask = Vector128.Create((ushort)0x1001).AsByte(); + int matchedLane = 0; + + Vector128 values = Vector128.Create(value); + while (lengthToExamine > offset) + { + Vector128 search = LoadVector128(ref searchSpace, offset); + Vector128 compareResult = AdvSimd.CompareEqual(values, search); + + if (!TryFindFirstMatchedLane(mask, compareResult, ref matchedLane)) + { + // Zero flags set so no matches + offset += (nuint)Vector128.Count; + continue; + } + + return (int)(offset + (uint)matchedLane); + } + + if (offset < (nuint)(uint)length) + { + lengthToExamine = ((nuint)(uint)length - offset); + goto SequentialScan; + } + } + } else if (Vector.IsHardwareAccelerated) { if (offset < (nuint)(uint)length) @@ -566,7 +601,7 @@ public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, int nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations nuint lengthToExamine = (nuint)(uint)length; - if (Avx2.IsSupported || Sse2.IsSupported) + if (Sse2.IsSupported) { // Avx2 branch also operates on Sse2 sizes, so check is combined. if (length >= Vector128.Count * 2) @@ -807,7 +842,7 @@ public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, byt nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations nuint lengthToExamine = (nuint)(uint)length; - if (Avx2.IsSupported || Sse2.IsSupported) + if (Sse2.IsSupported) { // Avx2 branch also operates on Sse2 sizes, so check is combined. if (length >= Vector128.Count * 2) @@ -1803,5 +1838,25 @@ private static unsafe nuint UnalignedCountVectorFromEnd(ref byte searchSpace, in nint unaligned = (nint)Unsafe.AsPointer(ref searchSpace) & (Vector.Count - 1); return (nuint)(uint)(((length & (Vector.Count - 1)) + unaligned) & (Vector.Count - 1)); } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryFindFirstMatchedLane(Vector128 mask, Vector128 compareResult, ref int matchedLane) + { + Debug.Assert(AdvSimd.Arm64.IsSupported); + + // Find the first lane that is set inside compareResult. + Vector128 maskedSelectedLanes = AdvSimd.And(compareResult, mask); + Vector128 pairwiseSelectedLane = AdvSimd.Arm64.AddPairwise(maskedSelectedLanes, maskedSelectedLanes); + ulong selectedLanes = pairwiseSelectedLane.AsUInt64().ToScalar(); + if (selectedLanes == 0) + { + // all lanes are zero, so nothing matched. + return false; + } + + // Find the first lane that is set inside compareResult. + matchedLane = BitOperations.TrailingZeroCount(selectedLanes) >> 2; + return true; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index 02f0ba9920724f..b5ee0d05554ced 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -6,6 +6,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; using Internal.Runtime.CompilerServices; @@ -222,7 +223,7 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) { // Input isn't char aligned, we won't be able to align it to a Vector } - else if (Sse2.IsSupported) + else if (Sse2.IsSupported || AdvSimd.Arm64.IsSupported) { // Avx2 branch also operates on Sse2 sizes, so check is combined. // Needs to be double length to allow us to align the data first. @@ -408,6 +409,44 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length) } } } + else if (AdvSimd.Arm64.IsSupported) + { + if (offset < length) + { + Debug.Assert(length - offset >= Vector128.Count); + + lengthToExamine = GetCharVector128SpanLength(offset, length); + if (lengthToExamine > 0) + { + Vector128 values = Vector128.Create((ushort)value); + int matchedLane = 0; + + do + { + Debug.Assert(lengthToExamine >= Vector128.Count); + + Vector128 search = LoadVector128(ref searchSpace, offset); + Vector128 compareResult = AdvSimd.CompareEqual(values, search); + + if (!TryFindFirstMatchedLane(compareResult, ref matchedLane)) + { + // Zero flags set so no matches + offset += Vector128.Count; + lengthToExamine -= Vector128.Count; + continue; + } + + return (int)(offset + matchedLane); + } while (lengthToExamine > 0); + } + + if (offset < length) + { + lengthToExamine = length - offset; + goto SequentialScan; + } + } + } else if (Vector.IsHardwareAccelerated) { if (offset < length) @@ -1050,5 +1089,24 @@ private static unsafe nint UnalignedCountVector128(ref char searchSpace) // isn't too important to pin to maintain the alignment. return (nint)(uint)(-(int)Unsafe.AsPointer(ref searchSpace) / ElementsPerByte) & (Vector128.Count - 1); } + + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryFindFirstMatchedLane(Vector128 compareResult, ref int matchedLane) + { + Debug.Assert(AdvSimd.Arm64.IsSupported); + + Vector128 pairwiseSelectedLane = AdvSimd.Arm64.AddPairwise(compareResult.AsByte(), compareResult.AsByte()); + ulong selectedLanes = pairwiseSelectedLane.AsUInt64().ToScalar(); + if (selectedLanes == 0) + { + // all lanes are zero, so nothing matched. + return false; + } + + // Find the first lane that is set inside compareResult. + matchedLane = BitOperations.TrailingZeroCount(selectedLanes) >> 3; + return true; + } } }