Skip to content

Remove Vector<T> fallbacks for Vector128<T> #85916

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 8, 2023
Merged

Remove Vector<T> fallbacks for Vector128<T> #85916

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f54a512
Remove Vector<T> fallbacks for Vector128<T>
stephentoub May 8, 2023
f4817f9
ifdef instead of deleting Vector<T> code paths
stephentoub Jun 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 47 additions & 36 deletions src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,10 +355,13 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace)
// Avx2 branch also operates on Sse2 sizes, so check is combined.
lengthToExamine = UnalignedCountVector128(searchSpace);
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
lengthToExamine = UnalignedCountVector(searchSpace);
}
#endif

SequentialScan:
while (lengthToExamine >= 8)
{
Expand Down Expand Up @@ -517,6 +520,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace)
}
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (offset < (nuint)(uint)Length)
Expand All @@ -543,6 +547,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace)
}
}
}
#endif

ThrowMustBeNullTerminatedString();
Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
Expand Down Expand Up @@ -690,6 +695,7 @@ public static unsafe bool SequenceEqual(ref byte first, ref byte second, nuint l
goto NotEqual;
}
}
#if MONO
else if (Vector.IsHardwareAccelerated && length >= (nuint)Vector<byte>.Count)
{
nuint offset = 0;
Expand Down Expand Up @@ -718,6 +724,7 @@ public static unsafe bool SequenceEqual(ref byte first, ref byte second, nuint l
// This becomes a conditional jmp forward to not favor it.
goto NotEqual;
}
#endif

#if TARGET_64BIT
if (Vector128.IsHardwareAccelerated)
Expand Down Expand Up @@ -767,27 +774,6 @@ public static unsafe bool SequenceEqual(ref byte first, ref byte second, nuint l
return false;
}

// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundByte(Vector<byte> match)
{
var vector64 = Vector.AsVectorUInt64(match);
ulong candidate = 0;
int i = 0;
// Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
for (; i < Vector<ulong>.Count; i++)
{
candidate = vector64[i];
if (candidate != 0)
{
break;
}
}

// Single LEA instruction with jitted const (using function result)
return i * 8 + LocateFirstFoundByte(candidate);
}

public static unsafe int SequenceCompareTo(ref byte first, int firstLength, ref byte second, int secondLength)
{
Debug.Assert(firstLength >= 0);
Expand Down Expand Up @@ -907,6 +893,7 @@ public static unsafe int SequenceCompareTo(ref byte first, int firstLength, ref
goto BytewiseCheck;
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (lengthToExamine > (nuint)Vector<byte>.Count)
Expand All @@ -923,6 +910,7 @@ public static unsafe int SequenceCompareTo(ref byte first, int firstLength, ref
goto BytewiseCheck;
}
}
#endif

if (lengthToExamine > (nuint)sizeof(nuint))
{
Expand Down Expand Up @@ -1044,6 +1032,28 @@ public static nuint CommonPrefixLength(ref byte first, ref byte second, nuint le
return i + uint.TrailingZeroCount(mask);
}

#if MONO
// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundByte(Vector<byte> match)
{
var vector64 = Vector.AsVectorUInt64(match);
ulong candidate = 0;
int i = 0;
// Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
for (; i < Vector<ulong>.Count; i++)
{
candidate = vector64[i];
if (candidate != 0)
{
break;
}
}

// Single LEA instruction with jitted const (using function result)
return i * 8 + LocateFirstFoundByte(candidate);
}

// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateLastFoundByte(Vector<byte> match)
Expand All @@ -1069,6 +1079,22 @@ private static int LocateLastFoundByte(Vector<byte> match)
return i * 8 + LocateLastFoundByte(candidate);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<byte> LoadVector(ref byte start, nuint offset)
=> Unsafe.ReadUnaligned<Vector<byte>>(ref Unsafe.AddByteOffset(ref start, offset));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint GetByteVectorSpanLength(nuint offset, int length)
=> (nuint)(uint)((length - (int)offset) & ~(Vector<byte>.Count - 1));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nuint UnalignedCountVector(byte* searchSpace)
{
nint unaligned = (nint)searchSpace & (Vector<byte>.Count - 1);
return (nuint)((Vector<byte>.Count - unaligned) & (Vector<byte>.Count - 1));
}
#endif

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundByte(ulong match)
=> BitOperations.TrailingZeroCount(match) >> 3;
Expand Down Expand Up @@ -1097,10 +1123,6 @@ private static nuint LoadNUInt(ref byte start)
private static nuint LoadNUInt(ref byte start, nuint offset)
=> Unsafe.ReadUnaligned<nuint>(ref Unsafe.AddByteOffset(ref start, offset));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<byte> LoadVector(ref byte start, nuint offset)
=> Unsafe.ReadUnaligned<Vector<byte>>(ref Unsafe.AddByteOffset(ref start, offset));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> LoadVector128(ref byte start, nuint offset)
=> Unsafe.ReadUnaligned<Vector128<byte>>(ref Unsafe.AddByteOffset(ref start, offset));
Expand All @@ -1109,10 +1131,6 @@ private static Vector128<byte> LoadVector128(ref byte start, nuint offset)
private static Vector256<byte> LoadVector256(ref byte start, nuint offset)
=> Unsafe.ReadUnaligned<Vector256<byte>>(ref Unsafe.AddByteOffset(ref start, offset));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint GetByteVectorSpanLength(nuint offset, int length)
=> (nuint)(uint)((length - (int)offset) & ~(Vector<byte>.Count - 1));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint GetByteVector128SpanLength(nuint offset, int length)
=> (nuint)(uint)((length - (int)offset) & ~(Vector128<byte>.Count - 1));
Expand All @@ -1121,13 +1139,6 @@ private static nuint GetByteVector128SpanLength(nuint offset, int length)
private static nuint GetByteVector256SpanLength(nuint offset, int length)
=> (nuint)(uint)((length - (int)offset) & ~(Vector256<byte>.Count - 1));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nuint UnalignedCountVector(byte* searchSpace)
{
nint unaligned = (nint)searchSpace & (Vector<byte>.Count - 1);
return (nuint)((Vector<byte>.Count - unaligned) & (Vector<byte>.Count - 1));
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nuint UnalignedCountVector128(byte* searchSpace)
{
Expand Down
28 changes: 17 additions & 11 deletions src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,11 +439,13 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
// Needs to be double length to allow us to align the data first.
lengthToExamine = UnalignedCountVector128(searchSpace);
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
// Needs to be double length to allow us to align the data first.
lengthToExamine = UnalignedCountVector(searchSpace);
}
#endif

SequentialScan:
// In the non-vector case lengthToExamine is the total length.
Expand Down Expand Up @@ -603,6 +605,7 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
}
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (offset < length)
Expand Down Expand Up @@ -637,6 +640,7 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
}
}
}
#endif

ThrowMustBeNullTerminatedString();
Found3:
Expand All @@ -649,6 +653,7 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
return (int)(offset);
}

#if MONO
// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundChar(Vector<ushort> match)
Expand All @@ -670,10 +675,6 @@ private static int LocateFirstFoundChar(Vector<ushort> match)
return i * 4 + LocateFirstFoundChar(candidate);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundChar(ulong match)
=> BitOperations.TrailingZeroCount(match) >> 4;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<ushort> LoadVector(ref char start, nint offset)
=> Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, offset)));
Expand All @@ -686,6 +687,18 @@ private static Vector<ushort> LoadVector(ref char start, nuint offset)
private static nint GetCharVectorSpanLength(nint offset, nint length)
=> (length - offset) & ~(Vector<ushort>.Count - 1);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nint UnalignedCountVector(char* searchSpace)
{
const int ElementsPerByte = sizeof(ushort) / sizeof(byte);
return (nint)(uint)(-(int)searchSpace / ElementsPerByte) & (Vector<ushort>.Count - 1);
}
#endif

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundChar(ulong match)
=> BitOperations.TrailingZeroCount(match) >> 4;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nint GetCharVector128SpanLength(nint offset, nint length)
=> (length - offset) & ~(Vector128<ushort>.Count - 1);
Expand All @@ -694,13 +707,6 @@ private static nint GetCharVector128SpanLength(nint offset, nint length)
private static nint GetCharVector256SpanLength(nint offset, nint length)
=> (length - offset) & ~(Vector256<ushort>.Count - 1);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nint UnalignedCountVector(char* searchSpace)
{
const int ElementsPerByte = sizeof(ushort) / sizeof(byte);
return (nint)(uint)(-(int)searchSpace / ElementsPerByte) & (Vector<ushort>.Count - 1);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nint UnalignedCountVector128(char* searchSpace)
{
Expand Down