Skip to content

[NativeAOT] Using the same CastCache implementation as in CoreClr #84430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Apr 7, 2023
Merged

[NativeAOT] Using the same CastCache implementation as in CoreClr #84430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d3ed7b4
Getter
VSadov Apr 4, 2023
8c43fd6
different limits on debug/release
VSadov Apr 5, 2023
de42edc
tweaks
VSadov Apr 5, 2023
9f6b993
remove now unnecessary CrstCastCache
VSadov Apr 5, 2023
eab1dee
implement flushing
VSadov Apr 5, 2023
335ac89
move coreclr castcache to a separate file
VSadov Apr 6, 2023
3831d61
Unified CastCache implementation
VSadov Apr 6, 2023
173b820
comments and cleanups
VSadov Apr 6, 2023
c6e1ba5
couple more cleanups
VSadov Apr 6, 2023
8fb20e5
trivial implementation of the cast cache for the Test.Corlib
VSadov Apr 6, 2023
410dca4
use Numerics.BitOperations for bit math
VSadov Apr 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 6 additions & 147 deletions src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,155 +3,14 @@

using System.Diagnostics;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;

namespace System.Runtime.CompilerServices
{
internal static unsafe class CastHelpers
{
private static int[]? s_table;

[DebuggerDisplay("Source = {_source}; Target = {_targetAndResult & ~1}; Result = {_targetAndResult & 1}; VersionNum = {_version & ((1 << 29) - 1)}; Distance = {_version >> 29};")]
[StructLayout(LayoutKind.Sequential)]
private struct CastCacheEntry
{
// version has the following structure:
// [ distance:3bit | versionNum:29bit ]
//
// distance is how many iterations the entry is from it ideal position.
// we use that for preemption.
//
// versionNum is a monotonicaly increasing numerical tag.
// Writer "claims" entry by atomically incrementing the tag. Thus odd number indicates an entry in progress.
// Upon completion of adding an entry the tag is incremented again making it even. Even number indicates a complete entry.
//
// Readers will read the version twice before and after retrieving the entry.
// To have a usable entry both reads must yield the same even version.
//
internal int _version;
internal nuint _source;
// pointers have unused lower bits due to alignment, we use one for the result
internal nuint _targetAndResult;
};

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int KeyToBucket(ref int tableData, nuint source, nuint target)
{
// upper bits of addresses do not vary much, so to reduce loss due to cancelling out,
// we do `rotl(source, <half-size>) ^ target` for mixing inputs.
// then we use fibonacci hashing to reduce the value to desired size.

int hashShift = HashShift(ref tableData);
#if TARGET_64BIT
ulong hash = BitOperations.RotateLeft((ulong)source, 32) ^ (ulong)target;
return (int)((hash * 11400714819323198485ul) >> hashShift);
#else
uint hash = BitOperations.RotateLeft((uint)source, 16) ^ (uint)target;
return (int)((hash * 2654435769u) >> hashShift);
#endif
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ref int TableData(int[] table)
{
// element 0 is used for embedded aux data
return ref MemoryMarshal.GetArrayDataReference(table);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ref CastCacheEntry Element(ref int tableData, int index)
{
// element 0 is used for embedded aux data, skip it
return ref Unsafe.Add(ref Unsafe.As<int, CastCacheEntry>(ref tableData), index + 1);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int HashShift(ref int tableData)
{
return tableData;
}

// TableMask is "size - 1"
// we need that more often that we need size
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int TableMask(ref int tableData)
{
return Unsafe.Add(ref tableData, 1);
}

private enum CastResult
{
CannotCast = 0,
CanCast = 1,
MaybeCast = 2
}

// NOTE!!
// This is a copy of C++ implementation in castcache.cpp
// Keep the copies, if possible, in sync.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static CastResult TryGet(nuint source, nuint target)
{
const int BUCKET_SIZE = 8;

// table is initialized and updated by native code that guarantees it is not null.
ref int tableData = ref TableData(s_table!);

int index = KeyToBucket(ref tableData, source, target);
for (int i = 0; i < BUCKET_SIZE;)
{
ref CastCacheEntry pEntry = ref Element(ref tableData, index);

// must read in this order: version -> [entry parts] -> version
// if version is odd or changes, the entry is inconsistent and thus ignored
int version = Volatile.Read(ref pEntry._version);
nuint entrySource = pEntry._source;

// mask the lower version bit to make it even.
// This way we can check if version is odd or changing in just one compare.
version &= ~1;

if (entrySource == source)
{
nuint entryTargetAndResult = pEntry._targetAndResult;
// target never has its lower bit set.
// a matching entryTargetAndResult would the have same bits, except for the lowest one, which is the result.
entryTargetAndResult ^= target;
if (entryTargetAndResult <= 1)
{
// make sure 'version' is loaded after 'source' and 'targetAndResults'
//
// We can either:
// - use acquires for both _source and _targetAndResults or
// - issue a load barrier before reading _version
// benchmarks on available hardware show that use of a read barrier is cheaper.
Interlocked.ReadMemoryBarrier();
if (version != pEntry._version)
{
// oh, so close, the entry is in inconsistent state.
// it is either changing or has changed while we were reading.
// treat it as a miss.
break;
}

return (CastResult)entryTargetAndResult;
}
}

if (version == 0)
{
// the rest of the bucket is unclaimed, no point to search further
break;
}

// quadratic reprobe
i++;
index = (index + i) & TableMask(ref tableData);
}
return CastResult.MaybeCast;
}

[MethodImpl(MethodImplOptions.InternalCall)]
private static extern object IsInstanceOfAny_NoCacheLookup(void* toTypeHnd, object obj);

Expand All @@ -177,7 +36,7 @@ private static CastResult TryGet(nuint source, nuint target)
void* mt = RuntimeHelpers.GetMethodTable(obj);
if (mt != toTypeHnd)
{
CastResult result = TryGet((nuint)mt, (nuint)toTypeHnd);
CastResult result = CastCache.TryGet((nuint)mt, (nuint)toTypeHnd);
if (result == CastResult.CanCast)
{
// do nothing
Expand Down Expand Up @@ -327,7 +186,7 @@ private static CastResult TryGet(nuint source, nuint target)
[MethodImpl(MethodImplOptions.NoInlining)]
private static object? IsInstance_Helper(void* toTypeHnd, object obj)
{
CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd);
CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd);
if (result == CastResult.CanCast)
{
return obj;
Expand Down Expand Up @@ -356,7 +215,7 @@ private static CastResult TryGet(nuint source, nuint target)
void* mt = RuntimeHelpers.GetMethodTable(obj);
if (mt != toTypeHnd)
{
result = TryGet((nuint)mt, (nuint)toTypeHnd);
result = CastCache.TryGet((nuint)mt, (nuint)toTypeHnd);
if (result != CastResult.CanCast)
{
goto slowPath;
Expand All @@ -380,7 +239,7 @@ private static CastResult TryGet(nuint source, nuint target)
[MethodImpl(MethodImplOptions.NoInlining)]
private static object? ChkCast_Helper(void* toTypeHnd, object obj)
{
CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd);
CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd);
if (result == CastResult.CanCast)
{
return obj;
Expand Down Expand Up @@ -597,7 +456,7 @@ private static void StelemRef(Array array, nint index, object? obj)
[MethodImpl(MethodImplOptions.NoInlining)]
private static void StelemRef_Helper(ref object? element, void* elementType, object obj)
{
CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType);
CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType);
if (result == CastResult.CanCast)
{
WriteBarrier(ref element, obj);
Expand Down
8 changes: 0 additions & 8 deletions src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,14 +306,6 @@ internal static extern unsafe IntPtr RhpCallPropagateExceptionCallback(
[UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })]
internal static extern void RhpSignalFinalizationComplete();

[DllImport(Redhawk.BaseName)]
[UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })]
internal static extern void RhpAcquireCastCacheLock();

[DllImport(Redhawk.BaseName)]
[UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })]
internal static extern void RhpReleaseCastCacheLock();

[DllImport(Redhawk.BaseName)]
[UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })]
internal static extern ulong RhpGetTickCount64();
Expand Down
Loading