From 26ad0a8de86288f5bd2f3b4b92ee0e2cc42e2b80 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 12 Apr 2022 16:07:41 +0200 Subject: [PATCH 01/59] Add support for storing method handle histograms in profiles Allow method handle histograms in .mibc files and in the PGO text format. Contributes to #44610. --- src/coreclr/inc/corjit.h | 16 +- src/coreclr/inc/pgo_formatprocessing.h | 22 +- src/coreclr/jit/compiler.h | 1 + src/coreclr/jit/fgbasic.cpp | 1 + src/coreclr/jit/fgprofile.cpp | 39 +++- src/coreclr/jit/likelyclass.cpp | 30 +-- .../tools/Common/JitInterface/CorInfoImpl.cs | 15 +- src/coreclr/tools/Common/Pgo/PgoFormat.cs | 71 +++++- .../Common/Pgo/TypeSystemEntityOrUnknown.cs | 5 + .../InstrumentationDataTableNode.cs | 67 +++++- .../Compiler/ProfileData.cs | 2 +- .../IBC/MIbcProfileParser.cs | 31 ++- .../PgoInfo.cs | 20 +- src/coreclr/tools/dotnet-pgo/MibcEmitter.cs | 24 +- src/coreclr/tools/dotnet-pgo/Program.cs | 43 +++- .../tools/superpmi/mcs/verbdumpmap.cpp | 8 +- .../tools/superpmi/mcs/verbjitflags.cpp | 8 +- .../superpmi-shared/methodcontext.cpp | 14 +- .../superpmi/superpmi-shared/methodcontext.h | 5 +- .../superpmi-shared/spmidumphelper.cpp | 1 + src/coreclr/vm/pgo.cpp | 210 +++++++++++++----- 21 files changed, 477 insertions(+), 156 deletions(-) diff --git a/src/coreclr/inc/corjit.h b/src/coreclr/inc/corjit.h index 122fcd0d7b0ba2..ac44b8a9c8f3db 100644 --- a/src/coreclr/inc/corjit.h +++ b/src/coreclr/inc/corjit.h @@ -368,6 +368,7 @@ class ICorJitInfo : public ICorDynamicInfo FourByte = 1, EightByte = 2, TypeHandle = 3, + MethodHandle = 4, // Mask of all schema data types MarshalMask = 0xF, @@ -385,9 +386,10 @@ class ICorJitInfo : public ICorDynamicInfo Done = None, // All instrumentation schemas must end with a record which is "Done" BasicBlockIntCount = (DescriptorMin * 1) | FourByte, // basic block counter using unsigned 4 byte int BasicBlockLongCount = (DescriptorMin * 1) | EightByte, // basic block counter using unsigned 8 byte int - TypeHandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. - TypeHandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram - TypeHandleHistogramTypeHandle = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram + HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. + HandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram + HandleHistogramTypes = (DescriptorMin * 3) | TypeHandle, // Histogram of type handles + HandleHistogramMethods = (DescriptorMin * 3) | MethodHandle, // Histogram of method handles Version = (DescriptorMin * 4) | None, // Version is encoded in the Other field of the schema NumRuns = (DescriptorMin * 5) | None, // Number of runs is encoded in the Other field of the schema EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int @@ -416,12 +418,12 @@ class ICorJitInfo : public ICorDynamicInfo }; #define DEFAULT_UNKNOWN_TYPEHANDLE 1 -#define UNKNOWN_TYPEHANDLE_MIN 1 -#define UNKNOWN_TYPEHANDLE_MAX 33 +#define UNKNOWN_HANDLE_MIN 1 +#define UNKNOWN_HANDLE_MAX 33 - static inline bool IsUnknownTypeHandle(intptr_t typeHandle) + static inline bool IsUnknownHandle(intptr_t typeHandle) { - return ((typeHandle >= UNKNOWN_TYPEHANDLE_MIN) && (typeHandle <= UNKNOWN_TYPEHANDLE_MAX)); + return ((typeHandle >= UNKNOWN_HANDLE_MIN) && (typeHandle <= UNKNOWN_HANDLE_MAX)); } // get profile information to be used for optimizing a current method. The format diff --git a/src/coreclr/inc/pgo_formatprocessing.h b/src/coreclr/inc/pgo_formatprocessing.h index 18fe5601c0d283..175d7d8786ba5e 100644 --- a/src/coreclr/inc/pgo_formatprocessing.h +++ b/src/coreclr/inc/pgo_formatprocessing.h @@ -8,15 +8,7 @@ #ifdef FEATURE_PGO -inline bool AddTypeHandleToUnknownTypeHandleMask(INT_PTR typeHandle, uint32_t *unknownTypeHandleMask) -{ - uint32_t bitMask = (uint32_t)(1 << (typeHandle - UNKNOWN_TYPEHANDLE_MIN)); - bool result = (bitMask & *unknownTypeHandleMask) == 0; - *unknownTypeHandleMask |= bitMask; - return result; -} - -inline INT_PTR HashToPgoUnknownTypeHandle(uint32_t hash) +inline INT_PTR HashToPgoUnknownHandle(uint32_t hash) { // Map from a 32bit hash to the 32 different unknown type handle values return (hash & 0x1F) + 1; @@ -53,6 +45,7 @@ inline uint32_t InstrumentationKindToSize(ICorJitInfo::PgoInstrumentationKind ki case ICorJitInfo::PgoInstrumentationKind::EightByte: return 8; case ICorJitInfo::PgoInstrumentationKind::TypeHandle: + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: return TARGET_POINTER_SIZE; default: _ASSERTE(FALSE); @@ -242,6 +235,7 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa bool done = false; int64_t lastDataValue = 0; int64_t lastTypeDataValue = 0; + int64_t lastMethodDataValue = 0; int32_t dataCountToRead = 0; ReadCompressedInts(pByte, cbDataMax, [&](int64_t curValue) @@ -267,8 +261,16 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa return false; } break; + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: + lastMethodDataValue += curValue; + + if (!handler(schemaHandler.GetSchema(), lastMethodDataValue, schemaHandler.GetSchema().Count - dataCountToRead)) + { + return false; + } + break; default: - assert(false); + assert(!"Unexpected PGO instrumentation data type"); break; } dataCountToRead--; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 10305239a81d56..a30c0c6113c91f 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6264,6 +6264,7 @@ class Compiler UINT32 fgPgoBlockCounts; UINT32 fgPgoEdgeCounts; UINT32 fgPgoClassProfiles; + UINT32 fgPgoMethodProfiles; unsigned fgPgoInlineePgo; unsigned fgPgoInlineeNoPgo; unsigned fgPgoInlineeNoPgoSingleBlock; diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 632978641b9ee1..e8881e283a44c0 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -180,6 +180,7 @@ void Compiler::fgInit() fgPgoBlockCounts = 0; fgPgoEdgeCounts = 0; fgPgoClassProfiles = 0; + fgPgoMethodProfiles = 0; fgPgoInlineePgo = 0; fgPgoInlineeNoPgo = 0; fgPgoInlineeNoPgoSingleBlock = 0; diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 25de6f11e07217..a5914e36965322 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1496,15 +1496,15 @@ class BuildClassProbeSchemaGen } schemaElem.InstrumentationKind = JitConfig.JitCollect64BitCounts() - ? ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount - : ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount; + ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount + : ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; schemaElem.ILOffset = (int32_t)call->gtClassProfileCandidateInfo->ilOffset; schemaElem.Offset = 0; m_schema.push_back(schemaElem); // Re-using ILOffset and Other fields from schema item for TypeHandleHistogramCount - schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle; + schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes; schemaElem.Count = ICorJitInfo::ClassProfile32::SIZE; m_schema.push_back(schemaElem); @@ -1551,9 +1551,9 @@ class ClassProbeInserter // assert(m_schema[*m_currentSchemaIndex].ILOffset == (int32_t)call->gtClassProfileCandidateInfo->ilOffset); bool is32 = m_schema[*m_currentSchemaIndex].InstrumentationKind == - ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount; + ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; bool is64 = m_schema[*m_currentSchemaIndex].InstrumentationKind == - ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount; + ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount; assert(is32 || is64); // Figure out where the table is located. @@ -2050,12 +2050,32 @@ PhaseStatus Compiler::fgIncorporateProfileData() fgPgoEdgeCounts++; break; - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount: - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount: case ICorJitInfo::PgoInstrumentationKind::GetLikelyClass: fgPgoClassProfiles++; break; + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount: + if (iSchema + 1 < fgPgoSchemaCount) + { + if (fgPgoSchema[iSchema + 1].InstrumentationKind == + ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes) + { + fgPgoClassProfiles++; + iSchema++; + break; + } + if (fgPgoSchema[iSchema + 1].InstrumentationKind == + ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods) + { + fgPgoMethodProfiles++; + iSchema++; + break; + } + } + + __fallthrough; + default: JITDUMP("Unknown PGO record type 0x%x in schema entry %u (offset 0x%x count 0x%x other 0x%x)\n", fgPgoSchema[iSchema].InstrumentationKind, iSchema, fgPgoSchema[iSchema].ILOffset, @@ -2070,8 +2090,9 @@ PhaseStatus Compiler::fgIncorporateProfileData() fgNumProfileRuns = 1; } - JITDUMP("Profile summary: %d runs, %d block probes, %d edge probes, %d class profiles, %d other records\n", - fgNumProfileRuns, fgPgoBlockCounts, fgPgoEdgeCounts, fgPgoClassProfiles, otherRecords); + JITDUMP("Profile summary: %d runs, %d block probes, %d edge probes, %d class profiles, %d method profiles, %d " + "other records\n", + fgNumProfileRuns, fgPgoBlockCounts, fgPgoEdgeCounts, fgPgoClassProfiles, fgPgoMethodProfiles, otherRecords); const bool haveBlockCounts = fgPgoBlockCounts > 0; const bool haveEdgeCounts = fgPgoEdgeCounts > 0; diff --git a/src/coreclr/jit/likelyclass.cpp b/src/coreclr/jit/likelyclass.cpp index fc6028c0ac4efe..632c9ce8b847b9 100644 --- a/src/coreclr/jit/likelyclass.cpp +++ b/src/coreclr/jit/likelyclass.cpp @@ -157,7 +157,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* (schema[i].Count == 1)) { INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset); - if (ICorJitInfo::IsUnknownTypeHandle(result)) + if (ICorJitInfo::IsUnknownHandle(result)) { return 0; } @@ -168,11 +168,11 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* } const bool isHistogramCount = - (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) || - (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount); + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) || + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount); if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && - (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle)) + (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes)) { // Form a histogram // @@ -191,7 +191,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* { LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0); // Fast path for monomorphic cases - if (ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt)) + if (ICorJitInfo::IsUnknownHandle(hist0.m_mt)) { return 0; } @@ -205,12 +205,12 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0); LikelyClassHistogramEntry const hist1 = h.HistogramEntryAt(1); // Fast path for two classes - if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt)) + if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt)) { pLikelyClasses[0].likelihood = (100 * hist0.m_count) / h.m_totalCount; pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt; - if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt)) + if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist1.m_mt)) { pLikelyClasses[1].likelihood = (100 * hist1.m_count) / h.m_totalCount; pLikelyClasses[1].clsHandle = (CORINFO_CLASS_HANDLE)hist1.m_mt; @@ -219,12 +219,12 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* return 1; } - if (!ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt)) + if (!ICorJitInfo::IsUnknownHandle(hist1.m_mt)) { pLikelyClasses[0].likelihood = (100 * hist1.m_count) / h.m_totalCount; pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist1.m_mt; - if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt)) + if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt)) { pLikelyClasses[1].likelihood = (100 * hist0.m_count) / h.m_totalCount; pLikelyClasses[1].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt; @@ -244,7 +244,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* for (unsigned m = 0; m < h.countHistogramElements; m++) { LikelyClassHistogramEntry const hist = h.HistogramEntryAt(m); - if (!ICorJitInfo::IsUnknownTypeHandle(hist.m_mt)) + if (!ICorJitInfo::IsUnknownHandle(hist.m_mt)) { sortedEntries[knownHandles++] = hist; } @@ -311,7 +311,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch (schema[i].Count == 1)) { INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset); - if (ICorJitInfo::IsUnknownTypeHandle(result)) + if (ICorJitInfo::IsUnknownHandle(result)) { return NO_CLASS_HANDLE; } @@ -322,11 +322,11 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch } bool isHistogramCount = - (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) || - (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount); + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) || + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount); if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && - (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle)) + (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes)) { // Form a histogram // @@ -342,7 +342,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch unsigned randomEntryIndex = random->Next(0, h.countHistogramElements); LikelyClassHistogramEntry randomEntry = h.HistogramEntryAt(randomEntryIndex); - if (ICorJitInfo::IsUnknownTypeHandle(randomEntry.m_mt)) + if (ICorJitInfo::IsUnknownHandle(randomEntry.m_mt)) { return NO_CLASS_HANDLE; } diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 8af0179381d0ce..aee2d053426d03 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -195,8 +195,7 @@ public static IEnumerable ConvertTypeHandleHistogramsToCompactTyp bool hasTypeHistogram = false; foreach (var elem in pgoData) { - if (elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramIntCount || - elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramLongCount) + if (elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) { // found histogram hasTypeHistogram = true; @@ -218,10 +217,12 @@ public static IEnumerable ConvertTypeHandleHistogramsToCompactTyp ComputeJitPgoInstrumentationSchema(LocalObjectToHandle, pgoData, out var nativeSchema, out var instrumentationData); - for (int i = 0; i < (pgoData.Length); i++) + for (int i = 0; i < pgoData.Length; i++) { - if (pgoData[i].InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramIntCount || - pgoData[i].InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramLongCount) + if ((i + 1 < pgoData.Length) && + (pgoData[i].InstrumentationKind == PgoInstrumentationKind.HandleHistogramIntCount || + pgoData[i].InstrumentationKind == PgoInstrumentationKind.HandleHistogramLongCount) && + (pgoData[i + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes)) { PgoSchemaElem? newElem = ComputeLikelyClass(i, handleToObject, nativeSchema, instrumentationData, compilationModuleGroup); if (newElem.HasValue) @@ -3863,6 +3864,10 @@ public static void ComputeJitPgoInstrumentationSchema(Func objec { ptrVal = (IntPtr)objectToHandle(typeVal.AsType); } + else if (typeVal.AsMethod != null) + { + ptrVal = (IntPtr)objectToHandle(typeVal.AsMethod); + } else { // The "Unknown types are the values from 1-33 diff --git a/src/coreclr/tools/Common/Pgo/PgoFormat.cs b/src/coreclr/tools/Common/Pgo/PgoFormat.cs index a7de96a03fd0fc..a4e23286be4c67 100644 --- a/src/coreclr/tools/Common/Pgo/PgoFormat.cs +++ b/src/coreclr/tools/Common/Pgo/PgoFormat.cs @@ -21,6 +21,7 @@ public enum PgoInstrumentationKind FourByte = 1, EightByte = 2, TypeHandle = 3, + MethodHandle = 4, // Mask of all schema data types MarshalMask = 0xF, @@ -39,9 +40,10 @@ public enum PgoInstrumentationKind Done = None, // All instrumentation schemas must end with a record which is "Done" BasicBlockIntCount = (DescriptorMin * 1) | FourByte, // basic block counter using unsigned 4 byte int BasicBlockLongCount = (DescriptorMin * 1) | EightByte, // basic block counter using unsigned 8 byte int - TypeHandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. - TypeHandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram - TypeHandleHistogramTypeHandle = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram + HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. + HandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram + HandleHistogramTypes = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram + HandleHistogramMethods = (DescriptorMin * 3) | MethodHandle, // TypeHandle that is part of a type histogram Version = (DescriptorMin * 4) | None, // Version is encoded in the Other field of the schema NumRuns = (DescriptorMin * 5) | None, // Number of runs is encoded in the Other field of the schema EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int @@ -49,14 +51,16 @@ public enum PgoInstrumentationKind GetLikelyClass = (DescriptorMin * 7) | TypeHandle, // Compressed get likely class data } - public interface IPgoSchemaDataLoader + public interface IPgoSchemaDataLoader { TType TypeFromLong(long input); + TMethod MethodFromLong(long input); } - public interface IPgoEncodedValueEmitter + public interface IPgoEncodedValueEmitter { void EmitType(TType type, TType previousValue); + void EmitMethod(TMethod method, TMethod previousValue); void EmitLong(long value, long previousValue); bool EmitDone(); } @@ -77,6 +81,8 @@ public struct PgoSchemaElem public bool DataHeldInDataLong => (Count == 1 && (((InstrumentationKind & PgoInstrumentationKind.MarshalMask) == PgoInstrumentationKind.FourByte) || ((InstrumentationKind & PgoInstrumentationKind.MarshalMask) == PgoInstrumentationKind.EightByte))); + + public override string ToString() => $"{InstrumentationKind} @ {ILOffset} Count = {Count} DataLong = {DataLong}"; } // Flags stored in 'Other' field of TypeHandleHistogram*Count entries. @@ -230,13 +236,14 @@ public static IEnumerable PgoEncodedCompressedLongGenerator(IEnumerable ParsePgoData(IPgoSchemaDataLoader dataProvider, IEnumerable inputDataStream, bool longsAreCompressed) + public static IEnumerable ParsePgoData(IPgoSchemaDataLoader dataProvider, IEnumerable inputDataStream, bool longsAreCompressed) { int dataCountToRead = 0; PgoSchemaElem curSchema = default(PgoSchemaElem); InstrumentationDataProcessingState processingState = InstrumentationDataProcessingState.UpdateProcessMaskFlag; long lastDataValue = 0; long lastTypeValue = 0; + long lastMethodValue = 0; foreach (long value in inputDataStream) { @@ -278,6 +285,14 @@ public static IEnumerable ParsePgoData(IPgoSchemaDataLoade lastTypeValue = value; ((TType[])curSchema.DataObject)[dataIndex] = dataProvider.TypeFromLong(lastTypeValue); break; + + case PgoInstrumentationKind.MethodHandle: + if (longsAreCompressed) + lastMethodValue += value; + else + lastMethodValue = value; + ((TMethod[])curSchema.DataObject)[dataIndex] = dataProvider.MethodFromLong(lastMethodValue); + break; } } dataCountToRead--; @@ -364,6 +379,10 @@ public static IEnumerable ParsePgoData(IPgoSchemaDataLoade curSchema.DataObject = new TType[curSchema.Count]; dataCountToRead = curSchema.Count; break; + case PgoInstrumentationKind.MethodHandle: + curSchema.DataObject = new TMethod[curSchema.Count]; + dataCountToRead = curSchema.Count; + break; default: throw new Exception("Unknown Type"); } @@ -373,10 +392,11 @@ public static IEnumerable ParsePgoData(IPgoSchemaDataLoade throw new Exception("Partial Instrumentation Data"); } - public static void EncodePgoData(IEnumerable schemas, IPgoEncodedValueEmitter valueEmitter, bool emitAllElementsUnconditionally) + public static void EncodePgoData(IEnumerable schemas, IPgoEncodedValueEmitter valueEmitter, bool emitAllElementsUnconditionally) { PgoSchemaElem prevSchema = default(PgoSchemaElem); TType prevEmittedType = default(TType); + TMethod prevEmittedMethod = default(TMethod); long prevEmittedIntData = 0; foreach (PgoSchemaElem schema in schemas) @@ -422,7 +442,8 @@ public static void EncodePgoData(IEnumerable schemas, IPgo for (int i = 0; i < schema.Count; i++) { - switch (schema.InstrumentationKind & PgoInstrumentationKind.MarshalMask) + PgoInstrumentationKind marshal = schema.InstrumentationKind & PgoInstrumentationKind.MarshalMask; + switch (marshal) { case PgoInstrumentationKind.None: break; @@ -463,6 +484,15 @@ public static void EncodePgoData(IEnumerable schemas, IPgo prevEmittedType = typeToEmit; break; } + case PgoInstrumentationKind.MethodHandle: + { + TMethod methodToEmit = ((TMethod[])schema.DataObject)[i]; + valueEmitter.EmitMethod(methodToEmit, prevEmittedMethod); + prevEmittedMethod = methodToEmit; + break; + } + default: + throw new ArgumentException("Unknown schema marshal " + marshal); } } @@ -512,7 +542,7 @@ public bool Equals(PgoSchemaElem x, PgoSchemaElem y) int IEqualityComparer.GetHashCode(PgoSchemaElem obj) => obj.ILOffset ^ ((int)(obj.InstrumentationKind & PgoInstrumentationKind.DescriptorMask) << 20); } - public static PgoSchemaElem[] Merge(ReadOnlySpan schemasToMerge) + public static PgoSchemaElem[] Merge(ReadOnlySpan schemasToMerge) { { // The merging algorithm will sort the schema data by iloffset, then by InstrumentationKind @@ -560,8 +590,8 @@ void MergeInSchemaElem(Dictionary dataMerger, PgoS case PgoInstrumentationKind.BasicBlockLongCount: case PgoInstrumentationKind.EdgeIntCount: case PgoInstrumentationKind.EdgeLongCount: - case PgoInstrumentationKind.TypeHandleHistogramIntCount: - case PgoInstrumentationKind.TypeHandleHistogramLongCount: + case PgoInstrumentationKind.HandleHistogramIntCount: + case PgoInstrumentationKind.HandleHistogramLongCount: if ((existingSchemaItem.Count != 1) || (schema.Count != 1)) { throw new Exception("Unable to merge pgo data. Invalid format"); @@ -569,7 +599,7 @@ void MergeInSchemaElem(Dictionary dataMerger, PgoS mergedElem.DataLong = existingSchemaItem.DataLong + schema.DataLong; break; - case PgoInstrumentationKind.TypeHandleHistogramTypeHandle: + case PgoInstrumentationKind.HandleHistogramTypes: { mergedElem.Count = existingSchemaItem.Count + schema.Count; TType[] newMergedTypeArray = new TType[mergedElem.Count]; @@ -586,6 +616,23 @@ void MergeInSchemaElem(Dictionary dataMerger, PgoS break; } + case PgoInstrumentationKind.HandleHistogramMethods: + { + mergedElem.Count = existingSchemaItem.Count + schema.Count; + TMethod[] newMergedMethodArray = new TMethod[mergedElem.Count]; + mergedElem.DataObject = newMergedMethodArray; + int i = 0; + foreach (TMethod meth in (TMethod[])existingSchemaItem.DataObject) + { + newMergedMethodArray[i++] = meth; + } + foreach (TMethod meth in (TMethod[])schema.DataObject) + { + newMergedMethodArray[i++] = meth; + } + break; + } + case PgoInstrumentationKind.Version: { mergedElem.Other = Math.Max(existingSchemaItem.Other, schema.Other); diff --git a/src/coreclr/tools/Common/Pgo/TypeSystemEntityOrUnknown.cs b/src/coreclr/tools/Common/Pgo/TypeSystemEntityOrUnknown.cs index e8355b9b4d0a98..6eabe55cf7cc52 100644 --- a/src/coreclr/tools/Common/Pgo/TypeSystemEntityOrUnknown.cs +++ b/src/coreclr/tools/Common/Pgo/TypeSystemEntityOrUnknown.cs @@ -18,6 +18,11 @@ public TypeSystemEntityOrUnknown(TypeDesc type) _data = type; } + public TypeSystemEntityOrUnknown(MethodDesc method) + { + _data = method; + } + readonly object _data; public TypeDesc AsType => _data as TypeDesc; public MethodDesc AsMethod => _data as MethodDesc; diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs index 90883bf0b073d4..c73888a49c357c 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/InstrumentationDataTableNode.cs @@ -38,7 +38,7 @@ public void Initialize(ReadyToRunSymbolNodeFactory symbolNodeFactory) _symbolNodeFactory = symbolNodeFactory; } - class PgoValueEmitter : IPgoEncodedValueEmitter + class PgoValueEmitter : IPgoEncodedValueEmitter { public PgoValueEmitter(CompilationModuleGroup compilationGroup, ReadyToRunSymbolNodeFactory factory, bool actuallyCaptureOutput) { @@ -52,14 +52,18 @@ public void Clear() _longs.Clear(); _imports.Clear(); _typeConversions.Clear(); + _methodConversions.Clear(); _unknownTypesFound = 0; + _unknownMethodsFound = 0; } public IReadOnlyList ReferencedImports => _imports; List _longs = new List(); List _imports = new List(); Dictionary _typeConversions = new Dictionary(); + Dictionary _methodConversions = new Dictionary(); int _unknownTypesFound = 0; + int _unknownMethodsFound = 0; CompilationModuleGroup _compilationGroup; ReadyToRunSymbolNodeFactory _symbolFactory; bool _actuallyCaptureOutput; @@ -84,18 +88,23 @@ public void EmitType(TypeSystemEntityOrUnknown type, TypeSystemEntityOrUnknown p EmitLong(TypeToInt(type), TypeToInt(previousValue)); } - private int TypeToInt(TypeSystemEntityOrUnknown type) + public void EmitMethod(TypeSystemEntityOrUnknown method, TypeSystemEntityOrUnknown previousValue) { - if (type.IsNull || (type.AsType == null && type.AsUnknown == 0)) + EmitLong(MethodToInt(method), MethodToInt(previousValue)); + } + + private int TypeToInt(TypeSystemEntityOrUnknown handle) + { + if (handle.IsNull || (handle.AsType == null && handle.AsUnknown == 0)) return 0; - if (_typeConversions.TryGetValue(type, out int computedInt)) + if (_typeConversions.TryGetValue(handle, out int computedInt)) { return computedInt; } - if (type.AsType != null && _compilationGroup.VersionsWithTypeReference(type.AsType)) + if (handle.AsType != null && _compilationGroup.VersionsWithTypeReference(handle.AsType)) { - Import typeHandleImport = (Import)_symbolFactory.CreateReadyToRunHelper(ReadyToRunHelperId.TypeHandle, type.AsType); + Import typeHandleImport = (Import)_symbolFactory.CreateReadyToRunHelper(ReadyToRunHelperId.TypeHandle, handle.AsType); _imports.Add(typeHandleImport); if (_actuallyCaptureOutput) @@ -120,7 +129,51 @@ private int TypeToInt(TypeSystemEntityOrUnknown type) { computedInt = ((++_unknownTypesFound) << 4) | 0xF; } - _typeConversions.Add(type, computedInt); + _typeConversions.Add(handle, computedInt); + return computedInt; + } + + private int MethodToInt(TypeSystemEntityOrUnknown handle) + { + if (handle.IsNull || (handle.AsMethod == null && handle.AsUnknown == 0)) + return 0; + + if (_methodConversions.TryGetValue(handle, out int computedInt)) + { + return computedInt; + } + if (handle.AsMethod != null && _compilationGroup.VersionsWithMethodBody(handle.AsMethod)) + { + EcmaMethod typicalMethod = (EcmaMethod)handle.AsMethod.GetTypicalMethodDefinition(); + ModuleToken moduleToken = new ModuleToken(typicalMethod.Module, typicalMethod.Handle); + + MethodWithToken tok = new MethodWithToken(handle.AsMethod, moduleToken, constrainedType: null, unboxing: false, context: null); + Import methodHandleImport = (Import)_symbolFactory.CreateReadyToRunHelper(ReadyToRunHelperId.MethodHandle, tok); + _imports.Add(methodHandleImport); + + if (_actuallyCaptureOutput) + { + if (methodHandleImport.Table.IndexFromBeginningOfArray >= 0xF) + { + // The current implementation of this table only allows for 15 different + // import tables to be used. This is probably enough for long term + // but this code will throw if we use more import tables and attempt + // to encode pgo data + throw new Exception("Unexpected high index for table import"); + } + + computedInt = (methodHandleImport.IndexFromBeginningOfArray << 4) | methodHandleImport.Table.IndexFromBeginningOfArray; + } + else + { + computedInt = _imports.Count << 1; + } + } + else + { + computedInt = ((++_unknownMethodsFound) << 4) | 0xF; + } + _methodConversions.Add(handle, computedInt); return computedInt; } } diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs index 5220e13f813eef..dc5641a83366d6 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ProfileData.cs @@ -111,7 +111,7 @@ public static void MergeProfileData(ref bool partialNgen, Dictionary(schemaElemMergerArray); + mergedSchemaData = PgoProcessor.Merge(schemaElemMergerArray); } mergedProfileData[data.Method] = new MethodProfileData(data.Method, dataToMerge.Flags | data.Flags, data.ExclusiveWeight + dataToMerge.ExclusiveWeight, mergedCallWeights, dataToMerge.ScenarioMask | data.ScenarioMask, mergedSchemaData); } diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IBC/MIbcProfileParser.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IBC/MIbcProfileParser.cs index 2f302d2edd6774..f0d7c5afc4100f 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IBC/MIbcProfileParser.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IBC/MIbcProfileParser.cs @@ -22,7 +22,7 @@ namespace ILCompiler.IBC { static class MIbcProfileParser { - private class MetadataLoaderForPgoData : IPgoSchemaDataLoader + private class MetadataLoaderForPgoData : IPgoSchemaDataLoader { private readonly EcmaMethodIL _ilBody; @@ -30,12 +30,12 @@ public MetadataLoaderForPgoData(EcmaMethodIL ilBody) { _ilBody = ilBody; } - TypeSystemEntityOrUnknown IPgoSchemaDataLoader.TypeFromLong(long token) + TypeSystemEntityOrUnknown IPgoSchemaDataLoader.TypeFromLong(long token) { try { if (token == 0) - return new TypeSystemEntityOrUnknown(null); + return new TypeSystemEntityOrUnknown((TypeDesc)null); if ((token & 0xFF000000) == 0) { // token type is 0, therefore it can't be a type @@ -53,6 +53,29 @@ TypeSystemEntityOrUnknown IPgoSchemaDataLoader.TypeFr return new TypeSystemEntityOrUnknown((int)token); } } + TypeSystemEntityOrUnknown IPgoSchemaDataLoader.MethodFromLong(long token) + { + try + { + if (token == 0) + return new TypeSystemEntityOrUnknown((MethodDesc)null); + if ((token & 0xFF000000) == 0) + { + // token type is 0, therefore it can't be a method + return new TypeSystemEntityOrUnknown((int)token); + } + MethodDesc foundMethod = _ilBody.GetObject((int)token, NotFoundBehavior.ReturnNull) as MethodDesc; + if (foundMethod == null) + { + return new TypeSystemEntityOrUnknown((int)token & 0x00FFFFFF); + } + return new TypeSystemEntityOrUnknown(foundMethod); + } + catch + { + return new TypeSystemEntityOrUnknown((int)token); + } + } } public static PEReader OpenMibcAsPEReader(string filename) @@ -413,7 +436,7 @@ static IEnumerable ReadMIbcGroup(TypeSystemContext tsc, EcmaM { instrumentationDataLongs.Add(2); // MarshalMask 2 (Type) instrumentationDataLongs.Add(0); // PgoInstrumentationKind.Done (0) - pgoSchemaData = PgoProcessor.ParsePgoData(metadataLoader, instrumentationDataLongs, false).ToArray(); + pgoSchemaData = PgoProcessor.ParsePgoData(metadataLoader, instrumentationDataLongs, false).ToArray(); } state = MibcGroupParseState.LookingForOptionalData; break; diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/PgoInfo.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/PgoInfo.cs index d07b5496a970a0..aff9b42f027e7f 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/PgoInfo.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/PgoInfo.cs @@ -31,7 +31,7 @@ public PgoInfo(PgoInfoKey key, ReadyToRunReader r2rReader, int pgoFormatVersion, PgoSchemaElem[] _pgoData; int _size; - class PgoDataLoader : IPgoSchemaDataLoader + class PgoDataLoader : IPgoSchemaDataLoader { ReadyToRunReader _r2rReader; SignatureFormattingOptions _formatOptions; @@ -42,7 +42,7 @@ public PgoDataLoader(ReadyToRunReader r2rReader, SignatureFormattingOptions form _r2rReader = r2rReader; } - string IPgoSchemaDataLoader.TypeFromLong(long input) + string IPgoSchemaDataLoader.TypeFromLong(long input) { int tableIndex = checked((int)(input & 0xF)); int fixupIndex = checked((int)(input >> 4)); @@ -55,6 +55,20 @@ string IPgoSchemaDataLoader.TypeFromLong(long input) return _r2rReader.ImportSections[tableIndex].Entries[fixupIndex].Signature.ToString(_formatOptions); } } + + string IPgoSchemaDataLoader.MethodFromLong(long input) + { + int tableIndex = checked((int)(input & 0xF)); + int fixupIndex = checked((int)(input >> 4)); + if (tableIndex == 0xF) + { + return $"Unknown method {fixupIndex}"; + } + else + { + return _r2rReader.ImportSections[tableIndex].Entries[fixupIndex].Signature.ToString(_formatOptions); + } + } } void EnsurePgoData() @@ -72,7 +86,7 @@ void EnsurePgoData() SignatureFormattingOptions formattingOptions = new SignatureFormattingOptions(); - _pgoData = PgoProcessor.ParsePgoData(new PgoDataLoader(_r2rReader, formattingOptions), compressedIntParser, true).ToArray(); + _pgoData = PgoProcessor.ParsePgoData(new PgoDataLoader(_r2rReader, formattingOptions), compressedIntParser, true).ToArray(); _size = compressedIntParser.Offset - Offset; } } diff --git a/src/coreclr/tools/dotnet-pgo/MibcEmitter.cs b/src/coreclr/tools/dotnet-pgo/MibcEmitter.cs index 997f6110c96bea..a498bfd9e21d5b 100644 --- a/src/coreclr/tools/dotnet-pgo/MibcEmitter.cs +++ b/src/coreclr/tools/dotnet-pgo/MibcEmitter.cs @@ -32,7 +32,7 @@ namespace Microsoft.Diagnostics.Tools.Pgo { static class MibcEmitter { - class MIbcGroup : IPgoEncodedValueEmitter + class MIbcGroup : IPgoEncodedValueEmitter { private static int s_emitCount = 0; @@ -100,7 +100,7 @@ public void AddProcessedMethodData(MethodProfileData processedMethodData) if (processedMethodData.SchemaData != null) { _il.LoadString(_emitter.GetUserStringHandle("InstrumentationDataStart")); - PgoProcessor.EncodePgoData(processedMethodData.SchemaData, this, true); + PgoProcessor.EncodePgoData(processedMethodData.SchemaData, this, true); } _il.OpCode(ILOpCode.Pop); } @@ -121,13 +121,13 @@ public MethodDefinitionHandle EmitMethod() return _emitter.AddGlobalMethod(methodName, _il, 8); } - bool IPgoEncodedValueEmitter.EmitDone() + bool IPgoEncodedValueEmitter.EmitDone() { _il.LoadString(_emitter.GetUserStringHandle("InstrumentationDataEnd")); return true; } - void IPgoEncodedValueEmitter.EmitLong(long value, long previousValue) + void IPgoEncodedValueEmitter.EmitLong(long value, long previousValue) { if ((value <= int.MaxValue) && (value >= int.MinValue)) { @@ -139,7 +139,7 @@ void IPgoEncodedValueEmitter.EmitLong(long value, lon } } - void IPgoEncodedValueEmitter.EmitType(TypeSystemEntityOrUnknown type, TypeSystemEntityOrUnknown previousValue) + void IPgoEncodedValueEmitter.EmitType(TypeSystemEntityOrUnknown type, TypeSystemEntityOrUnknown previousValue) { if (type.AsType != null) { @@ -153,6 +153,20 @@ void IPgoEncodedValueEmitter.EmitType(TypeSystemEntit } } + void IPgoEncodedValueEmitter.EmitMethod(TypeSystemEntityOrUnknown method, TypeSystemEntityOrUnknown previousValue) + { + if (method.AsMethod != null) + { + _il.OpCode(ILOpCode.Ldtoken); + _il.Token(_emitter.GetMethodRef(method.AsMethod)); + } + else + { + + _il.LoadConstantI4(method.AsUnknown & 0x00FFFFFF); + } + } + } private static string GetTypeDefiningAssembly(TypeDesc type) diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs index c210adca92148a..a9bfc1db4664ae 100644 --- a/src/coreclr/tools/dotnet-pgo/Program.cs +++ b/src/coreclr/tools/dotnet-pgo/Program.cs @@ -55,7 +55,7 @@ class MethodChunks public int LastChunk = -1; } - class PgoDataLoader : IPgoSchemaDataLoader + class PgoDataLoader : IPgoSchemaDataLoader { private TraceRuntimeDescToTypeSystemDesc _idParser; @@ -84,6 +84,27 @@ public TypeSystemEntityOrUnknown TypeFromLong(long input) // Unknown type, apply unique value, but keep the upper byte zeroed so that it can be distinguished from a token return new TypeSystemEntityOrUnknown(System.HashCode.Combine(input) & 0x7FFFFF | 0x800000); } + + public TypeSystemEntityOrUnknown MethodFromLong(long input) + { + if (input == 0) + return new TypeSystemEntityOrUnknown(0); + + MethodDesc method = null; + + try + { + method = _idParser.ResolveMethodID(input, false); + } + catch + { } + if (method != null) + { + return new TypeSystemEntityOrUnknown(method); + } + // Unknown type, apply unique value, but keep the upper byte zeroed so that it can be distinguished from a token + return new TypeSystemEntityOrUnknown(System.HashCode.Combine(input) & 0x7FFFFF | 0x800000); + } } struct ProcessedMethodData @@ -576,7 +597,7 @@ void PrintBar(string label, ref int curIndex, Func include, bool f List typeHandleHistogramCallSites = prof1.SchemaData.Concat(prof2.SchemaData) - .Where(e => e.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass || e.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle) + .Where(e => e.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass || e.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) .Select(e => e.ILOffset) .Distinct() .ToList(); @@ -781,8 +802,8 @@ static void PrintMibcStats(ProfileData data) PrintOutput($"# Methods with 64-bit block counts: {profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.BasicBlockLongCount))}"); PrintOutput($"# Methods with 32-bit edge counts: {profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.EdgeIntCount))}"); PrintOutput($"# Methods with 64-bit edge counts: {profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.EdgeLongCount))}"); - int numTypeHandleHistograms = profiledMethods.Sum(spd => spd.SchemaData.Count(elem => elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle)); - int methodsWithTypeHandleHistograms = profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle)); + int numTypeHandleHistograms = profiledMethods.Sum(spd => spd.SchemaData.Count(elem => elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes)); + int methodsWithTypeHandleHistograms = profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes)); PrintOutput($"# Type handle histograms: {numTypeHandleHistograms} in {methodsWithTypeHandleHistograms} methods"); int numGetLikelyClass = profiledMethods.Sum(spd => spd.SchemaData.Count(elem => elem.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass)); int methodsWithGetLikelyClass = profiledMethods.Count(spd => spd.SchemaData.Any(elem => elem.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass)); @@ -793,7 +814,7 @@ static void PrintMibcStats(ProfileData data) { var sites = mpd.SchemaData - .Where(e => e.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle || e.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass) + .Where(e => e.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes || e.InstrumentationKind == PgoInstrumentationKind.GetLikelyClass) .Select(e => e.ILOffset) .Distinct(); @@ -826,7 +847,7 @@ static bool CountersSumToZero(MethodProfileData data) PrintCallsitesByLikelyClassesChart(profiledMethods .SelectMany(m => m.SchemaData) - .Where(sd => sd.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle) + .Where(sd => sd.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) .Select(GetUniqueClassesSeen) .ToArray()); @@ -841,7 +862,7 @@ static int GetUniqueClassesSeen(PgoSchemaElem se) PrintLikelihoodHistogram(profiledMethods .SelectMany(m => m.SchemaData) - .Where(sd => sd.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle) + .Where(sd => sd.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) .Select(GetLikelihoodOfMostPopularType) .ToArray()); @@ -890,10 +911,10 @@ static bool IsUnknownTypeHandle(int handle) } bool isHistogramCount = - elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramIntCount || - elem.InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramLongCount; + elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramIntCount || + elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramLongCount; - if (isHistogramCount && elem.Count == 1 && i + 1 < schema.Length && schema[i + 1].InstrumentationKind == PgoInstrumentationKind.TypeHandleHistogramTypeHandle) + if (isHistogramCount && elem.Count == 1 && i + 1 < schema.Length && schema[i + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) { var handles = (TypeSystemEntityOrUnknown[])schema[i + 1].DataObject; var histogram = handles.Where(e => !e.IsNull).GroupBy(e => e).ToList(); @@ -1572,7 +1593,7 @@ void AddToInstrumentationData(int eventClrInstanceId, long methodID, int methodF } var intDecompressor = new PgoProcessor.PgoEncodedCompressedIntParser(instrumentationData, 0); - methodData.InstrumentationData = PgoProcessor.ParsePgoData(pgoDataLoader, intDecompressor, true).ToArray(); + methodData.InstrumentationData = PgoProcessor.ParsePgoData(pgoDataLoader, intDecompressor, true).ToArray(); } else { diff --git a/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp b/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp index c612b13f80de6e..1cfbc598dbee16 100644 --- a/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp +++ b/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp @@ -91,9 +91,10 @@ void DumpMap(int index, MethodContext* mc) // Add in the "fake" pgo flags bool hasEdgeProfile = false; bool hasClassProfile = false; + bool hasMethodProfile = false; bool hasLikelyClass = false; ICorJitInfo::PgoSource pgoSource = ICorJitInfo::PgoSource::Unknown; - if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasLikelyClass, pgoSource)) + if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasMethodProfile, hasLikelyClass, pgoSource)) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_PGO); @@ -107,6 +108,11 @@ void DumpMap(int index, MethodContext* mc) rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_CLASS_PROFILE); } + if (hasMethodProfile) + { + rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_METHOD_PROFILE); + } + if (hasLikelyClass) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); diff --git a/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp b/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp index eaf208508b7ddd..a3e31a1f73062f 100644 --- a/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp +++ b/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp @@ -29,9 +29,10 @@ int verbJitFlags::DoWork(const char* nameOfInput) // bool hasEdgeProfile = false; bool hasClassProfile = false; + bool hasMethodProfile = false; bool hasLikelyClass = false; ICorJitInfo::PgoSource pgoSource = ICorJitInfo::PgoSource::Unknown; - if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasLikelyClass, pgoSource)) + if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasMethodProfile, hasLikelyClass, pgoSource)) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_PGO); @@ -45,6 +46,11 @@ int verbJitFlags::DoWork(const char* nameOfInput) rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_CLASS_PROFILE); } + if (hasMethodProfile) + { + rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_METHOD_PROFILE); + } + if (hasLikelyClass) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 7cca8b9adc482a..a684eb5d405715 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -5587,13 +5587,14 @@ void MethodContext::dmpGetPgoInstrumentationResults(DWORDLONG key, const Agnosti case ICorJitInfo::PgoInstrumentationKind::EdgeLongCount: printf("E %llu", *(uint64_t*)(pInstrumentationData + pBuf[i].Offset)); break; - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount: printf("T %u", *(unsigned*)(pInstrumentationData + pBuf[i].Offset)); break; - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount: printf("T %llu", *(uint64_t*)(pInstrumentationData + pBuf[i].Offset)); break; - case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes: + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods: for (unsigned int j = 0; j < pBuf[i].Count; j++) { printf("[%u] %016llX ", j, CastHandle(*(uintptr_t*)(pInstrumentationData + pBuf[i].Offset + j * sizeof(uintptr_t)))); @@ -7065,10 +7066,11 @@ int MethodContext::dumpMD5HashToBuffer(BYTE* pBuffer, int bufLen, char* hash, in return m_hash.HashBuffer(pBuffer, bufLen, hash, hashLen); } -bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource) +bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasMethodProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource) { hasEdgeProfile = false; hasClassProfile = false; + hasMethodProfile = false; hasLikelyClass = false; // Obtain the Method Info structure for this method @@ -7091,8 +7093,8 @@ bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool { hasEdgeProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::EdgeIntCount); hasEdgeProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::EdgeLongCount); - hasClassProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount); - hasClassProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount); + hasClassProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes); + hasMethodProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods); hasLikelyClass |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyClass); if (hasEdgeProfile && hasClassProfile && hasLikelyClass) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index 589d6d85b2c65c..81c098d2924836 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -52,7 +52,8 @@ enum EXTRA_JIT_FLAGS HAS_CLASS_PROFILE = 61, HAS_LIKELY_CLASS = 60, HAS_STATIC_PROFILE = 59, - HAS_DYNAMIC_PROFILE = 58 + HAS_DYNAMIC_PROFILE = 58, + HAS_METHOD_PROFILE = 57, }; // Asserts to catch changes in corjit flags definitions. @@ -105,7 +106,7 @@ class MethodContext int dumpMethodIdentityInfoToBuffer(char* buff, int len, bool ignoreMethodName = false, CORINFO_METHOD_INFO* optInfo = nullptr, unsigned optFlags = 0); int dumpMethodMD5HashToBuffer(char* buff, int len, bool ignoreMethodName = false, CORINFO_METHOD_INFO* optInfo = nullptr, unsigned optFlags = 0); - bool hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource); + bool hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasMethodProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource); void recGlobalContext(const MethodContext& other); diff --git a/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp b/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp index ad4aee9fcd945e..1d1d4d53b1a845 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp @@ -283,6 +283,7 @@ std::string SpmiDumpHelper::DumpJitFlags(unsigned long long flags) AddFlagNumeric(HAS_PGO, EXTRA_JIT_FLAGS::HAS_PGO); AddFlagNumeric(HAS_EDGE_PROFILE, EXTRA_JIT_FLAGS::HAS_EDGE_PROFILE); AddFlagNumeric(HAS_CLASS_PROFILE, EXTRA_JIT_FLAGS::HAS_CLASS_PROFILE); + AddFlagNumeric(HAS_METHOD_PROFILE, EXTRA_JIT_FLAGS::HAS_METHOD_PROFILE); AddFlagNumeric(HAS_LIKELY_CLASS, EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); AddFlagNumeric(HAS_STATIC_PROFILE, EXTRA_JIT_FLAGS::HAS_STATIC_PROFILE); AddFlagNumeric(HAS_DYNAMIC_PROFILE, EXTRA_JIT_FLAGS::HAS_DYNAMIC_PROFILE); diff --git a/src/coreclr/vm/pgo.cpp b/src/coreclr/vm/pgo.cpp index c3f37a3226b981..d28e143cfe8693 100644 --- a/src/coreclr/vm/pgo.cpp +++ b/src/coreclr/vm/pgo.cpp @@ -254,6 +254,34 @@ void PgoManager::WritePgoData() } break; } + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: + { + MethodDesc* md = *(MethodDesc**)(data + entryOffset); + if (md == nullptr) + { + fprintf(pgoDataFile, "MethodHandle: NULL"); + } + else + { + SString garbage1, tMethodName, garbage2; + md->GetMethodInfo(garbage1, tMethodName, garbage2); + StackSString tTypeName; + TypeString::AppendType(tTypeName, TypeHandle(md->GetMethodTable()), TypeString::FormatNamespace | TypeString::FormatFullInst | TypeString::FormatAssembly); + // Format is: + // MethodName|@|fully_qualified_type_name + if (tTypeName.GetCount() + 1 + tMethodName.GetCount() > 8192) + { + fprintf(pgoDataFile, "MethodHandle: unknown"); + } + else + { + StackScratchBuffer methodNameBuffer; + StackScratchBuffer typeBuffer; + fprintf(pgoDataFile, "MethodHandle: %s|@|%s", tMethodName.GetUTF8(methodNameBuffer), tTypeName.GetUTF8(typeBuffer)); + } + } + break; + } default: break; } @@ -469,6 +497,41 @@ void PgoManager::ReadPgoData() ptrVal += 1; // Set low bit to indicate that this isn't actually a TypeHandle, but is instead a pointer } + uint8_t *rawBuffer = methodInstrumentationData.OpenRawBuffer(maxSize); + *(INT_PTR *)(rawBuffer + entryOffset) = ptrVal; + methodInstrumentationData.CloseRawBuffer(); + break; + } + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: + { + char* methodString; + if (strncmp(buffer, "MethodHandle: ", 14) != 0) + { + failed = true; + break; + } + methodString = buffer + 14; + size_t endOfString = strlen(methodString); + if (endOfString == 0 || (methodString[endOfString - 1] != '\n')) + { + failed = true; + break; + } + // Remove \n and replace will null + methodString[endOfString - 1] = '\0'; + + TypeHandle th; + INT_PTR ptrVal = 0; + if (strcmp(methodString, "NULL") != 0) + { + // As early type loading is likely problematic, simply drop the string into the data, and fix it up later + void* tempString = malloc(endOfString); + memcpy(tempString, methodString, endOfString); + + ptrVal = (INT_PTR)tempString; + ptrVal += 1; // Set low bit to indicate that this isn't actually a TypeHandle, but is instead a pointer + } + uint8_t *rawBuffer = methodInstrumentationData.OpenRawBuffer(maxSize); *(INT_PTR *)(rawBuffer + entryOffset) = ptrVal; methodInstrumentationData.CloseRawBuffer(); @@ -747,34 +810,53 @@ HRESULT PgoManager::getPgoInstrumentationResults(MethodDesc* pMD, BYTE** pAlloca for (unsigned iSchema = 0; iSchema < schemaArray.GetCount(); iSchema++) { ICorJitInfo::PgoInstrumentationSchema *schema = &(schemaArray)[iSchema]; - if ((schema->InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask) == ICorJitInfo::PgoInstrumentationKind::TypeHandle) + ICorJitInfo::PgoInstrumentationKind kind = schema->InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask; + if ((kind == ICorJitInfo::PgoInstrumentationKind::TypeHandle) || (kind == ICorJitInfo::PgoInstrumentationKind::MethodHandle)) { for (int iEntry = 0; iEntry < schema->Count; iEntry++) { - INT_PTR* typeHandleValueAddress = (INT_PTR*)(found->GetData() + schema->Offset + iEntry * InstrumentationKindToSize(schema->InstrumentationKind)); - INT_PTR initialTypeHandleValue = VolatileLoad(typeHandleValueAddress); - if (((initialTypeHandleValue & 1) == 1) && !ICorJitInfo::IsUnknownTypeHandle(initialTypeHandleValue)) + INT_PTR* handleValueAddress = (INT_PTR*)(found->GetData() + schema->Offset + iEntry * InstrumentationKindToSize(schema->InstrumentationKind)); + INT_PTR initialHandleValue = VolatileLoad(handleValueAddress); + if (((initialHandleValue & 1) == 1) && !ICorJitInfo::IsUnknownHandle(initialHandleValue)) { INT_PTR newPtr = 0; - TypeHandle th; - char* typeString = ((char *)initialTypeHandleValue) - 1; + char* string = ((char *)initialHandleValue) - 1; - // Don't attempt to load any types until the EE is started + // Don't attempt to load any types or methods until the EE is started if (g_fEEStarted) { - StackSString ss(SString::Utf8, typeString); - th = TypeName::GetTypeManaged(ss.GetUnicode(), NULL, FALSE, FALSE, FALSE, NULL, NULL); + if (kind == ICorJitInfo::PgoInstrumentationKind::TypeHandle) + { + StackSString ts(SString::Utf8, string); + TypeHandle th = TypeName::GetTypeManaged(ts.GetUnicode(), NULL, FALSE, FALSE, FALSE, NULL, NULL); + newPtr = (INT_PTR)th.AsPtr(); + } + else + { + assert(kind == ICorJitInfo::PgoInstrumentationKind::MethodHandle); + // Format is: + // MethodName|@|fully_qualified_type_name + char* sep = strstr(string, "|@|"); + if (sep != nullptr) + { + StackSString typeString(SString::Utf8, sep + 3); + StackSString methodString(SString::Utf8, string, (COUNT_T)(sep - string)); + TypeHandle th = TypeName::GetTypeManaged(typeString.GetUnicode(), NULL, FALSE, FALSE, FALSE, NULL, NULL); + if (!th.IsNull()) + { + MethodDesc* pMD = MemberLoader::FindMethodByName(th.GetMethodTable(), methodString.GetUTF8NoConvert()); + newPtr = (INT_PTR)pMD; + } + } + } } - if (th.IsNull()) + if (newPtr == 0) { - newPtr = HashToPgoUnknownTypeHandle(HashStringA(typeString)); + newPtr = HashToPgoUnknownHandle(HashStringA(string)); } - else - { - newPtr = (INT_PTR)th.AsPtr(); - } - InterlockedCompareExchangeT(typeHandleValueAddress, newPtr, initialTypeHandleValue); + + InterlockedCompareExchangeT(handleValueAddress, newPtr, initialHandleValue); } } } @@ -853,63 +935,77 @@ class R2RInstrumentationDataReader schemaArray[schemaArray.GetCount() - 1].Offset = instrumentationData.GetCount(); } - if ((schema.InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask) == ICorJitInfo::PgoInstrumentationKind::TypeHandle) + ICorJitInfo::PgoInstrumentationKind kind = schema.InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask; + switch (kind) { - intptr_t typeHandleData = 0; - if (dataItem != 0) + case ICorJitInfo::PgoInstrumentationKind::TypeHandle: + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: { - uint32_t importSection = dataItem & 0xF; - int64_t typeIndex = dataItem >> 4; - if (importSection != 0xF) + intptr_t handleData = 0; + if (dataItem != 0) { - COUNT_T countImportSections; - PTR_CORCOMPILE_IMPORT_SECTION pImportSections = m_pReadyToRunInfo->GetImportSections(&countImportSections); - - if (importSection >= countImportSections) + uint32_t importSection = dataItem & 0xF; + int64_t typeIndex = dataItem >> 4; + if (importSection != 0xF) { - _ASSERTE(!"Malformed pgo type handle data"); - return false; - } + COUNT_T countImportSections; + PTR_CORCOMPILE_IMPORT_SECTION pImportSections = m_pReadyToRunInfo->GetImportSections(&countImportSections); - PTR_CORCOMPILE_IMPORT_SECTION pImportSection = &pImportSections[importSection]; - COUNT_T cbData; - TADDR pData = m_pNativeImage->GetDirectoryData(&pImportSection->Section, &cbData); - uint32_t fixupIndex = (uint32_t)typeIndex; - PTR_SIZE_T fixupAddress = dac_cast(pData + fixupIndex * sizeof(TADDR)); - if (!m_pModule->FixupNativeEntry(pImportSections + importSection, fixupIndex, fixupAddress)) + if (importSection >= countImportSections) + { + _ASSERTE(!"Malformed PGO type or method handle data"); + return false; + } + + PTR_CORCOMPILE_IMPORT_SECTION pImportSection = &pImportSections[importSection]; + COUNT_T cbData; + TADDR pData = m_pNativeImage->GetDirectoryData(&pImportSection->Section, &cbData); + uint32_t fixupIndex = (uint32_t)typeIndex; + PTR_SIZE_T fixupAddress = dac_cast(pData + fixupIndex * sizeof(TADDR)); + if (!m_pModule->FixupNativeEntry(pImportSections + importSection, fixupIndex, fixupAddress)) + { + return false; + } + + handleData = *(intptr_t*)fixupAddress; + } + else { - return false; + handleData = HashToPgoUnknownHandle((uint32_t)typeIndex); } - - typeHandleData = *(intptr_t*)fixupAddress; } - else + + BYTE* pHandleData = (BYTE*)&handleData; + for (size_t i = 0; i < sizeof(intptr_t); i++) { - typeHandleData = HashToPgoUnknownTypeHandle((uint32_t)typeIndex); + instrumentationData.Append(pHandleData[i]); } - } - BYTE* pTypeHandleData = (BYTE*)&typeHandleData; - for (size_t i = 0; i < sizeof(intptr_t); i++) - { - instrumentationData.Append(pTypeHandleData[i]); + break; } - } - else if ((schema.InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask) == ICorJitInfo::PgoInstrumentationKind::FourByte) - { - BYTE* pFourByteData = (BYTE*)&dataItem; - for (int i = 0; i < 4; i++) + case ICorJitInfo::PgoInstrumentationKind::FourByte: { - instrumentationData.Append(pFourByteData[i]); + BYTE* pFourByteData = (BYTE*)&dataItem; + for (int i = 0; i < 4; i++) + { + instrumentationData.Append(pFourByteData[i]); + } + + break; } - } - else if ((schema.InstrumentationKind & ICorJitInfo::PgoInstrumentationKind::MarshalMask) == ICorJitInfo::PgoInstrumentationKind::EightByte) - { - BYTE* pEightByteData = (BYTE*)&dataItem; - for (int i = 0; i < 8; i++) + case ICorJitInfo::PgoInstrumentationKind::EightByte: { - instrumentationData.Append(pEightByteData[i]); + BYTE* pEightByteData = (BYTE*)&dataItem; + for (int i = 0; i < 8; i++) + { + instrumentationData.Append(pEightByteData[i]); + } + + break; } + default: + assert(!"Unexpected PGO instrumentation data type"); + break; } return true; From 6959656cb16ed797be001045296eedf7ed62fe44 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 12 Apr 2022 22:01:00 +0200 Subject: [PATCH 02/59] Fix R2R reader for PGO schema entries without data --- src/coreclr/vm/pgo.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/vm/pgo.cpp b/src/coreclr/vm/pgo.cpp index d28e143cfe8693..b3cc609ddec270 100644 --- a/src/coreclr/vm/pgo.cpp +++ b/src/coreclr/vm/pgo.cpp @@ -1003,6 +1003,8 @@ class R2RInstrumentationDataReader break; } + case ICorJitInfo::PgoInstrumentationKind::None: + break; default: assert(!"Unexpected PGO instrumentation data type"); break; From 3245442ab4cad7dc2ca76c95a1938683b7e7fda2 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 13 Apr 2022 16:35:51 +0200 Subject: [PATCH 03/59] Support instrumenting method handles for delegates and vtable calls --- src/coreclr/inc/corinfo.h | 3 + src/coreclr/inc/corjit.h | 18 +- src/coreclr/inc/jithelpers.h | 3 + src/coreclr/jit/block.h | 8 +- src/coreclr/jit/compiler.cpp | 8 +- src/coreclr/jit/compiler.h | 16 +- src/coreclr/jit/fgbasic.cpp | 2 +- src/coreclr/jit/fgprofile.cpp | 300 +++++++++++++++++----------- src/coreclr/jit/gentree.h | 4 +- src/coreclr/jit/importer.cpp | 131 ++++++++----- src/coreclr/jit/inline.h | 6 +- src/coreclr/jit/jitconfigvalues.h | 5 +- src/coreclr/jit/patchpoint.cpp | 2 +- src/coreclr/vm/jithelpers.cpp | 311 ++++++++++++++++++++++++++++-- 14 files changed, 619 insertions(+), 198 deletions(-) diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index ca792f63e89751..38f1993edb3e5f 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -646,6 +646,9 @@ enum CorInfoHelpFunc CORINFO_HELP_VALIDATE_INDIRECT_CALL, // CFG: Validate function pointer CORINFO_HELP_DISPATCH_INDIRECT_CALL, // CFG: Validate and dispatch to pointer + CORINFO_HELP_METHODPROFILE32, // Update 32-bit method profile for a call site + CORINFO_HELP_METHODPROFILE64, // Update 64-bit method profile for a call site + CORINFO_HELP_COUNT, }; diff --git a/src/coreclr/inc/corjit.h b/src/coreclr/inc/corjit.h index ac44b8a9c8f3db..a64a850061b91f 100644 --- a/src/coreclr/inc/corjit.h +++ b/src/coreclr/inc/corjit.h @@ -330,7 +330,8 @@ class ICorJitInfo : public ICorDynamicInfo // Data structure for a single class probe using 32-bit count. // - // CLASS_FLAG and INTERFACE_FLAG are placed into the Other field in the schema + // CLASS_FLAG, INTERFACE_FLAG and DELEGATE_FLAG are placed into the Other field in the schema. + // If CLASS_FLAG is set the handle table consists of type handles, and otherwise method handles. // // Count is the number of times a call was made at that call site. // @@ -339,24 +340,25 @@ class ICorJitInfo : public ICorDynamicInfo // SAMPLE_INTERVAL must be >= SIZE. SAMPLE_INTERVAL / SIZE // gives the average number of calls between table updates. // - struct ClassProfile32 + struct HandleHistogram32 { enum { SIZE = 8, SAMPLE_INTERVAL = 32, CLASS_FLAG = 0x80000000, INTERFACE_FLAG = 0x40000000, - OFFSET_MASK = 0x3FFFFFFF + DELEGATE_FLAG = 0x20000000, + OFFSET_MASK = 0x0FFFFFFF }; uint32_t Count; - CORINFO_CLASS_HANDLE ClassTable[SIZE]; + void* HandleTable[SIZE]; }; - struct ClassProfile64 + struct HandleHistogram64 { uint64_t Count; - CORINFO_CLASS_HANDLE ClassTable[ClassProfile32::SIZE]; + void* HandleTable[HandleHistogram32::SIZE]; }; enum class PgoInstrumentationKind @@ -386,7 +388,7 @@ class ICorJitInfo : public ICorDynamicInfo Done = None, // All instrumentation schemas must end with a record which is "Done" BasicBlockIntCount = (DescriptorMin * 1) | FourByte, // basic block counter using unsigned 4 byte int BasicBlockLongCount = (DescriptorMin * 1) | EightByte, // basic block counter using unsigned 8 byte int - HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment. + HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match HandleHistogram32's alignment. HandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram HandleHistogramTypes = (DescriptorMin * 3) | TypeHandle, // Histogram of type handles HandleHistogramMethods = (DescriptorMin * 3) | MethodHandle, // Histogram of method handles @@ -417,7 +419,7 @@ class ICorJitInfo : public ICorDynamicInfo Sampling= 6, // PGO data derived from sampling }; -#define DEFAULT_UNKNOWN_TYPEHANDLE 1 +#define DEFAULT_UNKNOWN_HANDLE 1 #define UNKNOWN_HANDLE_MIN 1 #define UNKNOWN_HANDLE_MAX 33 diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index e2c4e0e8fcbf06..57ffeaed51b66b 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -339,6 +339,9 @@ JITHELPER(CORINFO_HELP_DISPATCH_INDIRECT_CALL, NULL, CORINFO_HELP_SIG_REG_ONLY) #endif + JITHELPER(CORINFO_HELP_METHODPROFILE32, JIT_MethodProfile32, CORINFO_HELP_SIG_8_STACK) + JITHELPER(CORINFO_HELP_METHODPROFILE64, JIT_MethodProfile64, CORINFO_HELP_SIG_8_STACK) + #undef JITHELPER #undef DYNAMICJITHELPER #undef JITHELPER diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index 092bad7367e44a..0c4605be1f6a38 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -546,7 +546,7 @@ enum BasicBlockFlags : unsigned __int64 BBF_BACKWARD_JUMP_TARGET = MAKE_BBFLAG(35), // Block is a target of a backward jump BBF_PATCHPOINT = MAKE_BBFLAG(36), // Block is a patchpoint - BBF_HAS_CLASS_PROFILE = MAKE_BBFLAG(37), // BB contains a call needing a class profile + BBF_HAS_HISTOGRAM_PROFILE = MAKE_BBFLAG(37), // BB contains a call needing a class profile BBF_PARTIAL_COMPILATION_PATCHPOINT = MAKE_BBFLAG(38), // Block is a partial compilation patchpoint BBF_HAS_ALIGN = MAKE_BBFLAG(39), // BB ends with 'align' instruction BBF_TAILCALL_SUCCESSOR = MAKE_BBFLAG(40), // BB has pred that has potential tail call @@ -582,7 +582,7 @@ enum BasicBlockFlags : unsigned __int64 // TODO: Should BBF_RUN_RARELY be added to BBF_SPLIT_GAINED ? BBF_SPLIT_GAINED = BBF_DONT_REMOVE | BBF_HAS_JMP | BBF_BACKWARD_JUMP | BBF_HAS_IDX_LEN | BBF_HAS_NEWARRAY | BBF_PROF_WEIGHT | \ - BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK | BBF_HAS_CLASS_PROFILE, + BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK | BBF_HAS_HISTOGRAM_PROFILE, }; inline constexpr BasicBlockFlags operator ~(BasicBlockFlags a) @@ -918,8 +918,8 @@ struct BasicBlock : private LIR::Range }; union { - unsigned bbStkTempsOut; // base# for output stack temps - int bbClassSchemaIndex; // schema index for class instrumentation + unsigned bbStkTempsOut; // base# for output stack temps + int bbHistogramSchemaIndex; // schema index for histogram instrumentation }; #define MAX_XCPTN_INDEX (USHRT_MAX - 1) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index c7d21583d4b084..8975328e881c9b 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -6221,10 +6221,10 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr, compHndBBtabCount = 0; compHndBBtabAllocCount = 0; - info.compNativeCodeSize = 0; - info.compTotalHotCodeSize = 0; - info.compTotalColdCodeSize = 0; - info.compClassProbeCount = 0; + info.compNativeCodeSize = 0; + info.compTotalHotCodeSize = 0; + info.compTotalColdCodeSize = 0; + info.compHandleHistogramProbeCount = 0; compHasBackwardJump = false; compHasBackwardJumpInHandler = false; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index a30c0c6113c91f..9fcb5d343e3973 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4347,6 +4347,18 @@ class Compiler bool isExplicitTailCall, IL_OFFSET ilOffset = BAD_IL_OFFSET); + void impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset); + + enum class GDVProbeType + { + None, + ClassProfile, + MethodProfile, + MethodAndClassProfile, + }; + + GDVProbeType compClassifyGDVProbeType(GenTreeCall* call); + //========================================================================= // PROTECTED //========================================================================= @@ -6238,7 +6250,7 @@ class Compiler bool fgGetProfileWeightForBasicBlock(IL_OFFSET offset, weight_t* weight); Instrumentor* fgCountInstrumentor; - Instrumentor* fgClassInstrumentor; + Instrumentor* fgHistogramInstrumentor; PhaseStatus fgPrepareToInstrumentMethod(); PhaseStatus fgInstrumentMethod(); @@ -10319,7 +10331,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX unsigned genCPU; // What CPU are we running on // Number of class profile probes in this method - unsigned compClassProbeCount; + unsigned compHandleHistogramProbeCount; } info; diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index e8881e283a44c0..5224145e3abbd3 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -185,7 +185,7 @@ void Compiler::fgInit() fgPgoInlineeNoPgo = 0; fgPgoInlineeNoPgoSingleBlock = 0; fgCountInstrumentor = nullptr; - fgClassInstrumentor = nullptr; + fgHistogramInstrumentor = nullptr; fgPredListSortVector = nullptr; } diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index a5914e36965322..9e8cfb2174fb00 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1426,11 +1426,11 @@ void EfficientEdgeCountInstrumentor::Instrument(BasicBlock* block, Schema& schem } //------------------------------------------------------------------------ -// ClassProbeVisitor: invoke functor on each virtual call or cast-related +// HandleHistogramProbeVisitor: invoke functor on each virtual call or cast-related // helper calls in a tree // template -class ClassProbeVisitor final : public GenTreeVisitor> +class HandleHistogramProbeVisitor final : public GenTreeVisitor> { public: enum @@ -1441,26 +1441,16 @@ class ClassProbeVisitor final : public GenTreeVisitor(compiler), m_functor(functor), m_compiler(compiler) + HandleHistogramProbeVisitor(Compiler* compiler, TFunctor& functor) + : GenTreeVisitor(compiler), m_functor(functor), m_compiler(compiler) { } Compiler::fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) { GenTree* const node = *use; - if (node->IsCall() && (node->AsCall()->gtClassProfileCandidateInfo != nullptr)) + if (node->IsCall() && (node->AsCall()->gtHandleHistogramProfileCandidateInfo != nullptr)) { - GenTreeCall* const call = node->AsCall(); - if (call->IsVirtual() && (call->gtCallType != CT_INDIRECT)) - { - // virtual call - m_functor(m_compiler, call); - } - else if (m_compiler->impIsCastHelperEligibleForClassProbe(call)) - { - // isinst/cast helper - m_functor(m_compiler, call); - } + m_functor(m_compiler, node->AsCall()); } return Compiler::WALK_CONTINUE; @@ -1468,44 +1458,65 @@ class ClassProbeVisitor final : public GenTreeVisitorcompClassifyGDVProbeType(call); + + if ((probeType == Compiler::GDVProbeType::ClassProfile) || + (probeType == Compiler::GDVProbeType::MethodAndClassProfile)) + { + CreateHistogramEntriesIfNecessary(compiler, call, true /* isTypeHistogram */); + } + + if ((probeType == Compiler::GDVProbeType::MethodProfile) || + (probeType == Compiler::GDVProbeType::MethodAndClassProfile)) + { + CreateHistogramEntriesIfNecessary(compiler, call, false /* isTypeHistogram */); + } + } + + void CreateHistogramEntriesIfNecessary(Compiler* compiler, GenTreeCall* call, bool isTypeHistogram) + { + ICorJitInfo::PgoInstrumentationSchema schemaElem = {}; + schemaElem.Count = 1; + schemaElem.Other = isTypeHistogram ? ICorJitInfo::HandleHistogram32::CLASS_FLAG : 0; if (call->IsVirtualStub()) { - schemaElem.Other |= ICorJitInfo::ClassProfile32::INTERFACE_FLAG; + schemaElem.Other |= ICorJitInfo::HandleHistogram32::INTERFACE_FLAG; } - else + else if (call->IsDelegateInvoke()) { - assert(call->IsVirtualVtable() || compiler->impIsCastHelperEligibleForClassProbe(call)); + schemaElem.Other |= ICorJitInfo::HandleHistogram32::DELEGATE_FLAG; } schemaElem.InstrumentationKind = JitConfig.JitCollect64BitCounts() ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount : ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; - schemaElem.ILOffset = (int32_t)call->gtClassProfileCandidateInfo->ilOffset; + schemaElem.ILOffset = (int32_t)call->gtHandleHistogramProfileCandidateInfo->ilOffset; schemaElem.Offset = 0; m_schema.push_back(schemaElem); + m_schemaCount++; + // Re-using ILOffset and Other fields from schema item for TypeHandleHistogramCount - schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes; - schemaElem.Count = ICorJitInfo::ClassProfile32::SIZE; + schemaElem.InstrumentationKind = isTypeHistogram ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes + : ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods; + schemaElem.Count = ICorJitInfo::HandleHistogram32::SIZE; m_schema.push_back(schemaElem); m_schemaCount++; @@ -1513,9 +1524,9 @@ class BuildClassProbeSchemaGen }; //------------------------------------------------------------------------ -// ClassProbeInserter: functor that adds class probe instrumentation +// HandleHistogramProbeInserter: functor that adds class/method probe instrumentation // -class ClassProbeInserter +class HandleHistogramProbeInserter { Schema& m_schema; uint8_t* m_profileMemory; @@ -1523,7 +1534,7 @@ class ClassProbeInserter unsigned& m_instrCount; public: - ClassProbeInserter(Schema& schema, uint8_t* profileMemory, int* pCurrentSchemaIndex, unsigned& instrCount) + HandleHistogramProbeInserter(Schema& schema, uint8_t* profileMemory, int* pCurrentSchemaIndex, unsigned& instrCount) : m_schema(schema) , m_profileMemory(profileMemory) , m_currentSchemaIndex(pCurrentSchemaIndex) @@ -1534,7 +1545,8 @@ class ClassProbeInserter void operator()(Compiler* compiler, GenTreeCall* call) { JITDUMP("Found call [%06u] with probe index %d and ilOffset 0x%X\n", compiler->dspTreeID(call), - call->gtClassProfileCandidateInfo->probeIndex, call->gtClassProfileCandidateInfo->ilOffset); + call->gtHandleHistogramProfileCandidateInfo->probeIndex, + call->gtHandleHistogramProfileCandidateInfo->ilOffset); // We transform the call from (CALLVIRT obj, ... args ...) to // to @@ -1547,19 +1559,25 @@ class ClassProbeInserter // ... args ...) // - // Sanity check that we're looking at the right schema entry - // - assert(m_schema[*m_currentSchemaIndex].ILOffset == (int32_t)call->gtClassProfileCandidateInfo->ilOffset); - bool is32 = m_schema[*m_currentSchemaIndex].InstrumentationKind == - ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; - bool is64 = m_schema[*m_currentSchemaIndex].InstrumentationKind == - ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount; - assert(is32 || is64); - - // Figure out where the table is located. - // - uint8_t* classProfile = m_schema[*m_currentSchemaIndex].Offset + m_profileMemory; - *m_currentSchemaIndex += 2; // There are 2 schema entries per class probe + // Read histograms + void* typeHistogram = nullptr; + void* methodHistogram = nullptr; + + bool is32; + ReadHistogramAndAdvance(call->gtHandleHistogramProfileCandidateInfo->ilOffset, &typeHistogram, &methodHistogram, + &is32); + bool secondIs32; + ReadHistogramAndAdvance(call->gtHandleHistogramProfileCandidateInfo->ilOffset, &typeHistogram, &methodHistogram, + &secondIs32); + + assert(((typeHistogram != nullptr) || (methodHistogram != nullptr)) && + "Expected at least one handle histogram when inserting probes"); + + if ((typeHistogram != nullptr) && (methodHistogram != nullptr)) + { + // We expect both histograms to be 32-bit or 64-bit, not a mix. + assert(is32 == secondIs32); + } GenTreeCall::Use* objUse = nullptr; if (compiler->impIsCastHelperEligibleForClassProbe(call)) @@ -1577,22 +1595,36 @@ class ClassProbeInserter // Grab a temp to hold the 'this' object as it will be used three times // - unsigned const tmpNum = compiler->lvaGrabTemp(true DEBUGARG("class profile tmp")); + unsigned const tmpNum = compiler->lvaGrabTemp(true DEBUGARG("handle histogram profile tmp")); compiler->lvaTable[tmpNum].lvType = TYP_REF; + unsigned helper; + GenTreeCall::Use* args = nullptr; + if (methodHistogram != nullptr) + { + GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); + GenTree* const baseMethodNode = compiler->gtNewIconEmbMethHndNode(call->gtCallMethHnd); + GenTree* const methodProfileNode = compiler->gtNewIconNode((ssize_t)methodHistogram, TYP_I_IMPL); + GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); + args = compiler->gtNewCallArgs(tmpNode, baseMethodNode, methodProfileNode, classProfileNode); + helper = is32 ? CORINFO_HELP_METHODPROFILE32 : CORINFO_HELP_METHODPROFILE64; + } + else + { + GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); + GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); + args = compiler->gtNewCallArgs(tmpNode, classProfileNode); + helper = is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64; + } + // Generate the IR... // - GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)classProfile, TYP_I_IMPL); - GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); - GenTreeCall::Use* const args = compiler->gtNewCallArgs(tmpNode, classProfileNode); - GenTree* const helperCallNode = - compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64, TYP_VOID, - args); - GenTree* const tmpNode2 = compiler->gtNewLclvNode(tmpNum, TYP_REF); - GenTree* const callCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, helperCallNode, tmpNode2); - GenTree* const tmpNode3 = compiler->gtNewLclvNode(tmpNum, TYP_REF); - GenTree* const asgNode = compiler->gtNewOperNode(GT_ASG, TYP_REF, tmpNode3, objUse->GetNode()); - GenTree* const asgCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, asgNode, callCommaNode); + GenTree* const helperCallNode = compiler->gtNewHelperCallNode(helper, TYP_VOID, args); + GenTree* const tmpNode2 = compiler->gtNewLclvNode(tmpNum, TYP_REF); + GenTree* const callCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, helperCallNode, tmpNode2); + GenTree* const tmpNode3 = compiler->gtNewLclvNode(tmpNum, TYP_REF); + GenTree* const asgNode = compiler->gtNewOperNode(GT_ASG, TYP_REF, tmpNode3, objUse->GetNode()); + GenTree* const asgCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, asgNode, callCommaNode); // Update the call // @@ -1603,16 +1635,78 @@ class ClassProbeInserter m_instrCount++; } + +private: + void ReadHistogramAndAdvance(IL_OFFSET ilOffset, void** typeHistogram, void** methodHistogram, bool* histogramIs32) + { + if (*m_currentSchemaIndex >= m_schema.size()) + { + return; + } + + ICorJitInfo::PgoInstrumentationSchema& countEntry = m_schema[*m_currentSchemaIndex + 0]; + + bool is32 = countEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; + bool is64 = countEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount; + if (!is32 && !is64) + { + return; + } + + if (countEntry.ILOffset != static_cast(ilOffset)) + { + return; + } + + assert(*m_currentSchemaIndex + 2 <= m_schema.size()); + ICorJitInfo::PgoInstrumentationSchema& tableEntry = m_schema[*m_currentSchemaIndex + 1]; + assert((tableEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes) || + (tableEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods)); + + void** outHistogram; + if (tableEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes) + { + assert(*typeHistogram == nullptr); + outHistogram = typeHistogram; + } + else + { + assert(*methodHistogram == nullptr); + outHistogram = methodHistogram; + } + + *outHistogram = &m_profileMemory[countEntry.Offset]; + *histogramIs32 = is32; + +#ifdef DEBUG + if (is32) + { + ICorJitInfo::HandleHistogram32* h32 = + reinterpret_cast(&m_profileMemory[countEntry.Offset]); + assert(reinterpret_cast(&h32->Count) == &m_profileMemory[countEntry.Offset]); + assert(reinterpret_cast(h32->HandleTable) == &m_profileMemory[tableEntry.Offset]); + } + else + { + ICorJitInfo::HandleHistogram64* h64 = + reinterpret_cast(&m_profileMemory[countEntry.Offset]); + assert(reinterpret_cast(&h64->Count) == &m_profileMemory[countEntry.Offset]); + assert(reinterpret_cast(h64->HandleTable) == &m_profileMemory[tableEntry.Offset]); + } +#endif + + *m_currentSchemaIndex += 2; + } }; //------------------------------------------------------------------------ -// ClassProbeInstrumentor: instrumentor that adds a class probe to each +// HandleHistogramProbeInstrumentor: instrumentor that adds a class probe to each // virtual call in the basic block // -class ClassProbeInstrumentor : public Instrumentor +class HandleHistogramProbeInstrumentor : public Instrumentor { public: - ClassProbeInstrumentor(Compiler* comp) : Instrumentor(comp) + HandleHistogramProbeInstrumentor(Compiler* comp) : Instrumentor(comp) { } bool ShouldProcess(BasicBlock* block) override @@ -1625,13 +1719,13 @@ class ClassProbeInstrumentor : public Instrumentor }; //------------------------------------------------------------------------ -// ClassProbeInstrumentor::Prepare: prepare for class instrumentation +// HandleHistogramProbeInstrumentor::Prepare: prepare for class instrumentation // // Arguments: // preImport - true if this is the prepare call that happens before // importation // -void ClassProbeInstrumentor::Prepare(bool isPreImport) +void HandleHistogramProbeInstrumentor::Prepare(bool isPreImport) { if (isPreImport) { @@ -1643,33 +1737,33 @@ void ClassProbeInstrumentor::Prepare(bool isPreImport) // for (BasicBlock* const block : m_comp->Blocks()) { - block->bbClassSchemaIndex = -1; + block->bbHistogramSchemaIndex = -1; } #endif } //------------------------------------------------------------------------ -// ClassProbeInstrumentor::BuildSchemaElements: create schema elements for a class probe +// HandleHistogramProbeInstrumentor::BuildSchemaElements: create schema elements for a class probe // // Arguments: // block -- block to instrument // schema -- schema that we're building // -void ClassProbeInstrumentor::BuildSchemaElements(BasicBlock* block, Schema& schema) +void HandleHistogramProbeInstrumentor::BuildSchemaElements(BasicBlock* block, Schema& schema) { - if ((block->bbFlags & BBF_HAS_CLASS_PROFILE) == 0) + if ((block->bbFlags & BBF_HAS_HISTOGRAM_PROFILE) == 0) { return; } // Remember the schema index for this block. // - block->bbClassSchemaIndex = (int)schema.size(); + block->bbHistogramSchemaIndex = (int)schema.size(); // Scan the statements and identify the class probes // - BuildClassProbeSchemaGen schemaGen(schema, m_schemaCount); - ClassProbeVisitor visitor(m_comp, schemaGen); + BuildHandleHistogramProbeSchemaGen schemaGen(schema, m_schemaCount); + HandleHistogramProbeVisitor visitor(m_comp, schemaGen); for (Statement* const stmt : block->Statements()) { visitor.WalkTree(stmt->GetRootNodePointer(), nullptr); @@ -1677,16 +1771,16 @@ void ClassProbeInstrumentor::BuildSchemaElements(BasicBlock* block, Schema& sche } //------------------------------------------------------------------------ -// ClassProbeInstrumentor::Instrument: add class probes to block +// HandleHistogramProbeInstrumentor::Instrument: add class probes to block // // Arguments: // block -- block of interest // schema -- instrumentation schema // profileMemory -- profile data slab // -void ClassProbeInstrumentor::Instrument(BasicBlock* block, Schema& schema, uint8_t* profileMemory) +void HandleHistogramProbeInstrumentor::Instrument(BasicBlock* block, Schema& schema, uint8_t* profileMemory) { - if ((block->bbFlags & BBF_HAS_CLASS_PROFILE) == 0) + if ((block->bbFlags & BBF_HAS_HISTOGRAM_PROFILE) == 0) { return; } @@ -1698,11 +1792,11 @@ void ClassProbeInstrumentor::Instrument(BasicBlock* block, Schema& schema, uint8 // Scan the statements and add class probes // - int classSchemaIndex = block->bbClassSchemaIndex; - assert((classSchemaIndex >= 0) && (classSchemaIndex < (int)schema.size())); + int histogramSchemaIndex = block->bbHistogramSchemaIndex; + assert((histogramSchemaIndex >= 0) && (histogramSchemaIndex < (int)schema.size())); - ClassProbeInserter insertProbes(schema, profileMemory, &classSchemaIndex, m_instrCount); - ClassProbeVisitor visitor(m_comp, insertProbes); + HandleHistogramProbeInserter insertProbes(schema, profileMemory, &histogramSchemaIndex, m_instrCount); + HandleHistogramProbeVisitor visitor(m_comp, insertProbes); for (Statement* const stmt : block->Statements()) { visitor.WalkTree(stmt->GetRootNodePointer(), nullptr); @@ -1791,24 +1885,24 @@ PhaseStatus Compiler::fgPrepareToInstrumentMethod() // Enable class profiling by default, when jitting. // Todo: we may also want this on by default for prejitting. // - const bool useClassProfiles = (JitConfig.JitClassProfiling() > 0) && !prejit; - if (useClassProfiles) + const bool useClassProfiles = (JitConfig.JitClassProfiling() > 0); + const bool useMethodProfiles = (JitConfig.JitMethodProfiling() > 0); + if (!prejit && (useClassProfiles || useMethodProfiles)) { - fgClassInstrumentor = new (this, CMK_Pgo) ClassProbeInstrumentor(this); + fgHistogramInstrumentor = new (this, CMK_Pgo) HandleHistogramProbeInstrumentor(this); } else { - JITDUMP("Not doing class profiling, because %s\n", - (JitConfig.JitClassProfiling() > 0) ? "class profiles disabled" : "prejit"); + JITDUMP("Not doing class/method profiling, because %s\n", prejit ? "prejit" : "class/method profiles disabled"); - fgClassInstrumentor = new (this, CMK_Pgo) NonInstrumentor(this); + fgHistogramInstrumentor = new (this, CMK_Pgo) NonInstrumentor(this); } // Make pre-import preparations. // const bool isPreImport = true; fgCountInstrumentor->Prepare(isPreImport); - fgClassInstrumentor->Prepare(isPreImport); + fgHistogramInstrumentor->Prepare(isPreImport); return PhaseStatus::MODIFIED_NOTHING; } @@ -1837,7 +1931,7 @@ PhaseStatus Compiler::fgInstrumentMethod() // const bool isPreImport = false; fgCountInstrumentor->Prepare(isPreImport); - fgClassInstrumentor->Prepare(isPreImport); + fgHistogramInstrumentor->Prepare(isPreImport); // Walk the flow graph to build up the instrumentation schema. // @@ -1849,27 +1943,12 @@ PhaseStatus Compiler::fgInstrumentMethod() fgCountInstrumentor->BuildSchemaElements(block, schema); } - if (fgClassInstrumentor->ShouldProcess(block)) + if (fgHistogramInstrumentor->ShouldProcess(block)) { - fgClassInstrumentor->BuildSchemaElements(block, schema); + fgHistogramInstrumentor->BuildSchemaElements(block, schema); } } - // Verify we created schema for the calls needing class probes. - // (we counted those when importing) - // - // This is not true when we do partial compilation; it can/will erase class probes, - // and there's no easy way to figure out how many should be left. - // - if (doesMethodHavePartialCompilationPatchpoints()) - { - assert(fgClassInstrumentor->SchemaCount() <= info.compClassProbeCount); - } - else - { - assert(fgClassInstrumentor->SchemaCount() == info.compClassProbeCount); - } - // Optionally, when jitting, if there were no class probes and only one count probe, // suppress instrumentation. // @@ -1889,7 +1968,7 @@ PhaseStatus Compiler::fgInstrumentMethod() minimalProbeMode = (JitConfig.JitMinimalJitProfiling() > 0); } - if (minimalProbeMode && (fgCountInstrumentor->SchemaCount() == 1) && (fgClassInstrumentor->SchemaCount() == 0)) + if (minimalProbeMode && (fgCountInstrumentor->SchemaCount() == 1) && (fgHistogramInstrumentor->SchemaCount() == 0)) { JITDUMP( "Not instrumenting method: minimal probing enabled, and method has only one counter and no class probes\n"); @@ -1897,7 +1976,7 @@ PhaseStatus Compiler::fgInstrumentMethod() } JITDUMP("Instrumenting method: %d count probes and %d class probes\n", fgCountInstrumentor->SchemaCount(), - fgClassInstrumentor->SchemaCount()); + fgHistogramInstrumentor->SchemaCount()); assert(schema.size() > 0); @@ -1930,7 +2009,7 @@ PhaseStatus Compiler::fgInstrumentMethod() // Do any cleanup we might need to do... // fgCountInstrumentor->SuppressProbes(); - fgClassInstrumentor->SuppressProbes(); + fgHistogramInstrumentor->SuppressProbes(); // If we needed to create cheap preds, we're done with them now. // @@ -1941,7 +2020,7 @@ PhaseStatus Compiler::fgInstrumentMethod() // We may have modified control flow preparing for instrumentation. // - const bool modifiedFlow = fgCountInstrumentor->ModifiedFlow() || fgClassInstrumentor->ModifiedFlow(); + const bool modifiedFlow = fgCountInstrumentor->ModifiedFlow() || fgHistogramInstrumentor->ModifiedFlow(); return modifiedFlow ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } @@ -1956,22 +2035,25 @@ PhaseStatus Compiler::fgInstrumentMethod() fgCountInstrumentor->Instrument(block, schema, profileMemory); } - if (fgClassInstrumentor->ShouldProcess(block)) + if (fgHistogramInstrumentor->ShouldProcess(block)) { - fgClassInstrumentor->Instrument(block, schema, profileMemory); + fgHistogramInstrumentor->Instrument(block, schema, profileMemory); } } // Verify we instrumented everthing we created schemas for. // assert(fgCountInstrumentor->InstrCount() == fgCountInstrumentor->SchemaCount()); - assert(fgClassInstrumentor->InstrCount() == fgClassInstrumentor->SchemaCount()); + + // Verify we instrumented for each probe + // + assert(fgHistogramInstrumentor->InstrCount() == info.compHandleHistogramProbeCount); // Add any special entry instrumentation. This does not // use the schema mechanism. // fgCountInstrumentor->InstrumentMethodEntry(schema, profileMemory); - fgClassInstrumentor->InstrumentMethodEntry(schema, profileMemory); + fgHistogramInstrumentor->InstrumentMethodEntry(schema, profileMemory); // If we needed to create cheap preds, we're done with them now. // diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index bf064f08b6ae85..d597ee8bcce30d 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -150,7 +150,7 @@ struct BasicBlock; enum BasicBlockFlags : unsigned __int64; struct InlineCandidateInfo; struct GuardedDevirtualizationCandidateInfo; -struct ClassProfileCandidateInfo; +struct HandleHistogramProfileCandidateInfo; struct LateDevirtualizationInfo; typedef unsigned short AssertionIndex; @@ -4768,7 +4768,7 @@ struct GenTreeCall final : public GenTree // gtInlineCandidateInfo is only used when inlining methods InlineCandidateInfo* gtInlineCandidateInfo; GuardedDevirtualizationCandidateInfo* gtGuardedDevirtualizationCandidateInfo; - ClassProfileCandidateInfo* gtClassProfileCandidateInfo; + HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo; LateDevirtualizationInfo* gtLateDevirtualizationInfo; CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index aec26c053ebaa7..dabf0b198c9d0d 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -9627,6 +9627,8 @@ var_types Compiler::impImportCall(OPCODE opcode, } } + impConsiderCallProbe(call->AsCall(), rawILOffset); + //------------------------------------------------------------------------- // The "this" pointer for "newobj" @@ -10059,7 +10061,7 @@ var_types Compiler::impImportCall(OPCODE opcode, // important devirtualizations, we'll want to allow both a class probe and a captured context. // if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) && - (origCall->gtClassProfileCandidateInfo == nullptr)) + (origCall->gtHandleHistogramProfileCandidateInfo == nullptr)) { JITDUMP("\nSaving context %p for call [%06u]\n", exactContextHnd, dspTreeID(origCall)); origCall->gtCallMoreFlags |= GTF_CALL_M_LATE_DEVIRT; @@ -11737,11 +11739,11 @@ GenTree* Compiler::impCastClassOrIsInstToTree( GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, gtNewCallArgs(op2, op1)); if (impIsCastHelperEligibleForClassProbe(call) && !impIsClassExact(pResolvedToken->hClass)) { - ClassProfileCandidateInfo* pInfo = new (this, CMK_Inlining) ClassProfileCandidateInfo; - pInfo->ilOffset = ilOffset; - pInfo->probeIndex = info.compClassProbeCount++; - call->gtClassProfileCandidateInfo = pInfo; - compCurBB->bbFlags |= BBF_HAS_CLASS_PROFILE; + HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; + pInfo->ilOffset = ilOffset; + pInfo->probeIndex = info.compHandleHistogramProbeCount++; + call->gtHandleHistogramProfileCandidateInfo = pInfo; + compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; } return call; } @@ -20976,45 +20978,6 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, // assert(call->IsVirtual()); - // Possibly instrument. Note for OSR+PGO we will instrument when - // optimizing and (currently) won't devirtualize. We may want - // to revisit -- if we can devirtualize we should be able to - // suppress the probe. - // - // We strip BBINSTR from inlinees currently, so we'll only - // do this for the root method calls. - // - if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR)) - { - assert(opts.OptimizationDisabled() || opts.IsOSR()); - assert(!compIsForInlining()); - - // During importation, optionally flag this block as one that - // contains calls requiring class profiling. Ideally perhaps - // we'd just keep track of the calls themselves, so we don't - // have to search for them later. - // - if ((call->gtCallType != CT_INDIRECT) && opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && - !opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT) && (JitConfig.JitClassProfiling() > 0) && - !isLateDevirtualization) - { - JITDUMP("\n ... marking [%06u] in " FMT_BB " for class profile instrumentation\n", dspTreeID(call), - compCurBB->bbNum); - ClassProfileCandidateInfo* pInfo = new (this, CMK_Inlining) ClassProfileCandidateInfo; - - // Record some info needed for the class profiling probe. - // - pInfo->ilOffset = ilOffset; - pInfo->probeIndex = info.compClassProbeCount++; - call->gtClassProfileCandidateInfo = pInfo; - - // Flag block as needing scrutiny - // - compCurBB->bbFlags |= BBF_HAS_CLASS_PROFILE; - } - return; - } - // Bail if optimizations are disabled. if (opts.OptimizationDisabled()) { @@ -21682,6 +21645,84 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, #endif // FEATURE_READYTORUN } +void Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) +{ + // Possibly instrument. Note for OSR+PGO we will instrument when + // optimizing and (currently) won't devirtualize. We may want + // to revisit -- if we can devirtualize we should be able to + // suppress the probe. + // + // We strip BBINSTR from inlinees currently, so we'll only + // do this for the root method calls. + // + if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR)) + { + return; + } + + assert(opts.OptimizationDisabled() || opts.IsOSR()); + assert(!compIsForInlining()); + + // During importation, optionally flag this block as one that + // contains calls requiring class profiling. Ideally perhaps + // we'd just keep track of the calls themselves, so we don't + // have to search for them later. + // + if ((compClassifyGDVProbeType(call) != GDVProbeType::None)) + { + JITDUMP("\n ... marking [%06u] in " FMT_BB " for method/class profile instrumentation\n", dspTreeID(call), + compCurBB->bbNum); + HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; + + // Record some info needed for the class profiling probe. + // + pInfo->ilOffset = ilOffset; + pInfo->probeIndex = info.compHandleHistogramProbeCount++; + call->gtHandleHistogramProfileCandidateInfo = pInfo; + + // Flag block as needing scrutiny + // + compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; + } +} + +Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) +{ + if (call->gtCallType == CT_INDIRECT) + { + return GDVProbeType::None; + } + + if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) || opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT)) + { + return GDVProbeType::None; + } + + bool createTypeHistogram = + (JitConfig.JitClassProfiling() > 0) && + (call->IsVirtualStub() || call->IsVirtualVtable() || impIsCastHelperEligibleForClassProbe(call)); + + bool createMethodHistogram = + (JitConfig.JitMethodProfiling() > 0) && (call->IsVirtualVtable() || call->IsDelegateInvoke()); + + if (createTypeHistogram && createMethodHistogram) + { + return GDVProbeType::MethodAndClassProfile; + } + + if (createTypeHistogram) + { + return GDVProbeType::ClassProfile; + } + + if (createMethodHistogram) + { + return GDVProbeType::MethodProfile; + } + + return GDVProbeType::None; +} + //------------------------------------------------------------------------ // impGetSpecialIntrinsicExactReturnType: Look for special cases where a call // to an intrinsic returns an exact type diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index 7182bdf78fa92a..80fa3cf6dbe666 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -547,10 +547,10 @@ class InlineResult bool m_Reported; }; -// ClassProfileCandidateInfo provides information about +// HandleHistogramProfileCandidateInfo provides information about // profiling an indirect or virtual call. // -struct ClassProfileCandidateInfo +struct HandleHistogramProfileCandidateInfo { IL_OFFSET ilOffset; unsigned probeIndex; @@ -559,7 +559,7 @@ struct ClassProfileCandidateInfo // GuardedDevirtualizationCandidateInfo provides information about // a potential target of a virtual or interface call. // -struct GuardedDevirtualizationCandidateInfo : ClassProfileCandidateInfo +struct GuardedDevirtualizationCandidateInfo : HandleHistogramProfileCandidateInfo { CORINFO_CLASS_HANDLE guardedClassHandle; CORINFO_METHOD_HANDLE guardedMethodHandle; diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index f3bef3cf014755..df807c0614b0ea 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -503,8 +503,9 @@ CONFIG_INTEGER(JitProfileCasts, W("JitProfileCasts"), 0) // CONFIG_INTEGER(JitConsumeProfileForCasts, W("JitConsumeProfileForCasts"), 0) // Consume profile data (if any) for // castclass/isinst -CONFIG_INTEGER(JitClassProfiling, W("JitClassProfiling"), 1) // Profile virtual and interface calls -CONFIG_INTEGER(JitEdgeProfiling, W("JitEdgeProfiling"), 1) // Profile edges instead of blocks +CONFIG_INTEGER(JitClassProfiling, W("JitClassProfiling"), 1) // Profile virtual and interface calls +CONFIG_INTEGER(JitMethodProfiling, W("JitMethodProfiling"), 0) // Profile resolved delegate and vtable call targets +CONFIG_INTEGER(JitEdgeProfiling, W("JitEdgeProfiling"), 1) // Profile edges instead of blocks CONFIG_INTEGER(JitCollect64BitCounts, W("JitCollect64BitCounts"), 0) // Collect counts as 64-bit values. // Profile consumption options diff --git a/src/coreclr/jit/patchpoint.cpp b/src/coreclr/jit/patchpoint.cpp index 15a37150326183..53719cfeb9a22a 100644 --- a/src/coreclr/jit/patchpoint.cpp +++ b/src/coreclr/jit/patchpoint.cpp @@ -78,7 +78,7 @@ class PatchpointTransformer // If we're instrumenting, we should not have decided to // put class probes here, as that is driven by looking at IL. // - assert((block->bbFlags & BBF_HAS_CLASS_PROFILE) == 0); + assert((block->bbFlags & BBF_HAS_HISTOGRAM_PROFILE) == 0); // Clear the partial comp flag. // diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 11b9ff7bf7f5d7..0def558c4a9621 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -5313,7 +5313,7 @@ void JIT_PartialCompilationPatchpoint(int* counter, int ilOffset) #endif // FEATURE_ON_STACK_REPLACEMENT -static unsigned ClassProfileRand() +static unsigned HandleHistogramProfileRand() { // generate a random number (xorshift32) // @@ -5330,7 +5330,7 @@ static unsigned ClassProfileRand() return x; } -HCIMPL2(void, JIT_ClassProfile32, Object *obj, void* tableAddress) +HCIMPL2(void, JIT_ClassProfile32, Object *obj, ICorJitInfo::HandleHistogram32* classProfile) { FCALL_CONTRACT; FC_GC_POLL_NOT_NEEDED(); @@ -5338,11 +5338,10 @@ HCIMPL2(void, JIT_ClassProfile32, Object *obj, void* tableAddress) OBJECTREF objRef = ObjectToOBJECTREF(obj); VALIDATEOBJECTREF(objRef); - ICorJitInfo::ClassProfile32* const classProfile = (ICorJitInfo::ClassProfile32*) tableAddress; volatile unsigned* pCount = (volatile unsigned*) &classProfile->Count; const unsigned count = (*pCount)++; - const unsigned S = ICorJitInfo::ClassProfile32::SIZE; - const unsigned N = ICorJitInfo::ClassProfile32::SAMPLE_INTERVAL; + const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; + const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; _ASSERTE(N >= S); if (objRef == NULL) @@ -5358,7 +5357,7 @@ HCIMPL2(void, JIT_ClassProfile32, Object *obj, void* tableAddress) // if (pMT->GetLoaderAllocator()->IsCollectible()) { - pMT = (MethodTable*)DEFAULT_UNKNOWN_TYPEHANDLE; + pMT = (MethodTable*)DEFAULT_UNKNOWN_HANDLE; } #ifdef _DEBUG @@ -5370,11 +5369,11 @@ HCIMPL2(void, JIT_ClassProfile32, Object *obj, void* tableAddress) // if (count < S) { - classProfile->ClassTable[count] = (CORINFO_CLASS_HANDLE)pMT; + classProfile->HandleTable[count] = (CORINFO_CLASS_HANDLE)pMT; } else { - unsigned x = ClassProfileRand(); + unsigned x = HandleHistogramProfileRand(); // N is the sampling window size, // it should be larger than the table size. @@ -5391,14 +5390,14 @@ HCIMPL2(void, JIT_ClassProfile32, Object *obj, void* tableAddress) if ((x % N) < S) { unsigned i = x % S; - classProfile->ClassTable[i] = (CORINFO_CLASS_HANDLE)pMT; + classProfile->HandleTable[i] = (CORINFO_CLASS_HANDLE)pMT; } } } HCIMPLEND // Version of helper above used when the count is 64-bit -HCIMPL2(void, JIT_ClassProfile64, Object *obj, void* tableAddress) +HCIMPL2(void, JIT_ClassProfile64, Object *obj, ICorJitInfo::HandleHistogram64* classProfile) { FCALL_CONTRACT; FC_GC_POLL_NOT_NEEDED(); @@ -5406,11 +5405,10 @@ HCIMPL2(void, JIT_ClassProfile64, Object *obj, void* tableAddress) OBJECTREF objRef = ObjectToOBJECTREF(obj); VALIDATEOBJECTREF(objRef); - ICorJitInfo::ClassProfile64* const classProfile = (ICorJitInfo::ClassProfile64*) tableAddress; volatile uint64_t* pCount = (volatile uint64_t*) &classProfile->Count; const uint64_t count = (*pCount)++; - const unsigned S = ICorJitInfo::ClassProfile32::SIZE; - const unsigned N = ICorJitInfo::ClassProfile32::SAMPLE_INTERVAL; + const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; + const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; _ASSERTE(N >= S); if (objRef == NULL) @@ -5422,7 +5420,7 @@ HCIMPL2(void, JIT_ClassProfile64, Object *obj, void* tableAddress) if (pMT->GetLoaderAllocator()->IsCollectible()) { - pMT = (MethodTable*)DEFAULT_UNKNOWN_TYPEHANDLE; + pMT = (MethodTable*)DEFAULT_UNKNOWN_HANDLE; } #ifdef _DEBUG @@ -5432,16 +5430,295 @@ HCIMPL2(void, JIT_ClassProfile64, Object *obj, void* tableAddress) if (count < S) { - classProfile->ClassTable[count] = (CORINFO_CLASS_HANDLE)pMT; + classProfile->HandleTable[count] = (CORINFO_CLASS_HANDLE)pMT; } else { - unsigned x = ClassProfileRand(); + unsigned x = HandleHistogramProfileRand(); if ((x % N) < S) { unsigned i = x % S; - classProfile->ClassTable[i] = (CORINFO_CLASS_HANDLE)pMT; + classProfile->HandleTable[i] = (CORINFO_CLASS_HANDLE)pMT; + } + } +} +HCIMPLEND + +HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod, ICorJitInfo::HandleHistogram32* methodProfile, ICorJitInfo::HandleHistogram32* typeProfile) +{ + FCALL_CONTRACT; + FC_GC_POLL_NOT_NEEDED(); + + OBJECTREF objRef = ObjectToOBJECTREF(obj); + VALIDATEOBJECTREF(objRef); + + volatile unsigned* pMethodCount = (volatile unsigned*) &methodProfile->Count; + const unsigned methodCount = (*pMethodCount)++; + const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; + const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; + _ASSERTE(N >= S); + + unsigned typeCount = 0; + if (typeProfile != nullptr) + { + volatile unsigned* pTypeCount = (volatile unsigned*) &typeProfile->Count; + typeCount = (*pTypeCount)++; + } + + if (objRef == NULL) + { + return; + } + + MethodDesc* pBaseMD = GetMethod(baseMethod); + + // Method better be virtual + _ASSERTE(pBaseMD->IsVirtual()); + + // We currently do not expect to see interface methods here as we cannot + // efficiently use method handle information for these anyway. + _ASSERTE(!pBaseMD->IsInterface()); + + // Shouldn't be doing this for instantiated methods as they live elsewhere + _ASSERTE(!pBaseMD->HasMethodInstantiation()); + + MethodTable* pMT = objRef->GetMethodTable(); + + // Resolve method + MethodDesc* pMD = NULL; + if (pMT->IsDelegate()) + { + // We handle only the common "direct" delegate as that is in any case + // the only one we can reasonably do GDV for. For instance, open + // delegates are filtered out here, and many cases with inner + // "complicated" logic as well (e.g. multicast, unmanaged functions). + // + // Note that this covers all cases of delegates that can be created + // without the use of reflection. + // + DELEGATEREF del = (DELEGATEREF)objRef; + if (del->GetInvocationCount() == 0 && del->GetMethodPtrAux() == NULL) + { + PCODE code = del->GetMethodPtr(); + pMD = NonVirtualEntry2MethodDesc(code); + } + + if (pMD == NULL) + { + return; + } + } + else + { + WORD slot = pBaseMD->GetSlot(); + _ASSERTE(slot < pBaseMD->GetMethodTable()->GetNumVirtuals()); + + pMD = pMT->GetMethodDescForSlot(slot); + } + + // If the object class is collectible, record an unknown handle. + // We do this instead of recording NULL so that we won't over-estimate + // the likelihood of known handles. + // + if (pMD->GetLoaderAllocator()->IsCollectible()) + { + pMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + } + +#ifdef _DEBUG + PgoManager::VerifyAddress(methodProfile); + PgoManager::VerifyAddress(methodProfile + 1); +#endif + + // If table is not yet full, just add entries in. + // + if (methodCount < S) + { + methodProfile->HandleTable[methodCount] = (CORINFO_METHOD_HANDLE)pMD; + } + else + { + unsigned x = HandleHistogramProfileRand(); + + // N is the sampling window size, + // it should be larger than the table size. + // + // If we let N == count then we are building an entire + // run sample -- probability of update decreases over time. + // Would be a good strategy for an AOT profiler. + // + // But for TieredPGO we would prefer something that is more + // weighted to recent observations. + // + // For S=4, N=128, we'll sample (on average) every 32nd call. + // + if ((x % N) < S) + { + unsigned i = x % S; + methodProfile->HandleTable[i] = (CORINFO_METHOD_HANDLE)pMD; + } + } + + if (typeProfile != nullptr) + { + if (pMT->GetLoaderAllocator()->IsCollectible()) + { + pMT = (MethodTable*)DEFAULT_UNKNOWN_HANDLE; + } + + if (typeCount < S) + { + typeProfile->HandleTable[typeCount] = (CORINFO_CLASS_HANDLE)pMT; + } + else + { + unsigned x = HandleHistogramProfileRand(); + if ((x % N) < S) + { + unsigned i = x % S; + typeProfile->HandleTable[i] = (CORINFO_CLASS_HANDLE)pMT; + } + } + } +} +HCIMPLEND + +// Version of helper above used when the count is 64-bit +HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod, ICorJitInfo::HandleHistogram64* methodProfile, ICorJitInfo::HandleHistogram64* typeProfile) +{ + FCALL_CONTRACT; + FC_GC_POLL_NOT_NEEDED(); + + OBJECTREF objRef = ObjectToOBJECTREF(obj); + VALIDATEOBJECTREF(objRef); + + volatile uint64_t* pMethodCount = (volatile uint64_t*) &methodProfile->Count; + const uint64_t methodCount = (*pMethodCount)++; + const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; + const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; + _ASSERTE(N >= S); + + unsigned typeCount = 0; + if (typeProfile != nullptr) + { + volatile unsigned* pTypeCount = (volatile unsigned*) &typeProfile->Count; + typeCount = (*pTypeCount)++; + } + + if (objRef == NULL) + { + return; + } + + MethodDesc* pBaseMD = GetMethod(baseMethod); + + // Method better be virtual + _ASSERTE(pBaseMD->IsVirtual()); + + // We currently do not expect to see interface methods here as we cannot + // efficiently use method handle information for these anyway. + _ASSERTE(!pBaseMD->IsInterface()); + + // Shouldn't be doing this for instantiated methods as they live elsewhere + _ASSERTE(!pBaseMD->HasMethodInstantiation()); + + MethodTable* pMT = objRef->GetMethodTable(); + + // Resolve method + MethodDesc* pMD = NULL; + if (pMT->IsDelegate()) + { + // We handle only the common "direct" delegate as that is in any case + // the only one we can reasonably do GDV for. For instance, open + // delegates are filtered out here, and many cases with inner + // "complicated" logic as well (e.g. multicast, unmanaged functions). + // + // Note that this covers all cases of delegates that can be created + // without the use of reflection. + // + DELEGATEREF del = (DELEGATEREF)objRef; + if (del->GetInvocationCount() == 0 && del->GetMethodPtrAux() == NULL) + { + PCODE code = del->GetMethodPtr(); + pMD = NonVirtualEntry2MethodDesc(code); + } + + if (pMD == NULL) + { + return; + } + } + else + { + WORD slot = pBaseMD->GetSlot(); + _ASSERTE(slot < pBaseMD->GetMethodTable()->GetNumVirtuals()); + + pMD = pMT->GetMethodDescForSlot(slot); + } + + // If the object class is collectible, record an unknown handle. + // We do this instead of recording NULL so that we won't over-estimate + // the likelihood of known handles. + // + if (pMD->GetLoaderAllocator()->IsCollectible()) + { + pMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + } + +#ifdef _DEBUG + PgoManager::VerifyAddress(methodProfile); + PgoManager::VerifyAddress(methodProfile + 1); +#endif + + // If table is not yet full, just add entries in. + // + if (methodCount < S) + { + methodProfile->HandleTable[methodCount] = (CORINFO_METHOD_HANDLE)pMD; + } + else + { + unsigned x = HandleHistogramProfileRand(); + + // N is the sampling window size, + // it should be larger than the table size. + // + // If we let N == count then we are building an entire + // run sample -- probability of update decreases over time. + // Would be a good strategy for an AOT profiler. + // + // But for TieredPGO we would prefer something that is more + // weighted to recent observations. + // + // For S=4, N=128, we'll sample (on average) every 32nd call. + // + if ((x % N) < S) + { + unsigned i = x % S; + methodProfile->HandleTable[i] = (CORINFO_METHOD_HANDLE)pMD; + } + } + + if (typeProfile != nullptr) + { + if (pMT->GetLoaderAllocator()->IsCollectible()) + { + pMT = (MethodTable*)DEFAULT_UNKNOWN_HANDLE; + } + + if (typeCount < S) + { + typeProfile->HandleTable[typeCount] = (CORINFO_CLASS_HANDLE)pMT; + } + else + { + unsigned x = HandleHistogramProfileRand(); + if ((x % N) < S) + { + unsigned i = x % S; + typeProfile->HandleTable[i] = (CORINFO_CLASS_HANDLE)pMT; + } } } } From 2cbe83f5c094931bbf5f7cc911cd65cdf81cd2ab Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 14 Apr 2022 13:37:25 +0200 Subject: [PATCH 04/59] Handle static methods and also move fcall handling --- src/coreclr/vm/jithelpers.cpp | 28 ++++++++++++++++++++++------ src/coreclr/vm/method.cpp | 12 +++++++----- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 0def558c4a9621..d1d5b0cd884613 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -5498,10 +5498,18 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod // without the use of reflection. // DELEGATEREF del = (DELEGATEREF)objRef; - if (del->GetInvocationCount() == 0 && del->GetMethodPtrAux() == NULL) + if (del->GetInvocationCount() == 0) { - PCODE code = del->GetMethodPtr(); - pMD = NonVirtualEntry2MethodDesc(code); + PCODE code = del->GetMethodPtrAux(); + if (code != NULL) + { + pMD = MethodTable::GetMethodDescForSlotAddress(code); + } + else + { + code = del->GetMethodPtr(); + pMD = NonVirtualEntry2MethodDesc(code); + } } if (pMD == NULL) @@ -5638,10 +5646,18 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod // without the use of reflection. // DELEGATEREF del = (DELEGATEREF)objRef; - if (del->GetInvocationCount() == 0 && del->GetMethodPtrAux() == NULL) + if (del->GetInvocationCount() == 0) { - PCODE code = del->GetMethodPtr(); - pMD = NonVirtualEntry2MethodDesc(code); + PCODE code = del->GetMethodPtrAux(); + if (code != NULL) + { + pMD = MethodTable::GetMethodDescForSlotAddress(code); + } + else + { + code = del->GetMethodPtr(); + pMD = NonVirtualEntry2MethodDesc(code); + } } if (pMD == NULL) diff --git a/src/coreclr/vm/method.cpp b/src/coreclr/vm/method.cpp index 08e4a3a278da6e..15f3d8d64aa88e 100644 --- a/src/coreclr/vm/method.cpp +++ b/src/coreclr/vm/method.cpp @@ -2137,6 +2137,13 @@ MethodDesc* NonVirtualEntry2MethodDesc(PCODE entryPoint) return (MethodDesc*)((FixupPrecode*)pInstr)->GetMethodDesc(); } + // Is it an FCALL? + MethodDesc* pFCallMD = ECall::MapTargetBackToMethod(entryPoint); + if (pFCallMD != NULL) + { + return pFCallMD; + } + return NULL; } @@ -2173,11 +2180,6 @@ MethodDesc* Entry2MethodDesc(PCODE entryPoint, MethodTable *pMT) if (pMD != NULL) RETURN(pMD); - // Is it an FCALL? - pMD = ECall::MapTargetBackToMethod(entryPoint); - if (pMD != NULL) - RETURN(pMD); - // We should never get here _ASSERTE(!"Entry2MethodDesc failed"); RETURN (NULL); From 5b31bee3c029ad5b9efe120da54aa8126fbb221c Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 14 Apr 2022 13:56:50 +0200 Subject: [PATCH 05/59] Fix writing out methods --- src/coreclr/inc/corjit.h | 4 ++-- src/coreclr/inc/eventtracebase.h | 4 ++-- src/coreclr/inc/pgo_formatprocessing.h | 19 +++++++++++++++++-- src/coreclr/vm/eventtrace.cpp | 14 +++++++++++--- src/coreclr/vm/pgo.cpp | 20 +++++++++++++++++--- 5 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/coreclr/inc/corjit.h b/src/coreclr/inc/corjit.h index ac44b8a9c8f3db..5a16650bbe938d 100644 --- a/src/coreclr/inc/corjit.h +++ b/src/coreclr/inc/corjit.h @@ -421,9 +421,9 @@ class ICorJitInfo : public ICorDynamicInfo #define UNKNOWN_HANDLE_MIN 1 #define UNKNOWN_HANDLE_MAX 33 - static inline bool IsUnknownHandle(intptr_t typeHandle) + static inline bool IsUnknownHandle(intptr_t handle) { - return ((typeHandle >= UNKNOWN_HANDLE_MIN) && (typeHandle <= UNKNOWN_HANDLE_MAX)); + return ((handle >= UNKNOWN_HANDLE_MIN) && (handle <= UNKNOWN_HANDLE_MAX)); } // get profile information to be used for optimizing a current method. The format diff --git a/src/coreclr/inc/eventtracebase.h b/src/coreclr/inc/eventtracebase.h index f29f9fa27c7a08..dd1332581e8f68 100644 --- a/src/coreclr/inc/eventtracebase.h +++ b/src/coreclr/inc/eventtracebase.h @@ -939,7 +939,7 @@ namespace ETW static VOID MethodRestored(MethodDesc * pMethodDesc); static VOID MethodTableRestored(MethodTable * pMethodTable); static VOID DynamicMethodDestroyed(MethodDesc *pMethodDesc); - static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t typeHandles); + static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t numTypeHandles, MethodDesc** pMethods, uint32_t numMethods); #else // FEATURE_EVENT_TRACE public: static VOID GetR2RGetEntryPointStart(MethodDesc *pMethodDesc) {}; @@ -951,7 +951,7 @@ namespace ETW static VOID MethodRestored(MethodDesc * pMethodDesc) {}; static VOID MethodTableRestored(MethodTable * pMethodTable) {}; static VOID DynamicMethodDestroyed(MethodDesc *pMethodDesc) {}; - static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t typeHandles) {}; + static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t numTypeHandles, MethodDesc** pMethods, uint32_t numMethods) {}; #endif // FEATURE_EVENT_TRACE }; diff --git a/src/coreclr/inc/pgo_formatprocessing.h b/src/coreclr/inc/pgo_formatprocessing.h index 175d7d8786ba5e..d761ad6bcbe483 100644 --- a/src/coreclr/inc/pgo_formatprocessing.h +++ b/src/coreclr/inc/pgo_formatprocessing.h @@ -518,6 +518,7 @@ class SchemaAndDataWriter ICorJitInfo::PgoInstrumentationSchema prevSchema = {}; int64_t lastIntDataWritten = 0; int64_t lastTypeDataWritten = 0; + int64_t lastMethodDataWritten = 0; public: SchemaAndDataWriter(const ByteWriter& byteWriter, uint8_t* pInstrumentationData) : @@ -534,8 +535,8 @@ class SchemaAndDataWriter return true; } - template - bool AppendDataFromLastSchema(TypeHandleProcessor& thProcessor) + template + bool AppendDataFromLastSchema(TypeHandleProcessor& thProcessor, MethodHandleProcessor& mhProcessor) { uint8_t *pData = (pInstrumentationData + prevSchema.Offset); for (int32_t iDataElem = 0; iDataElem < prevSchema.Count; iDataElem++) @@ -579,6 +580,20 @@ class SchemaAndDataWriter pData += sizeof(intptr_t); break; } + case ICorJitInfo::PgoInstrumentationKind::MethodHandle: + { + logicalDataToWrite = *(volatile intptr_t*)pData; + + // As there could be tearing otherwise, inform the caller of exactly what value was written. + mhProcessor(logicalDataToWrite); + + bool returnValue = WriteCompressedIntToBytes(logicalDataToWrite - lastMethodDataWritten, byteWriter); + lastMethodDataWritten = logicalDataToWrite; + if (!returnValue) + return false; + pData += sizeof(intptr_t); + break; + } default: _ASSERTE(!"Unexpected type"); return false; diff --git a/src/coreclr/vm/eventtrace.cpp b/src/coreclr/vm/eventtrace.cpp index 3c5d8ded8c574e..69753e49857572 100644 --- a/src/coreclr/vm/eventtrace.cpp +++ b/src/coreclr/vm/eventtrace.cpp @@ -5407,7 +5407,7 @@ VOID ETW::MethodLog::GetR2RGetEntryPointStart(MethodDesc *pMethodDesc) } } -VOID ETW::MethodLog::LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t typeHandles) +VOID ETW::MethodLog::LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t numTypeHandles, MethodDesc** pMethods, uint32_t numMethods) { CONTRACTL{ NOTHROW; @@ -5424,17 +5424,25 @@ VOID ETW::MethodLog::LogMethodInstrumentationData(MethodDesc* method, uint32_t c SendMethodDetailsEvent(method); // If there are any type handles, fire the BulkType events to describe them - if (typeHandles != 0) + if (numTypeHandles != 0) { BulkTypeEventLogger typeLogger; - for (uint32_t iTypeHandle = 0; iTypeHandle < typeHandles; iTypeHandle++) + for (uint32_t iTypeHandle = 0; iTypeHandle < numTypeHandles; iTypeHandle++) { ETW::TypeSystemLog::LogTypeAndParametersIfNecessary(&typeLogger, (ULONGLONG)pTypeHandles[iTypeHandle].AsPtr(), ETW::TypeSystemLog::kTypeLogBehaviorAlwaysLog); } typeLogger.FireBulkTypeEvent(); } + if (numMethods != 0) + { + for (uint32_t iMethod = 0; iMethod < numMethods; iMethod++) + { + ETW::MethodLog::SendMethodDetailsEvent(pMethods[iMethod]); + } + } + ULONG ulMethodToken=0; auto pModule = method->GetModule_NoLogging(); bool bIsDynamicMethod = method->IsDynamicMethod(); diff --git a/src/coreclr/vm/pgo.cpp b/src/coreclr/vm/pgo.cpp index b3cc609ddec270..ff9634e694c505 100644 --- a/src/coreclr/vm/pgo.cpp +++ b/src/coreclr/vm/pgo.cpp @@ -102,6 +102,7 @@ class SchemaWriterFunctor public: StackSArray byteData; StackSArray typeHandlesEncountered; + StackSArray methodsEncountered; private: const PgoManager::HeaderList *pgoData; SArrayByteWriterFunctor byteWriter; @@ -119,7 +120,7 @@ class SchemaWriterFunctor if (!writer.AppendSchema(schema)) return false; - auto lambda = [&](int64_t thWritten) + auto thProcessor = [&](intptr_t thWritten) { if (thWritten != 0) { @@ -131,7 +132,16 @@ class SchemaWriterFunctor } }; - if (!writer.AppendDataFromLastSchema(lambda)) + auto mhProcessor = [&](intptr_t mhWritten) + { + if (mhWritten == 0 || ICorJitInfo::IsUnknownHandle(mhWritten)) + return; + + MethodDesc* pMD = reinterpret_cast(mhWritten); + methodsEncountered.Append(pMD); + }; + + if (!writer.AppendDataFromLastSchema(thProcessor, mhProcessor)) { return false; } @@ -151,7 +161,11 @@ void PgoManager::WritePgoData() { if (!schemaWriter.writer.Finish()) return false; - ETW::MethodLog::LogMethodInstrumentationData(pgoData->header.method, schemaWriter.byteData.GetCount(), schemaWriter.byteData.GetElements(), schemaWriter.typeHandlesEncountered.GetElements(), schemaWriter.typeHandlesEncountered.GetCount()); + ETW::MethodLog::LogMethodInstrumentationData( + pgoData->header.method, + schemaWriter.byteData.GetCount(), schemaWriter.byteData.GetElements(), + schemaWriter.typeHandlesEncountered.GetElements(), schemaWriter.typeHandlesEncountered.GetCount(), + schemaWriter.methodsEncountered.GetElements(), schemaWriter.methodsEncountered.GetCount()); } return true; From 759614628d8085b06389841e5323c78ab696ef5e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 15 Apr 2022 13:53:30 +0200 Subject: [PATCH 06/59] Remove an unnecessary check --- src/coreclr/vm/eventtrace.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/coreclr/vm/eventtrace.cpp b/src/coreclr/vm/eventtrace.cpp index 69753e49857572..ebf8d8ee36311c 100644 --- a/src/coreclr/vm/eventtrace.cpp +++ b/src/coreclr/vm/eventtrace.cpp @@ -5435,12 +5435,9 @@ VOID ETW::MethodLog::LogMethodInstrumentationData(MethodDesc* method, uint32_t c typeLogger.FireBulkTypeEvent(); } - if (numMethods != 0) + for (uint32_t iMethod = 0; iMethod < numMethods; iMethod++) { - for (uint32_t iMethod = 0; iMethod < numMethods; iMethod++) - { - ETW::MethodLog::SendMethodDetailsEvent(pMethods[iMethod]); - } + ETW::MethodLog::SendMethodDetailsEvent(pMethods[iMethod]); } ULONG ulMethodToken=0; From 5114f2cb3b2709b9ac83c219a9e91eed9218d7b5 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 15 Apr 2022 14:00:12 +0200 Subject: [PATCH 07/59] Fix build and some logic inversions --- src/coreclr/vm/pgo.cpp | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/coreclr/vm/pgo.cpp b/src/coreclr/vm/pgo.cpp index 3fcfcef8bdbff5..fd2f2c540a4e7c 100644 --- a/src/coreclr/vm/pgo.cpp +++ b/src/coreclr/vm/pgo.cpp @@ -122,15 +122,13 @@ class SchemaWriterFunctor auto thProcessor = [&](intptr_t thWritten) { - if (ICorJitInfo::IsUnknownTypeHandle(thWritten)) return; + if (thWritten == 0 || ICorJitInfo::IsUnknownHandle(thWritten)) + return; - if (thWritten != 0) + TypeHandle th = *(TypeHandle*)&thWritten; + if (!th.IsNull()) { - TypeHandle th = *(TypeHandle*)&thWritten; - if (!th.IsNull()) - { - typeHandlesEncountered.Append(th); - } + typeHandlesEncountered.Append(th); } return; }; @@ -257,7 +255,7 @@ void PgoManager::WritePgoData() { fprintf(pgoDataFile, s_TypeHandle, "NULL"); } - else if (ICorJitInfo::IsUnknownTypeHandle(typehandleData)) + else if (ICorJitInfo::IsUnknownHandle(typehandleData)) { fprintf(pgoDataFile, s_TypeHandle, "UNKNOWN"); } @@ -268,7 +266,7 @@ void PgoManager::WritePgoData() TypeString::AppendType(ss, th, TypeString::FormatNamespace | TypeString::FormatFullInst | TypeString::FormatAssembly); if (ss.GetCount() > 8192) { - fprintf(pgoDataFile, s_TypeHandle, "unknown"); + fprintf(pgoDataFile, s_TypeHandle, "UNKNOWN"); } else { @@ -279,11 +277,16 @@ void PgoManager::WritePgoData() } case ICorJitInfo::PgoInstrumentationKind::MethodHandle: { - MethodDesc* md = *(MethodDesc**)(data + entryOffset); + intptr_t methodHandleData = *(intptr_t*)(data + entryOffset); + MethodDesc* md = reinterpret_cast(methodHandleData); if (md == nullptr) { fprintf(pgoDataFile, "MethodHandle: NULL"); } + else if (ICorJitInfo::IsUnknownHandle(methodHandleData)) + { + fprintf(pgoDataFile, "MethodHandle: UNKNOWN"); + } else { SString garbage1, tMethodName, garbage2; @@ -294,7 +297,7 @@ void PgoManager::WritePgoData() // MethodName|@|fully_qualified_type_name if (tTypeName.GetCount() + 1 + tMethodName.GetCount() > 8192) { - fprintf(pgoDataFile, "MethodHandle: unknown"); + fprintf(pgoDataFile, "MethodHandle: UNKNOWN"); } else { @@ -511,7 +514,7 @@ void PgoManager::ReadPgoData() TypeHandle th; INT_PTR ptrVal = 0; - if ((strcmp(typeString, "NULL") != 0) || (strcmp(typeString, "UNKNOWN") != 0)) + if ((strcmp(typeString, "NULL") != 0) && (strcmp(typeString, "UNKNOWN") != 0)) { // As early type loading is likely problematic, simply drop the string into the data, and fix it up later void* tempString = malloc(endOfString); @@ -546,7 +549,7 @@ void PgoManager::ReadPgoData() TypeHandle th; INT_PTR ptrVal = 0; - if (strcmp(methodString, "NULL") != 0) + if ((strcmp(methodString, "NULL") != 0) && (strcmp(methodString, "UNKNOWN") != 0)) { // As early type loading is likely problematic, simply drop the string into the data, and fix it up later void* tempString = malloc(endOfString); From decb84d4fa5ba1f55e1aafcea29449544c957771 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 18 Apr 2022 23:00:09 +0200 Subject: [PATCH 08/59] Bump R2R minor version and JIT-EE GUID --- src/coreclr/inc/jiteeversionguid.h | 10 +++++----- src/coreclr/inc/readytorun.h | 2 +- .../tools/Common/Internal/Runtime/ModuleHeaders.cs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index b1b0f06a4ece9c..013fa160ce7370 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID; #define GUID_DEFINED #endif // !GUID_DEFINED -constexpr GUID JITEEVersionIdentifier = { /* 206a7aa6-9f5c-47c1-b63b-54f4cb169ee3 */ - 0x206a7aa6, - 0x9f5c, - 0x47c1, - {0xb6, 0x3b, 0x54, 0xf4, 0xcb, 0x16, 0x9e, 0xe3} +constexpr GUID JITEEVersionIdentifier = { /* 7503fe09-4852-40f6-829a-ff91402c9604 */ + 0x7503fe09, + 0x4852, + 0x40f6, + {0x82, 0x9a, 0xff, 0x91, 0x40, 0x2c, 0x96, 0x04} }; ////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h index 25bd45d376e8fa..e376e29c2cc7cc 100644 --- a/src/coreclr/inc/readytorun.h +++ b/src/coreclr/inc/readytorun.h @@ -16,7 +16,7 @@ // Keep these in sync with src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs #define READYTORUN_MAJOR_VERSION 0x0006 -#define READYTORUN_MINOR_VERSION 0x0000 +#define READYTORUN_MINOR_VERSION 0x0001 #define MINIMUM_READYTORUN_MAJOR_VERSION 0x006 diff --git a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs index 12a9ba0327f74e..47238ec82aa491 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs @@ -15,7 +15,7 @@ internal struct ReadyToRunHeaderConstants public const uint Signature = 0x00525452; // 'RTR' public const ushort CurrentMajorVersion = 6; - public const ushort CurrentMinorVersion = 0; + public const ushort CurrentMinorVersion = 1; } #pragma warning disable 0169 From 01353ed8497950fd7ca27d00b7fee092c4a82f8f Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Apr 2022 19:13:39 +0200 Subject: [PATCH 09/59] Fix after merge --- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/compiler.hpp | 8 +++++++- src/coreclr/jit/fgprofile.cpp | 18 +++++++++++------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 0b5fd0cee96838..3028a9cb3d908e 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -2318,7 +2318,7 @@ class Compiler GenTreeCall* gtNewIndCallNode(GenTree* addr, var_types type, const DebugInfo& di = DebugInfo()); GenTreeCall* gtNewHelperCallNode( - unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr); + unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr, GenTree* arg4 = nullptr); GenTreeCall* gtNewRuntimeLookupHelperCallNode(CORINFO_RUNTIME_LOOKUP* pRuntimeLookup, GenTree* ctxTree, diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index c1040b1f3c275d..a2e53e93fca842 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -997,7 +997,7 @@ inline GenTree* Compiler::gtNewIconEmbFldHndNode(CORINFO_FIELD_HANDLE fldHnd) // New CT_HELPER node inline GenTreeCall* Compiler::gtNewHelperCallNode( - unsigned helper, var_types type, GenTree* arg1, GenTree* arg2, GenTree* arg3) + unsigned helper, var_types type, GenTree* arg1, GenTree* arg2, GenTree* arg3, GenTree* arg4) { GenTreeFlags flags = s_helperCallProperties.NoThrow((CorInfoHelpFunc)helper) ? GTF_EMPTY : GTF_EXCEPT; GenTreeCall* result = gtNewCallNode(CT_HELPER, eeFindHelper(helper), type); @@ -1009,6 +1009,12 @@ inline GenTreeCall* Compiler::gtNewHelperCallNode( result->gtInlineObservation = InlineObservation::CALLSITE_IS_CALL_TO_HELPER; #endif + if (arg4 != nullptr) + { + result->gtArgs.PushFront(this, arg4); + result->gtFlags |= arg4->gtFlags & GTF_ALL_EFFECT; + } + if (arg3 != nullptr) { result->gtArgs.PushFront(this, arg3); diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index cecd5eb43db07a..5291d87ba70b68 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1597,28 +1597,32 @@ class HandleHistogramProbeInserter unsigned const tmpNum = compiler->lvaGrabTemp(true DEBUGARG("handle histogram profile tmp")); compiler->lvaTable[tmpNum].lvType = TYP_REF; - unsigned helper; - GenTreeCall::Use* args = nullptr; + GenTree* helperCallNode; if (methodHistogram != nullptr) { GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); GenTree* const baseMethodNode = compiler->gtNewIconEmbMethHndNode(call->gtCallMethHnd); GenTree* const methodProfileNode = compiler->gtNewIconNode((ssize_t)methodHistogram, TYP_I_IMPL); GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); - args = compiler->gtNewCallArgs(tmpNode, baseMethodNode, methodProfileNode, classProfileNode); - helper = is32 ? CORINFO_HELP_METHODPROFILE32 : CORINFO_HELP_METHODPROFILE64; + helperCallNode = + compiler->gtNewHelperCallNode( + is32 ? CORINFO_HELP_METHODPROFILE32 : CORINFO_HELP_METHODPROFILE64, + TYP_VOID, + tmpNode, baseMethodNode, methodProfileNode, classProfileNode); } else { GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); - args = compiler->gtNewCallArgs(tmpNode, classProfileNode); - helper = is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64; + helperCallNode = + compiler->gtNewHelperCallNode( + is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64, + TYP_VOID, + tmpNode, classProfileNode); } // Generate the IR... // - GenTree* const helperCallNode = compiler->gtNewHelperCallNode(helper, TYP_VOID, args); GenTree* const tmpNode2 = compiler->gtNewLclvNode(tmpNum, TYP_REF); GenTree* const callCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, helperCallNode, tmpNode2); GenTree* const tmpNode3 = compiler->gtNewLclvNode(tmpNum, TYP_REF); From a3b04a38e114d9e70f40b9b950c4219ef31d7e11 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 28 Apr 2022 17:06:29 +0200 Subject: [PATCH 10/59] Change to static_assert and remove a comment --- src/coreclr/vm/jithelpers.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index b4c3c1e8a26b34..6d88069d9c098e 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -5340,7 +5340,7 @@ HCIMPL2(void, JIT_ClassProfile32, Object *obj, ICorJitInfo::HandleHistogram32* c const unsigned count = (*pCount)++; const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; - _ASSERTE(N >= S); + static_assert_no_msg(N >= S); if (objRef == NULL) { @@ -5407,7 +5407,7 @@ HCIMPL2(void, JIT_ClassProfile64, Object *obj, ICorJitInfo::HandleHistogram64* c const uint64_t count = (*pCount)++; const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; - _ASSERTE(N >= S); + static_assert_no_msg(N >= S); if (objRef == NULL) { @@ -5455,7 +5455,7 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod const unsigned methodCount = (*pMethodCount)++; const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; - _ASSERTE(N >= S); + static_assert_no_msg(N >= S); unsigned typeCount = 0; if (typeProfile != nullptr) @@ -5492,9 +5492,6 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod // delegates are filtered out here, and many cases with inner // "complicated" logic as well (e.g. multicast, unmanaged functions). // - // Note that this covers all cases of delegates that can be created - // without the use of reflection. - // DELEGATEREF del = (DELEGATEREF)objRef; if (del->GetInvocationCount() == 0) { @@ -5603,7 +5600,7 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod const uint64_t methodCount = (*pMethodCount)++; const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; - _ASSERTE(N >= S); + static_assert_no_msg(N >= S); unsigned typeCount = 0; if (typeProfile != nullptr) @@ -5640,9 +5637,6 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod // delegates are filtered out here, and many cases with inner // "complicated" logic as well (e.g. multicast, unmanaged functions). // - // Note that this covers all cases of delegates that can be created - // without the use of reflection. - // DELEGATEREF del = (DELEGATEREF)objRef; if (del->GetInvocationCount() == 0) { From 8dc5c6a3da361846be6a1caedc31caf570ec6562 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 29 Apr 2022 14:39:51 +0200 Subject: [PATCH 11/59] Foo --- src/coreclr/inc/corjit.h | 1 + src/coreclr/jit/compiler.h | 6 +- src/coreclr/jit/fgprofile.cpp | 3 +- src/coreclr/jit/importer.cpp | 292 ++++++++++++-------- src/coreclr/jit/indirectcalltransformer.cpp | 230 +++++++++++++-- src/coreclr/jit/jit.h | 16 +- src/coreclr/jit/likelyclass.cpp | 199 +++++++------ src/coreclr/jit/morph.cpp | 6 +- 8 files changed, 521 insertions(+), 232 deletions(-) diff --git a/src/coreclr/inc/corjit.h b/src/coreclr/inc/corjit.h index 76fdd304ee535f..6f6241d08480ca 100644 --- a/src/coreclr/inc/corjit.h +++ b/src/coreclr/inc/corjit.h @@ -397,6 +397,7 @@ class ICorJitInfo : public ICorDynamicInfo EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int EdgeLongCount = (DescriptorMin * 6) | EightByte, // edge counter using unsigned 8 byte int GetLikelyClass = (DescriptorMin * 7) | TypeHandle, // Compressed get likely class data + GetLikelyMethod = (DescriptorMin * 7) | MethodHandle, // Compressed get likely method data }; struct PgoInstrumentationSchema diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 3028a9cb3d908e..6b199abb4bf4f5 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -1814,6 +1814,7 @@ class Compiler friend class MorphInitBlockHelper; friend class MorphCopyBlockHelper; friend class CallArgs; + friend class IndirectCallTransformer; #ifdef FEATURE_HW_INTRINSICS friend struct HWIntrinsicInfo; @@ -5671,6 +5672,7 @@ class Compiler Statement* paramAssignmentInsertionPoint); GenTree* fgMorphCall(GenTreeCall* call); GenTree* fgExpandVirtualVtableCallTarget(GenTreeCall* call); + void fgMorphCallInline(GenTreeCall* call, InlineResult* result); void fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result, InlineContext** createdContext); #if DEBUG @@ -6794,13 +6796,13 @@ class Compiler optMethodFlags &= ~OMF_HAS_GUARDEDDEVIRT; } + void pickGDV(GenTreeCall* call, IL_OFFSET ilOffset, bool isInterface, CORINFO_CLASS_HANDLE* classGuess, CORINFO_METHOD_HANDLE* methodGuess, unsigned* likelihood); void considerGuardedDevirtualization(GenTreeCall* call, IL_OFFSET ilOffset, bool isInterface, CORINFO_METHOD_HANDLE baseMethod, CORINFO_CLASS_HANDLE baseClass, - CORINFO_CONTEXT_HANDLE* pContextHandle DEBUGARG(CORINFO_CLASS_HANDLE objClass) - DEBUGARG(const char* objClassName)); + CORINFO_CONTEXT_HANDLE* pContextHandle); void addGuardedDevirtualizationCandidate(GenTreeCall* call, CORINFO_METHOD_HANDLE methodHandle, diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 5291d87ba70b68..7d4a4cf72d4ac5 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1447,8 +1447,9 @@ class HandleHistogramProbeVisitor final : public GenTreeVisitorIsCall() && (node->AsCall()->gtHandleHistogramProfileCandidateInfo != nullptr)) + if (node->IsCall() && (m_compiler->compClassifyGDVProbeType(node->AsCall()) != Compiler::GDVProbeType::None)) { + assert(node->AsCall()->gtHandleHistogramProfileCandidateInfo != nullptr); m_functor(m_compiler, node->AsCall()); } diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index cd77a2f5fb4dfc..3b299431846185 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -9536,6 +9536,10 @@ var_types Compiler::impImportCall(OPCODE opcode, // methHnd = callInfo->hMethod; } + else if (call->AsCall()->IsDelegateInvoke() && opts.OptimizationEnabled() && !opts.IsOSR()) + { + considerGuardedDevirtualization(call->AsCall(), rawILOffset, false, NO_METHOD_HANDLE, NO_CLASS_HANDLE, nullptr); + } if (impIsThis(obj)) { @@ -11591,7 +11595,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( bool doRandomDevirt = false; const int maxLikelyClasses = 32; int likelyClassCount = 0; - LikelyClassRecord likelyClasses[maxLikelyClasses]; + LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; #ifdef DEBUG // Optional stress mode to pick a random known class, rather than // the most likely known class. @@ -11602,9 +11606,9 @@ GenTree* Compiler::impCastClassOrIsInstToTree( // Reuse the random inliner's random state. CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); - likelyClasses[0].clsHandle = getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); + likelyClasses[0].handle = (intptr_t)getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); likelyClasses[0].likelihood = 100; - if (likelyClasses[0].clsHandle != NO_CLASS_HANDLE) + if (likelyClasses[0].handle != (intptr_t)NO_CLASS_HANDLE) { likelyClassCount = 1; } @@ -11618,8 +11622,8 @@ GenTree* Compiler::impCastClassOrIsInstToTree( if (likelyClassCount > 0) { - LikelyClassRecord likelyClass = likelyClasses[0]; - CORINFO_CLASS_HANDLE likelyCls = likelyClass.clsHandle; + LikelyClassMethodRecord likelyClass = likelyClasses[0]; + CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle; if ((likelyCls != NO_CLASS_HANDLE) && (likelyClass.likelihood > (UINT32)JitConfig.JitGuardedDevirtualizationChainLikelihood())) @@ -20497,7 +20501,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, // Delegate Invoke method doesn't have a body and gets special cased instead. // Don't even bother trying to inline it. - if (call->IsDelegateInvoke()) + if (call->IsDelegateInvoke() && !call->IsGuardedDevirtualizationCandidate()) { inlineResult.NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY); return; @@ -20979,8 +20983,7 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, return; } - considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, - pContextHandle DEBUGARG(objClass) DEBUGARG("unknown")); + considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); return; } @@ -21030,8 +21033,7 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, return; } - considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, - pContextHandle DEBUGARG(objClass) DEBUGARG(objClassName)); + considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); return; } @@ -21147,8 +21149,7 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, return; } - considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, - pContextHandle DEBUGARG(objClass) DEBUGARG(objClassName)); + considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); return; } @@ -21209,13 +21210,13 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, { // We only can handle a single likely class for now const int maxLikelyClasses = 1; - LikelyClassRecord likelyClasses[maxLikelyClasses]; + LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; UINT32 numberOfClasses = getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); UINT32 likelihood = likelyClasses[0].likelihood; - CORINFO_CLASS_HANDLE likelyClass = likelyClasses[0].clsHandle; + CORINFO_CLASS_HANDLE likelyClass = (CORINFO_CLASS_HANDLE)likelyClasses[0].handle; if (numberOfClasses > 0) { @@ -21755,6 +21756,134 @@ void Compiler::addFatPointerCandidate(GenTreeCall* call) helper.StoreRetExprResultsInArgs(call); } +void Compiler::pickGDV( + GenTreeCall* call, + IL_OFFSET ilOffset, + bool isInterface, + CORINFO_CLASS_HANDLE* classGuess, + CORINFO_METHOD_HANDLE* methodGuess, + unsigned* likelihood) +{ + *classGuess = NO_CLASS_HANDLE; + *methodGuess = NO_METHOD_HANDLE; + *likelihood = 0; + + const unsigned likelihoodThreshold = isInterface ? 25 : 30; + +#ifdef DEBUG + // Optional stress mode to pick a random known class, rather than + // the most likely known class. + // + if (JitConfig.JitRandomGuardedDevirtualization() != 0) + { + // Reuse the random inliner's random state. + // + CLRRandom* const random = + impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + *classGuess = getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); + if (*classGuess != NO_CLASS_HANDLE) + { + JITDUMP("Picked random class for GDV: %p (%s)\n", *classGuess, eeGetClassName(*classGuess)); + return; + } + } +#endif + + const int maxLikelyClasses = 32; + LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; + unsigned numberOfClasses = getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + + const int maxLikelyMethods = 32; + LikelyClassMethodRecord likelyMethods[maxLikelyMethods]; + unsigned numberOfMethods = 0; + if (!isInterface) + { + numberOfMethods = getLikelyMethods(likelyMethods, maxLikelyMethods, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + } + + if ((numberOfClasses < 1) && (numberOfMethods < 1)) + { + JITDUMP("No likely class or method, sorry\n"); + return; + } + + if (numberOfClasses > 0) + { + bool isExact; + bool isNonNull; + CallArg* thisArg = call->gtArgs.GetThisArg(); + CORINFO_CLASS_HANDLE declaredThisClsHnd = gtGetClassHandle(thisArg->GetNode(), &isExact, &isNonNull); + JITDUMP("Likely classes for call [%06u]", dspTreeID(call)); + if (declaredThisClsHnd != NO_CLASS_HANDLE) + { + JITDUMP(" on class %p (%s)", declaredThisClsHnd, eeGetClassName(declaredThisClsHnd)); + } + JITDUMP("\n"); + + for (UINT32 i = 0; i < numberOfClasses; i++) + { + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, + eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle), likelyClasses[i].likelihood); + } + } + + if (numberOfMethods > 0) + { + assert(call->gtCallType == CT_USER_FUNC); + JITDUMP("Likely methods for call [%06u] to method %s\n", dspTreeID(call), eeGetMethodFullName(call->gtCallMethHnd)); + + for (UINT32 i = 0; i < numberOfMethods; i++) + { + CORINFO_CONST_LOOKUP lookup; + info.compCompHnd->getFunctionFixedEntryPoint((CORINFO_METHOD_HANDLE)likelyMethods[i].handle, false, &lookup); + + const char* methName = eeGetMethodFullName((CORINFO_METHOD_HANDLE)likelyMethods[i].handle); + switch (lookup.accessType) + { + case IAT_VALUE: + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, likelyMethods[i].likelihood); + break; + case IAT_PVALUE: + JITDUMP(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, likelyMethods[i].likelihood); + break; + case IAT_PPVALUE: + JITDUMP(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, likelyMethods[i].likelihood); + break; + default: + JITDUMP(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); + break; + } + } + } + + // Prefer class guess as it is cheaper + if ((numberOfClasses > 0) && (isInterface || call->IsVirtualVtable())) + { + unsigned likelihoodThreshold = isInterface ? 25 : 30; + if (likelyClasses[0].likelihood >= likelihoodThreshold) + { + *classGuess = (CORINFO_CLASS_HANDLE)likelyClasses[0].handle; + *likelihood = likelyClasses[0].likelihood; + return; + } + + JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", isInterface ? "interface" : "virtual", likelihoodThreshold); + } + + if ((numberOfMethods > 0) && (call->IsDelegateInvoke() || call->IsVirtualVtable())) + { + unsigned likelihoodThreshold = 75; + if (likelyMethods[0].likelihood >= likelihoodThreshold) + { + *methodGuess = (CORINFO_METHOD_HANDLE)likelyMethods[0].handle; + *likelihood = likelyMethods[0].likelihood; + return; + } + + JITDUMP("Not guessing for method; likelihood is below %s call threshold %u\n", call->IsDelegateInvoke() ? "delegate" : "virtual", likelihoodThreshold); + } +} + //------------------------------------------------------------------------ // considerGuardedDevirtualization: see if we can profitably guess at the // class involved in an interface or virtual call. @@ -21780,7 +21909,7 @@ void Compiler::considerGuardedDevirtualization( bool isInterface, CORINFO_METHOD_HANDLE baseMethod, CORINFO_CLASS_HANDLE baseClass, - CORINFO_CONTEXT_HANDLE* pContextHandle DEBUGARG(CORINFO_CLASS_HANDLE objClass) DEBUGARG(const char* objClassName)) + CORINFO_CONTEXT_HANDLE* pContextHandle) { #if defined(DEBUG) const char* callKind = isInterface ? "interface" : "virtual"; @@ -21791,113 +21920,58 @@ void Compiler::considerGuardedDevirtualization( // We currently only get likely class guesses when there is PGO data // with class profiles. // - if (fgPgoClassProfiles == 0) + if ((fgPgoClassProfiles == 0) && (fgPgoMethodProfiles == 0)) { - JITDUMP("Not guessing for class: no class profile pgo data, or pgo disabled\n"); + JITDUMP("Not guessing for class or method: no GDV profile pgo data, or pgo disabled\n"); return; } - // See if there's a likely guess for the class. - // - const unsigned likelihoodThreshold = isInterface ? 25 : 30; - unsigned likelihood = 0; - unsigned numberOfClasses = 0; - - CORINFO_CLASS_HANDLE likelyClass = NO_CLASS_HANDLE; - - bool doRandomDevirt = false; - - const int maxLikelyClasses = 32; - LikelyClassRecord likelyClasses[maxLikelyClasses]; - -#ifdef DEBUG - // Optional stress mode to pick a random known class, rather than - // the most likely known class. - // - doRandomDevirt = JitConfig.JitRandomGuardedDevirtualization() != 0; + CORINFO_CLASS_HANDLE likelyClass; + CORINFO_METHOD_HANDLE likelyMethod; + unsigned likelihood; + pickGDV(call, ilOffset, isInterface, &likelyClass, &likelyMethod, &likelihood); - if (doRandomDevirt) + if ((likelyClass == NO_CLASS_HANDLE) && (likelyMethod == NO_METHOD_HANDLE)) { - // Reuse the random inliner's random state. - // - CLRRandom* const random = - impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); - likelyClasses[0].clsHandle = getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); - likelyClasses[0].likelihood = 100; - if (likelyClasses[0].clsHandle != NO_CLASS_HANDLE) - { - numberOfClasses = 1; - } - } - else -#endif - { - numberOfClasses = - getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); - } - - // For now we only use the most popular type - - likelihood = likelyClasses[0].likelihood; - likelyClass = likelyClasses[0].clsHandle; - - if (numberOfClasses < 1) - { - JITDUMP("No likely class, sorry\n"); return; } - assert(likelyClass != NO_CLASS_HANDLE); - - // Print all likely classes - JITDUMP("%s classes for %p (%s):\n", doRandomDevirt ? "Random" : "Likely", dspPtr(objClass), objClassName) - for (UINT32 i = 0; i < numberOfClasses; i++) + uint32_t likelyClassAttribs = 0; + if (likelyClass != NO_CLASS_HANDLE) { - JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].clsHandle, - eeGetClassName(likelyClasses[i].clsHandle), likelyClasses[i].likelihood); - } + // TODO: Should we do this inside pickGDV and use the remaining + // profile, or discard it if we find something like this? + likelyClassAttribs = info.compCompHnd->getClassAttribs(likelyClass); - // Todo: a more advanced heuristic using likelihood, number of - // classes, and the profile count for this block. - // - // For now we will guess if the likelihood is at least 25%/30% (intfc/virt), as studies - // have shown this transformation should pay off even if we guess wrong sometimes. - // - if (likelihood < likelihoodThreshold) - { - JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", callKind, likelihoodThreshold); - return; - } - - uint32_t const likelyClassAttribs = info.compCompHnd->getClassAttribs(likelyClass); + if ((likelyClassAttribs & CORINFO_FLG_ABSTRACT) != 0) + { + // We may see an abstract likely class, if we have a stale profile. + // No point guessing for this. + // + JITDUMP("Not guessing for class; abstract (stale profile)\n"); + return; + } - if ((likelyClassAttribs & CORINFO_FLG_ABSTRACT) != 0) - { - // We may see an abstract likely class, if we have a stale profile. - // No point guessing for this. + // Figure out which method will be called. // - JITDUMP("Not guessing for class; abstract (stale profile)\n"); - return; - } + CORINFO_DEVIRTUALIZATION_INFO dvInfo; + dvInfo.virtualMethod = baseMethod; + dvInfo.objClass = likelyClass; + dvInfo.context = *pContextHandle; + dvInfo.exactContext = *pContextHandle; + dvInfo.pResolvedTokenVirtualMethod = nullptr; - // Figure out which method will be called. - // - CORINFO_DEVIRTUALIZATION_INFO dvInfo; - dvInfo.virtualMethod = baseMethod; - dvInfo.objClass = likelyClass; - dvInfo.context = *pContextHandle; - dvInfo.exactContext = *pContextHandle; - dvInfo.pResolvedTokenVirtualMethod = nullptr; + const bool canResolve = info.compCompHnd->resolveVirtualMethod(&dvInfo); - const bool canResolve = info.compCompHnd->resolveVirtualMethod(&dvInfo); + if (!canResolve) + { + JITDUMP("Can't figure out which method would be invoked, sorry\n"); + return; + } - if (!canResolve) - { - JITDUMP("Can't figure out which method would be invoked, sorry\n"); - return; + likelyMethod = dvInfo.devirtualizedMethod; } - CORINFO_METHOD_HANDLE likelyMethod = dvInfo.devirtualizedMethod; JITDUMP("%s call would invoke method %s\n", callKind, eeGetMethodName(likelyMethod, nullptr)); // Add this as a potential candidate. @@ -21935,8 +22009,8 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, unsigned classAttr, unsigned likelihood) { - // This transformation only makes sense for virtual calls - assert(call->IsVirtual()); + // This transformation only makes sense for delegate and virtual calls + assert(call->IsDelegateInvoke() || call->IsVirtual()); // Only mark calls if the feature is enabled. const bool isEnabled = JitConfig.JitEnableGuardedDevirtualization() > 0; @@ -21986,8 +22060,10 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, // We're all set, proceed with candidate creation. // - JITDUMP("Marking call [%06u] as guarded devirtualization candidate; will guess for class %s\n", dspTreeID(call), - eeGetClassName(classHandle)); + JITDUMP("Marking call [%06u] as guarded devirtualization candidate; will guess for %s %s\n", + dspTreeID(call), + classHandle != NO_CLASS_HANDLE ? "class" : "method", + classHandle != NO_CLASS_HANDLE ? eeGetClassName(classHandle) : eeGetMethodFullName(methodHandle)); setMethodHasGuardedDevirtualization(); call->SetGuardedDevirtualizationCandidate(); diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index a76bac72b8d8b9..859b977fb71b39 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -538,14 +538,16 @@ class IndirectCallTransformer checkBlock = currBlock; checkBlock->bbJumpKind = BBJ_COND; - // Fetch method table from object arg to call. - GenTree* thisTree = compiler->gtCloneExpr(origCall->gtArgs.GetThisArg()->GetNode()); + GenTree* thisTree = origCall->gtArgs.GetThisArg()->GetNode(); // Create temp for this if the tree is costly. - if (!thisTree->IsLocal()) + if (thisTree->IsLocal()) + { + thisTree = compiler->gtCloneExpr(thisTree); + } + else { const unsigned thisTempNum = compiler->lvaGrabTemp(true DEBUGARG("guarded devirt this temp")); - // lvaSetClass(thisTempNum, ...); GenTree* asgTree = compiler->gtNewTempAssign(thisTempNum, thisTree); Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); @@ -565,17 +567,138 @@ class IndirectCallTransformer // lastStmt = checkBlock->lastStmt(); - // Find target method table - // - GenTree* methodTable = compiler->gtNewMethodTableLookup(thisTree); GuardedDevirtualizationCandidateInfo* guardedInfo = origCall->gtGuardedDevirtualizationCandidateInfo; - CORINFO_CLASS_HANDLE clsHnd = guardedInfo->guardedClassHandle; - GenTree* targetMethodTable = compiler->gtNewIconEmbClsHndNode(clsHnd); - // Compare and jump to else (which does the indirect call) if NOT equal - // - GenTree* methodTableCompare = compiler->gtNewOperNode(GT_NE, TYP_INT, targetMethodTable, methodTable); - GenTree* jmpTree = compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, methodTableCompare); + GenTree* compare; + if (guardedInfo->guardedClassHandle != NO_CLASS_HANDLE) + { + + // Find target method table + // + GenTree* methodTable = compiler->gtNewMethodTableLookup(thisTree); + CORINFO_CLASS_HANDLE clsHnd = guardedInfo->guardedClassHandle; + GenTree* targetMethodTable = compiler->gtNewIconEmbClsHndNode(clsHnd); + + // Compare and jump to else (which does the indirect call) if NOT equal + // + compare = compiler->gtNewOperNode(GT_NE, TYP_INT, targetMethodTable, methodTable); + } + else + { + assert(origCall->IsVirtualVtable() || origCall->IsDelegateInvoke()); + if (origCall->IsVirtualVtable()) + { + const unsigned addrTempNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); + compiler->lvaGetDesc(addrTempNum)->lvSingleDef = 1; + + GenTree* tarTree = compiler->fgExpandVirtualVtableCallTarget(origCall); + GenTree* asgTree = compiler->gtNewTempAssign(addrTempNum, tarTree); + Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); + compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); + + origCall->gtControlExpr = compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL); + CORINFO_METHOD_HANDLE methHnd = guardedInfo->guardedMethodHandle; + CORINFO_CONST_LOOKUP lookup; + compiler->info.compCompHnd->getFunctionEntryPoint(methHnd, &lookup); + + GenTree* compareTarTree = nullptr; + switch (lookup.accessType) + { + case IAT_VALUE: + { + compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + break; + } + case IAT_PVALUE: + { + compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + compareTarTree = compiler->gtNewIndir(TYP_I_IMPL, compareTarTree); + compareTarTree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; + compareTarTree->gtFlags &= ~GTF_EXCEPT; + break; + } + case IAT_PPVALUE: + { + noway_assert(!"Unexpected IAT_PPVALUE"); + break; + } + case IAT_RELPVALUE: + { + GenTree* addr = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + compareTarTree = compiler->gtNewIndir(TYP_I_IMPL, compareTarTree); + compareTarTree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; + compareTarTree->gtFlags &= ~GTF_EXCEPT; + compareTarTree = compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, compareTarTree, addr); + break; + } + default: + { + noway_assert(!"Bad accessType"); + break; + } + } + compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)); + } + else + { + //const unsigned addrTempNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); + //compiler->lvaGetDesc(addrTempNum)->lvSingleDef = 1; + + GenTree* offset = compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateFirstTarget, TYP_I_IMPL); + GenTree* tarTree = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, thisTree, offset); + tarTree = compiler->gtNewIndir(TYP_I_IMPL, tarTree); + //GenTree* asgTree = compiler->gtNewTempAssign(addrTempNum, tarTree); + //Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); + //compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); + + //origCall->gtControlExpr = compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL); + CORINFO_METHOD_HANDLE methHnd = guardedInfo->guardedMethodHandle; + CORINFO_CONST_LOOKUP lookup; + compiler->info.compCompHnd->getFunctionFixedEntryPoint(methHnd, false, &lookup); + + GenTree* compareTarTree = nullptr; + switch (lookup.accessType) + { + case IAT_VALUE: + { + compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + break; + } + case IAT_PVALUE: + { + compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + compareTarTree = compiler->gtNewIndir(TYP_I_IMPL, compareTarTree); + compareTarTree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; + compareTarTree->gtFlags &= ~GTF_EXCEPT; + break; + } + case IAT_PPVALUE: + { + noway_assert(!"Unexpected IAT_PPVALUE"); + break; + } + case IAT_RELPVALUE: + { + GenTree* addr = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + compareTarTree = compiler->gtNewIndir(TYP_I_IMPL, compareTarTree); + compareTarTree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; + compareTarTree->gtFlags &= ~GTF_EXCEPT; + compareTarTree = compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, compareTarTree, addr); + break; + } + default: + { + noway_assert(!"Bad accessType"); + break; + } + } + compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, tarTree /*compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)*/); + } + } + + GenTree* jmpTree = compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, compare); Statement* jmpStmt = compiler->fgNewStmtFromTree(jmpTree, stmt->GetDebugInfo()); compiler->fgInsertStmtAtEnd(checkBlock, jmpStmt); } @@ -682,11 +805,31 @@ class IndirectCallTransformer InlineCandidateInfo* inlineInfo = origCall->gtInlineCandidateInfo; CORINFO_CLASS_HANDLE clsHnd = inlineInfo->guardedClassHandle; - // copy 'this' to temp with exact type. + // copy new 'this' to temp with exact type. const unsigned thisTemp = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt this exact temp")); GenTree* clonedObj = compiler->gtCloneExpr(origCall->gtArgs.GetThisArg()->GetNode()); - GenTree* assign = compiler->gtNewTempAssign(thisTemp, clonedObj); - compiler->lvaSetClass(thisTemp, clsHnd, true); + GenTree* newThisObj; + if (origCall->IsDelegateInvoke()) + { + GenTree* offset = compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateInstance, TYP_I_IMPL); + newThisObj = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, clonedObj, offset); + newThisObj = compiler->gtNewIndir(TYP_REF, newThisObj); + } + else + { + newThisObj = clonedObj; + } + GenTree* assign = compiler->gtNewTempAssign(thisTemp, newThisObj); + + if (clsHnd != NO_CLASS_HANDLE) + { + compiler->lvaSetClass(thisTemp, clsHnd, true); + } + else + { + compiler->lvaSetClass(thisTemp, compiler->info.compCompHnd->getMethodClass(inlineInfo->guardedMethodHandle)); + } + compiler->fgNewStmtAtEnd(thenBlock, assign); // Clone call. Note we must use the special candidate helper. @@ -696,16 +839,49 @@ class IndirectCallTransformer JITDUMP("Direct call [%06u] in block " FMT_BB "\n", compiler->dspTreeID(call), thenBlock->bbNum); - // Then invoke impDevirtualizeCall to actually transform the call for us, - // given the original (base) method and the exact guarded class. It should succeed. - // - CORINFO_METHOD_HANDLE methodHnd = call->gtCallMethHnd; - unsigned methodFlags = compiler->info.compCompHnd->getMethodAttribs(methodHnd); - CORINFO_CONTEXT_HANDLE context = inlineInfo->exactContextHnd; - const bool isLateDevirtualization = true; - const bool explicitTailCall = (call->AsCall()->gtCallMoreFlags & GTF_CALL_M_EXPLICIT_TAILCALL) != 0; - compiler->impDevirtualizeCall(call, nullptr, &methodHnd, &methodFlags, &context, nullptr, - isLateDevirtualization, explicitTailCall); + CORINFO_METHOD_HANDLE methodHnd = call->gtCallMethHnd; + CORINFO_CONTEXT_HANDLE context = inlineInfo->exactContextHnd; + if (clsHnd != NO_CLASS_HANDLE) + { + // Then invoke impDevirtualizeCall to actually transform the call for us, + // given the original (base) method and the exact guarded class. It should succeed. + // + unsigned methodFlags = compiler->info.compCompHnd->getMethodAttribs(methodHnd); + const bool isLateDevirtualization = true; + const bool explicitTailCall = (call->AsCall()->gtCallMoreFlags & GTF_CALL_M_EXPLICIT_TAILCALL) != 0; + compiler->impDevirtualizeCall(call, nullptr, &methodHnd, &methodFlags, &context, nullptr, + isLateDevirtualization, explicitTailCall); + } + else + { + // Make the updates. + call->gtFlags &= ~GTF_CALL_VIRT_KIND_MASK; + call->gtCallMethHnd = methodHnd = inlineInfo->guardedMethodHandle; + call->gtCallType = CT_USER_FUNC; + call->gtCallMoreFlags |= GTF_CALL_M_DEVIRTUALIZED; + call->gtCallMoreFlags &= ~GTF_CALL_M_DELEGATE_INV; + // TODO: R2R entry point + // Vtable target could have been expanded early. + call->gtControlExpr = nullptr; + + bool isExact; + bool objIsNonNull; + compiler->gtGetClassHandle(newThisObj, &isExact, &objIsNonNull); + + // Virtual calls include an implicit null check, which we may + // now need to make explicit. + if (!objIsNonNull) + { + call->gtFlags |= GTF_CALL_NULLCHECK; + } + + // Clear the inline candidate info (may be non-null since + // it's a union field used for other things by virtual + // stubs) + call->gtInlineCandidateInfo = nullptr; + + context = MAKE_METHODCONTEXT(methodHnd); + } // We know this call can devirtualize or we would not have set up GDV here. // So impDevirtualizeCall should succeed in devirtualizing. @@ -959,7 +1135,7 @@ class IndirectCallTransformer // GenTree* const root = nextStmt->GetRootNode(); - if (root->IsCall()) + if (root->IsCall() && !root->AsCall()->IsDelegateInvoke()) { GenTreeCall* const call = root->AsCall(); diff --git a/src/coreclr/jit/jit.h b/src/coreclr/jit/jit.h index d4ea7ab0dcb910..9a4fa196743b1d 100644 --- a/src/coreclr/jit/jit.h +++ b/src/coreclr/jit/jit.h @@ -329,6 +329,7 @@ typedef class ICorJitInfo* COMP_HANDLE; const CORINFO_CLASS_HANDLE NO_CLASS_HANDLE = nullptr; const CORINFO_FIELD_HANDLE NO_FIELD_HANDLE = nullptr; +const CORINFO_METHOD_HANDLE NO_METHOD_HANDLE = nullptr; /*****************************************************************************/ @@ -838,19 +839,26 @@ T dspOffset(T o) #endif // !defined(DEBUG) -struct LikelyClassRecord +struct LikelyClassMethodRecord { - CORINFO_CLASS_HANDLE clsHandle; - UINT32 likelihood; + intptr_t handle; + UINT32 likelihood; }; -extern "C" UINT32 WINAPI getLikelyClasses(LikelyClassRecord* pLikelyClasses, +extern "C" UINT32 WINAPI getLikelyClasses(LikelyClassMethodRecord* pLikelyClasses, UINT32 maxLikelyClasses, ICorJitInfo::PgoInstrumentationSchema* schema, UINT32 countSchemaItems, BYTE* pInstrumentationData, int32_t ilOffset); +extern "C" UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* pLikelyMethods, + UINT32 maxLikelyMethods, + ICorJitInfo::PgoInstrumentationSchema* schema, + UINT32 countSchemaItems, + BYTE* pInstrumentationData, + int32_t ilOffset); + /*****************************************************************************/ #endif //_JIT_H_ /*****************************************************************************/ diff --git a/src/coreclr/jit/likelyclass.cpp b/src/coreclr/jit/likelyclass.cpp index 632c9ce8b847b9..636e5650e2e9b2 100644 --- a/src/coreclr/jit/likelyclass.cpp +++ b/src/coreclr/jit/likelyclass.cpp @@ -26,45 +26,45 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // Data item in class profile histogram // -struct LikelyClassHistogramEntry +struct LikelyClassMethodHistogramEntry { - // Class that was observed at runtime - INT_PTR m_mt; // This may be an "unknown type handle" + // Handle that was observed at runtime + INT_PTR m_handle; // This may be an "unknown handle" // Number of observations in the table unsigned m_count; }; // Summarizes a ClassProfile table by forming a Histogram // -struct LikelyClassHistogram +struct LikelyClassMethodHistogram { - LikelyClassHistogram(INT_PTR* histogramEntries, unsigned entryCount); + LikelyClassMethodHistogram(INT_PTR* histogramEntries, unsigned entryCount); // Sum of counts from all entries in the histogram. This includes "unknown" entries which are not captured in // m_histogram unsigned m_totalCount; - // Rough guess at count of unknown types - unsigned m_unknownTypes; + // Rough guess at count of unknown handles + unsigned m_unknownHandles; // Histogram entries, in no particular order. - LikelyClassHistogramEntry m_histogram[HISTOGRAM_MAX_SIZE_COUNT]; + LikelyClassMethodHistogramEntry m_histogram[HISTOGRAM_MAX_SIZE_COUNT]; UINT32 countHistogramElements = 0; - LikelyClassHistogramEntry HistogramEntryAt(unsigned index) + LikelyClassMethodHistogramEntry HistogramEntryAt(unsigned index) { return m_histogram[index]; } }; //------------------------------------------------------------------------ -// LikelyClassHistogram::LikelyClassHistgram: construct a new histogram +// LikelyClassMethodHistogram::LikelyClassMethodHistgram: construct a new histogram // // Arguments: // histogramEntries - pointer to the table portion of a ClassProfile* object (see corjit.h) // entryCount - number of entries in the table to examine // -LikelyClassHistogram::LikelyClassHistogram(INT_PTR* histogramEntries, unsigned entryCount) +LikelyClassMethodHistogram::LikelyClassMethodHistogram(INT_PTR* histogramEntries, unsigned entryCount) { - m_unknownTypes = 0; + m_unknownHandles = 0; m_totalCount = 0; uint32_t unknownTypeHandleMask = 0; @@ -83,7 +83,7 @@ LikelyClassHistogram::LikelyClassHistogram(INT_PTR* histogramEntries, unsigned e unsigned h = 0; for (; h < countHistogramElements; h++) { - if (m_histogram[h].m_mt == currentEntry) + if (m_histogram[h].m_handle == currentEntry) { m_histogram[h].m_count++; found = true; @@ -97,51 +97,27 @@ LikelyClassHistogram::LikelyClassHistogram(INT_PTR* histogramEntries, unsigned e { continue; } - LikelyClassHistogramEntry newEntry; - newEntry.m_mt = currentEntry; + LikelyClassMethodHistogramEntry newEntry; + newEntry.m_handle = currentEntry; newEntry.m_count = 1; m_histogram[countHistogramElements++] = newEntry; } } } -//------------------------------------------------------------------------ -// getLikelyClasses: find class profile data for an IL offset, and return the most likely classes -// -// Arguments: -// pLikelyClasses - [OUT] array of likely classes sorted by likelihood (descending). It must be -// at least of 'maxLikelyClasses' (next argument) length. -// The array consists of pairs "clsHandle - likelihood" ordered by likelihood -// (descending) where likelihood can be any value in [0..100] range. clsHandle -// is never null for [0..) range, Items in -// [..maxLikelyClasses) are zeroed if the number -// of classes seen is less than maxLikelyClasses provided. -// maxLikelyClasses - limit for likely classes to output -// schema - profile schema -// countSchemaItems - number of items in the schema -// pInstrumentationData - associated data -// ilOffset - il offset of the callvirt -// -// Returns: -// Estimated number of classes seen at runtime -// -// Notes: -// A "monomorphic" call site will return likelihood 100 and number of entries = 1. -// -// This is used by the devirtualization logic below, and by crossgen2 when producing -// the R2R image (to reduce the sizecost of carrying the type histogram) -// -// This code can runs without a jit instance present, so JITDUMP and related -// cannot be used. -// -extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* pLikelyClasses, - UINT32 maxLikelyClasses, - ICorJitInfo::PgoInstrumentationSchema* schema, - UINT32 countSchemaItems, - BYTE* pInstrumentationData, - int32_t ilOffset) +static unsigned getLikelyClassesOrMethods( + LikelyClassMethodRecord* pLikelyEntries, + UINT32 maxLikelyClasses, + ICorJitInfo::PgoInstrumentationSchema* schema, + UINT32 countSchemaItems, + BYTE* pInstrumentationData, + int32_t ilOffset, + bool types) { - ZeroMemory(pLikelyClasses, maxLikelyClasses * sizeof(*pLikelyClasses)); + ICorJitInfo::PgoInstrumentationKind histogramKind = types ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes : ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods; + ICorJitInfo::PgoInstrumentationKind compressedKind = types ? ICorJitInfo::PgoInstrumentationKind::GetLikelyClass : ICorJitInfo::PgoInstrumentationKind::GetLikelyMethod; + + memset(pLikelyEntries, 0, maxLikelyClasses * sizeof(*pLikelyEntries)); if (schema == nullptr) { @@ -153,17 +129,17 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* if (schema[i].ILOffset != ilOffset) continue; - if ((schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyClass) && + if ((schema[i].InstrumentationKind == compressedKind) && (schema[i].Count == 1)) { - INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset); + intptr_t result = *(intptr_t*)(pInstrumentationData + schema[i].Offset); if (ICorJitInfo::IsUnknownHandle(result)) { return 0; } assert(result != 0); // we don't expect zero in GetLikelyClass - pLikelyClasses[0].likelihood = (UINT32)(schema[i].Other & 0xFF); - pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)result; + pLikelyEntries[0].likelihood = (UINT32)(schema[i].Other & 0xFF); + pLikelyEntries[0].handle = result; return 1; } @@ -171,12 +147,11 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) || (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount); - if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && - (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes)) + if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && (schema[i + 1].InstrumentationKind == histogramKind)) { // Form a histogram // - LikelyClassHistogram h((INT_PTR*)(pInstrumentationData + schema[i + 1].Offset), schema[i + 1].Count); + LikelyClassMethodHistogram h((INT_PTR*)(pInstrumentationData + schema[i + 1].Offset), schema[i + 1].Count); // Use histogram count as number of classes estimate // Report back what we've learned @@ -189,45 +164,45 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* case 1: { - LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0); + LikelyClassMethodHistogramEntry const hist0 = h.HistogramEntryAt(0); // Fast path for monomorphic cases - if (ICorJitInfo::IsUnknownHandle(hist0.m_mt)) + if (ICorJitInfo::IsUnknownHandle(hist0.m_handle)) { return 0; } - pLikelyClasses[0].likelihood = 100; - pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt; + pLikelyEntries[0].likelihood = 100; + pLikelyEntries[0].handle = hist0.m_handle; return 1; } case 2: { - LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0); - LikelyClassHistogramEntry const hist1 = h.HistogramEntryAt(1); // Fast path for two classes - if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt)) + LikelyClassMethodHistogramEntry const hist0 = h.HistogramEntryAt(0); + LikelyClassMethodHistogramEntry const hist1 = h.HistogramEntryAt(1); + if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownHandle(hist0.m_handle)) { - pLikelyClasses[0].likelihood = (100 * hist0.m_count) / h.m_totalCount; - pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt; + pLikelyEntries[0].likelihood = (100 * hist0.m_count) / h.m_totalCount; + pLikelyEntries[0].handle = hist0.m_handle; - if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist1.m_mt)) + if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist1.m_handle)) { - pLikelyClasses[1].likelihood = (100 * hist1.m_count) / h.m_totalCount; - pLikelyClasses[1].clsHandle = (CORINFO_CLASS_HANDLE)hist1.m_mt; + pLikelyEntries[1].likelihood = (100 * hist1.m_count) / h.m_totalCount; + pLikelyEntries[1].handle = hist1.m_handle; return 2; } return 1; } - if (!ICorJitInfo::IsUnknownHandle(hist1.m_mt)) + if (!ICorJitInfo::IsUnknownHandle(hist1.m_handle)) { - pLikelyClasses[0].likelihood = (100 * hist1.m_count) / h.m_totalCount; - pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist1.m_mt; + pLikelyEntries[0].likelihood = (100 * hist1.m_count) / h.m_totalCount; + pLikelyEntries[0].handle = hist1.m_handle; - if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt)) + if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist0.m_handle)) { - pLikelyClasses[1].likelihood = (100 * hist0.m_count) / h.m_totalCount; - pLikelyClasses[1].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt; + pLikelyEntries[1].likelihood = (100 * hist0.m_count) / h.m_totalCount; + pLikelyEntries[1].handle = hist0.m_handle; return 2; } return 1; @@ -237,14 +212,14 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* default: { - LikelyClassHistogramEntry sortedEntries[HISTOGRAM_MAX_SIZE_COUNT]; + LikelyClassMethodHistogramEntry sortedEntries[HISTOGRAM_MAX_SIZE_COUNT]; // Since this method can be invoked without a jit instance we can't use any existing allocators unsigned knownHandles = 0; for (unsigned m = 0; m < h.countHistogramElements; m++) { - LikelyClassHistogramEntry const hist = h.HistogramEntryAt(m); - if (!ICorJitInfo::IsUnknownHandle(hist.m_mt)) + LikelyClassMethodHistogramEntry const hist = h.HistogramEntryAt(m); + if (!ICorJitInfo::IsUnknownHandle(hist.m_handle)) { sortedEntries[knownHandles++] = hist; } @@ -252,7 +227,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* // sort by m_count (descending) jitstd::sort(sortedEntries, sortedEntries + knownHandles, - [](const LikelyClassHistogramEntry& h1, const LikelyClassHistogramEntry& h2) -> bool { + [](const LikelyClassMethodHistogramEntry& h1, const LikelyClassMethodHistogramEntry& h2) -> bool { return h1.m_count > h2.m_count; }); @@ -260,9 +235,9 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* for (size_t hIdx = 0; hIdx < numberOfClasses; hIdx++) { - LikelyClassHistogramEntry const hc = sortedEntries[hIdx]; - pLikelyClasses[hIdx].clsHandle = (CORINFO_CLASS_HANDLE)hc.m_mt; - pLikelyClasses[hIdx].likelihood = hc.m_count * 100 / h.m_totalCount; + LikelyClassMethodHistogramEntry const hc = sortedEntries[hIdx]; + pLikelyEntries[hIdx].handle = hc.m_handle; + pLikelyEntries[hIdx].likelihood = hc.m_count * 100 / h.m_totalCount; } return numberOfClasses; } @@ -273,6 +248,56 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord* // Failed to find histogram data for this method // return 0; + +} + +//------------------------------------------------------------------------ +// getLikelyClasses: find class profile data for an IL offset, and return the most likely classes +// +// Arguments: +// pLikelyClasses - [OUT] array of likely classes sorted by likelihood (descending). It must be +// at least of 'maxLikelyClasses' (next argument) length. +// The array consists of pairs "clsHandle - likelihood" ordered by likelihood +// (descending) where likelihood can be any value in [0..100] range. clsHandle +// is never null for [0..) range, Items in +// [..maxLikelyClasses) are zeroed if the number +// of classes seen is less than maxLikelyClasses provided. +// maxLikelyClasses - limit for likely classes to output +// schema - profile schema +// countSchemaItems - number of items in the schema +// pInstrumentationData - associated data +// ilOffset - il offset of the callvirt +// +// Returns: +// Estimated number of classes seen at runtime +// +// Notes: +// A "monomorphic" call site will return likelihood 100 and number of entries = 1. +// +// This is used by the devirtualization logic below, and by crossgen2 when producing +// the R2R image (to reduce the sizecost of carrying the type histogram) +// +// This code can runs without a jit instance present, so JITDUMP and related +// cannot be used. +// +extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassMethodRecord* pLikelyClasses, + UINT32 maxLikelyClasses, + ICorJitInfo::PgoInstrumentationSchema* schema, + UINT32 countSchemaItems, + BYTE* pInstrumentationData, + int32_t ilOffset) +{ + return getLikelyClassesOrMethods(pLikelyClasses, maxLikelyClasses, schema, countSchemaItems, pInstrumentationData, ilOffset, true); +} + +extern "C" DLLEXPORT UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* pLikelyMethods, + UINT32 maxLikelyMethods, + ICorJitInfo::PgoInstrumentationSchema* schema, + UINT32 countSchemaItems, + BYTE* pInstrumentationData, + int32_t ilOffset) +{ + return getLikelyClassesOrMethods(pLikelyMethods, maxLikelyMethods, schema, countSchemaItems, pInstrumentationData, ilOffset, false); } //------------------------------------------------------------------------ @@ -330,7 +355,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch { // Form a histogram // - LikelyClassHistogram h((INT_PTR*)(pInstrumentationData + schema[i + 1].Offset), schema[i + 1].Count); + LikelyClassMethodHistogram h((INT_PTR*)(pInstrumentationData + schema[i + 1].Offset), schema[i + 1].Count); if (h.countHistogramElements == 0) { @@ -340,14 +365,14 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch // Choose an entry at random. // unsigned randomEntryIndex = random->Next(0, h.countHistogramElements); - LikelyClassHistogramEntry randomEntry = h.HistogramEntryAt(randomEntryIndex); + LikelyClassMethodHistogramEntry randomEntry = h.HistogramEntryAt(randomEntryIndex); - if (ICorJitInfo::IsUnknownHandle(randomEntry.m_mt)) + if (ICorJitInfo::IsUnknownHandle(randomEntry.m_handle)) { return NO_CLASS_HANDLE; } - return (CORINFO_CLASS_HANDLE)randomEntry.m_mt; + return (CORINFO_CLASS_HANDLE)randomEntry.m_handle; } } diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 6ce7e9bfb0e9b8..938347711bcba7 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -8660,10 +8660,10 @@ GenTree* Compiler::fgMorphCall(GenTreeCall* call) // if (call->IsExpandedEarly() && call->IsVirtualVtable()) { - // We only expand the Vtable Call target once in the global morph phase - if (fgGlobalMorph) + // We expand the Vtable Call target either in the global morph phase or + // in guarded devirt if we need it for the guard. + if (fgGlobalMorph && (call->gtControlExpr == nullptr)) { - assert(call->gtControlExpr == nullptr); // We only call this method and assign gtControlExpr once call->gtControlExpr = fgExpandVirtualVtableCallTarget(call); } // We always have to morph or re-morph the control expr From da05869d0eb67e6574e408254b346cf21dc346b8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 29 Apr 2022 15:08:13 +0200 Subject: [PATCH 12/59] Few cleanups --- src/coreclr/jit/importer.cpp | 27 ++--- src/coreclr/jit/indirectcalltransformer.cpp | 115 +++++++------------- 2 files changed, 51 insertions(+), 91 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 3b299431846185..d785bf93881073 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -21530,7 +21530,7 @@ void Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) // we'd just keep track of the calls themselves, so we don't // have to search for them later. // - if ((compClassifyGDVProbeType(call) != GDVProbeType::None)) + if (compClassifyGDVProbeType(call) != GDVProbeType::None) { JITDUMP("\n ... marking [%06u] in " FMT_BB " for method/class profile instrumentation\n", dspTreeID(call), compCurBB->bbNum); @@ -21768,8 +21768,6 @@ void Compiler::pickGDV( *methodGuess = NO_METHOD_HANDLE; *likelihood = 0; - const unsigned likelihoodThreshold = isInterface ? 25 : 30; - #ifdef DEBUG // Optional stress mode to pick a random known class, rather than // the most likely known class. @@ -21791,12 +21789,16 @@ void Compiler::pickGDV( const int maxLikelyClasses = 32; LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; - unsigned numberOfClasses = getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + unsigned numberOfClasses = 0; + if (call->IsVirtualStub() || call->IsVirtualVtable()) + { + numberOfClasses = getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + } const int maxLikelyMethods = 32; LikelyClassMethodRecord likelyMethods[maxLikelyMethods]; unsigned numberOfMethods = 0; - if (!isInterface) + if (call->IsVirtualVtable() || call->IsDelegateInvoke()) { numberOfMethods = getLikelyMethods(likelyMethods, maxLikelyMethods, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); } @@ -21857,7 +21859,7 @@ void Compiler::pickGDV( } // Prefer class guess as it is cheaper - if ((numberOfClasses > 0) && (isInterface || call->IsVirtualVtable())) + if (numberOfClasses > 0) { unsigned likelihoodThreshold = isInterface ? 25 : 30; if (likelyClasses[0].likelihood >= likelihoodThreshold) @@ -21870,7 +21872,7 @@ void Compiler::pickGDV( JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", isInterface ? "interface" : "virtual", likelihoodThreshold); } - if ((numberOfMethods > 0) && (call->IsDelegateInvoke() || call->IsVirtualVtable())) + if (numberOfMethods > 0) { unsigned likelihoodThreshold = 75; if (likelyMethods[0].likelihood >= likelihoodThreshold) @@ -21892,12 +21894,9 @@ void Compiler::pickGDV( // // call - potential guarded devirtualization candidate // ilOffset - IL ofset of the call instruction -// isInterface - true if this is an interface call // baseMethod - target method of the call // baseClass - class that introduced the target method // pContextHandle - context handle for the call -// objClass - class of 'this' in the call -// objClassName - name of the obj Class // // Notes: // Consults with VM to see if there's a likely class at runtime, @@ -21911,10 +21910,6 @@ void Compiler::considerGuardedDevirtualization( CORINFO_CLASS_HANDLE baseClass, CORINFO_CONTEXT_HANDLE* pContextHandle) { -#if defined(DEBUG) - const char* callKind = isInterface ? "interface" : "virtual"; -#endif - JITDUMP("Considering guarded devirtualization at IL offset %u (0x%x)\n", ilOffset, ilOffset); // We currently only get likely class guesses when there is PGO data @@ -21939,8 +21934,6 @@ void Compiler::considerGuardedDevirtualization( uint32_t likelyClassAttribs = 0; if (likelyClass != NO_CLASS_HANDLE) { - // TODO: Should we do this inside pickGDV and use the remaining - // profile, or discard it if we find something like this? likelyClassAttribs = info.compCompHnd->getClassAttribs(likelyClass); if ((likelyClassAttribs & CORINFO_FLG_ABSTRACT) != 0) @@ -21972,7 +21965,7 @@ void Compiler::considerGuardedDevirtualization( likelyMethod = dvInfo.devirtualizedMethod; } - JITDUMP("%s call would invoke method %s\n", callKind, eeGetMethodName(likelyMethod, nullptr)); + JITDUMP("%s call would invoke method %s\n", isInterface ? "interface" : call->IsDelegateInvoke() ? "delegate" : "virtual", eeGetMethodName(likelyMethod, nullptr)); // Add this as a potential candidate. // diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 859b977fb71b39..072a3a535771a8 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -601,43 +601,7 @@ class IndirectCallTransformer CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionEntryPoint(methHnd, &lookup); - GenTree* compareTarTree = nullptr; - switch (lookup.accessType) - { - case IAT_VALUE: - { - compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - break; - } - case IAT_PVALUE: - { - compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - compareTarTree = compiler->gtNewIndir(TYP_I_IMPL, compareTarTree); - compareTarTree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; - compareTarTree->gtFlags &= ~GTF_EXCEPT; - break; - } - case IAT_PPVALUE: - { - noway_assert(!"Unexpected IAT_PPVALUE"); - break; - } - case IAT_RELPVALUE: - { - GenTree* addr = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - compareTarTree = compiler->gtNewIndir(TYP_I_IMPL, compareTarTree); - compareTarTree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; - compareTarTree->gtFlags &= ~GTF_EXCEPT; - compareTarTree = compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, compareTarTree, addr); - break; - } - default: - { - noway_assert(!"Bad accessType"); - break; - } - } + GenTree* compareTarTree = CreateLookup(lookup);; compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)); } else @@ -657,43 +621,7 @@ class IndirectCallTransformer CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionFixedEntryPoint(methHnd, false, &lookup); - GenTree* compareTarTree = nullptr; - switch (lookup.accessType) - { - case IAT_VALUE: - { - compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - break; - } - case IAT_PVALUE: - { - compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - compareTarTree = compiler->gtNewIndir(TYP_I_IMPL, compareTarTree); - compareTarTree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; - compareTarTree->gtFlags &= ~GTF_EXCEPT; - break; - } - case IAT_PPVALUE: - { - noway_assert(!"Unexpected IAT_PPVALUE"); - break; - } - case IAT_RELPVALUE: - { - GenTree* addr = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - compareTarTree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - compareTarTree = compiler->gtNewIndir(TYP_I_IMPL, compareTarTree); - compareTarTree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; - compareTarTree->gtFlags &= ~GTF_EXCEPT; - compareTarTree = compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, compareTarTree, addr); - break; - } - default: - { - noway_assert(!"Bad accessType"); - break; - } - } + GenTree* compareTarTree = CreateLookup(lookup); compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, tarTree /*compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)*/); } } @@ -1181,6 +1109,45 @@ class IndirectCallTransformer private: unsigned returnTemp; Statement* lastStmt; + + GenTree* CreateLookup(const CORINFO_CONST_LOOKUP& lookup) + { + switch (lookup.accessType) + { + case IAT_VALUE: + { + return compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + } + case IAT_PVALUE: + { + GenTree* tree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + tree = compiler->gtNewIndir(TYP_I_IMPL, tree); + tree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; + tree->gtFlags &= ~GTF_EXCEPT; + return tree; + } + case IAT_PPVALUE: + { + noway_assert(!"Unexpected IAT_PPVALUE"); + return nullptr; + } + case IAT_RELPVALUE: + { + GenTree* addr = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + GenTree* tree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + tree = compiler->gtNewIndir(TYP_I_IMPL, tree); + tree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; + tree->gtFlags &= ~GTF_EXCEPT; + tree = compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, tree, addr); + return tree; + } + default: + { + noway_assert(!"Bad accessType"); + return nullptr; + } + } + } }; // Runtime lookup with dynamic dictionary expansion transformer, From af7232b89859543233bfa8611f71f2bbf9cf8045 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 29 Apr 2022 15:09:51 +0200 Subject: [PATCH 13/59] Better comment --- src/coreclr/jit/indirectcalltransformer.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 072a3a535771a8..c02044cdebf04f 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -733,7 +733,11 @@ class IndirectCallTransformer InlineCandidateInfo* inlineInfo = origCall->gtInlineCandidateInfo; CORINFO_CLASS_HANDLE clsHnd = inlineInfo->guardedClassHandle; - // copy new 'this' to temp with exact type. + // Copy the 'this' for the devirtualized call to a new temp. For + // class-based GDV this will allow us to set the exact type on that + // temp. For delegate GDV, this will be the actual 'this' object + // stored in the delegate. + const unsigned thisTemp = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt this exact temp")); GenTree* clonedObj = compiler->gtCloneExpr(origCall->gtArgs.GetThisArg()->GetNode()); GenTree* newThisObj; From 12d1d863c4d1b9b9c1974ef455c7a7872151baa1 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 29 Apr 2022 15:10:15 +0200 Subject: [PATCH 14/59] Run jit-format --- src/coreclr/jit/compiler.h | 15 ++- src/coreclr/jit/fgprofile.cpp | 22 ++- src/coreclr/jit/importer.cpp | 136 ++++++++++--------- src/coreclr/jit/indirectcalltransformer.cpp | 141 +++++++++++--------- src/coreclr/jit/jit.h | 12 +- src/coreclr/jit/likelyclass.cpp | 66 ++++----- 6 files changed, 210 insertions(+), 182 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 6b199abb4bf4f5..20c3a39013969b 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -2318,8 +2318,12 @@ class Compiler GenTreeCall* gtNewIndCallNode(GenTree* addr, var_types type, const DebugInfo& di = DebugInfo()); - GenTreeCall* gtNewHelperCallNode( - unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr, GenTree* arg4 = nullptr); + GenTreeCall* gtNewHelperCallNode(unsigned helper, + var_types type, + GenTree* arg1 = nullptr, + GenTree* arg2 = nullptr, + GenTree* arg3 = nullptr, + GenTree* arg4 = nullptr); GenTreeCall* gtNewRuntimeLookupHelperCallNode(CORINFO_RUNTIME_LOOKUP* pRuntimeLookup, GenTree* ctxTree, @@ -6796,7 +6800,12 @@ class Compiler optMethodFlags &= ~OMF_HAS_GUARDEDDEVIRT; } - void pickGDV(GenTreeCall* call, IL_OFFSET ilOffset, bool isInterface, CORINFO_CLASS_HANDLE* classGuess, CORINFO_METHOD_HANDLE* methodGuess, unsigned* likelihood); + void pickGDV(GenTreeCall* call, + IL_OFFSET ilOffset, + bool isInterface, + CORINFO_CLASS_HANDLE* classGuess, + CORINFO_METHOD_HANDLE* methodGuess, + unsigned* likelihood); void considerGuardedDevirtualization(GenTreeCall* call, IL_OFFSET ilOffset, bool isInterface, diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 7d4a4cf72d4ac5..1f121c083f7440 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1606,29 +1606,25 @@ class HandleHistogramProbeInserter GenTree* const methodProfileNode = compiler->gtNewIconNode((ssize_t)methodHistogram, TYP_I_IMPL); GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); helperCallNode = - compiler->gtNewHelperCallNode( - is32 ? CORINFO_HELP_METHODPROFILE32 : CORINFO_HELP_METHODPROFILE64, - TYP_VOID, - tmpNode, baseMethodNode, methodProfileNode, classProfileNode); + compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_METHODPROFILE32 : CORINFO_HELP_METHODPROFILE64, + TYP_VOID, tmpNode, baseMethodNode, methodProfileNode, classProfileNode); } else { GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); helperCallNode = - compiler->gtNewHelperCallNode( - is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64, - TYP_VOID, - tmpNode, classProfileNode); + compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64, + TYP_VOID, tmpNode, classProfileNode); } // Generate the IR... // - GenTree* const tmpNode2 = compiler->gtNewLclvNode(tmpNum, TYP_REF); - GenTree* const callCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, helperCallNode, tmpNode2); - GenTree* const tmpNode3 = compiler->gtNewLclvNode(tmpNum, TYP_REF); - GenTree* const asgNode = compiler->gtNewOperNode(GT_ASG, TYP_REF, tmpNode3, objUse->GetNode()); - GenTree* const asgCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, asgNode, callCommaNode); + GenTree* const tmpNode2 = compiler->gtNewLclvNode(tmpNum, TYP_REF); + GenTree* const callCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, helperCallNode, tmpNode2); + GenTree* const tmpNode3 = compiler->gtNewLclvNode(tmpNum, TYP_REF); + GenTree* const asgNode = compiler->gtNewOperNode(GT_ASG, TYP_REF, tmpNode3, objUse->GetNode()); + GenTree* const asgCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, asgNode, callCommaNode); // Update the call // diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index d785bf93881073..f670b0659a4b79 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -9538,7 +9538,8 @@ var_types Compiler::impImportCall(OPCODE opcode, } else if (call->AsCall()->IsDelegateInvoke() && opts.OptimizationEnabled() && !opts.IsOSR()) { - considerGuardedDevirtualization(call->AsCall(), rawILOffset, false, NO_METHOD_HANDLE, NO_CLASS_HANDLE, nullptr); + considerGuardedDevirtualization(call->AsCall(), rawILOffset, false, NO_METHOD_HANDLE, NO_CLASS_HANDLE, + nullptr); } if (impIsThis(obj)) @@ -11592,9 +11593,9 @@ GenTree* Compiler::impCastClassOrIsInstToTree( // Check if this cast helper have some profile data if (impIsCastHelperMayHaveProfileData(helper)) { - bool doRandomDevirt = false; - const int maxLikelyClasses = 32; - int likelyClassCount = 0; + bool doRandomDevirt = false; + const int maxLikelyClasses = 32; + int likelyClassCount = 0; LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; #ifdef DEBUG // Optional stress mode to pick a random known class, rather than @@ -11606,7 +11607,8 @@ GenTree* Compiler::impCastClassOrIsInstToTree( // Reuse the random inliner's random state. CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); - likelyClasses[0].handle = (intptr_t)getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); + likelyClasses[0].handle = + (intptr_t)getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); likelyClasses[0].likelihood = 100; if (likelyClasses[0].handle != (intptr_t)NO_CLASS_HANDLE) { @@ -11622,8 +11624,8 @@ GenTree* Compiler::impCastClassOrIsInstToTree( if (likelyClassCount > 0) { - LikelyClassMethodRecord likelyClass = likelyClasses[0]; - CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle; + LikelyClassMethodRecord likelyClass = likelyClasses[0]; + CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle; if ((likelyCls != NO_CLASS_HANDLE) && (likelyClass.likelihood > (UINT32)JitConfig.JitGuardedDevirtualizationChainLikelihood())) @@ -21209,7 +21211,7 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, if (JitConfig.JitCrossCheckDevirtualizationAndPGO() && canSensiblyCheck) { // We only can handle a single likely class for now - const int maxLikelyClasses = 1; + const int maxLikelyClasses = 1; LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; UINT32 numberOfClasses = @@ -21756,17 +21758,16 @@ void Compiler::addFatPointerCandidate(GenTreeCall* call) helper.StoreRetExprResultsInArgs(call); } -void Compiler::pickGDV( - GenTreeCall* call, - IL_OFFSET ilOffset, - bool isInterface, - CORINFO_CLASS_HANDLE* classGuess, - CORINFO_METHOD_HANDLE* methodGuess, - unsigned* likelihood) +void Compiler::pickGDV(GenTreeCall* call, + IL_OFFSET ilOffset, + bool isInterface, + CORINFO_CLASS_HANDLE* classGuess, + CORINFO_METHOD_HANDLE* methodGuess, + unsigned* likelihood) { - *classGuess = NO_CLASS_HANDLE; + *classGuess = NO_CLASS_HANDLE; *methodGuess = NO_METHOD_HANDLE; - *likelihood = 0; + *likelihood = 0; #ifdef DEBUG // Optional stress mode to pick a random known class, rather than @@ -21787,20 +21788,22 @@ void Compiler::pickGDV( } #endif - const int maxLikelyClasses = 32; + const int maxLikelyClasses = 32; LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; - unsigned numberOfClasses = 0; + unsigned numberOfClasses = 0; if (call->IsVirtualStub() || call->IsVirtualVtable()) { - numberOfClasses = getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + numberOfClasses = + getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); } - const int maxLikelyMethods = 32; + const int maxLikelyMethods = 32; LikelyClassMethodRecord likelyMethods[maxLikelyMethods]; - unsigned numberOfMethods = 0; + unsigned numberOfMethods = 0; if (call->IsVirtualVtable() || call->IsDelegateInvoke()) { - numberOfMethods = getLikelyMethods(likelyMethods, maxLikelyMethods, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + numberOfMethods = + getLikelyMethods(likelyMethods, maxLikelyMethods, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); } if ((numberOfClasses < 1) && (numberOfMethods < 1)) @@ -21811,9 +21814,9 @@ void Compiler::pickGDV( if (numberOfClasses > 0) { - bool isExact; - bool isNonNull; - CallArg* thisArg = call->gtArgs.GetThisArg(); + bool isExact; + bool isNonNull; + CallArg* thisArg = call->gtArgs.GetThisArg(); CORINFO_CLASS_HANDLE declaredThisClsHnd = gtGetClassHandle(thisArg->GetNode(), &isExact, &isNonNull); JITDUMP("Likely classes for call [%06u]", dspTreeID(call)); if (declaredThisClsHnd != NO_CLASS_HANDLE) @@ -21825,35 +21828,40 @@ void Compiler::pickGDV( for (UINT32 i = 0; i < numberOfClasses; i++) { JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, - eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle), likelyClasses[i].likelihood); + eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle), likelyClasses[i].likelihood); } } if (numberOfMethods > 0) { assert(call->gtCallType == CT_USER_FUNC); - JITDUMP("Likely methods for call [%06u] to method %s\n", dspTreeID(call), eeGetMethodFullName(call->gtCallMethHnd)); + JITDUMP("Likely methods for call [%06u] to method %s\n", dspTreeID(call), + eeGetMethodFullName(call->gtCallMethHnd)); for (UINT32 i = 0; i < numberOfMethods; i++) { CORINFO_CONST_LOOKUP lookup; - info.compCompHnd->getFunctionFixedEntryPoint((CORINFO_METHOD_HANDLE)likelyMethods[i].handle, false, &lookup); + info.compCompHnd->getFunctionFixedEntryPoint((CORINFO_METHOD_HANDLE)likelyMethods[i].handle, false, + &lookup); const char* methName = eeGetMethodFullName((CORINFO_METHOD_HANDLE)likelyMethods[i].handle); switch (lookup.accessType) { - case IAT_VALUE: - JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, likelyMethods[i].likelihood); - break; - case IAT_PVALUE: - JITDUMP(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, likelyMethods[i].likelihood); - break; - case IAT_PPVALUE: - JITDUMP(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, likelyMethods[i].likelihood); - break; - default: - JITDUMP(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); - break; + case IAT_VALUE: + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); + break; + case IAT_PVALUE: + JITDUMP(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); + break; + case IAT_PPVALUE: + JITDUMP(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); + break; + default: + JITDUMP(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); + break; } } } @@ -21869,7 +21877,8 @@ void Compiler::pickGDV( return; } - JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", isInterface ? "interface" : "virtual", likelihoodThreshold); + JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", + isInterface ? "interface" : "virtual", likelihoodThreshold); } if (numberOfMethods > 0) @@ -21878,11 +21887,12 @@ void Compiler::pickGDV( if (likelyMethods[0].likelihood >= likelihoodThreshold) { *methodGuess = (CORINFO_METHOD_HANDLE)likelyMethods[0].handle; - *likelihood = likelyMethods[0].likelihood; + *likelihood = likelyMethods[0].likelihood; return; } - JITDUMP("Not guessing for method; likelihood is below %s call threshold %u\n", call->IsDelegateInvoke() ? "delegate" : "virtual", likelihoodThreshold); + JITDUMP("Not guessing for method; likelihood is below %s call threshold %u\n", + call->IsDelegateInvoke() ? "delegate" : "virtual", likelihoodThreshold); } } @@ -21902,13 +21912,12 @@ void Compiler::pickGDV( // Consults with VM to see if there's a likely class at runtime, // if so, adds a candidate for guarded devirtualization. // -void Compiler::considerGuardedDevirtualization( - GenTreeCall* call, - IL_OFFSET ilOffset, - bool isInterface, - CORINFO_METHOD_HANDLE baseMethod, - CORINFO_CLASS_HANDLE baseClass, - CORINFO_CONTEXT_HANDLE* pContextHandle) +void Compiler::considerGuardedDevirtualization(GenTreeCall* call, + IL_OFFSET ilOffset, + bool isInterface, + CORINFO_METHOD_HANDLE baseMethod, + CORINFO_CLASS_HANDLE baseClass, + CORINFO_CONTEXT_HANDLE* pContextHandle) { JITDUMP("Considering guarded devirtualization at IL offset %u (0x%x)\n", ilOffset, ilOffset); @@ -21921,9 +21930,9 @@ void Compiler::considerGuardedDevirtualization( return; } - CORINFO_CLASS_HANDLE likelyClass; + CORINFO_CLASS_HANDLE likelyClass; CORINFO_METHOD_HANDLE likelyMethod; - unsigned likelihood; + unsigned likelihood; pickGDV(call, ilOffset, isInterface, &likelyClass, &likelyMethod, &likelihood); if ((likelyClass == NO_CLASS_HANDLE) && (likelyMethod == NO_METHOD_HANDLE)) @@ -21931,7 +21940,7 @@ void Compiler::considerGuardedDevirtualization( return; } - uint32_t likelyClassAttribs = 0; + uint32_t likelyClassAttribs = 0; if (likelyClass != NO_CLASS_HANDLE) { likelyClassAttribs = info.compCompHnd->getClassAttribs(likelyClass); @@ -21948,10 +21957,10 @@ void Compiler::considerGuardedDevirtualization( // Figure out which method will be called. // CORINFO_DEVIRTUALIZATION_INFO dvInfo; - dvInfo.virtualMethod = baseMethod; - dvInfo.objClass = likelyClass; - dvInfo.context = *pContextHandle; - dvInfo.exactContext = *pContextHandle; + dvInfo.virtualMethod = baseMethod; + dvInfo.objClass = likelyClass; + dvInfo.context = *pContextHandle; + dvInfo.exactContext = *pContextHandle; dvInfo.pResolvedTokenVirtualMethod = nullptr; const bool canResolve = info.compCompHnd->resolveVirtualMethod(&dvInfo); @@ -21965,7 +21974,9 @@ void Compiler::considerGuardedDevirtualization( likelyMethod = dvInfo.devirtualizedMethod; } - JITDUMP("%s call would invoke method %s\n", isInterface ? "interface" : call->IsDelegateInvoke() ? "delegate" : "virtual", eeGetMethodName(likelyMethod, nullptr)); + JITDUMP("%s call would invoke method %s\n", + isInterface ? "interface" : call->IsDelegateInvoke() ? "delegate" : "virtual", + eeGetMethodName(likelyMethod, nullptr)); // Add this as a potential candidate. // @@ -22053,10 +22064,9 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, // We're all set, proceed with candidate creation. // - JITDUMP("Marking call [%06u] as guarded devirtualization candidate; will guess for %s %s\n", - dspTreeID(call), - classHandle != NO_CLASS_HANDLE ? "class" : "method", - classHandle != NO_CLASS_HANDLE ? eeGetClassName(classHandle) : eeGetMethodFullName(methodHandle)); + JITDUMP("Marking call [%06u] as guarded devirtualization candidate; will guess for %s %s\n", dspTreeID(call), + classHandle != NO_CLASS_HANDLE ? "class" : "method", + classHandle != NO_CLASS_HANDLE ? eeGetClassName(classHandle) : eeGetMethodFullName(methodHandle)); setMethodHasGuardedDevirtualization(); call->SetGuardedDevirtualizationCandidate(); diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index c02044cdebf04f..fb5feaf1a80f3f 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -548,8 +548,8 @@ class IndirectCallTransformer else { const unsigned thisTempNum = compiler->lvaGrabTemp(true DEBUGARG("guarded devirt this temp")); - GenTree* asgTree = compiler->gtNewTempAssign(thisTempNum, thisTree); - Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); + GenTree* asgTree = compiler->gtNewTempAssign(thisTempNum, thisTree); + Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); thisTree = compiler->gtNewLclvNode(thisTempNum, TYP_REF); @@ -567,7 +567,7 @@ class IndirectCallTransformer // lastStmt = checkBlock->lastStmt(); - GuardedDevirtualizationCandidateInfo* guardedInfo = origCall->gtGuardedDevirtualizationCandidateInfo; + GuardedDevirtualizationCandidateInfo* guardedInfo = origCall->gtGuardedDevirtualizationCandidateInfo; GenTree* compare; if (guardedInfo->guardedClassHandle != NO_CLASS_HANDLE) @@ -575,9 +575,9 @@ class IndirectCallTransformer // Find target method table // - GenTree* methodTable = compiler->gtNewMethodTableLookup(thisTree); - CORINFO_CLASS_HANDLE clsHnd = guardedInfo->guardedClassHandle; - GenTree* targetMethodTable = compiler->gtNewIconEmbClsHndNode(clsHnd); + GenTree* methodTable = compiler->gtNewMethodTableLookup(thisTree); + CORINFO_CLASS_HANDLE clsHnd = guardedInfo->guardedClassHandle; + GenTree* targetMethodTable = compiler->gtNewIconEmbClsHndNode(clsHnd); // Compare and jump to else (which does the indirect call) if NOT equal // @@ -588,46 +588,53 @@ class IndirectCallTransformer assert(origCall->IsVirtualVtable() || origCall->IsDelegateInvoke()); if (origCall->IsVirtualVtable()) { - const unsigned addrTempNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); + const unsigned addrTempNum = + compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); compiler->lvaGetDesc(addrTempNum)->lvSingleDef = 1; - GenTree* tarTree = compiler->fgExpandVirtualVtableCallTarget(origCall); - GenTree* asgTree = compiler->gtNewTempAssign(addrTempNum, tarTree); + GenTree* tarTree = compiler->fgExpandVirtualVtableCallTarget(origCall); + GenTree* asgTree = compiler->gtNewTempAssign(addrTempNum, tarTree); Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); - origCall->gtControlExpr = compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL); + origCall->gtControlExpr = compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL); CORINFO_METHOD_HANDLE methHnd = guardedInfo->guardedMethodHandle; - CORINFO_CONST_LOOKUP lookup; + CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionEntryPoint(methHnd, &lookup); - GenTree* compareTarTree = CreateLookup(lookup);; - compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)); + GenTree* compareTarTree = CreateLookup(lookup); + ; + compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, + compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)); } else { - //const unsigned addrTempNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); - //compiler->lvaGetDesc(addrTempNum)->lvSingleDef = 1; + // const unsigned addrTempNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target + // temp")); + // compiler->lvaGetDesc(addrTempNum)->lvSingleDef = 1; - GenTree* offset = compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateFirstTarget, TYP_I_IMPL); + GenTree* offset = + compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateFirstTarget, + TYP_I_IMPL); GenTree* tarTree = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, thisTree, offset); - tarTree = compiler->gtNewIndir(TYP_I_IMPL, tarTree); - //GenTree* asgTree = compiler->gtNewTempAssign(addrTempNum, tarTree); - //Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); - //compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); + tarTree = compiler->gtNewIndir(TYP_I_IMPL, tarTree); + // GenTree* asgTree = compiler->gtNewTempAssign(addrTempNum, tarTree); + // Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); + // compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); - //origCall->gtControlExpr = compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL); + // origCall->gtControlExpr = compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL); CORINFO_METHOD_HANDLE methHnd = guardedInfo->guardedMethodHandle; - CORINFO_CONST_LOOKUP lookup; + CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionFixedEntryPoint(methHnd, false, &lookup); GenTree* compareTarTree = CreateLookup(lookup); - compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, tarTree /*compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)*/); + compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, + tarTree /*compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)*/); } } - GenTree* jmpTree = compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, compare); - Statement* jmpStmt = compiler->fgNewStmtFromTree(jmpTree, stmt->GetDebugInfo()); + GenTree* jmpTree = compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, compare); + Statement* jmpStmt = compiler->fgNewStmtFromTree(jmpTree, stmt->GetDebugInfo()); compiler->fgInsertStmtAtEnd(checkBlock, jmpStmt); } @@ -740,10 +747,11 @@ class IndirectCallTransformer const unsigned thisTemp = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt this exact temp")); GenTree* clonedObj = compiler->gtCloneExpr(origCall->gtArgs.GetThisArg()->GetNode()); - GenTree* newThisObj; + GenTree* newThisObj; if (origCall->IsDelegateInvoke()) { - GenTree* offset = compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateInstance, TYP_I_IMPL); + GenTree* offset = + compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateInstance, TYP_I_IMPL); newThisObj = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, clonedObj, offset); newThisObj = compiler->gtNewIndir(TYP_REF, newThisObj); } @@ -751,7 +759,7 @@ class IndirectCallTransformer { newThisObj = clonedObj; } - GenTree* assign = compiler->gtNewTempAssign(thisTemp, newThisObj); + GenTree* assign = compiler->gtNewTempAssign(thisTemp, newThisObj); if (clsHnd != NO_CLASS_HANDLE) { @@ -759,7 +767,8 @@ class IndirectCallTransformer } else { - compiler->lvaSetClass(thisTemp, compiler->info.compCompHnd->getMethodClass(inlineInfo->guardedMethodHandle)); + compiler->lvaSetClass(thisTemp, + compiler->info.compCompHnd->getMethodClass(inlineInfo->guardedMethodHandle)); } compiler->fgNewStmtAtEnd(thenBlock, assign); @@ -772,24 +781,24 @@ class IndirectCallTransformer JITDUMP("Direct call [%06u] in block " FMT_BB "\n", compiler->dspTreeID(call), thenBlock->bbNum); CORINFO_METHOD_HANDLE methodHnd = call->gtCallMethHnd; - CORINFO_CONTEXT_HANDLE context = inlineInfo->exactContextHnd; + CORINFO_CONTEXT_HANDLE context = inlineInfo->exactContextHnd; if (clsHnd != NO_CLASS_HANDLE) { // Then invoke impDevirtualizeCall to actually transform the call for us, // given the original (base) method and the exact guarded class. It should succeed. // - unsigned methodFlags = compiler->info.compCompHnd->getMethodAttribs(methodHnd); - const bool isLateDevirtualization = true; + unsigned methodFlags = compiler->info.compCompHnd->getMethodAttribs(methodHnd); + const bool isLateDevirtualization = true; const bool explicitTailCall = (call->AsCall()->gtCallMoreFlags & GTF_CALL_M_EXPLICIT_TAILCALL) != 0; compiler->impDevirtualizeCall(call, nullptr, &methodHnd, &methodFlags, &context, nullptr, - isLateDevirtualization, explicitTailCall); + isLateDevirtualization, explicitTailCall); } else { // Make the updates. call->gtFlags &= ~GTF_CALL_VIRT_KIND_MASK; call->gtCallMethHnd = methodHnd = inlineInfo->guardedMethodHandle; - call->gtCallType = CT_USER_FUNC; + call->gtCallType = CT_USER_FUNC; call->gtCallMoreFlags |= GTF_CALL_M_DEVIRTUALIZED; call->gtCallMoreFlags &= ~GTF_CALL_M_DELEGATE_INV; // TODO: R2R entry point @@ -1118,38 +1127,38 @@ class IndirectCallTransformer { switch (lookup.accessType) { - case IAT_VALUE: - { - return compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - } - case IAT_PVALUE: - { - GenTree* tree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - tree = compiler->gtNewIndir(TYP_I_IMPL, tree); - tree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; - tree->gtFlags &= ~GTF_EXCEPT; - return tree; - } - case IAT_PPVALUE: - { - noway_assert(!"Unexpected IAT_PPVALUE"); - return nullptr; - } - case IAT_RELPVALUE: - { - GenTree* addr = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - GenTree* tree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - tree = compiler->gtNewIndir(TYP_I_IMPL, tree); - tree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; - tree->gtFlags &= ~GTF_EXCEPT; - tree = compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, tree, addr); - return tree; - } - default: - { - noway_assert(!"Bad accessType"); - return nullptr; - } + case IAT_VALUE: + { + return compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + } + case IAT_PVALUE: + { + GenTree* tree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + tree = compiler->gtNewIndir(TYP_I_IMPL, tree); + tree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; + tree->gtFlags &= ~GTF_EXCEPT; + return tree; + } + case IAT_PPVALUE: + { + noway_assert(!"Unexpected IAT_PPVALUE"); + return nullptr; + } + case IAT_RELPVALUE: + { + GenTree* addr = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + GenTree* tree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + tree = compiler->gtNewIndir(TYP_I_IMPL, tree); + tree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; + tree->gtFlags &= ~GTF_EXCEPT; + tree = compiler->gtNewOperNode(GT_ADD, TYP_I_IMPL, tree, addr); + return tree; + } + default: + { + noway_assert(!"Bad accessType"); + return nullptr; + } } } }; diff --git a/src/coreclr/jit/jit.h b/src/coreclr/jit/jit.h index 9a4fa196743b1d..64612e6a96ade5 100644 --- a/src/coreclr/jit/jit.h +++ b/src/coreclr/jit/jit.h @@ -327,8 +327,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX typedef class ICorJitInfo* COMP_HANDLE; -const CORINFO_CLASS_HANDLE NO_CLASS_HANDLE = nullptr; -const CORINFO_FIELD_HANDLE NO_FIELD_HANDLE = nullptr; +const CORINFO_CLASS_HANDLE NO_CLASS_HANDLE = nullptr; +const CORINFO_FIELD_HANDLE NO_FIELD_HANDLE = nullptr; const CORINFO_METHOD_HANDLE NO_METHOD_HANDLE = nullptr; /*****************************************************************************/ @@ -841,18 +841,18 @@ T dspOffset(T o) struct LikelyClassMethodRecord { - intptr_t handle; - UINT32 likelihood; + intptr_t handle; + UINT32 likelihood; }; -extern "C" UINT32 WINAPI getLikelyClasses(LikelyClassMethodRecord* pLikelyClasses, +extern "C" UINT32 WINAPI getLikelyClasses(LikelyClassMethodRecord* pLikelyClasses, UINT32 maxLikelyClasses, ICorJitInfo::PgoInstrumentationSchema* schema, UINT32 countSchemaItems, BYTE* pInstrumentationData, int32_t ilOffset); -extern "C" UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* pLikelyMethods, +extern "C" UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* pLikelyMethods, UINT32 maxLikelyMethods, ICorJitInfo::PgoInstrumentationSchema* schema, UINT32 countSchemaItems, diff --git a/src/coreclr/jit/likelyclass.cpp b/src/coreclr/jit/likelyclass.cpp index 636e5650e2e9b2..c31e3b428c2c6c 100644 --- a/src/coreclr/jit/likelyclass.cpp +++ b/src/coreclr/jit/likelyclass.cpp @@ -47,7 +47,7 @@ struct LikelyClassMethodHistogram unsigned m_unknownHandles; // Histogram entries, in no particular order. LikelyClassMethodHistogramEntry m_histogram[HISTOGRAM_MAX_SIZE_COUNT]; - UINT32 countHistogramElements = 0; + UINT32 countHistogramElements = 0; LikelyClassMethodHistogramEntry HistogramEntryAt(unsigned index) { @@ -64,7 +64,7 @@ struct LikelyClassMethodHistogram // LikelyClassMethodHistogram::LikelyClassMethodHistogram(INT_PTR* histogramEntries, unsigned entryCount) { - m_unknownHandles = 0; + m_unknownHandles = 0; m_totalCount = 0; uint32_t unknownTypeHandleMask = 0; @@ -98,24 +98,26 @@ LikelyClassMethodHistogram::LikelyClassMethodHistogram(INT_PTR* histogramEntries continue; } LikelyClassMethodHistogramEntry newEntry; - newEntry.m_handle = currentEntry; + newEntry.m_handle = currentEntry; newEntry.m_count = 1; m_histogram[countHistogramElements++] = newEntry; } } } -static unsigned getLikelyClassesOrMethods( - LikelyClassMethodRecord* pLikelyEntries, - UINT32 maxLikelyClasses, - ICorJitInfo::PgoInstrumentationSchema* schema, - UINT32 countSchemaItems, - BYTE* pInstrumentationData, - int32_t ilOffset, - bool types) +static unsigned getLikelyClassesOrMethods(LikelyClassMethodRecord* pLikelyEntries, + UINT32 maxLikelyClasses, + ICorJitInfo::PgoInstrumentationSchema* schema, + UINT32 countSchemaItems, + BYTE* pInstrumentationData, + int32_t ilOffset, + bool types) { - ICorJitInfo::PgoInstrumentationKind histogramKind = types ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes : ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods; - ICorJitInfo::PgoInstrumentationKind compressedKind = types ? ICorJitInfo::PgoInstrumentationKind::GetLikelyClass : ICorJitInfo::PgoInstrumentationKind::GetLikelyMethod; + ICorJitInfo::PgoInstrumentationKind histogramKind = + types ? ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes + : ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods; + ICorJitInfo::PgoInstrumentationKind compressedKind = types ? ICorJitInfo::PgoInstrumentationKind::GetLikelyClass + : ICorJitInfo::PgoInstrumentationKind::GetLikelyMethod; memset(pLikelyEntries, 0, maxLikelyClasses * sizeof(*pLikelyEntries)); @@ -129,8 +131,7 @@ static unsigned getLikelyClassesOrMethods( if (schema[i].ILOffset != ilOffset) continue; - if ((schema[i].InstrumentationKind == compressedKind) && - (schema[i].Count == 1)) + if ((schema[i].InstrumentationKind == compressedKind) && (schema[i].Count == 1)) { intptr_t result = *(intptr_t*)(pInstrumentationData + schema[i].Offset); if (ICorJitInfo::IsUnknownHandle(result)) @@ -139,7 +140,7 @@ static unsigned getLikelyClassesOrMethods( } assert(result != 0); // we don't expect zero in GetLikelyClass pLikelyEntries[0].likelihood = (UINT32)(schema[i].Other & 0xFF); - pLikelyEntries[0].handle = result; + pLikelyEntries[0].handle = result; return 1; } @@ -147,7 +148,8 @@ static unsigned getLikelyClassesOrMethods( (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) || (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount); - if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && (schema[i + 1].InstrumentationKind == histogramKind)) + if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && + (schema[i + 1].InstrumentationKind == histogramKind)) { // Form a histogram // @@ -171,7 +173,7 @@ static unsigned getLikelyClassesOrMethods( return 0; } pLikelyEntries[0].likelihood = 100; - pLikelyEntries[0].handle = hist0.m_handle; + pLikelyEntries[0].handle = hist0.m_handle; return 1; } @@ -183,12 +185,12 @@ static unsigned getLikelyClassesOrMethods( if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownHandle(hist0.m_handle)) { pLikelyEntries[0].likelihood = (100 * hist0.m_count) / h.m_totalCount; - pLikelyEntries[0].handle = hist0.m_handle; + pLikelyEntries[0].handle = hist0.m_handle; if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist1.m_handle)) { pLikelyEntries[1].likelihood = (100 * hist1.m_count) / h.m_totalCount; - pLikelyEntries[1].handle = hist1.m_handle; + pLikelyEntries[1].handle = hist1.m_handle; return 2; } return 1; @@ -197,12 +199,12 @@ static unsigned getLikelyClassesOrMethods( if (!ICorJitInfo::IsUnknownHandle(hist1.m_handle)) { pLikelyEntries[0].likelihood = (100 * hist1.m_count) / h.m_totalCount; - pLikelyEntries[0].handle = hist1.m_handle; + pLikelyEntries[0].handle = hist1.m_handle; if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist0.m_handle)) { pLikelyEntries[1].likelihood = (100 * hist0.m_count) / h.m_totalCount; - pLikelyEntries[1].handle = hist0.m_handle; + pLikelyEntries[1].handle = hist0.m_handle; return 2; } return 1; @@ -227,7 +229,8 @@ static unsigned getLikelyClassesOrMethods( // sort by m_count (descending) jitstd::sort(sortedEntries, sortedEntries + knownHandles, - [](const LikelyClassMethodHistogramEntry& h1, const LikelyClassMethodHistogramEntry& h2) -> bool { + [](const LikelyClassMethodHistogramEntry& h1, + const LikelyClassMethodHistogramEntry& h2) -> bool { return h1.m_count > h2.m_count; }); @@ -236,8 +239,8 @@ static unsigned getLikelyClassesOrMethods( for (size_t hIdx = 0; hIdx < numberOfClasses; hIdx++) { LikelyClassMethodHistogramEntry const hc = sortedEntries[hIdx]; - pLikelyEntries[hIdx].handle = hc.m_handle; - pLikelyEntries[hIdx].likelihood = hc.m_count * 100 / h.m_totalCount; + pLikelyEntries[hIdx].handle = hc.m_handle; + pLikelyEntries[hIdx].likelihood = hc.m_count * 100 / h.m_totalCount; } return numberOfClasses; } @@ -248,7 +251,6 @@ static unsigned getLikelyClassesOrMethods( // Failed to find histogram data for this method // return 0; - } //------------------------------------------------------------------------ @@ -280,24 +282,26 @@ static unsigned getLikelyClassesOrMethods( // This code can runs without a jit instance present, so JITDUMP and related // cannot be used. // -extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassMethodRecord* pLikelyClasses, +extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassMethodRecord* pLikelyClasses, UINT32 maxLikelyClasses, ICorJitInfo::PgoInstrumentationSchema* schema, UINT32 countSchemaItems, BYTE* pInstrumentationData, int32_t ilOffset) { - return getLikelyClassesOrMethods(pLikelyClasses, maxLikelyClasses, schema, countSchemaItems, pInstrumentationData, ilOffset, true); + return getLikelyClassesOrMethods(pLikelyClasses, maxLikelyClasses, schema, countSchemaItems, pInstrumentationData, + ilOffset, true); } -extern "C" DLLEXPORT UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* pLikelyMethods, +extern "C" DLLEXPORT UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* pLikelyMethods, UINT32 maxLikelyMethods, ICorJitInfo::PgoInstrumentationSchema* schema, UINT32 countSchemaItems, BYTE* pInstrumentationData, int32_t ilOffset) { - return getLikelyClassesOrMethods(pLikelyMethods, maxLikelyMethods, schema, countSchemaItems, pInstrumentationData, ilOffset, false); + return getLikelyClassesOrMethods(pLikelyMethods, maxLikelyMethods, schema, countSchemaItems, pInstrumentationData, + ilOffset, false); } //------------------------------------------------------------------------ @@ -364,7 +368,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch // Choose an entry at random. // - unsigned randomEntryIndex = random->Next(0, h.countHistogramElements); + unsigned randomEntryIndex = random->Next(0, h.countHistogramElements); LikelyClassMethodHistogramEntry randomEntry = h.HistogramEntryAt(randomEntryIndex); if (ICorJitInfo::IsUnknownHandle(randomEntry.m_handle)) From 5a09741a7e46bbe68fed413954298da09b5e6ec4 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 29 Apr 2022 15:17:16 +0200 Subject: [PATCH 15/59] More cleanup --- src/coreclr/jit/indirectcalltransformer.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index fb5feaf1a80f3f..2d57d4aea4e944 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -773,7 +773,8 @@ class IndirectCallTransformer compiler->fgNewStmtAtEnd(thenBlock, assign); - // Clone call. Note we must use the special candidate helper. + // Clone call for the devirtualized case. Note we must use the + // special candidate helper and we need to use the new 'this'. GenTreeCall* call = compiler->gtCloneCandidateCall(origCall); call->gtArgs.GetThisArg()->SetEarlyNode(compiler->gtNewLclvNode(thisTemp, TYP_REF)); call->SetIsGuarded(); @@ -795,7 +796,8 @@ class IndirectCallTransformer } else { - // Make the updates. + // Otherwise we know the exact method already, so just change + // the call as necessary here. call->gtFlags &= ~GTF_CALL_VIRT_KIND_MASK; call->gtCallMethHnd = methodHnd = inlineInfo->guardedMethodHandle; call->gtCallType = CT_USER_FUNC; @@ -825,9 +827,9 @@ class IndirectCallTransformer } // We know this call can devirtualize or we would not have set up GDV here. - // So impDevirtualizeCall should succeed in devirtualizing. + // So above code should succeed in devirtualizing. // - assert(!call->IsVirtual()); + assert(!call->IsVirtual() && !call->IsDelegateInvoke()); // If the devirtualizer was unable to transform the call to invoke the unboxed entry, the inline info // we set up may be invalid. We won't be able to inline anyways. So demote the call as an inline candidate. From cd077098fe43956ba9e59f1e593b15ca7aff8e89 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 29 Apr 2022 17:34:50 +0200 Subject: [PATCH 16/59] Do not null check delegate GDV calls --- src/coreclr/jit/indirectcalltransformer.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 2d57d4aea4e944..9c74fe43c44917 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -807,15 +807,18 @@ class IndirectCallTransformer // Vtable target could have been expanded early. call->gtControlExpr = nullptr; - bool isExact; - bool objIsNonNull; - compiler->gtGetClassHandle(newThisObj, &isExact, &objIsNonNull); - - // Virtual calls include an implicit null check, which we may - // now need to make explicit. - if (!objIsNonNull) + if (origCall->IsVirtual()) { - call->gtFlags |= GTF_CALL_NULLCHECK; + // Virtual calls include an implicit null check, which we may + // now need to make explicit. + bool isExact; + bool objIsNonNull; + compiler->gtGetClassHandle(newThisObj, &isExact, &objIsNonNull); + + if (!objIsNonNull) + { + call->gtFlags |= GTF_CALL_NULLCHECK; + } } // Clear the inline candidate info (may be non-null since From 8456a11598192756bf53ab5ff6734c74cc761ed7 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 29 Apr 2022 23:12:49 +0200 Subject: [PATCH 17/59] Avoid devirt for probed calls --- src/coreclr/jit/compiler.h | 2 +- src/coreclr/jit/importer.cpp | 88 ++++++++++++++++++++---------------- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 20c3a39013969b..ddce583b90e042 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3523,7 +3523,7 @@ class Compiler bool isExplicitTailCall, IL_OFFSET ilOffset = BAD_IL_OFFSET); - void impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset); + bool impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset); enum class GDVProbeType { diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index f670b0659a4b79..1a193944ba8a7a 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -9516,40 +9516,44 @@ var_types Compiler::impImportCall(OPCODE opcode, call->gtFlags |= obj->gtFlags & GTF_GLOB_EFFECT; call->AsCall()->gtArgs.PushFront(this, obj, WellKnownArg::ThisPointer); - // Is this a virtual or interface call? + if (impIsThis(obj)) + { + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_NONVIRT_SAME_THIS; + } + } + + bool probing; + probing = impConsiderCallProbe(call->AsCall(), rawILOffset); + + // See if we can devirt if we aren't probing. + if (!probing && opts.OptimizationEnabled()) + { if (call->AsCall()->IsVirtual()) { // only true object pointers can be virtual - assert(obj->gtType == TYP_REF); + assert(call->AsCall()->gtArgs.HasThisPointer() && call->AsCall()->gtArgs.GetThisArg()->GetNode()->TypeIs(TYP_REF)); // See if we can devirtualize. - const bool isExplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_EXPLICIT) != 0; + const bool isExplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_EXPLICIT) != 0; const bool isLateDevirtualization = false; impDevirtualizeCall(call->AsCall(), pResolvedToken, &callInfo->hMethod, &callInfo->methodFlags, - &callInfo->contextHandle, &exactContextHnd, isLateDevirtualization, isExplicitTailCall, - // Take care to pass raw IL offset here as the 'debug info' might be different for - // inlinees. - rawILOffset); + &callInfo->contextHandle, &exactContextHnd, isLateDevirtualization, isExplicitTailCall, + // Take care to pass raw IL offset here as the 'debug info' might be different for + // inlinees. + rawILOffset); // Devirtualization may change which method gets invoked. Update our local cache. // methHnd = callInfo->hMethod; } - else if (call->AsCall()->IsDelegateInvoke() && opts.OptimizationEnabled() && !opts.IsOSR()) + else if (call->AsCall()->IsDelegateInvoke()) { considerGuardedDevirtualization(call->AsCall(), rawILOffset, false, NO_METHOD_HANDLE, NO_CLASS_HANDLE, - nullptr); - } - - if (impIsThis(obj)) - { - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_NONVIRT_SAME_THIS; + nullptr); } } - impConsiderCallProbe(call->AsCall(), rawILOffset); - //------------------------------------------------------------------------- // The "this" pointer for "newobj" @@ -20888,12 +20892,7 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, // This should be a virtual vtable or virtual stub call. // assert(call->IsVirtual()); - - // Bail if optimizations are disabled. - if (opts.OptimizationDisabled()) - { - return; - } + assert(opts.OptimizationEnabled()); #if defined(DEBUG) // Bail if devirt is disabled. @@ -21509,7 +21508,18 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, #endif // FEATURE_READYTORUN } -void Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) +//------------------------------------------------------------------------ +// impConsiderCallProbe: Consider whether a call should get a histogram probe +// and mark it if so. +// +// Arguments: +// call - The call +// ilOffset - The precise IL offset of the call +// +// Returns: +// True if the call was marked such that we will add a class or method probe for it. +// +bool Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) { // Possibly instrument. Note for OSR+PGO we will instrument when // optimizing and (currently) won't devirtualize. We may want @@ -21521,7 +21531,7 @@ void Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) // if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR)) { - return; + return false; } assert(opts.OptimizationDisabled() || opts.IsOSR()); @@ -21532,22 +21542,25 @@ void Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) // we'd just keep track of the calls themselves, so we don't // have to search for them later. // - if (compClassifyGDVProbeType(call) != GDVProbeType::None) + if (compClassifyGDVProbeType(call) == GDVProbeType::None) { - JITDUMP("\n ... marking [%06u] in " FMT_BB " for method/class profile instrumentation\n", dspTreeID(call), - compCurBB->bbNum); - HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; + return false; + } - // Record some info needed for the class profiling probe. - // - pInfo->ilOffset = ilOffset; - pInfo->probeIndex = info.compHandleHistogramProbeCount++; - call->gtHandleHistogramProfileCandidateInfo = pInfo; + JITDUMP("\n ... marking [%06u] in " FMT_BB " for method/class profile instrumentation\n", dspTreeID(call), + compCurBB->bbNum); + HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; - // Flag block as needing scrutiny - // - compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; - } + // Record some info needed for the class profiling probe. + // + pInfo->ilOffset = ilOffset; + pInfo->probeIndex = info.compHandleHistogramProbeCount++; + call->gtHandleHistogramProfileCandidateInfo = pInfo; + + // Flag block as needing scrutiny + // + compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; + return true; } Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) @@ -21597,7 +21610,6 @@ Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) // Returns: // Exact class handle returned by the intrinsic call, if known. // Nullptr if not known, or not likely to lead to beneficial optimization. - CORINFO_CLASS_HANDLE Compiler::impGetSpecialIntrinsicExactReturnType(CORINFO_METHOD_HANDLE methodHnd) { JITDUMP("Special intrinsic: looking for exact type returned by %s\n", eeGetMethodFullName(methodHnd)); From e4017111961215e664c64bf9b8f506c97e7b3f40 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 30 Apr 2022 12:40:35 +0200 Subject: [PATCH 18/59] Cleanup and format --- src/coreclr/jit/importer.cpp | 15 ++++++++------- src/coreclr/jit/indirectcalltransformer.cpp | 17 ++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 1a193944ba8a7a..75873f674afe70 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -9531,17 +9531,18 @@ var_types Compiler::impImportCall(OPCODE opcode, if (call->AsCall()->IsVirtual()) { // only true object pointers can be virtual - assert(call->AsCall()->gtArgs.HasThisPointer() && call->AsCall()->gtArgs.GetThisArg()->GetNode()->TypeIs(TYP_REF)); + assert(call->AsCall()->gtArgs.HasThisPointer() && + call->AsCall()->gtArgs.GetThisArg()->GetNode()->TypeIs(TYP_REF)); // See if we can devirtualize. - const bool isExplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_EXPLICIT) != 0; + const bool isExplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_EXPLICIT) != 0; const bool isLateDevirtualization = false; impDevirtualizeCall(call->AsCall(), pResolvedToken, &callInfo->hMethod, &callInfo->methodFlags, - &callInfo->contextHandle, &exactContextHnd, isLateDevirtualization, isExplicitTailCall, - // Take care to pass raw IL offset here as the 'debug info' might be different for - // inlinees. - rawILOffset); + &callInfo->contextHandle, &exactContextHnd, isLateDevirtualization, isExplicitTailCall, + // Take care to pass raw IL offset here as the 'debug info' might be different for + // inlinees. + rawILOffset); // Devirtualization may change which method gets invoked. Update our local cache. // @@ -9550,7 +9551,7 @@ var_types Compiler::impImportCall(OPCODE opcode, else if (call->AsCall()->IsDelegateInvoke()) { considerGuardedDevirtualization(call->AsCall(), rawILOffset, false, NO_METHOD_HANDLE, NO_CLASS_HANDLE, - nullptr); + nullptr); } } diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 9c74fe43c44917..030a61dfcb0234 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -569,18 +569,16 @@ class IndirectCallTransformer GuardedDevirtualizationCandidateInfo* guardedInfo = origCall->gtGuardedDevirtualizationCandidateInfo; + // Create comparison. On success we will jump to do the indirect call. GenTree* compare; if (guardedInfo->guardedClassHandle != NO_CLASS_HANDLE) { - // Find target method table // GenTree* methodTable = compiler->gtNewMethodTableLookup(thisTree); CORINFO_CLASS_HANDLE clsHnd = guardedInfo->guardedClassHandle; GenTree* targetMethodTable = compiler->gtNewIconEmbClsHndNode(clsHnd); - // Compare and jump to else (which does the indirect call) if NOT equal - // compare = compiler->gtNewOperNode(GT_NE, TYP_INT, targetMethodTable, methodTable); } else @@ -602,13 +600,13 @@ class IndirectCallTransformer CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionEntryPoint(methHnd, &lookup); - GenTree* compareTarTree = CreateLookup(lookup); - ; - compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, + GenTree* compareTarTree = CreateTreeForLookup(lookup); + compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)); } else { + // TODO: Change original call to CT_INDIRECT call here to reuse the call target. // const unsigned addrTempNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target // temp")); // compiler->lvaGetDesc(addrTempNum)->lvSingleDef = 1; @@ -627,7 +625,7 @@ class IndirectCallTransformer CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionFixedEntryPoint(methHnd, false, &lookup); - GenTree* compareTarTree = CreateLookup(lookup); + GenTree* compareTarTree = CreateTreeForLookup(lookup); compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, tarTree /*compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)*/); } @@ -740,11 +738,12 @@ class IndirectCallTransformer InlineCandidateInfo* inlineInfo = origCall->gtInlineCandidateInfo; CORINFO_CLASS_HANDLE clsHnd = inlineInfo->guardedClassHandle; + // // Copy the 'this' for the devirtualized call to a new temp. For // class-based GDV this will allow us to set the exact type on that // temp. For delegate GDV, this will be the actual 'this' object // stored in the delegate. - + // const unsigned thisTemp = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt this exact temp")); GenTree* clonedObj = compiler->gtCloneExpr(origCall->gtArgs.GetThisArg()->GetNode()); GenTree* newThisObj; @@ -1128,7 +1127,7 @@ class IndirectCallTransformer unsigned returnTemp; Statement* lastStmt; - GenTree* CreateLookup(const CORINFO_CONST_LOOKUP& lookup) + GenTree* CreateTreeForLookup(const CORINFO_CONST_LOOKUP& lookup) { switch (lookup.accessType) { From 7bc4374dab0894608d58c1e81f5839bea6ce0754 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 30 Apr 2022 12:40:40 +0200 Subject: [PATCH 19/59] Fix 32-bit build --- src/coreclr/jit/fgprofile.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 1f121c083f7440..c16a910ad6e4db 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1639,7 +1639,7 @@ class HandleHistogramProbeInserter private: void ReadHistogramAndAdvance(IL_OFFSET ilOffset, void** typeHistogram, void** methodHistogram, bool* histogramIs32) { - if (*m_currentSchemaIndex >= m_schema.size()) + if (*m_currentSchemaIndex >= (int)m_schema.size()) { return; } @@ -1658,7 +1658,7 @@ class HandleHistogramProbeInserter return; } - assert(*m_currentSchemaIndex + 2 <= m_schema.size()); + assert(*m_currentSchemaIndex + 2 <= (int)m_schema.size()); ICorJitInfo::PgoInstrumentationSchema& tableEntry = m_schema[*m_currentSchemaIndex + 1]; assert((tableEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes) || (tableEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods)); From 5ee82e264e3b2bc1e9440c917f5594bfeff993ed Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 2 May 2022 21:53:09 +0200 Subject: [PATCH 20/59] Reuse target from temp in cold call --- src/coreclr/jit/indirectcalltransformer.cpp | 87 ++++++++++++++++----- 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 030a61dfcb0234..1356514e9f9b2f 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -450,9 +450,10 @@ class IndirectCallTransformer class GuardedDevirtualizationTransformer final : public Transformer { + unsigned m_targetLclNum; public: GuardedDevirtualizationTransformer(Compiler* compiler, BasicBlock* block, Statement* stmt) - : Transformer(compiler, block, stmt), returnTemp(BAD_VAR_NUM) + : Transformer(compiler, block, stmt), m_targetLclNum(BAD_VAR_NUM), returnTemp(BAD_VAR_NUM) { } @@ -538,7 +539,8 @@ class IndirectCallTransformer checkBlock = currBlock; checkBlock->bbJumpKind = BBJ_COND; - GenTree* thisTree = origCall->gtArgs.GetThisArg()->GetNode(); + CallArg* thisArg = origCall->gtArgs.GetThisArg(); + GenTree* thisTree = thisArg->GetNode(); // Create temp for this if the tree is costly. if (thisTree->IsLocal()) @@ -556,7 +558,7 @@ class IndirectCallTransformer // Propagate the new this to the call. Must be a new expr as the call // will live on in the else block and thisTree is used below. - origCall->gtArgs.GetThisArg()->SetEarlyNode(compiler->gtNewLclvNode(thisTempNum, TYP_REF)); + thisArg->SetEarlyNode(compiler->gtNewLclvNode(thisTempNum, TYP_REF)); } // Remember the current last statement. If we're doing a chained GDV, we'll clone/copy @@ -586,48 +588,62 @@ class IndirectCallTransformer assert(origCall->IsVirtualVtable() || origCall->IsDelegateInvoke()); if (origCall->IsVirtualVtable()) { - const unsigned addrTempNum = + m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); - compiler->lvaGetDesc(addrTempNum)->lvSingleDef = 1; + compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; GenTree* tarTree = compiler->fgExpandVirtualVtableCallTarget(origCall); - GenTree* asgTree = compiler->gtNewTempAssign(addrTempNum, tarTree); + GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); - origCall->gtControlExpr = compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL); CORINFO_METHOD_HANDLE methHnd = guardedInfo->guardedMethodHandle; CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionEntryPoint(methHnd, &lookup); GenTree* compareTarTree = CreateTreeForLookup(lookup); compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, - compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)); + compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL)); } else { - // TODO: Change original call to CT_INDIRECT call here to reuse the call target. - // const unsigned addrTempNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target - // temp")); - // compiler->lvaGetDesc(addrTempNum)->lvSingleDef = 1; + // Since the guard requires the target address we will + // expand the cold call to reuse the target call. + // Essentially we will do the transformation done in + // LowerDelegateInvoke here by converting the call to + // CT_INDIRECT and reusing the target address. + bool expandOrigCall = true; + +#ifdef TARGET_ARM + // Not impossible to support, but would additionally + // require us to load the wrapper delegate cell when + // expanding. + expandOrigCall &= (origCall->gtCallMoreFlags & GTF_CALL_M_WRAPPER_DELEGATE_INV) == 0; +#endif GenTree* offset = compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateFirstTarget, TYP_I_IMPL); GenTree* tarTree = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, thisTree, offset); tarTree = compiler->gtNewIndir(TYP_I_IMPL, tarTree); - // GenTree* asgTree = compiler->gtNewTempAssign(addrTempNum, tarTree); - // Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); - // compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); - // origCall->gtControlExpr = compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL); + if (expandOrigCall) + { + m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); + compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; + + GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); + Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); + compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); + tarTree = compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL); + } + CORINFO_METHOD_HANDLE methHnd = guardedInfo->guardedMethodHandle; CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionFixedEntryPoint(methHnd, false, &lookup); GenTree* compareTarTree = CreateTreeForLookup(lookup); - compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, - tarTree /*compiler->gtNewLclvNode(addrTempNum, TYP_I_IMPL)*/); + compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, tarTree); } } @@ -803,8 +819,6 @@ class IndirectCallTransformer call->gtCallMoreFlags |= GTF_CALL_M_DEVIRTUALIZED; call->gtCallMoreFlags &= ~GTF_CALL_M_DELEGATE_INV; // TODO: R2R entry point - // Vtable target could have been expanded early. - call->gtControlExpr = nullptr; if (origCall->IsVirtual()) { @@ -897,7 +911,7 @@ class IndirectCallTransformer } //------------------------------------------------------------------------ - // CreateElse: create else block. This executes the unaltered indirect call. + // CreateElse: create else block. This executes the original indirect call. // virtual void CreateElse() { @@ -917,6 +931,37 @@ class IndirectCallTransformer newStmt->SetRootNode(assign); } + if (m_targetLclNum != BAD_VAR_NUM) + { + if (call->IsVirtualVtable()) + { + // We already loaded the target once for the check, so reuse it from the temp. + call->gtControlExpr = compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL); + } + else if (call->IsDelegateInvoke()) + { + // Target was saved into a temp during check. We expand the + // delegate call to a CT_INDIRECT call that uses the target + // directly, somewhat similarly to LowerDelegateInvoke. + call->gtCallType = CT_INDIRECT; + call->gtCallAddr = compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL); + call->gtCallCookie = nullptr; + call->gtCallMoreFlags &= ~GTF_CALL_M_DELEGATE_INV; + + GenTree* thisOffset = compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateInstance, TYP_I_IMPL); + CallArg* thisArg = call->gtArgs.GetThisArg(); + GenTree* delegateObj = thisArg->GetNode(); + // TODO: Is it worth it to create a local for this before + // the check as well? In many cases it will end up unused + // after inlining. + assert(delegateObj->OperIsLocal()); + GenTree* newThis = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, compiler->gtCloneExpr(delegateObj), thisOffset); + newThis = compiler->gtNewIndir(TYP_REF, newThis); + + thisArg->SetEarlyNode(newThis); + } + } + compiler->fgInsertStmtAtEnd(elseBlock, newStmt); // Set the original statement to a nop. From b175511ffe85a7a3f1c4cc57f25f71d7fb73e162 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 18 May 2022 10:25:10 +0200 Subject: [PATCH 21/59] Fix build --- src/coreclr/jit/compiler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index c06b14abf24e1a..f4d9398dda998c 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1008,7 +1008,7 @@ inline GenTreeCall* Compiler::gtNewHelperCallNode( if (arg4 != nullptr) { - result->gtArgs.PushFront(this, arg4); + result->gtArgs.PushFront(this, NewCallArg::Primitive(arg4)); result->gtFlags |= arg4->gtFlags & GTF_ALL_EFFECT; } From 95af330145369101c7b21abe6d997a52b292e91d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 18 May 2022 11:48:55 +0200 Subject: [PATCH 22/59] Add MethodProfiling PGO scenarios --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 4 +++- eng/pipelines/libraries/run-test-job.yml | 2 ++ src/tests/Common/testenvironment.proj | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index f7f310dfb97429..d040b523ce02ae 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -397,7 +397,7 @@ jobs: ${{ if eq(parameters.runtimeFlavor, 'mono') }}: # tiered compilation isn't done on mono yet scenarios: - - normal + - normal ${{ elseif eq(variables['Build.Reason'], 'PullRequest') }}: scenarios: - no_tiered_compilation @@ -545,7 +545,9 @@ jobs: - defaultpgo - dynamicpgo - fullpgo + - fullpgo_methodprofiling - fullpgo_random_gdv + - fullpgo_random_gdv_methodprofiling_only - fullpgo_random_edge - fullpgo_random_gdv_edge ${{ if in(parameters.testGroup, 'gc-longrunning') }}: diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 17e3a63875d176..61697db4c25778 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -173,7 +173,9 @@ jobs: - defaultpgo - dynamicpgo - fullpgo + - fullpgo_methodprofiling - fullpgo_random_gdv + - fullpgo_random_gdv_methodprofiling_only - fullpgo_random_edge - fullpgo_random_gdv_edge - jitosr diff --git a/src/tests/Common/testenvironment.proj b/src/tests/Common/testenvironment.proj index fcc3e3926febc8..d2ccdcf35261d1 100644 --- a/src/tests/Common/testenvironment.proj +++ b/src/tests/Common/testenvironment.proj @@ -58,6 +58,7 @@ COMPlus_JitObjectStackAllocation; COMPlus_JitInlinePolicyProfile; COMPlus_JitClassProfiling; + COMPlus_JitMethodProfiling; COMPlus_JitEdgeProfiling; COMPlus_JitRandomGuardedDevirtualization; COMPlus_JitRandomEdgeCounts; @@ -204,7 +205,9 @@ + + From fa016648df1376926affbf107a55add21bd1eda9 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 18 May 2022 11:49:19 +0200 Subject: [PATCH 23/59] Support randomized GDVs for method handle histograms --- src/coreclr/jit/compiler.h | 12 +++-- src/coreclr/jit/importer.cpp | 19 +++++--- src/coreclr/jit/likelyclass.cpp | 84 ++++++++++++++++++++------------- 3 files changed, 70 insertions(+), 45 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 5a6cef6b23f09b..cfaae9aae5d0b5 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5441,11 +5441,13 @@ class Compiler void fgIncorporateBlockCounts(); void fgIncorporateEdgeCounts(); - CORINFO_CLASS_HANDLE getRandomClass(ICorJitInfo::PgoInstrumentationSchema* schema, - UINT32 countSchemaItems, - BYTE* pInstrumentationData, - int32_t ilOffset, - CLRRandom* random); + void getRandomGDV(ICorJitInfo::PgoInstrumentationSchema* schema, + UINT32 countSchemaItems, + BYTE* pInstrumentationData, + int32_t ilOffset, + CLRRandom* random, + CORINFO_CLASS_HANDLE* classGuess, + CORINFO_METHOD_HANDLE* methodGuess); public: const char* fgPgoFailReason; diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 38567133db84c6..4b2df5a489ae36 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -12190,12 +12190,14 @@ GenTree* Compiler::impCastClassOrIsInstToTree( // Reuse the random inliner's random state. CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); - likelyClasses[0].handle = - (intptr_t)getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); - likelyClasses[0].likelihood = 100; - if (likelyClasses[0].handle != (intptr_t)NO_CLASS_HANDLE) + CORINFO_CLASS_HANDLE clsGuess; + CORINFO_METHOD_HANDLE methGuess; + getRandomGDV(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random, &clsGuess, &methGuess); + if (clsGuess != NO_CLASS_HANDLE) { - likelyClassCount = 1; + likelyClasses[0].likelihood = 100; + likelyClasses[0].handle = (intptr_t)clsGuess; + likelyClassCount = 1; } } else @@ -22400,12 +22402,17 @@ void Compiler::pickGDV(GenTreeCall* call, // CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); - *classGuess = getRandomClass(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random); + getRandomGDV(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random, classGuess, methodGuess); if (*classGuess != NO_CLASS_HANDLE) { JITDUMP("Picked random class for GDV: %p (%s)\n", *classGuess, eeGetClassName(*classGuess)); return; } + if (*methodGuess != NO_METHOD_HANDLE) + { + JITDUMP("Picked random method for GDV: %p (%s)\n", *methodGuess, eeGetMethodFullName(*methodGuess)); + return; + } } #endif diff --git a/src/coreclr/jit/likelyclass.cpp b/src/coreclr/jit/likelyclass.cpp index c31e3b428c2c6c..f293a8f164ac48 100644 --- a/src/coreclr/jit/likelyclass.cpp +++ b/src/coreclr/jit/likelyclass.cpp @@ -305,8 +305,8 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* } //------------------------------------------------------------------------ -// getRandomClass: find class profile data for an IL offset, and return -// one of the possible classes at random +// getRandomGDV: find GDV profile data for an IL offset, and return +// one of the possible methods/classes at random // // Arguments: // schema - profile schema @@ -318,17 +318,48 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* // Returns: // Randomly observed class, or nullptr. // -CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSchema* schema, - UINT32 countSchemaItems, - BYTE* pInstrumentationData, - int32_t ilOffset, - CLRRandom* random) +void Compiler::getRandomGDV(ICorJitInfo::PgoInstrumentationSchema* schema, + UINT32 countSchemaItems, + BYTE* pInstrumentationData, + int32_t ilOffset, + CLRRandom* random, + CORINFO_CLASS_HANDLE* classGuess, + CORINFO_METHOD_HANDLE* methodGuess) { + *classGuess = NO_CLASS_HANDLE; + *methodGuess = NO_METHOD_HANDLE; + if (schema == nullptr) { - return NO_CLASS_HANDLE; + return; } + // We can have multiple histograms for the same IL offset. Use reservoir + // sampling to pick an entry at random. + int numElementsSeen = 0; + auto addElement = [random, classGuess, methodGuess, &numElementsSeen](intptr_t handle, bool isClass) { + if (ICorJitInfo::IsUnknownHandle(handle)) + { + return; + } + + numElementsSeen++; + bool replace = (numElementsSeen == 1) || (random->Next(numElementsSeen) == 0); + if (replace) + { + if (isClass) + { + *classGuess = (CORINFO_CLASS_HANDLE)handle; + *methodGuess = NO_METHOD_HANDLE; + } + else + { + *classGuess = NO_CLASS_HANDLE; + *methodGuess = (CORINFO_METHOD_HANDLE)handle; + } + } + }; + for (COUNT_T i = 0; i < countSchemaItems; i++) { if (schema[i].ILOffset != (int32_t)ilOffset) @@ -340,14 +371,8 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch (schema[i].Count == 1)) { INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset); - if (ICorJitInfo::IsUnknownHandle(result)) - { - return NO_CLASS_HANDLE; - } - else - { - return (CORINFO_CLASS_HANDLE)result; - } + addElement(result, true); + continue; } bool isHistogramCount = @@ -355,30 +380,21 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount); if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) && - (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes)) + ((schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes) || + (schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods))) { - // Form a histogram + // Form a histogram. Note that even though we use reservoir + // sampling we want to weigh distinct handles equally, regardless + // of count. // LikelyClassMethodHistogram h((INT_PTR*)(pInstrumentationData + schema[i + 1].Offset), schema[i + 1].Count); - if (h.countHistogramElements == 0) + bool isClass = + schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes; + for (UINT32 i = 0; i < h.countHistogramElements; i++) { - return NO_CLASS_HANDLE; + addElement(h.HistogramEntryAt(i).m_handle, isClass); } - - // Choose an entry at random. - // - unsigned randomEntryIndex = random->Next(0, h.countHistogramElements); - LikelyClassMethodHistogramEntry randomEntry = h.HistogramEntryAt(randomEntryIndex); - - if (ICorJitInfo::IsUnknownHandle(randomEntry.m_handle)) - { - return NO_CLASS_HANDLE; - } - - return (CORINFO_CLASS_HANDLE)randomEntry.m_handle; } } - - return NO_CLASS_HANDLE; } From 7a5386e7a8b98ad03e7a3131c4c557d6c08af8ec Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 18 May 2022 11:50:49 +0200 Subject: [PATCH 24/59] Run jit-format --- src/coreclr/jit/indirectcalltransformer.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index adca1d5e1c91eb..cf414a3b9f5d91 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -451,6 +451,7 @@ class IndirectCallTransformer class GuardedDevirtualizationTransformer final : public Transformer { unsigned m_targetLclNum; + public: GuardedDevirtualizationTransformer(Compiler* compiler, BasicBlock* block, Statement* stmt) : Transformer(compiler, block, stmt), m_targetLclNum(BAD_VAR_NUM), returnTemp(BAD_VAR_NUM) @@ -539,7 +540,7 @@ class IndirectCallTransformer checkBlock = currBlock; checkBlock->bbJumpKind = BBJ_COND; - CallArg* thisArg = origCall->gtArgs.GetThisArg(); + CallArg* thisArg = origCall->gtArgs.GetThisArg(); GenTree* thisTree = thisArg->GetNode(); // Create temp for this if the tree is costly. @@ -588,8 +589,7 @@ class IndirectCallTransformer assert(origCall->IsVirtualVtable() || origCall->IsDelegateInvoke()); if (origCall->IsVirtualVtable()) { - m_targetLclNum = - compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); + m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; GenTree* tarTree = compiler->fgExpandVirtualVtableCallTarget(origCall); @@ -632,7 +632,7 @@ class IndirectCallTransformer m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; - GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); + GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); tarTree = compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL); @@ -943,19 +943,21 @@ class IndirectCallTransformer // Target was saved into a temp during check. We expand the // delegate call to a CT_INDIRECT call that uses the target // directly, somewhat similarly to LowerDelegateInvoke. - call->gtCallType = CT_INDIRECT; - call->gtCallAddr = compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL); + call->gtCallType = CT_INDIRECT; + call->gtCallAddr = compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL); call->gtCallCookie = nullptr; call->gtCallMoreFlags &= ~GTF_CALL_M_DELEGATE_INV; - GenTree* thisOffset = compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateInstance, TYP_I_IMPL); - CallArg* thisArg = call->gtArgs.GetThisArg(); + GenTree* thisOffset = + compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateInstance, TYP_I_IMPL); + CallArg* thisArg = call->gtArgs.GetThisArg(); GenTree* delegateObj = thisArg->GetNode(); // TODO: Is it worth it to create a local for this before // the check as well? In many cases it will end up unused // after inlining. assert(delegateObj->OperIsLocal()); - GenTree* newThis = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, compiler->gtCloneExpr(delegateObj), thisOffset); + GenTree* newThis = + compiler->gtNewOperNode(GT_ADD, TYP_BYREF, compiler->gtCloneExpr(delegateObj), thisOffset); newThis = compiler->gtNewIndir(TYP_REF, newThis); thisArg->SetEarlyNode(newThis); From edd2db799a95898a7caea459250ce2b737ddba17 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 23 May 2022 13:20:11 +0200 Subject: [PATCH 25/59] Fix an asertion failure If we manage to always devirt in the cold path, then we could leave the control expression causing issues downstream. --- src/coreclr/jit/importer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 4b2df5a489ae36..4818891d6758b5 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -21780,6 +21780,7 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, call->gtFlags &= ~GTF_CALL_VIRT_STUB; call->gtCallMethHnd = derivedMethod; call->gtCallType = CT_USER_FUNC; + call->gtControlExpr = nullptr; call->gtCallMoreFlags |= GTF_CALL_M_DEVIRTUALIZED; // Virtual calls include an implicit null check, which we may From e4093c9ee556c9509f0c7e5a8275b41fdea5e2f1 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 23 May 2022 13:57:59 +0200 Subject: [PATCH 26/59] Make sure we only log in DEBUG --- src/coreclr/jit/importer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 4818891d6758b5..b650d78c663b09 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22441,6 +22441,7 @@ void Compiler::pickGDV(GenTreeCall* call, return; } +#ifdef DEBUG if (numberOfClasses > 0) { bool isExact; @@ -22494,6 +22495,7 @@ void Compiler::pickGDV(GenTreeCall* call, } } } +#endif // Prefer class guess as it is cheaper if (numberOfClasses > 0) From 2c1ebe30e58778fd9de2c248afd935c2a269e936 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 23 May 2022 13:58:19 +0200 Subject: [PATCH 27/59] Avoid GDV for static calls --- src/coreclr/jit/importer.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index b650d78c663b09..6d609d8bc36a99 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22605,13 +22605,20 @@ void Compiler::considerGuardedDevirtualization(GenTreeCall* call, likelyMethod = dvInfo.devirtualizedMethod; } + uint32_t likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); + if ((likelyMethodAttribs & CORINFO_FLG_STATIC) != 0) + { + assert(call->IsDelegateInvoke()); + JITDUMP("Cannot currently handle devirtualizing static delegate calls, sorry\n"); + return; + } + JITDUMP("%s call would invoke method %s\n", isInterface ? "interface" : call->IsDelegateInvoke() ? "delegate" : "virtual", eeGetMethodName(likelyMethod, nullptr)); // Add this as a potential candidate. // - uint32_t const likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); addGuardedDevirtualizationCandidate(call, likelyMethod, likelyClass, likelyMethodAttribs, likelyClassAttribs, likelihood); } From fbbac89d88dbc578851f2778d8ef32242ac1009e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 23 May 2022 15:54:57 +0200 Subject: [PATCH 28/59] Fix interaction with chained GDV For chained GDV candidates, we cannot rely on the 'check' having been executed, so we cannot reuse the target that was already loaded by the check. --- src/coreclr/jit/indirectcalltransformer.cpp | 40 ++++++++++++--------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index cf414a3b9f5d91..e427307de7e433 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -587,38 +587,46 @@ class IndirectCallTransformer else { assert(origCall->IsVirtualVtable() || origCall->IsDelegateInvoke()); + // We reuse the target except if this is a chained GDV, in + // which case the check will be moved into the success case of + // a previous GDV and thus may not execute when we hit the cold + // path. + bool reuseTarget = (origCall->gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_CHAIN) == 0; if (origCall->IsVirtualVtable()) { - m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); - compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; + GenTree* tarTree = compiler->fgExpandVirtualVtableCallTarget(origCall); - GenTree* tarTree = compiler->fgExpandVirtualVtableCallTarget(origCall); - GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); - Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); - compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); + if (reuseTarget) + { + m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); + compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; + + GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); + Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); + compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); + + tarTree = compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL); + } CORINFO_METHOD_HANDLE methHnd = guardedInfo->guardedMethodHandle; CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionEntryPoint(methHnd, &lookup); GenTree* compareTarTree = CreateTreeForLookup(lookup); - compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, - compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL)); + compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, tarTree); } else { - // Since the guard requires the target address we will - // expand the cold call to reuse the target call. - // Essentially we will do the transformation done in + // Reusing the call target for delegates is more complicated. + // Essentially we need to do the transformation done in // LowerDelegateInvoke here by converting the call to // CT_INDIRECT and reusing the target address. - bool expandOrigCall = true; - + CLANG_FORMAT_COMMENT_ANCHOR; #ifdef TARGET_ARM // Not impossible to support, but would additionally // require us to load the wrapper delegate cell when // expanding. - expandOrigCall &= (origCall->gtCallMoreFlags & GTF_CALL_M_WRAPPER_DELEGATE_INV) == 0; + reuseTarget &= (origCall->gtCallMoreFlags & GTF_CALL_M_WRAPPER_DELEGATE_INV) == 0; #endif GenTree* offset = @@ -627,7 +635,7 @@ class IndirectCallTransformer GenTree* tarTree = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, thisTree, offset); tarTree = compiler->gtNewIndir(TYP_I_IMPL, tarTree); - if (expandOrigCall) + if (reuseTarget) { m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; @@ -1127,7 +1135,7 @@ class IndirectCallTransformer // GenTree* const root = nextStmt->GetRootNode(); - if (root->IsCall() && !root->AsCall()->IsDelegateInvoke()) + if (root->IsCall()) { GenTreeCall* const call = root->AsCall(); From d622b647d37667b7956f74795a907af0af690482 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 23 May 2022 16:50:15 +0200 Subject: [PATCH 29/59] Fix cast helper probes * Do not insert them when class profiling is turned off * Use the probe information to determine the probe time. Just eligibility is not enough as they are conditionally added. --- src/coreclr/jit/importer.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 6d609d8bc36a99..4fbe54ff3aaff2 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -12245,7 +12245,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( op2->gtFlags |= GTF_DONT_CSE; GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); - if (impIsCastHelperEligibleForClassProbe(call) && !impIsClassExact(pResolvedToken->hClass)) + if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && !impIsClassExact(pResolvedToken->hClass)) { HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; pInfo->ilOffset = ilOffset; @@ -22187,9 +22187,17 @@ Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) return GDVProbeType::None; } - bool createTypeHistogram = - (JitConfig.JitClassProfiling() > 0) && - (call->IsVirtualStub() || call->IsVirtualVtable() || impIsCastHelperEligibleForClassProbe(call)); + bool createTypeHistogram = false; + if (JitConfig.JitClassProfiling() > 0) + { + createTypeHistogram = call->IsVirtualStub() || call->IsVirtualVtable(); + + // Cast helpers may conditionally (depending on whether the class is + // exact or not) have probes. For those helpers we do not use this + // function to classify the probe type until after we have decided on + // whether we probe them or not. + createTypeHistogram = createTypeHistogram || (impIsCastHelperEligibleForClassProbe(call) && (call->gtHandleHistogramProfileCandidateInfo != nullptr)); + } bool createMethodHistogram = (JitConfig.JitMethodProfiling() > 0) && (call->IsVirtualVtable() || call->IsDelegateInvoke()); From 8ce47f81354efabcfc55c2b366bddbf8c328fbb4 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 23 May 2022 17:02:20 +0200 Subject: [PATCH 30/59] Run jit-format --- src/coreclr/jit/importer.cpp | 6 ++++-- src/coreclr/jit/indirectcalltransformer.cpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 4fbe54ff3aaff2..9699b132db2bbb 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -12245,7 +12245,8 @@ GenTree* Compiler::impCastClassOrIsInstToTree( op2->gtFlags |= GTF_DONT_CSE; GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); - if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && !impIsClassExact(pResolvedToken->hClass)) + if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && + !impIsClassExact(pResolvedToken->hClass)) { HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; pInfo->ilOffset = ilOffset; @@ -22196,7 +22197,8 @@ Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) // exact or not) have probes. For those helpers we do not use this // function to classify the probe type until after we have decided on // whether we probe them or not. - createTypeHistogram = createTypeHistogram || (impIsCastHelperEligibleForClassProbe(call) && (call->gtHandleHistogramProfileCandidateInfo != nullptr)); + createTypeHistogram = createTypeHistogram || (impIsCastHelperEligibleForClassProbe(call) && + (call->gtHandleHistogramProfileCandidateInfo != nullptr)); } bool createMethodHistogram = diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index e427307de7e433..c31101dee814dc 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -601,7 +601,7 @@ class IndirectCallTransformer m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; - GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); + GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); compiler->fgInsertStmtAtEnd(checkBlock, asgStmt); From 39ac90a00f6a3c29ebea89f1317bab69557d0d0c Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 24 May 2022 12:16:43 +0200 Subject: [PATCH 31/59] Pick random GDV after logging histogram --- src/coreclr/jit/importer.cpp | 46 +++++++++++++++++------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 9699b132db2bbb..662a8dae7441d3 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22403,30 +22403,6 @@ void Compiler::pickGDV(GenTreeCall* call, *methodGuess = NO_METHOD_HANDLE; *likelihood = 0; -#ifdef DEBUG - // Optional stress mode to pick a random known class, rather than - // the most likely known class. - // - if (JitConfig.JitRandomGuardedDevirtualization() != 0) - { - // Reuse the random inliner's random state. - // - CLRRandom* const random = - impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); - getRandomGDV(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random, classGuess, methodGuess); - if (*classGuess != NO_CLASS_HANDLE) - { - JITDUMP("Picked random class for GDV: %p (%s)\n", *classGuess, eeGetClassName(*classGuess)); - return; - } - if (*methodGuess != NO_METHOD_HANDLE) - { - JITDUMP("Picked random method for GDV: %p (%s)\n", *methodGuess, eeGetMethodFullName(*methodGuess)); - return; - } - } -#endif - const int maxLikelyClasses = 32; LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; unsigned numberOfClasses = 0; @@ -22505,6 +22481,28 @@ void Compiler::pickGDV(GenTreeCall* call, } } } + + // Optional stress mode to pick a random known class, rather than + // the most likely known class. + // + if (JitConfig.JitRandomGuardedDevirtualization() != 0) + { + // Reuse the random inliner's random state. + // + CLRRandom* const random = + impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + getRandomGDV(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random, classGuess, methodGuess); + if (*classGuess != NO_CLASS_HANDLE) + { + JITDUMP("Picked random class for GDV: %p (%s)\n", *classGuess, eeGetClassName(*classGuess)); + return; + } + if (*methodGuess != NO_METHOD_HANDLE) + { + JITDUMP("Picked random method for GDV: %p (%s)\n", *methodGuess, eeGetMethodFullName(*methodGuess)); + return; + } + } #endif // Prefer class guess as it is cheaper From 0ab00aec1ad387f02328efd693956d369a869eed Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 24 May 2022 12:27:04 +0200 Subject: [PATCH 32/59] Fix GDV for vtable calls on System.Delegate --- src/coreclr/vm/jithelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 2df02775fba496..3eb15a10e242fd 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -5475,7 +5475,7 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod // Resolve method MethodDesc* pMD = NULL; - if (pMT->IsDelegate()) + if (pMT->IsDelegate() && (pBaseMD == ((DelegateEEClass*)pMT->GetClass())->GetInvokeMethod())) { // We handle only the common "direct" delegate as that is in any case // the only one we can reasonably do GDV for. For instance, open From 93efde4c8ffeeb772fcbb3dc85aa0e20b000cdf6 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 24 May 2022 14:12:10 +0200 Subject: [PATCH 33/59] Mark early expanded vtable calls as such On arm32 we may not already have set this flag. --- src/coreclr/jit/indirectcalltransformer.cpp | 1 + src/coreclr/jit/morph.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index c31101dee814dc..382a7da4e8c404 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -945,6 +945,7 @@ class IndirectCallTransformer { // We already loaded the target once for the check, so reuse it from the temp. call->gtControlExpr = compiler->gtNewLclvNode(m_targetLclNum, TYP_I_IMPL); + call->SetExpandedEarly(); } else if (call->IsDelegateInvoke()) { diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 8bfd53fa6eac7b..f68ca9ac8760b4 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -6643,9 +6643,11 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call) (call->gtCallType == CT_USER_FUNC) ? call->gtCallMethHnd : nullptr, call->IsTailPrefixedCall(), tailCallResult, nullptr); - // Are we currently planning to expand the gtControlExpr as an early virtual call target? + // Do some profitability checks for whether we should expand a vtable call + // target early. Note that we may already have expanded it due to GDV at + // this point, so make sure we do not undo that work. // - if (call->IsExpandedEarly() && call->IsVirtualVtable()) + if (call->IsExpandedEarly() && call->IsVirtualVtable() && (call->gtControlExpr == nullptr)) { assert(call->gtArgs.HasThisPointer()); // It isn't alway profitable to expand a virtual call early From 4b44c8d33f6509be5a77c95f2ab14e72e0b6e54b Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 24 May 2022 15:14:58 +0200 Subject: [PATCH 34/59] Pull reservoir index calculation up in profile helpers --- src/coreclr/vm/jithelpers.cpp | 238 +++++++++++++--------------------- 1 file changed, 93 insertions(+), 145 deletions(-) diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 43b93b45e716b4..c1428e8a5e7dd8 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -5316,6 +5316,42 @@ static unsigned HandleHistogramProfileRand() return x; } +template +static int CheckSample(T index) +{ + const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; + const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; + static_assert_no_msg(N >= S); + static_assert_no_msg((std::is_same::value || std::is_same::value)); + + // If table is not yet full, just add entries in. + // + if (index < S) + { + return static_cast(index); + } + + unsigned x = HandleHistogramProfileRand(); + // N is the sampling window size, + // it should be larger than the table size. + // + // If we let N == count then we are building an entire + // run sample -- probability of update decreases over time. + // Would be a good strategy for an AOT profiler. + // + // But for TieredPGO we would prefer something that is more + // weighted to recent observations. + // + // For S=4, N=128, we'll sample (on average) every 32nd call. + // + if ((x % N) >= S) + { + return -1; + } + + return static_cast(x % S); +} + HCIMPL2(void, JIT_ClassProfile32, Object *obj, ICorJitInfo::HandleHistogram32* classProfile) { FCALL_CONTRACT; @@ -5325,10 +5361,13 @@ HCIMPL2(void, JIT_ClassProfile32, Object *obj, ICorJitInfo::HandleHistogram32* c VALIDATEOBJECTREF(objRef); volatile unsigned* pCount = (volatile unsigned*) &classProfile->Count; - const unsigned count = (*pCount)++; - const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; - const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; - static_assert_no_msg(N >= S); + const unsigned callIndex = (*pCount)++; + + int sampleIndex = CheckSample(callIndex); + if (sampleIndex == -1) + { + return; + } if (objRef == NULL) { @@ -5351,34 +5390,7 @@ HCIMPL2(void, JIT_ClassProfile32, Object *obj, ICorJitInfo::HandleHistogram32* c PgoManager::VerifyAddress(classProfile + 1); #endif - // If table is not yet full, just add entries in. - // - if (count < S) - { - classProfile->HandleTable[count] = (CORINFO_CLASS_HANDLE)pMT; - } - else - { - unsigned x = HandleHistogramProfileRand(); - - // N is the sampling window size, - // it should be larger than the table size. - // - // If we let N == count then we are building an entire - // run sample -- probability of update decreases over time. - // Would be a good strategy for an AOT profiler. - // - // But for TieredPGO we would prefer something that is more - // weighted to recent observations. - // - // For S=4, N=128, we'll sample (on average) every 32nd call. - // - if ((x % N) < S) - { - unsigned i = x % S; - classProfile->HandleTable[i] = (CORINFO_CLASS_HANDLE)pMT; - } - } + classProfile->HandleTable[sampleIndex] = (CORINFO_CLASS_HANDLE)pMT; } HCIMPLEND @@ -5392,10 +5404,13 @@ HCIMPL2(void, JIT_ClassProfile64, Object *obj, ICorJitInfo::HandleHistogram64* c VALIDATEOBJECTREF(objRef); volatile uint64_t* pCount = (volatile uint64_t*) &classProfile->Count; - const uint64_t count = (*pCount)++; - const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; - const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; - static_assert_no_msg(N >= S); + const uint64_t callIndex = (*pCount)++; + + int sampleIndex = CheckSample(callIndex); + if (sampleIndex == -1) + { + return; + } if (objRef == NULL) { @@ -5414,20 +5429,7 @@ HCIMPL2(void, JIT_ClassProfile64, Object *obj, ICorJitInfo::HandleHistogram64* c PgoManager::VerifyAddress(classProfile + 1); #endif - if (count < S) - { - classProfile->HandleTable[count] = (CORINFO_CLASS_HANDLE)pMT; - } - else - { - unsigned x = HandleHistogramProfileRand(); - - if ((x % N) < S) - { - unsigned i = x % S; - classProfile->HandleTable[i] = (CORINFO_CLASS_HANDLE)pMT; - } - } + classProfile->HandleTable[sampleIndex] = (CORINFO_CLASS_HANDLE)pMT; } HCIMPLEND @@ -5440,16 +5442,27 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod VALIDATEOBJECTREF(objRef); volatile unsigned* pMethodCount = (volatile unsigned*) &methodProfile->Count; - const unsigned methodCount = (*pMethodCount)++; - const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; - const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; - static_assert_no_msg(N >= S); + const unsigned methodCallIndex = (*pMethodCount)++; + int methodSampleIndex = CheckSample(methodCallIndex); - unsigned typeCount = 0; + int typeSampleIndex = -1; if (typeProfile != nullptr) { volatile unsigned* pTypeCount = (volatile unsigned*) &typeProfile->Count; - typeCount = (*pTypeCount)++; + const unsigned typeCallIndex = (*pTypeCount)++; + typeSampleIndex = CheckSample(typeCallIndex); + + if ((methodSampleIndex == -1) && (typeSampleIndex == -1)) + { + return; + } + } + else + { + if (methodSampleIndex == -1) + { + return; + } } if (objRef == NULL) @@ -5524,53 +5537,16 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod // If table is not yet full, just add entries in. // - if (methodCount < S) - { - methodProfile->HandleTable[methodCount] = (CORINFO_METHOD_HANDLE)pMD; - } - else - { - unsigned x = HandleHistogramProfileRand(); + methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pMD; - // N is the sampling window size, - // it should be larger than the table size. - // - // If we let N == count then we are building an entire - // run sample -- probability of update decreases over time. - // Would be a good strategy for an AOT profiler. - // - // But for TieredPGO we would prefer something that is more - // weighted to recent observations. - // - // For S=4, N=128, we'll sample (on average) every 32nd call. - // - if ((x % N) < S) - { - unsigned i = x % S; - methodProfile->HandleTable[i] = (CORINFO_METHOD_HANDLE)pMD; - } - } - - if (typeProfile != nullptr) + if (typeSampleIndex != -1) { if (pMT->GetLoaderAllocator()->IsCollectible()) { pMT = (MethodTable*)DEFAULT_UNKNOWN_HANDLE; } - if (typeCount < S) - { - typeProfile->HandleTable[typeCount] = (CORINFO_CLASS_HANDLE)pMT; - } - else - { - unsigned x = HandleHistogramProfileRand(); - if ((x % N) < S) - { - unsigned i = x % S; - typeProfile->HandleTable[i] = (CORINFO_CLASS_HANDLE)pMT; - } - } + typeProfile->HandleTable[typeSampleIndex] = (CORINFO_CLASS_HANDLE)pMT; } } HCIMPLEND @@ -5585,16 +5561,27 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod VALIDATEOBJECTREF(objRef); volatile uint64_t* pMethodCount = (volatile uint64_t*) &methodProfile->Count; - const uint64_t methodCount = (*pMethodCount)++; - const unsigned S = ICorJitInfo::HandleHistogram32::SIZE; - const unsigned N = ICorJitInfo::HandleHistogram32::SAMPLE_INTERVAL; - static_assert_no_msg(N >= S); + const uint64_t methodCallIndex = (*pMethodCount)++; + int methodSampleIndex = CheckSample(methodCallIndex); - unsigned typeCount = 0; + int typeSampleIndex = -1; if (typeProfile != nullptr) { - volatile unsigned* pTypeCount = (volatile unsigned*) &typeProfile->Count; - typeCount = (*pTypeCount)++; + volatile uint64_t* pTypeCount = (volatile uint64_t*) &typeProfile->Count; + const uint64_t typeCallIndex = (*pTypeCount)++; + typeSampleIndex = CheckSample(typeCallIndex); + + if ((methodSampleIndex == -1) && (typeSampleIndex == -1)) + { + return; + } + } + else + { + if (methodSampleIndex == -1) + { + return; + } } if (objRef == NULL) @@ -5667,55 +5654,16 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod PgoManager::VerifyAddress(methodProfile + 1); #endif - // If table is not yet full, just add entries in. - // - if (methodCount < S) - { - methodProfile->HandleTable[methodCount] = (CORINFO_METHOD_HANDLE)pMD; - } - else - { - unsigned x = HandleHistogramProfileRand(); + methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pMD; - // N is the sampling window size, - // it should be larger than the table size. - // - // If we let N == count then we are building an entire - // run sample -- probability of update decreases over time. - // Would be a good strategy for an AOT profiler. - // - // But for TieredPGO we would prefer something that is more - // weighted to recent observations. - // - // For S=4, N=128, we'll sample (on average) every 32nd call. - // - if ((x % N) < S) - { - unsigned i = x % S; - methodProfile->HandleTable[i] = (CORINFO_METHOD_HANDLE)pMD; - } - } - - if (typeProfile != nullptr) + if (typeSampleIndex != -1) { if (pMT->GetLoaderAllocator()->IsCollectible()) { pMT = (MethodTable*)DEFAULT_UNKNOWN_HANDLE; } - if (typeCount < S) - { - typeProfile->HandleTable[typeCount] = (CORINFO_CLASS_HANDLE)pMT; - } - else - { - unsigned x = HandleHistogramProfileRand(); - if ((x % N) < S) - { - unsigned i = x % S; - typeProfile->HandleTable[i] = (CORINFO_CLASS_HANDLE)pMT; - } - } + typeProfile->HandleTable[typeSampleIndex] = (CORINFO_CLASS_HANDLE)pMT; } } HCIMPLEND From c30a9512d4e127604d78533eb07b3f7d64330b39 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 25 May 2022 16:43:54 +0200 Subject: [PATCH 35/59] Record only delegate cases that we can actually GDV for We would record instance methods for open delegate call sites here, which would result in a mismatch in arg counts during inlining the devirtualized call. This case is indistinguishable from delegates pointing at static methods as both require a shuffle thunk that drops the 'this' arg stored in the delegate, so just record unknown handles for both these cases. This is not a huge loss as we currently cannot GDV for delegates pointing to static methods anyway. --- src/coreclr/vm/jithelpers.cpp | 80 ++++++++++++----------------------- 1 file changed, 27 insertions(+), 53 deletions(-) diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index c1428e8a5e7dd8..a2582cea8aa165 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -5485,32 +5485,23 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod MethodTable* pMT = objRef->GetMethodTable(); // Resolve method - MethodDesc* pMD = NULL; + MethodDesc* pRecordedMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; if (pMT->IsDelegate() && (pBaseMD == ((DelegateEEClass*)pMT->GetClass())->GetInvokeMethod())) { // We handle only the common "direct" delegate as that is in any case // the only one we can reasonably do GDV for. For instance, open // delegates are filtered out here, and many cases with inner - // "complicated" logic as well (e.g. multicast, unmanaged functions). + // "complicated" logic as well (e.g. static functions, multicast, + // unmanaged functions). // DELEGATEREF del = (DELEGATEREF)objRef; - if (del->GetInvocationCount() == 0) + if ((del->GetInvocationCount() == 0) && (del->GetMethodPtrAux() == NULL)) { - PCODE code = del->GetMethodPtrAux(); - if (code != NULL) + MethodDesc* pMD = NonVirtualEntry2MethodDesc(del->GetMethodPtr()); + if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible()) { - pMD = MethodTable::GetMethodDescForSlotAddress(code); + pRecordedMD = pMD; } - else - { - code = del->GetMethodPtr(); - pMD = NonVirtualEntry2MethodDesc(code); - } - } - - if (pMD == NULL) - { - return; } } else @@ -5518,16 +5509,12 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod WORD slot = pBaseMD->GetSlot(); _ASSERTE(slot < pBaseMD->GetMethodTable()->GetNumVirtuals()); - pMD = pMT->GetMethodDescForSlot(slot); - } + MethodDesc* pMD = pMT->GetMethodDescForSlot(slot); - // If the object class is collectible, record an unknown handle. - // We do this instead of recording NULL so that we won't over-estimate - // the likelihood of known handles. - // - if (pMD->GetLoaderAllocator()->IsCollectible()) - { - pMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + if (!pMD->GetLoaderAllocator()->IsCollectible()) + { + pRecordedMD = pMD; + } } #ifdef _DEBUG @@ -5537,7 +5524,7 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod // If table is not yet full, just add entries in. // - methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pMD; + methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pRecordedMD; if (typeSampleIndex != -1) { @@ -5604,32 +5591,23 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod MethodTable* pMT = objRef->GetMethodTable(); // Resolve method - MethodDesc* pMD = NULL; - if (pMT->IsDelegate()) + MethodDesc* pRecordedMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + if (pMT->IsDelegate() && (pBaseMD == ((DelegateEEClass*)pMT->GetClass())->GetInvokeMethod())) { // We handle only the common "direct" delegate as that is in any case // the only one we can reasonably do GDV for. For instance, open // delegates are filtered out here, and many cases with inner - // "complicated" logic as well (e.g. multicast, unmanaged functions). + // "complicated" logic as well (e.g. static functions, multicast, + // unmanaged functions). // DELEGATEREF del = (DELEGATEREF)objRef; - if (del->GetInvocationCount() == 0) + if ((del->GetInvocationCount() == 0) && (del->GetMethodPtrAux() == NULL)) { - PCODE code = del->GetMethodPtrAux(); - if (code != NULL) + MethodDesc* pMD = NonVirtualEntry2MethodDesc(del->GetMethodPtr()); + if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible()) { - pMD = MethodTable::GetMethodDescForSlotAddress(code); + pRecordedMD = pMD; } - else - { - code = del->GetMethodPtr(); - pMD = NonVirtualEntry2MethodDesc(code); - } - } - - if (pMD == NULL) - { - return; } } else @@ -5637,16 +5615,12 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod WORD slot = pBaseMD->GetSlot(); _ASSERTE(slot < pBaseMD->GetMethodTable()->GetNumVirtuals()); - pMD = pMT->GetMethodDescForSlot(slot); - } + MethodDesc* pMD = pMT->GetMethodDescForSlot(slot); - // If the object class is collectible, record an unknown handle. - // We do this instead of recording NULL so that we won't over-estimate - // the likelihood of known handles. - // - if (pMD->GetLoaderAllocator()->IsCollectible()) - { - pMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + if (!pMD->GetLoaderAllocator()->IsCollectible()) + { + pRecordedMD = pMD; + } } #ifdef _DEBUG @@ -5654,7 +5628,7 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod PgoManager::VerifyAddress(methodProfile + 1); #endif - methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pMD; + methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pRecordedMD; if (typeSampleIndex != -1) { From f6e0827b3aa1ae7b58ceb673f174aa0058433f72 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 27 May 2022 14:36:59 +0200 Subject: [PATCH 36/59] Fix pesky x86 GTF_CALL_POP_ARGS bug We would propagate *all* flags from the control expression into the call when morphing early-expanded vtable calls. For the cold vtable calls in the control expression is usually a local variable to reuse the already loaded call target. We would end up remorphing the call in assertion prop which happens after liveness that sets GTF_VAR_DEATH on the control expression. This flag conflicts with GTF_CALL_POP_ARGS that on x86 would cause us to incorrectly believe the call was caller-cleaned and unbalance the stack. --- src/coreclr/jit/morph.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 4ac9982acff290..23d589b56e0093 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -8363,8 +8363,8 @@ GenTree* Compiler::fgMorphCall(GenTreeCall* call) // call->gtControlExpr = fgMorphTree(call->gtControlExpr); - // Propagate any gtFlags into the call - call->gtFlags |= call->gtControlExpr->gtFlags; + // Propagate any side effect flags into the call + call->gtFlags |= call->gtControlExpr->gtFlags & GTF_ALL_EFFECT; } // Morph stelem.ref helper call to store a null value, into a store into an array without the helper. From 27aff20ace516410303f904aa2b561728946cc82 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 28 May 2022 00:09:48 +0200 Subject: [PATCH 37/59] Exclude dynamic methods as well as methods in collectible ALCs --- src/coreclr/vm/jithelpers.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index a2582cea8aa165..6ac1d8d9ab398f 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -5498,7 +5498,7 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod if ((del->GetInvocationCount() == 0) && (del->GetMethodPtrAux() == NULL)) { MethodDesc* pMD = NonVirtualEntry2MethodDesc(del->GetMethodPtr()); - if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible()) + if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) { pRecordedMD = pMD; } @@ -5511,7 +5511,7 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod MethodDesc* pMD = pMT->GetMethodDescForSlot(slot); - if (!pMD->GetLoaderAllocator()->IsCollectible()) + if (!pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) { pRecordedMD = pMD; } @@ -5604,7 +5604,7 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod if ((del->GetInvocationCount() == 0) && (del->GetMethodPtrAux() == NULL)) { MethodDesc* pMD = NonVirtualEntry2MethodDesc(del->GetMethodPtr()); - if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible()) + if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) { pRecordedMD = pMD; } @@ -5617,7 +5617,7 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod MethodDesc* pMD = pMT->GetMethodDescForSlot(slot); - if (!pMD->GetLoaderAllocator()->IsCollectible()) + if (!pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) { pRecordedMD = pMD; } From 8444cee05e0129a5be70be3b68f5a3a7b8a1dcf4 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 28 May 2022 00:11:11 +0200 Subject: [PATCH 38/59] Remove excessive JIT-EE calls for debug logging in non-verbose --- src/coreclr/jit/importer.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 05116e0d2c86aa..5ad4973e674ccc 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22428,31 +22428,31 @@ void Compiler::pickGDV(GenTreeCall* call, } #ifdef DEBUG - if (numberOfClasses > 0) + if (verbose && (numberOfClasses > 0)) { bool isExact; bool isNonNull; CallArg* thisArg = call->gtArgs.GetThisArg(); CORINFO_CLASS_HANDLE declaredThisClsHnd = gtGetClassHandle(thisArg->GetNode(), &isExact, &isNonNull); - JITDUMP("Likely classes for call [%06u]", dspTreeID(call)); + printf("Likely classes for call [%06u]", dspTreeID(call)); if (declaredThisClsHnd != NO_CLASS_HANDLE) { - JITDUMP(" on class %p (%s)", declaredThisClsHnd, eeGetClassName(declaredThisClsHnd)); + printf(" on class %p (%s)", declaredThisClsHnd, eeGetClassName(declaredThisClsHnd)); } - JITDUMP("\n"); + printf("\n"); for (UINT32 i = 0; i < numberOfClasses; i++) { - JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, - eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle), likelyClasses[i].likelihood); + printf(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, + eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle), likelyClasses[i].likelihood); } } - if (numberOfMethods > 0) + if (verbose && (numberOfMethods > 0)) { assert(call->gtCallType == CT_USER_FUNC); - JITDUMP("Likely methods for call [%06u] to method %s\n", dspTreeID(call), - eeGetMethodFullName(call->gtCallMethHnd)); + printf("Likely methods for call [%06u] to method %s\n", dspTreeID(call), + eeGetMethodFullName(call->gtCallMethHnd)); for (UINT32 i = 0; i < numberOfMethods; i++) { @@ -22464,19 +22464,19 @@ void Compiler::pickGDV(GenTreeCall* call, switch (lookup.accessType) { case IAT_VALUE: - JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); + printf(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); break; case IAT_PVALUE: - JITDUMP(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); + printf(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); break; case IAT_PPVALUE: - JITDUMP(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); + printf(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); break; default: - JITDUMP(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); + printf(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); break; } } From cbc494b7d53945cd876b20035bb281f81a53109e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 2 Jun 2022 12:56:11 +0200 Subject: [PATCH 39/59] Increase some timeouts to get a full test run --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 5 ++--- eng/pipelines/coreclr/libraries-pgo.yml | 2 +- eng/pipelines/libraries/run-test-job.yml | 1 - src/tests/Common/testenvironment.proj | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index d040b523ce02ae..84d72603e6e8b5 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -227,13 +227,13 @@ jobs: timeoutInMinutes: 300 ${{ else }}: timeoutInMinutes: 200 - ${{ if in(parameters.testGroup, 'outerloop', 'jit-experimental', 'pgo', 'jit-cfg') }}: + ${{ if in(parameters.testGroup, 'outerloop', 'jit-experimental', 'jit-cfg') }}: timeoutInMinutes: 270 ${{ if in(parameters.testGroup, 'gc-longrunning', 'gc-simulator') }}: timeoutInMinutes: 480 ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc', 'ilasm') }}: timeoutInMinutes: 390 - ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra', 'clrinterpreter') }}: + ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra', 'clrinterpreter', 'pgo') }}: timeoutInMinutes: 510 ${{ if eq(parameters.testGroup, 'jitstress-isas-x86') }}: timeoutInMinutes: 960 @@ -570,7 +570,6 @@ jobs: - jitelthookenabled_tiered ${{ if in(parameters.testGroup, 'jit-experimental') }}: scenarios: - - jitosr - jitosr_stress - jitosr_pgo - jitosr_stress_random diff --git a/eng/pipelines/coreclr/libraries-pgo.yml b/eng/pipelines/coreclr/libraries-pgo.yml index 0914451b55ec66..0a3346141744f9 100644 --- a/eng/pipelines/coreclr/libraries-pgo.yml +++ b/eng/pipelines/coreclr/libraries-pgo.yml @@ -47,7 +47,7 @@ jobs: helixQueueGroup: libraries helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - timeoutInMinutes: 150 + timeoutInMinutes: 600 testScope: innerloop liveRuntimeBuildConfig: checked dependsOnTestBuildConfiguration: Release diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 61697db4c25778..c6fbe831a4d9bd 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -178,7 +178,6 @@ jobs: - fullpgo_random_gdv_methodprofiling_only - fullpgo_random_edge - fullpgo_random_gdv_edge - - jitosr - jitosr_stress - jitosr_stress_random - jitosr_pgo diff --git a/src/tests/Common/testenvironment.proj b/src/tests/Common/testenvironment.proj index d2ccdcf35261d1..b8d1da92359ed0 100644 --- a/src/tests/Common/testenvironment.proj +++ b/src/tests/Common/testenvironment.proj @@ -186,7 +186,6 @@ - From 09370049781d96e060cec18cad771995bb8cbaa8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 2 Jun 2022 13:09:32 +0200 Subject: [PATCH 40/59] Add missing helper in CorInfoHelpFunc.cs --- src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs index c161947fb8323c..4dcf92f4982390 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs @@ -289,6 +289,7 @@ which is the right helper to use to allocate an object of a given type. */ CORINFO_HELP_CLASSPROFILE64, // Update 64-bit class profile for a call site CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, // Notify runtime that code has reached a part of the method that wasn't originally jitted. + CORINFO_HELP_STATIC_VIRTUAL_AMBIGUOUS_RESOLUTION, // Throw AmbiguousResolutionException for failed static virtual method resolution CORINFO_HELP_VALIDATE_INDIRECT_CALL, // CFG: Validate function pointer CORINFO_HELP_DISPATCH_INDIRECT_CALL, // CFG: Validate and dispatch to pointer From 56019b7ad5aec6d9b2f3dd23975f88194fdf7835 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 2 Jun 2022 13:13:59 +0200 Subject: [PATCH 41/59] Fix conflict resolution and add profiling helpers on crossgen2 side --- src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs index 4dcf92f4982390..ac10f8a22a7149 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs @@ -289,10 +289,14 @@ which is the right helper to use to allocate an object of a given type. */ CORINFO_HELP_CLASSPROFILE64, // Update 64-bit class profile for a call site CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, // Notify runtime that code has reached a part of the method that wasn't originally jitted. - CORINFO_HELP_STATIC_VIRTUAL_AMBIGUOUS_RESOLUTION, // Throw AmbiguousResolutionException for failed static virtual method resolution CORINFO_HELP_VALIDATE_INDIRECT_CALL, // CFG: Validate function pointer CORINFO_HELP_DISPATCH_INDIRECT_CALL, // CFG: Validate and dispatch to pointer + CORINFO_HELP_STATIC_VIRTUAL_AMBIGUOUS_RESOLUTION, // Throw AmbiguousResolutionException for failed static virtual method resolution + + CORINFO_HELP_METHODPROFILE32, // Update 32-bit method profile for a call site + CORINFO_HELP_METHODPROFILE64, // Update 64-bit method profile for a call site + CORINFO_HELP_COUNT, } } From 1bd5203e7cbb63c07c1044bb1e8fe93766ef2634 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 4 Jun 2022 16:11:25 +0200 Subject: [PATCH 42/59] Lower likelihood threshold to 30%, like vtable calls --- src/coreclr/jit/importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 8cc26b4728e2f9..3abf1c6495c519 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22509,7 +22509,7 @@ void Compiler::pickGDV(GenTreeCall* call, if (numberOfMethods > 0) { - unsigned likelihoodThreshold = 75; + unsigned likelihoodThreshold = 30; if (likelyMethods[0].likelihood >= likelihoodThreshold) { *methodGuess = (CORINFO_METHOD_HANDLE)likelyMethods[0].handle; From 924d0b1b3eae32ca0999e1640585a5ec609d006b Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Jun 2022 13:33:08 +0200 Subject: [PATCH 43/59] Revert "Increase some timeouts to get a full test run" This reverts commit cbc494b7d53945cd876b20035bb281f81a53109e. --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 5 +++-- eng/pipelines/coreclr/libraries-pgo.yml | 2 +- eng/pipelines/libraries/run-test-job.yml | 1 + src/tests/Common/testenvironment.proj | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 84d72603e6e8b5..d040b523ce02ae 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -227,13 +227,13 @@ jobs: timeoutInMinutes: 300 ${{ else }}: timeoutInMinutes: 200 - ${{ if in(parameters.testGroup, 'outerloop', 'jit-experimental', 'jit-cfg') }}: + ${{ if in(parameters.testGroup, 'outerloop', 'jit-experimental', 'pgo', 'jit-cfg') }}: timeoutInMinutes: 270 ${{ if in(parameters.testGroup, 'gc-longrunning', 'gc-simulator') }}: timeoutInMinutes: 480 ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc', 'ilasm') }}: timeoutInMinutes: 390 - ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra', 'clrinterpreter', 'pgo') }}: + ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra', 'clrinterpreter') }}: timeoutInMinutes: 510 ${{ if eq(parameters.testGroup, 'jitstress-isas-x86') }}: timeoutInMinutes: 960 @@ -570,6 +570,7 @@ jobs: - jitelthookenabled_tiered ${{ if in(parameters.testGroup, 'jit-experimental') }}: scenarios: + - jitosr - jitosr_stress - jitosr_pgo - jitosr_stress_random diff --git a/eng/pipelines/coreclr/libraries-pgo.yml b/eng/pipelines/coreclr/libraries-pgo.yml index 0a3346141744f9..0914451b55ec66 100644 --- a/eng/pipelines/coreclr/libraries-pgo.yml +++ b/eng/pipelines/coreclr/libraries-pgo.yml @@ -47,7 +47,7 @@ jobs: helixQueueGroup: libraries helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - timeoutInMinutes: 600 + timeoutInMinutes: 150 testScope: innerloop liveRuntimeBuildConfig: checked dependsOnTestBuildConfiguration: Release diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index c6fbe831a4d9bd..61697db4c25778 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -178,6 +178,7 @@ jobs: - fullpgo_random_gdv_methodprofiling_only - fullpgo_random_edge - fullpgo_random_gdv_edge + - jitosr - jitosr_stress - jitosr_stress_random - jitosr_pgo diff --git a/src/tests/Common/testenvironment.proj b/src/tests/Common/testenvironment.proj index b8d1da92359ed0..d2ccdcf35261d1 100644 --- a/src/tests/Common/testenvironment.proj +++ b/src/tests/Common/testenvironment.proj @@ -186,6 +186,7 @@ + From dfbcc868bce52658edf64b21c55463c99a496362 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Jun 2022 19:08:55 +0200 Subject: [PATCH 44/59] Add signature compatibility check --- src/coreclr/jit/compiler.h | 2 + src/coreclr/jit/importer.cpp | 98 ++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index c3a6a346293ed1..dc2e9e51a28662 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6826,6 +6826,8 @@ class Compiler CORINFO_CLASS_HANDLE baseClass, CORINFO_CONTEXT_HANDLE* pContextHandle); + bool isCompatibleMethodGDV(GenTreeCall* call, CORINFO_METHOD_HANDLE gdvTarget); + void addGuardedDevirtualizationCandidate(GenTreeCall* call, CORINFO_METHOD_HANDLE methodHandle, CORINFO_CLASS_HANDLE classHandle, diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 3abf1c6495c519..c8667a5c457c24 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22522,6 +22522,94 @@ void Compiler::pickGDV(GenTreeCall* call, } } +//------------------------------------------------------------------------ +// isCompatibleMethodGDV: +// Check if devirtualizing a call node as a specified target method call is +// reasonable. +// +// Arguments: +// call - the call +// gdvTarget - the target method that we want to guess for and devirtualize to +// +// Returns: +// true if we can proceed with GDV. +// +// Notes: +// This implements a small simplified signature-compatibility check to +// verify that a guess is reasonable. The main goal here is to avoid blowing +// up the JIT on PGO data with stale GDV candidates; if they are not +// compatible in the ECMA sense then we do not expect the guard to ever pass +// at runtime, so we can get by with simplified rules here. +// +bool Compiler::isCompatibleMethodGDV(GenTreeCall* call, CORINFO_METHOD_HANDLE gdvTarget) +{ + CORINFO_SIG_INFO sig; + info.compCompHnd->getMethodSig(gdvTarget, &sig); + + CORINFO_ARG_LIST_HANDLE sigParam = sig.args; + unsigned numParams = sig.numArgs; + unsigned numArgs = 0; + for (CallArg& arg : call->gtArgs.Args()) + { + switch (arg.GetWellKnownArg()) + { + case WellKnownArg::RetBuffer: + case WellKnownArg::ThisPointer: + // Not part of signature but we still expect to see it here + continue; + case WellKnownArg::None: + break; + default: + assert(!"Unexpected well known arg to method GDV candidate"); + continue; + } + + numArgs++; + if (numArgs > numParams) + { + JITDUMP("Incompatible method GDV: call [%06u] has more arguments than signature (sig has %d parameters)\n", + dspTreeID(call), numParams); + return false; + } + + CORINFO_CLASS_HANDLE classHnd = NO_CLASS_HANDLE; + CorInfoType corType = strip(info.compCompHnd->getArgType(&sig, sigParam, &classHnd)); + var_types sigType = JITtype2varType(corType); + + if (!impCheckImplicitArgumentCoercion(sigType, arg.GetNode()->TypeGet())) + { + JITDUMP("Incompatible method GDV: arg [%06u] is type-incompatible with signature of target\n", + dspTreeID(arg.GetNode())); + return false; + } + + // Best-effort check for struct compatibility here. + if (varTypeIsStruct(sigType) && (arg.GetSignatureClassHandle() != classHnd)) + { + ClassLayout* callLayout = typGetObjLayout(arg.GetSignatureClassHandle()); + ClassLayout* tarLayout = typGetObjLayout(classHnd); + + if (!ClassLayout::AreCompatible(callLayout, tarLayout)) + { + JITDUMP("Incompatible method GDV: struct arg [%06u] is layout-incompatible with signature of target\n", + dspTreeID(arg.GetNode())); + return false; + } + } + + sigParam = info.compCompHnd->getArgNext(sigParam); + } + + if (numArgs < numParams) + { + JITDUMP("Incompatible method GDV: call [%06u] has fewer arguments (%d) than signature (%d)\n", dspTreeID(call), + numArgs, numParams); + return false; + } + + return true; +} + //------------------------------------------------------------------------ // considerGuardedDevirtualization: see if we can profitably guess at the // class involved in an interface or virtual call. @@ -22599,6 +22687,16 @@ void Compiler::considerGuardedDevirtualization(GenTreeCall* call, likelyMethod = dvInfo.devirtualizedMethod; } + else + { + // Verify that this is a reasonable looking call target. + if (!isCompatibleMethodGDV(call, likelyMethod)) + { + JITDUMP("Target for method-based GDV is incompatible (stale profile?)\n"); + assert((fgPgoSource != ICorJitInfo::PgoSource::Dynamic) && "Unexpected stale profile in dynamic PGO data"); + return; + } + } uint32_t likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); if ((likelyMethodAttribs & CORINFO_FLG_STATIC) != 0) From 1f7a1be623d9ed5ad0bc79221b2fc473e34f76e0 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Jun 2022 19:09:15 +0200 Subject: [PATCH 45/59] Increase some timeouts to get a full test run" --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 5 ++--- eng/pipelines/coreclr/libraries-pgo.yml | 2 +- eng/pipelines/libraries/run-test-job.yml | 1 - src/tests/Common/testenvironment.proj | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index d040b523ce02ae..84d72603e6e8b5 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -227,13 +227,13 @@ jobs: timeoutInMinutes: 300 ${{ else }}: timeoutInMinutes: 200 - ${{ if in(parameters.testGroup, 'outerloop', 'jit-experimental', 'pgo', 'jit-cfg') }}: + ${{ if in(parameters.testGroup, 'outerloop', 'jit-experimental', 'jit-cfg') }}: timeoutInMinutes: 270 ${{ if in(parameters.testGroup, 'gc-longrunning', 'gc-simulator') }}: timeoutInMinutes: 480 ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc', 'ilasm') }}: timeoutInMinutes: 390 - ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra', 'clrinterpreter') }}: + ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra', 'clrinterpreter', 'pgo') }}: timeoutInMinutes: 510 ${{ if eq(parameters.testGroup, 'jitstress-isas-x86') }}: timeoutInMinutes: 960 @@ -570,7 +570,6 @@ jobs: - jitelthookenabled_tiered ${{ if in(parameters.testGroup, 'jit-experimental') }}: scenarios: - - jitosr - jitosr_stress - jitosr_pgo - jitosr_stress_random diff --git a/eng/pipelines/coreclr/libraries-pgo.yml b/eng/pipelines/coreclr/libraries-pgo.yml index 0914451b55ec66..0a3346141744f9 100644 --- a/eng/pipelines/coreclr/libraries-pgo.yml +++ b/eng/pipelines/coreclr/libraries-pgo.yml @@ -47,7 +47,7 @@ jobs: helixQueueGroup: libraries helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - timeoutInMinutes: 150 + timeoutInMinutes: 600 testScope: innerloop liveRuntimeBuildConfig: checked dependsOnTestBuildConfiguration: Release diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 61697db4c25778..c6fbe831a4d9bd 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -178,7 +178,6 @@ jobs: - fullpgo_random_gdv_methodprofiling_only - fullpgo_random_edge - fullpgo_random_gdv_edge - - jitosr - jitosr_stress - jitosr_stress_random - jitosr_pgo diff --git a/src/tests/Common/testenvironment.proj b/src/tests/Common/testenvironment.proj index d2ccdcf35261d1..b8d1da92359ed0 100644 --- a/src/tests/Common/testenvironment.proj +++ b/src/tests/Common/testenvironment.proj @@ -186,7 +186,6 @@ - From 0363bf2b26f640eb62386361bad71b24d9bb75a0 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Jun 2022 19:16:29 +0200 Subject: [PATCH 46/59] Fix after merge --- src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs index 6cb591c0a27c1d..921f46cd24e89c 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs @@ -293,8 +293,6 @@ which is the right helper to use to allocate an object of a given type. */ CORINFO_HELP_VALIDATE_INDIRECT_CALL, // CFG: Validate function pointer CORINFO_HELP_DISPATCH_INDIRECT_CALL, // CFG: Validate and dispatch to pointer - CORINFO_HELP_STATIC_VIRTUAL_AMBIGUOUS_RESOLUTION, // Throw AmbiguousResolutionException for failed static virtual method resolution - CORINFO_HELP_METHODPROFILE32, // Update 32-bit method profile for a call site CORINFO_HELP_METHODPROFILE64, // Update 64-bit method profile for a call site From ed1a0738befadb06e893a4548d1fa398f7b52764 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Jun 2022 13:31:30 +0200 Subject: [PATCH 47/59] Separate delegate and vtable profiling --- src/coreclr/inc/corinfo.h | 10 +- src/coreclr/inc/jithelpers.h | 10 +- src/coreclr/jit/fgprofile.cpp | 45 +++- src/coreclr/jit/importer.cpp | 4 +- src/coreclr/jit/jitconfigvalues.h | 7 +- .../Common/JitInterface/CorInfoHelpFunc.cs | 10 +- src/coreclr/vm/jithelpers.cpp | 238 +++++++++--------- src/tests/Common/testenvironment.proj | 7 +- 8 files changed, 187 insertions(+), 144 deletions(-) diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index 5b661902ecc0d2..5e3e5c918d36f6 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -640,16 +640,18 @@ enum CorInfoHelpFunc CORINFO_HELP_STACK_PROBE, // Probes each page of the allocated stack frame CORINFO_HELP_PATCHPOINT, // Notify runtime that code has reached a patchpoint + CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, // Notify runtime that code has reached a part of the method that wasn't originally jitted. + CORINFO_HELP_CLASSPROFILE32, // Update 32-bit class profile for a call site CORINFO_HELP_CLASSPROFILE64, // Update 64-bit class profile for a call site - CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, // Notify runtime that code has reached a part of the method that wasn't originally jitted. + CORINFO_HELP_DELEGATEPROFILE32, // Update 32-bit method profile for a delegate call site + CORINFO_HELP_DELEGATEPROFILE64, // Update 64-bit method profile for a delegate call site + CORINFO_HELP_VTABLEPROFILE32, // Update 32-bit method profile for a vtable call site + CORINFO_HELP_VTABLEPROFILE64, // Update 64-bit method profile for a vtable call site CORINFO_HELP_VALIDATE_INDIRECT_CALL, // CFG: Validate function pointer CORINFO_HELP_DISPATCH_INDIRECT_CALL, // CFG: Validate and dispatch to pointer - CORINFO_HELP_METHODPROFILE32, // Update 32-bit method profile for a call site - CORINFO_HELP_METHODPROFILE64, // Update 64-bit method profile for a call site - CORINFO_HELP_COUNT, }; diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index 6ca8c23d82be62..a500c298978b67 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -328,9 +328,14 @@ #endif JITHELPER(CORINFO_HELP_PATCHPOINT, JIT_Patchpoint, CORINFO_HELP_SIG_REG_ONLY) + JITHELPER(CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, JIT_PartialCompilationPatchpoint, CORINFO_HELP_SIG_REG_ONLY) + JITHELPER(CORINFO_HELP_CLASSPROFILE32, JIT_ClassProfile32, CORINFO_HELP_SIG_REG_ONLY) JITHELPER(CORINFO_HELP_CLASSPROFILE64, JIT_ClassProfile64, CORINFO_HELP_SIG_REG_ONLY) - JITHELPER(CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, JIT_PartialCompilationPatchpoint, CORINFO_HELP_SIG_REG_ONLY) + JITHELPER(CORINFO_HELP_DELEGATEPROFILE32, JIT_DelegateProfile32, CORINFO_HELP_SIG_REG_ONLY) + JITHELPER(CORINFO_HELP_DELEGATEPROFILE64, JIT_DelegateProfile64, CORINFO_HELP_SIG_REG_ONLY) + JITHELPER(CORINFO_HELP_VTABLEPROFILE32, JIT_VTableProfile32, CORINFO_HELP_SIG_4_STACK) + JITHELPER(CORINFO_HELP_VTABLEPROFILE64, JIT_VTableProfile64, CORINFO_HELP_SIG_4_STACK) #if defined(TARGET_AMD64) || defined(TARGET_ARM64) JITHELPER(CORINFO_HELP_VALIDATE_INDIRECT_CALL, JIT_ValidateIndirectCall, CORINFO_HELP_SIG_REG_ONLY) @@ -344,9 +349,6 @@ JITHELPER(CORINFO_HELP_DISPATCH_INDIRECT_CALL, NULL, CORINFO_HELP_SIG_REG_ONLY) #endif - JITHELPER(CORINFO_HELP_METHODPROFILE32, JIT_MethodProfile32, CORINFO_HELP_SIG_8_STACK) - JITHELPER(CORINFO_HELP_METHODPROFILE64, JIT_MethodProfile64, CORINFO_HELP_SIG_8_STACK) - #undef JITHELPER #undef DYNAMICJITHELPER #undef JITHELPER diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 574cecdaa1d2e4..0c240b77f1eafb 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1549,7 +1549,7 @@ class HandleHistogramProbeInserter call->gtHandleHistogramProfileCandidateInfo->ilOffset); // We transform the call from (CALLVIRT obj, ... args ...) to - // to + // // (CALLVIRT // (COMMA // (ASG tmp, obj) @@ -1598,24 +1598,44 @@ class HandleHistogramProbeInserter unsigned const tmpNum = compiler->lvaGrabTemp(true DEBUGARG("handle histogram profile tmp")); compiler->lvaTable[tmpNum].lvType = TYP_REF; - GenTree* helperCallNode; + GenTree* helperCallNode = nullptr; if (methodHistogram != nullptr) { GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); - GenTree* const baseMethodNode = compiler->gtNewIconEmbMethHndNode(call->gtCallMethHnd); GenTree* const methodProfileNode = compiler->gtNewIconNode((ssize_t)methodHistogram, TYP_I_IMPL); - GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); - helperCallNode = - compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_METHODPROFILE32 : CORINFO_HELP_METHODPROFILE64, - TYP_VOID, tmpNode, baseMethodNode, methodProfileNode, classProfileNode); + + if (call->IsDelegateInvoke()) + { + helperCallNode = compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_DELEGATEPROFILE32 + : CORINFO_HELP_DELEGATEPROFILE64, + TYP_VOID, tmpNode, methodProfileNode); + } + else + { + assert(call->IsVirtualVtable()); + GenTree* const baseMethodNode = compiler->gtNewIconEmbMethHndNode(call->gtCallMethHnd); + helperCallNode = + compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_VTABLEPROFILE32 : CORINFO_HELP_VTABLEPROFILE64, + TYP_VOID, tmpNode, baseMethodNode, methodProfileNode); + } } - else + + if (typeHistogram != nullptr) { GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); - helperCallNode = + GenTree* classProfileCall = compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64, TYP_VOID, tmpNode, classProfileNode); + + if (helperCallNode == nullptr) + { + helperCallNode = classProfileCall; + } + else + { + helperCallNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, classProfileCall, helperCallNode); + } } // Generate the IR... @@ -1885,9 +1905,10 @@ PhaseStatus Compiler::fgPrepareToInstrumentMethod() // Enable class profiling by default, when jitting. // Todo: we may also want this on by default for prejitting. // - const bool useClassProfiles = (JitConfig.JitClassProfiling() > 0); - const bool useMethodProfiles = (JitConfig.JitMethodProfiling() > 0); - if (!prejit && (useClassProfiles || useMethodProfiles)) + const bool useClassProfiles = (JitConfig.JitClassProfiling() > 0); + const bool useDelegateProfiles = (JitConfig.JitDelegateProfiling() > 0); + const bool useVTableProfiles = (JitConfig.JitVTableProfiling() > 0); + if (!prejit && (useClassProfiles || useDelegateProfiles || useVTableProfiles)) { fgHistogramInstrumentor = new (this, CMK_Pgo) HandleHistogramProbeInstrumentor(this); } diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index ae21f852f13d7e..b65cfefba55e94 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22099,8 +22099,8 @@ Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) (call->gtHandleHistogramProfileCandidateInfo != nullptr)); } - bool createMethodHistogram = - (JitConfig.JitMethodProfiling() > 0) && (call->IsVirtualVtable() || call->IsDelegateInvoke()); + bool createMethodHistogram = ((JitConfig.JitDelegateProfiling() > 0) && call->IsDelegateInvoke()) || + ((JitConfig.JitVTableProfiling() > 0) && call->IsVirtualVtable()); if (createTypeHistogram && createMethodHistogram) { diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 6daf4f1411240e..38bab75ed50f74 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -551,9 +551,10 @@ CONFIG_INTEGER(JitProfileCasts, W("JitProfileCasts"), 0) // CONFIG_INTEGER(JitConsumeProfileForCasts, W("JitConsumeProfileForCasts"), 0) // Consume profile data (if any) for // castclass/isinst -CONFIG_INTEGER(JitClassProfiling, W("JitClassProfiling"), 1) // Profile virtual and interface calls -CONFIG_INTEGER(JitMethodProfiling, W("JitMethodProfiling"), 0) // Profile resolved delegate and vtable call targets -CONFIG_INTEGER(JitEdgeProfiling, W("JitEdgeProfiling"), 1) // Profile edges instead of blocks +CONFIG_INTEGER(JitClassProfiling, W("JitClassProfiling"), 1) // Profile virtual and interface calls +CONFIG_INTEGER(JitDelegateProfiling, W("JitDelegateProfiling"), 0) // Profile resolved delegate call targets +CONFIG_INTEGER(JitVTableProfiling, W("JitVTableProfiling"), 0) // Profile resolved vtable call targets +CONFIG_INTEGER(JitEdgeProfiling, W("JitEdgeProfiling"), 1) // Profile edges instead of blocks CONFIG_INTEGER(JitCollect64BitCounts, W("JitCollect64BitCounts"), 0) // Collect counts as 64-bit values. // Profile consumption options diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs index 921f46cd24e89c..3bb774ed11b500 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs @@ -286,16 +286,18 @@ which is the right helper to use to allocate an object of a given type. */ CORINFO_HELP_STACK_PROBE, // Probes each page of the allocated stack frame CORINFO_HELP_PATCHPOINT, // Notify runtime that code has reached a patchpoint + CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, // Notify runtime that code has reached a part of the method that wasn't originally jitted. + CORINFO_HELP_CLASSPROFILE32, // Update 32-bit class profile for a call site CORINFO_HELP_CLASSPROFILE64, // Update 64-bit class profile for a call site - CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, // Notify runtime that code has reached a part of the method that wasn't originally jitted. + CORINFO_HELP_DELEGATEPROFILE32, // Update 32-bit method profile for a delegate call site + CORINFO_HELP_DELEGATEPROFILE64, // Update 64-bit method profile for a delegate call site + CORINFO_HELP_VTABLEPROFILE32, // Update 32-bit method profile for a vtable call site + CORINFO_HELP_VTABLEPROFILE64, // Update 64-bit method profile for a vtable call site CORINFO_HELP_VALIDATE_INDIRECT_CALL, // CFG: Validate function pointer CORINFO_HELP_DISPATCH_INDIRECT_CALL, // CFG: Validate and dispatch to pointer - CORINFO_HELP_METHODPROFILE32, // Update 32-bit method profile for a call site - CORINFO_HELP_METHODPROFILE64, // Update 64-bit method profile for a call site - CORINFO_HELP_COUNT, } } diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 77673855c23540..96819036fe63df 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -5462,7 +5462,7 @@ HCIMPL2(void, JIT_ClassProfile64, Object *obj, ICorJitInfo::HandleHistogram64* c } HCIMPLEND -HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod, ICorJitInfo::HandleHistogram32* methodProfile, ICorJitInfo::HandleHistogram32* typeProfile) +HCIMPL2(void, JIT_DelegateProfile32, Object *obj, ICorJitInfo::HandleHistogram32* methodProfile) { FCALL_CONTRACT; FC_GC_POLL_NOT_NEEDED(); @@ -5474,26 +5474,120 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod const unsigned methodCallIndex = (*pMethodCount)++; int methodSampleIndex = CheckSample(methodCallIndex); - int typeSampleIndex = -1; - if (typeProfile != nullptr) + if (methodSampleIndex == -1) { - volatile unsigned* pTypeCount = (volatile unsigned*) &typeProfile->Count; - const unsigned typeCallIndex = (*pTypeCount)++; - typeSampleIndex = CheckSample(typeCallIndex); + return; + } + + if (objRef == NULL) + { + return; + } + + MethodTable* pMT = objRef->GetMethodTable(); - if ((methodSampleIndex == -1) && (typeSampleIndex == -1)) + _ASSERTE(pMT->IsDelegate()); + + // Resolve method. We handle only the common "direct" delegate as that is + // in any case the only one we can reasonably do GDV for. For instance, + // open delegates are filtered out here, and many cases with inner + // "complicated" logic as well (e.g. static functions, multicast, unmanaged + // functions). + // + MethodDesc* pRecordedMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + DELEGATEREF del = (DELEGATEREF)objRef; + if ((del->GetInvocationCount() == 0) && (del->GetMethodPtrAux() == NULL)) + { + MethodDesc* pMD = NonVirtualEntry2MethodDesc(del->GetMethodPtr()); + if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) { - return; + pRecordedMD = pMD; } } - else + +#ifdef _DEBUG + PgoManager::VerifyAddress(methodProfile); + PgoManager::VerifyAddress(methodProfile + 1); +#endif + + // If table is not yet full, just add entries in. + // + methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pRecordedMD; +} +HCIMPLEND + +// Version of helper above used when the count is 64-bit +HCIMPL3(void, JIT_DelegateProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod, ICorJitInfo::HandleHistogram64* methodProfile) +{ + FCALL_CONTRACT; + FC_GC_POLL_NOT_NEEDED(); + + OBJECTREF objRef = ObjectToOBJECTREF(obj); + VALIDATEOBJECTREF(objRef); + + volatile uint64_t* pMethodCount = (volatile uint64_t*) &methodProfile->Count; + const uint64_t methodCallIndex = (*pMethodCount)++; + int methodSampleIndex = CheckSample(methodCallIndex); + + if (methodSampleIndex == -1) + { + return; + } + + if (objRef == NULL) + { + return; + } + + MethodTable* pMT = objRef->GetMethodTable(); + + _ASSERTE(pMT->IsDelegate()); + + // Resolve method. We handle only the common "direct" delegate as that is + // in any case the only one we can reasonably do GDV for. For instance, + // open delegates are filtered out here, and many cases with inner + // "complicated" logic as well (e.g. static functions, multicast, unmanaged + // functions). + // + MethodDesc* pRecordedMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + DELEGATEREF del = (DELEGATEREF)objRef; + if ((del->GetInvocationCount() == 0) && (del->GetMethodPtrAux() == NULL)) { - if (methodSampleIndex == -1) + MethodDesc* pMD = NonVirtualEntry2MethodDesc(del->GetMethodPtr()); + if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) { - return; + pRecordedMD = pMD; } } +#ifdef _DEBUG + PgoManager::VerifyAddress(methodProfile); + PgoManager::VerifyAddress(methodProfile + 1); +#endif + + // If table is not yet full, just add entries in. + // + methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pRecordedMD; +} +HCIMPLEND + +HCIMPL3(void, JIT_VTableProfile32, Object* obj, CORINFO_METHOD_HANDLE baseMethod, ICorJitInfo::HandleHistogram32* methodProfile) +{ + FCALL_CONTRACT; + FC_GC_POLL_NOT_NEEDED(); + + OBJECTREF objRef = ObjectToOBJECTREF(obj); + VALIDATEOBJECTREF(objRef); + + volatile unsigned* pMethodCount = (volatile unsigned*) &methodProfile->Count; + const unsigned methodCallIndex = (*pMethodCount)++; + int methodSampleIndex = CheckSample(methodCallIndex); + + if (methodSampleIndex == -1) + { + return; + } + if (objRef == NULL) { return; @@ -5504,8 +5598,8 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod // Method better be virtual _ASSERTE(pBaseMD->IsVirtual()); - // We currently do not expect to see interface methods here as we cannot - // efficiently use method handle information for these anyway. + // We do not expect to see interface methods here as we cannot efficiently + // use method handle information for these anyway. _ASSERTE(!pBaseMD->IsInterface()); // Shouldn't be doing this for instantiated methods as they live elsewhere @@ -5514,36 +5608,15 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod MethodTable* pMT = objRef->GetMethodTable(); // Resolve method - MethodDesc* pRecordedMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; - if (pMT->IsDelegate() && (pBaseMD == ((DelegateEEClass*)pMT->GetClass())->GetInvokeMethod())) - { - // We handle only the common "direct" delegate as that is in any case - // the only one we can reasonably do GDV for. For instance, open - // delegates are filtered out here, and many cases with inner - // "complicated" logic as well (e.g. static functions, multicast, - // unmanaged functions). - // - DELEGATEREF del = (DELEGATEREF)objRef; - if ((del->GetInvocationCount() == 0) && (del->GetMethodPtrAux() == NULL)) - { - MethodDesc* pMD = NonVirtualEntry2MethodDesc(del->GetMethodPtr()); - if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) - { - pRecordedMD = pMD; - } - } - } - else - { - WORD slot = pBaseMD->GetSlot(); - _ASSERTE(slot < pBaseMD->GetMethodTable()->GetNumVirtuals()); + WORD slot = pBaseMD->GetSlot(); + _ASSERTE(slot < pBaseMD->GetMethodTable()->GetNumVirtuals()); - MethodDesc* pMD = pMT->GetMethodDescForSlot(slot); + MethodDesc* pMD = pMT->GetMethodDescForSlot(slot); - if (!pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) - { - pRecordedMD = pMD; - } + MethodDesc* pRecordedMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + if (!pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) + { + pRecordedMD = pMD; } #ifdef _DEBUG @@ -5551,24 +5624,11 @@ HCIMPL4(void, JIT_MethodProfile32, Object *obj, CORINFO_METHOD_HANDLE baseMethod PgoManager::VerifyAddress(methodProfile + 1); #endif - // If table is not yet full, just add entries in. - // methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pRecordedMD; - - if (typeSampleIndex != -1) - { - if (pMT->GetLoaderAllocator()->IsCollectible()) - { - pMT = (MethodTable*)DEFAULT_UNKNOWN_HANDLE; - } - - typeProfile->HandleTable[typeSampleIndex] = (CORINFO_CLASS_HANDLE)pMT; - } } HCIMPLEND -// Version of helper above used when the count is 64-bit -HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod, ICorJitInfo::HandleHistogram64* methodProfile, ICorJitInfo::HandleHistogram64* typeProfile) +HCIMPL3(void, JIT_VTableProfile64, Object* obj, CORINFO_METHOD_HANDLE baseMethod, ICorJitInfo::HandleHistogram64* methodProfile) { FCALL_CONTRACT; FC_GC_POLL_NOT_NEEDED(); @@ -5580,24 +5640,9 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod const uint64_t methodCallIndex = (*pMethodCount)++; int methodSampleIndex = CheckSample(methodCallIndex); - int typeSampleIndex = -1; - if (typeProfile != nullptr) - { - volatile uint64_t* pTypeCount = (volatile uint64_t*) &typeProfile->Count; - const uint64_t typeCallIndex = (*pTypeCount)++; - typeSampleIndex = CheckSample(typeCallIndex); - - if ((methodSampleIndex == -1) && (typeSampleIndex == -1)) - { - return; - } - } - else + if (methodSampleIndex == -1) { - if (methodSampleIndex == -1) - { - return; - } + return; } if (objRef == NULL) @@ -5610,8 +5655,8 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod // Method better be virtual _ASSERTE(pBaseMD->IsVirtual()); - // We currently do not expect to see interface methods here as we cannot - // efficiently use method handle information for these anyway. + // We do not expect to see interface methods here as we cannot efficiently + // use method handle information for these anyway. _ASSERTE(!pBaseMD->IsInterface()); // Shouldn't be doing this for instantiated methods as they live elsewhere @@ -5620,36 +5665,15 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod MethodTable* pMT = objRef->GetMethodTable(); // Resolve method - MethodDesc* pRecordedMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; - if (pMT->IsDelegate() && (pBaseMD == ((DelegateEEClass*)pMT->GetClass())->GetInvokeMethod())) - { - // We handle only the common "direct" delegate as that is in any case - // the only one we can reasonably do GDV for. For instance, open - // delegates are filtered out here, and many cases with inner - // "complicated" logic as well (e.g. static functions, multicast, - // unmanaged functions). - // - DELEGATEREF del = (DELEGATEREF)objRef; - if ((del->GetInvocationCount() == 0) && (del->GetMethodPtrAux() == NULL)) - { - MethodDesc* pMD = NonVirtualEntry2MethodDesc(del->GetMethodPtr()); - if ((pMD != nullptr) && !pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) - { - pRecordedMD = pMD; - } - } - } - else - { - WORD slot = pBaseMD->GetSlot(); - _ASSERTE(slot < pBaseMD->GetMethodTable()->GetNumVirtuals()); + WORD slot = pBaseMD->GetSlot(); + _ASSERTE(slot < pBaseMD->GetMethodTable()->GetNumVirtuals()); - MethodDesc* pMD = pMT->GetMethodDescForSlot(slot); + MethodDesc* pMD = pMT->GetMethodDescForSlot(slot); - if (!pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) - { - pRecordedMD = pMD; - } + MethodDesc* pRecordedMD = (MethodDesc*)DEFAULT_UNKNOWN_HANDLE; + if (!pMD->GetLoaderAllocator()->IsCollectible() && !pMD->IsDynamicMethod()) + { + pRecordedMD = pMD; } #ifdef _DEBUG @@ -5658,16 +5682,6 @@ HCIMPL4(void, JIT_MethodProfile64, Object *obj, CORINFO_METHOD_HANDLE baseMethod #endif methodProfile->HandleTable[methodSampleIndex] = (CORINFO_METHOD_HANDLE)pRecordedMD; - - if (typeSampleIndex != -1) - { - if (pMT->GetLoaderAllocator()->IsCollectible()) - { - pMT = (MethodTable*)DEFAULT_UNKNOWN_HANDLE; - } - - typeProfile->HandleTable[typeSampleIndex] = (CORINFO_CLASS_HANDLE)pMT; - } } HCIMPLEND diff --git a/src/tests/Common/testenvironment.proj b/src/tests/Common/testenvironment.proj index 6f713382725e17..f60eb9b8394ac7 100644 --- a/src/tests/Common/testenvironment.proj +++ b/src/tests/Common/testenvironment.proj @@ -60,7 +60,8 @@ COMPlus_JitObjectStackAllocation; COMPlus_JitInlinePolicyProfile; COMPlus_JitClassProfiling; - COMPlus_JitMethodProfiling; + COMPlus_JitDelegateProfiling; + COMPlus_JitVTableProfiling; COMPlus_JitEdgeProfiling; COMPlus_JitRandomGuardedDevirtualization; COMPlus_JitRandomEdgeCounts; @@ -207,9 +208,9 @@ - + - + From f0010e65132654dad56c23c64133d1ea3ea1487d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Jun 2022 15:44:02 +0200 Subject: [PATCH 48/59] Set gtTargetHandle for GDV function addrs --- src/coreclr/jit/indirectcalltransformer.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index e44daba1b7ee79..5f214520745eb9 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -612,7 +612,7 @@ class IndirectCallTransformer CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionEntryPoint(methHnd, &lookup); - GenTree* compareTarTree = CreateTreeForLookup(lookup); + GenTree* compareTarTree = CreateTreeForLookup(methHnd, lookup); compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, tarTree); } else @@ -650,7 +650,7 @@ class IndirectCallTransformer CORINFO_CONST_LOOKUP lookup; compiler->info.compCompHnd->getFunctionFixedEntryPoint(methHnd, false, &lookup); - GenTree* compareTarTree = CreateTreeForLookup(lookup); + GenTree* compareTarTree = CreateTreeForLookup(methHnd, lookup); compare = compiler->gtNewOperNode(GT_NE, TYP_INT, compareTarTree, tarTree); } } @@ -1183,17 +1183,17 @@ class IndirectCallTransformer unsigned returnTemp; Statement* lastStmt; - GenTree* CreateTreeForLookup(const CORINFO_CONST_LOOKUP& lookup) + GenTree* CreateTreeForLookup(CORINFO_METHOD_HANDLE methHnd, const CORINFO_CONST_LOOKUP& lookup) { switch (lookup.accessType) { case IAT_VALUE: { - return compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + return CreateFunctionTargetAddr(methHnd, lookup); } case IAT_PVALUE: { - GenTree* tree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + GenTree* tree = CreateFunctionTargetAddr(methHnd, lookup); tree = compiler->gtNewIndir(TYP_I_IMPL, tree); tree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; tree->gtFlags &= ~GTF_EXCEPT; @@ -1206,8 +1206,8 @@ class IndirectCallTransformer } case IAT_RELPVALUE: { - GenTree* addr = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - GenTree* tree = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + GenTree* addr = CreateFunctionTargetAddr(methHnd, lookup); + GenTree* tree = CreateFunctionTargetAddr(methHnd, lookup); tree = compiler->gtNewIndir(TYP_I_IMPL, tree); tree->gtFlags |= GTF_IND_NONFAULTING | GTF_IND_INVARIANT; tree->gtFlags &= ~GTF_EXCEPT; @@ -1221,6 +1221,13 @@ class IndirectCallTransformer } } } + + GenTreeIntCon* CreateFunctionTargetAddr(CORINFO_METHOD_HANDLE methHnd, const CORINFO_CONST_LOOKUP& lookup) + { + GenTreeIntCon* con = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + INDEBUG(con->gtTargetHandle = (size_t)methHnd); + return con; + } }; // Runtime lookup with dynamic dictionary expansion transformer, From 2b2781dc8120032ad2853b6a18784ad8e53e431f Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 9 Jun 2022 12:18:04 +0200 Subject: [PATCH 49/59] Fix build --- src/coreclr/jit/indirectcalltransformer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 5f214520745eb9..586c7d74fb0aac 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -1222,10 +1222,10 @@ class IndirectCallTransformer } } - GenTreeIntCon* CreateFunctionTargetAddr(CORINFO_METHOD_HANDLE methHnd, const CORINFO_CONST_LOOKUP& lookup) + GenTree* CreateFunctionTargetAddr(CORINFO_METHOD_HANDLE methHnd, const CORINFO_CONST_LOOKUP& lookup) { - GenTreeIntCon* con = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); - INDEBUG(con->gtTargetHandle = (size_t)methHnd); + GenTree* con = compiler->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR); + INDEBUG(con->AsIntCon()->gtTargetHandle = (size_t)methHnd); return con; } }; From 993c4c2e805f72d87930874590f039a3bd2e0d08 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 9 Jun 2022 12:23:51 +0200 Subject: [PATCH 50/59] Disable method GDV for R2R --- src/coreclr/jit/importer.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index b65cfefba55e94..c71adc31c75125 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22313,7 +22313,14 @@ void Compiler::pickGDV(GenTreeCall* call, const int maxLikelyMethods = 32; LikelyClassMethodRecord likelyMethods[maxLikelyMethods]; unsigned numberOfMethods = 0; - if (call->IsVirtualVtable() || call->IsDelegateInvoke()) + + // TODO-GDV: R2R support requires additional work to reacquire the + // entrypoint, similar to what happens at the end of impDevirtualizeCall. + // As part of supporting this we should merge the tail of + // impDevirtualizeCall and what happens in + // GuardedDevirtualizationTransformer::CreateThen for method GDV. + // + if (!opts.IsReadyToRun() && (call->IsVirtualVtable() || call->IsDelegateInvoke())) { numberOfMethods = getLikelyMethods(likelyMethods, maxLikelyMethods, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); From 54b5a0b13f120323ff5ab0ee12d325d1da352693 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 10 Jun 2022 13:49:43 +0200 Subject: [PATCH 51/59] Make extra SPMI queries for GDV debugging output purposes --- src/coreclr/jit/importer.cpp | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index c71adc31c75125..b27799abbd654c 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22333,35 +22333,37 @@ void Compiler::pickGDV(GenTreeCall* call, } #ifdef DEBUG - if (verbose && (numberOfClasses > 0)) + if ((verbose || JitConfig.EnableExtraSuperPmiQueries()) && (numberOfClasses > 0)) { bool isExact; bool isNonNull; CallArg* thisArg = call->gtArgs.GetThisArg(); CORINFO_CLASS_HANDLE declaredThisClsHnd = gtGetClassHandle(thisArg->GetNode(), &isExact, &isNonNull); - printf("Likely classes for call [%06u]", dspTreeID(call)); + JITDUMP("Likely classes for call [%06u]", dspTreeID(call)); if (declaredThisClsHnd != NO_CLASS_HANDLE) { - printf(" on class %p (%s)", declaredThisClsHnd, eeGetClassName(declaredThisClsHnd)); + const char* baseClassName = eeGetClassName(declaredThisClsHnd); + JITDUMP(" on class %p (%s)", declaredThisClsHnd, baseClassName); } - printf("\n"); + JITDUMP("\n"); for (UINT32 i = 0; i < numberOfClasses; i++) { - printf(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, - eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle), likelyClasses[i].likelihood); + const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, + className, likelyClasses[i].likelihood); } } - if (verbose && (numberOfMethods > 0)) + if ((verbose || JitConfig.EnableExtraSuperPmiQueries()) && (numberOfMethods > 0)) { assert(call->gtCallType == CT_USER_FUNC); - printf("Likely methods for call [%06u] to method %s\n", dspTreeID(call), - eeGetMethodFullName(call->gtCallMethHnd)); + const char* baseMethName = eeGetMethodFullName(call->gtCallMethHnd); + JITDUMP("Likely methods for call [%06u] to method %s\n", dspTreeID(call), baseMethName); for (UINT32 i = 0; i < numberOfMethods; i++) { - CORINFO_CONST_LOOKUP lookup; + CORINFO_CONST_LOOKUP lookup = {}; info.compCompHnd->getFunctionFixedEntryPoint((CORINFO_METHOD_HANDLE)likelyMethods[i].handle, false, &lookup); @@ -22369,19 +22371,19 @@ void Compiler::pickGDV(GenTreeCall* call, switch (lookup.accessType) { case IAT_VALUE: - printf(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); break; case IAT_PVALUE: - printf(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); + JITDUMP(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); break; case IAT_PPVALUE: - printf(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); + JITDUMP(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); break; default: - printf(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); + JITDUMP(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); break; } } From 901d4d18a67e5942a5d662ae4056e24ad67860f3 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 10 Jun 2022 14:24:40 +0200 Subject: [PATCH 52/59] Reorder static and signature check Delegate instances can point to a static method with the first argument bound; in this case the delegate instance is indistinguishable from one for an instance method, but the target signature has an extra argument compared to the call itself. --- src/coreclr/jit/importer.cpp | 42 +++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index b27799abbd654c..a108f5c713e9fb 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22350,8 +22350,8 @@ void Compiler::pickGDV(GenTreeCall* call, for (UINT32 i = 0; i < numberOfClasses; i++) { const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); - JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, - className, likelyClasses[i].likelihood); + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, + likelyClasses[i].likelihood); } } @@ -22607,9 +22607,35 @@ void Compiler::considerGuardedDevirtualization(GenTreeCall* call, likelyMethod = dvInfo.devirtualizedMethod; } - else + + uint32_t likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); + + if (likelyClass == NO_CLASS_HANDLE) { - // Verify that this is a reasonable looking call target. + // For method GDV do a few more checks that we get for free in the + // resolve call above for class-based GDV. + if ((likelyMethodAttribs & CORINFO_FLG_STATIC) != 0) + { + assert(call->IsDelegateInvoke()); + JITDUMP("Cannot currently handle devirtualizing static delegate calls, sorry\n"); + return; + } + + // Verify that the call target and args look reasonable so that the JIT + // does not blow up during inlining/call morphing. + // + // NOTE: Once we want to support devirtualization of delegate calls to + // static methods and remove the check above we will start failing here + // for delegates pointing to static methods that have the first arg + // bound. For example: + // + // public static void E(this C c) ... + // Action a = new C().E; + // + // The delegate instance looks exactly like one pointing to an instance + // method in this case and the call will have zero args while the + // signature has 1 arg. + // if (!isCompatibleMethodGDV(call, likelyMethod)) { JITDUMP("Target for method-based GDV is incompatible (stale profile?)\n"); @@ -22618,14 +22644,6 @@ void Compiler::considerGuardedDevirtualization(GenTreeCall* call, } } - uint32_t likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); - if ((likelyMethodAttribs & CORINFO_FLG_STATIC) != 0) - { - assert(call->IsDelegateInvoke()); - JITDUMP("Cannot currently handle devirtualizing static delegate calls, sorry\n"); - return; - } - JITDUMP("%s call would invoke method %s\n", isInterface ? "interface" : call->IsDelegateInvoke() ? "delegate" : "virtual", eeGetMethodName(likelyMethod, nullptr)); From 12f36546ff53b736f57ef59d20d4eb91d85fd582 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 10 Jun 2022 17:09:20 +0200 Subject: [PATCH 53/59] Misc changes --- src/coreclr/jit/compiler.h | 8 ++--- src/coreclr/jit/compiler.hpp | 8 +---- src/coreclr/jit/fgprofile.cpp | 34 +++++++++++---------- src/coreclr/jit/importer.cpp | 23 ++++++++++++++ src/coreclr/jit/indirectcalltransformer.cpp | 31 +++++++++++++------ src/coreclr/jit/likelyclass.cpp | 13 ++++++++ 6 files changed, 79 insertions(+), 38 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 37aa9d06350801..81afa964855097 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -2327,12 +2327,8 @@ class Compiler GenTreeCall* gtNewIndCallNode(GenTree* addr, var_types type, const DebugInfo& di = DebugInfo()); - GenTreeCall* gtNewHelperCallNode(unsigned helper, - var_types type, - GenTree* arg1 = nullptr, - GenTree* arg2 = nullptr, - GenTree* arg3 = nullptr, - GenTree* arg4 = nullptr); + GenTreeCall* gtNewHelperCallNode( + unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr); GenTreeCall* gtNewRuntimeLookupHelperCallNode(CORINFO_RUNTIME_LOOKUP* pRuntimeLookup, GenTree* ctxTree, diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index b916fdcc2bf574..2fe7c10574c6b0 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -994,7 +994,7 @@ inline GenTree* Compiler::gtNewIconEmbFldHndNode(CORINFO_FIELD_HANDLE fldHnd) // New CT_HELPER node // inline GenTreeCall* Compiler::gtNewHelperCallNode( - unsigned helper, var_types type, GenTree* arg1, GenTree* arg2, GenTree* arg3, GenTree* arg4) + unsigned helper, var_types type, GenTree* arg1, GenTree* arg2, GenTree* arg3) { GenTreeFlags flags = s_helperCallProperties.NoThrow((CorInfoHelpFunc)helper) ? GTF_EMPTY : GTF_EXCEPT; GenTreeCall* result = gtNewCallNode(CT_HELPER, eeFindHelper(helper), type); @@ -1006,12 +1006,6 @@ inline GenTreeCall* Compiler::gtNewHelperCallNode( result->gtInlineObservation = InlineObservation::CALLSITE_IS_CALL_TO_HELPER; #endif - if (arg4 != nullptr) - { - result->gtArgs.PushFront(this, NewCallArg::Primitive(arg4)); - result->gtFlags |= arg4->gtFlags & GTF_ALL_EFFECT; - } - if (arg3 != nullptr) { result->gtArgs.PushFront(this, NewCallArg::Primitive(arg3)); diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 0c240b77f1eafb..19d3146fce7c47 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1599,42 +1599,44 @@ class HandleHistogramProbeInserter compiler->lvaTable[tmpNum].lvType = TYP_REF; GenTree* helperCallNode = nullptr; + + if (typeHistogram != nullptr) + { + GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); + GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); + helperCallNode = + compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64, + TYP_VOID, tmpNode, classProfileNode); + } + if (methodHistogram != nullptr) { GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); GenTree* const methodProfileNode = compiler->gtNewIconNode((ssize_t)methodHistogram, TYP_I_IMPL); + GenTree* methodProfileCallNode; if (call->IsDelegateInvoke()) { - helperCallNode = compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_DELEGATEPROFILE32 - : CORINFO_HELP_DELEGATEPROFILE64, - TYP_VOID, tmpNode, methodProfileNode); + methodProfileCallNode = compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_DELEGATEPROFILE32 + : CORINFO_HELP_DELEGATEPROFILE64, + TYP_VOID, tmpNode, methodProfileNode); } else { assert(call->IsVirtualVtable()); GenTree* const baseMethodNode = compiler->gtNewIconEmbMethHndNode(call->gtCallMethHnd); - helperCallNode = + methodProfileCallNode = compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_VTABLEPROFILE32 : CORINFO_HELP_VTABLEPROFILE64, TYP_VOID, tmpNode, baseMethodNode, methodProfileNode); } - } - - if (typeHistogram != nullptr) - { - GenTree* const tmpNode = compiler->gtNewLclvNode(tmpNum, TYP_REF); - GenTree* const classProfileNode = compiler->gtNewIconNode((ssize_t)typeHistogram, TYP_I_IMPL); - GenTree* classProfileCall = - compiler->gtNewHelperCallNode(is32 ? CORINFO_HELP_CLASSPROFILE32 : CORINFO_HELP_CLASSPROFILE64, - TYP_VOID, tmpNode, classProfileNode); if (helperCallNode == nullptr) { - helperCallNode = classProfileCall; + helperCallNode = methodProfileCallNode; } else { - helperCallNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, classProfileCall, helperCallNode); + helperCallNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, helperCallNode, methodProfileCallNode); } } @@ -1664,7 +1666,7 @@ class HandleHistogramProbeInserter return; } - ICorJitInfo::PgoInstrumentationSchema& countEntry = m_schema[*m_currentSchemaIndex + 0]; + ICorJitInfo::PgoInstrumentationSchema& countEntry = m_schema[*m_currentSchemaIndex]; bool is32 = countEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount; bool is64 = countEntry.InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount; diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index a108f5c713e9fb..0465ed7dde7191 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -22074,6 +22074,16 @@ bool Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) return true; } +//------------------------------------------------------------------------ +// compClassifyGDVProbeType: +// Classify the type of GDV probe to use for a call site. +// +// Arguments: +// call - The call +// +// Returns: +// The type of probe to use. +// Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) { if (call->gtCallType == CT_INDIRECT) @@ -22290,6 +22300,17 @@ void Compiler::addFatPointerCandidate(GenTreeCall* call) helper.StoreRetExprResultsInArgs(call); } +//------------------------------------------------------------------------ +// pickGDV: Use profile information to pick a GDV candidate for a call site. +// +// Arguments: +// call - the call +// ilOffset - exact IL offset of the call +// isInterface - whether or not the call target is defined on an interface +// classGuess - [out] the class to guess for (mutually exclusive with methodGuess) +// methodGuess - [out] the method to guess for (mutually exclusive with classGuess) +// likelihood - [out] an estimate of the likelihood that the guess will succeed +// void Compiler::pickGDV(GenTreeCall* call, IL_OFFSET ilOffset, bool isInterface, @@ -22398,6 +22419,8 @@ void Compiler::pickGDV(GenTreeCall* call, // CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + // TODO-GDV: This can be simplified to just use likelyClasses and + // likelyMethods now that we have multiple candidates here. getRandomGDV(fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset, random, classGuess, methodGuess); if (*classGuess != NO_CLASS_HANDLE) { diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 586c7d74fb0aac..2385cc12fd5a7c 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -591,6 +591,9 @@ class IndirectCallTransformer // which case the check will be moved into the success case of // a previous GDV and thus may not execute when we hit the cold // path. + // TODO-GDV: Consider duplicating the store at the end of the + // cold case for the previous GDV. Then we can reuse the target + // if the second check of a chained GDV fails. bool reuseTarget = (origCall->gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_CHAIN) == 0; if (origCall->IsVirtualVtable()) { @@ -617,10 +620,12 @@ class IndirectCallTransformer } else { - // Reusing the call target for delegates is more complicated. - // Essentially we need to do the transformation done in - // LowerDelegateInvoke here by converting the call to - // CT_INDIRECT and reusing the target address. + // Reusing the call target for delegates is more + // complicated. Essentially we need to do the + // transformation done in LowerDelegateInvoke by converting + // the call to CT_INDIRECT and reusing the target address. + // We will do that transformation in CreateElse, but here + // we need to stash the target. CLANG_FORMAT_COMMENT_ANCHOR; #ifdef TARGET_ARM // Not impossible to support, but would additionally @@ -638,7 +643,6 @@ class IndirectCallTransformer if (reuseTarget) { m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); - compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); @@ -826,7 +830,8 @@ class IndirectCallTransformer call->gtCallType = CT_USER_FUNC; call->gtCallMoreFlags |= GTF_CALL_M_DEVIRTUALIZED; call->gtCallMoreFlags &= ~GTF_CALL_M_DELEGATE_INV; - // TODO: R2R entry point + // TODO-GDV: To support R2R we need to get the entry point + // here. We should unify with the tail of impDevirtualizeCall. if (origCall->IsVirtual()) { @@ -961,9 +966,7 @@ class IndirectCallTransformer compiler->gtNewIconNode((ssize_t)compiler->eeGetEEInfo()->offsetOfDelegateInstance, TYP_I_IMPL); CallArg* thisArg = call->gtArgs.GetThisArg(); GenTree* delegateObj = thisArg->GetNode(); - // TODO: Is it worth it to create a local for this before - // the check as well? In many cases it will end up unused - // after inlining. + assert(delegateObj->OperIsLocal()); GenTree* newThis = compiler->gtNewOperNode(GT_ADD, TYP_BYREF, compiler->gtCloneExpr(delegateObj), thisOffset); @@ -1183,6 +1186,16 @@ class IndirectCallTransformer unsigned returnTemp; Statement* lastStmt; + //------------------------------------------------------------------------ + // CreateTreeForLookup: Create a tree representing a lookup of a method address. + // + // Arguments: + // methHnd - the handle for the method the lookup is for + // lookup - lookup information for the address + // + // Returns: + // A node representing the lookup. + // GenTree* CreateTreeForLookup(CORINFO_METHOD_HANDLE methHnd, const CORINFO_CONST_LOOKUP& lookup) { switch (lookup.accessType) diff --git a/src/coreclr/jit/likelyclass.cpp b/src/coreclr/jit/likelyclass.cpp index f293a8f164ac48..a9c415f1fd4475 100644 --- a/src/coreclr/jit/likelyclass.cpp +++ b/src/coreclr/jit/likelyclass.cpp @@ -105,6 +105,14 @@ LikelyClassMethodHistogram::LikelyClassMethodHistogram(INT_PTR* histogramEntries } } +//------------------------------------------------------------------------ +// getLikelyClassesOrMethods: +// Find class/method profile data for an IL offset, and return the most +// likely classes/methods. +// +// This is a common entrypoint for getLikelyClasses and getLikelyMethods. +// See documentation for those for more information. +// static unsigned getLikelyClassesOrMethods(LikelyClassMethodRecord* pLikelyEntries, UINT32 maxLikelyClasses, ICorJitInfo::PgoInstrumentationSchema* schema, @@ -293,6 +301,11 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassMethodRecord* ilOffset, true); } +//------------------------------------------------------------------------ +// getLikelyMethods: find method profile data for an IL offset, and return the most likely methods +// +// See documentation on getLikelyClasses above. +// extern "C" DLLEXPORT UINT32 WINAPI getLikelyMethods(LikelyClassMethodRecord* pLikelyMethods, UINT32 maxLikelyMethods, ICorJitInfo::PgoInstrumentationSchema* schema, From 6a9fbfed7cc8681df113994da4911438d8fe7969 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 10 Jun 2022 17:28:51 +0200 Subject: [PATCH 54/59] Enable delegate profiling by default --- src/coreclr/jit/jitconfigvalues.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 38bab75ed50f74..5ff1d72eecf44f 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -552,7 +552,7 @@ CONFIG_INTEGER(JitConsumeProfileForCasts, W("JitConsumeProfileForCasts"), 0) // // castclass/isinst CONFIG_INTEGER(JitClassProfiling, W("JitClassProfiling"), 1) // Profile virtual and interface calls -CONFIG_INTEGER(JitDelegateProfiling, W("JitDelegateProfiling"), 0) // Profile resolved delegate call targets +CONFIG_INTEGER(JitDelegateProfiling, W("JitDelegateProfiling"), 1) // Profile resolved delegate call targets CONFIG_INTEGER(JitVTableProfiling, W("JitVTableProfiling"), 0) // Profile resolved vtable call targets CONFIG_INTEGER(JitEdgeProfiling, W("JitEdgeProfiling"), 1) // Profile edges instead of blocks CONFIG_INTEGER(JitCollect64BitCounts, W("JitCollect64BitCounts"), 0) // Collect counts as 64-bit values. From c46ebaa2effc2919ca4c9be4e3c117daeea01993 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 14 Jun 2022 13:56:26 +0200 Subject: [PATCH 55/59] Fix newlines when printing test profile --- src/coreclr/vm/pgo.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/vm/pgo.cpp b/src/coreclr/vm/pgo.cpp index 7f7115d1f3c560..a4bad1f5c6dffa 100644 --- a/src/coreclr/vm/pgo.cpp +++ b/src/coreclr/vm/pgo.cpp @@ -281,11 +281,11 @@ void PgoManager::WritePgoData() MethodDesc* md = reinterpret_cast(methodHandleData); if (md == nullptr) { - fprintf(pgoDataFile, "MethodHandle: NULL"); + fprintf(pgoDataFile, "MethodHandle: NULL\n"); } else if (ICorJitInfo::IsUnknownHandle(methodHandleData)) { - fprintf(pgoDataFile, "MethodHandle: UNKNOWN"); + fprintf(pgoDataFile, "MethodHandle: UNKNOWN\n"); } else { @@ -297,13 +297,13 @@ void PgoManager::WritePgoData() // MethodName|@|fully_qualified_type_name if (tTypeName.GetCount() + 1 + tMethodName.GetCount() > 8192) { - fprintf(pgoDataFile, "MethodHandle: UNKNOWN"); + fprintf(pgoDataFile, "MethodHandle: UNKNOWN\n"); } else { StackScratchBuffer methodNameBuffer; StackScratchBuffer typeBuffer; - fprintf(pgoDataFile, "MethodHandle: %s|@|%s", tMethodName.GetUTF8(methodNameBuffer), tTypeName.GetUTF8(typeBuffer)); + fprintf(pgoDataFile, "MethodHandle: %s|@|%s\n", tMethodName.GetUTF8(methodNameBuffer), tTypeName.GetUTF8(typeBuffer)); } } break; From 60f140ea83e7c4b152523ac82aa283ef23ec670d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 15 Jun 2022 17:30:27 +0200 Subject: [PATCH 56/59] Add GetLikelyMethod and hook it up through R2R This makes us support generating and consuming the method handle histograms in the same way as the type handle histograms by compressing them before they are put in the R2R format. Also finish some SPMI support. --- src/coreclr/inc/readytorun.h | 2 +- src/coreclr/jit/ClrJit.PAL.exports | 1 + src/coreclr/jit/ClrJit.exports | 1 + src/coreclr/jit/fgprofile.cpp | 4 + src/coreclr/jit/likelyclass.cpp | 5 +- .../Common/Internal/Runtime/ModuleHeaders.cs | 2 +- .../tools/Common/JitInterface/CorInfoImpl.cs | 77 ++++++++++++++----- src/coreclr/tools/Common/Pgo/PgoFormat.cs | 1 + .../tools/superpmi/mcs/verbdumpmap.cpp | 8 +- .../tools/superpmi/mcs/verbjitflags.cpp | 8 +- .../superpmi-shared/methodcontext.cpp | 11 ++- .../superpmi/superpmi-shared/methodcontext.h | 5 +- .../superpmi-shared/spmidumphelper.cpp | 1 + 13 files changed, 94 insertions(+), 32 deletions(-) diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h index 76354362d57d83..20db29298cfba1 100644 --- a/src/coreclr/inc/readytorun.h +++ b/src/coreclr/inc/readytorun.h @@ -16,7 +16,7 @@ // Keep these in sync with src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs #define READYTORUN_MAJOR_VERSION 0x0006 -#define READYTORUN_MINOR_VERSION 0x0001 +#define READYTORUN_MINOR_VERSION 0x0002 #define MINIMUM_READYTORUN_MAJOR_VERSION 0x006 diff --git a/src/coreclr/jit/ClrJit.PAL.exports b/src/coreclr/jit/ClrJit.PAL.exports index 2625e98bc421e7..e4e6064db84e89 100644 --- a/src/coreclr/jit/ClrJit.PAL.exports +++ b/src/coreclr/jit/ClrJit.PAL.exports @@ -1,4 +1,5 @@ getJit jitStartup getLikelyClasses +getLikelyMethods jitBuildString diff --git a/src/coreclr/jit/ClrJit.exports b/src/coreclr/jit/ClrJit.exports index c6a22db4cae403..5430f7b165929d 100644 --- a/src/coreclr/jit/ClrJit.exports +++ b/src/coreclr/jit/ClrJit.exports @@ -5,4 +5,5 @@ EXPORTS getJit jitStartup getLikelyClasses + getLikelyMethods jitBuildString diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 19d3146fce7c47..3c00b547965c7d 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -2159,6 +2159,10 @@ PhaseStatus Compiler::fgIncorporateProfileData() fgPgoClassProfiles++; break; + case ICorJitInfo::PgoInstrumentationKind::GetLikelyMethod: + fgPgoMethodProfiles++; + break; + case ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount: case ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount: if (iSchema + 1 < fgPgoSchemaCount) diff --git a/src/coreclr/jit/likelyclass.cpp b/src/coreclr/jit/likelyclass.cpp index a9c415f1fd4475..277d38201448de 100644 --- a/src/coreclr/jit/likelyclass.cpp +++ b/src/coreclr/jit/likelyclass.cpp @@ -146,7 +146,7 @@ static unsigned getLikelyClassesOrMethods(LikelyClassMethodRecord* { return 0; } - assert(result != 0); // we don't expect zero in GetLikelyClass + assert(result != 0); // we don't expect zero in GetLikelyClass/GetLikelyMethod pLikelyEntries[0].likelihood = (UINT32)(schema[i].Other & 0xFF); pLikelyEntries[0].handle = result; return 1; @@ -380,7 +380,8 @@ void Compiler::getRandomGDV(ICorJitInfo::PgoInstrumentationSchema* schema, continue; } - if ((schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyClass) && + if (((schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyClass) || + (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyMethod)) && (schema[i].Count == 1)) { INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset); diff --git a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs index 30d8af4efb23c7..d9671ef058ae13 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs @@ -15,7 +15,7 @@ internal struct ReadyToRunHeaderConstants public const uint Signature = 0x00525452; // 'RTR' public const ushort CurrentMajorVersion = 6; - public const ushort CurrentMinorVersion = 1; + public const ushort CurrentMinorVersion = 2; } #pragma warning disable 0169 diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 3c1eb891ac4f81..27dbbfb44b9f8e 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -101,20 +101,23 @@ public static IntPtr Get() private static readonly IntPtr s_jit; } - private struct LikelyClassRecord + private struct LikelyClassMethodRecord { - public IntPtr clsHandle; + public IntPtr handle; public uint likelihood; - public LikelyClassRecord(IntPtr clsHandle, uint likelihood) + public LikelyClassMethodRecord(IntPtr handle, uint likelihood) { - this.clsHandle = clsHandle; + this.handle = handle; this.likelihood = likelihood; } } [DllImport(JitLibrary)] - private extern static uint getLikelyClasses(LikelyClassRecord* pLikelyClasses, uint maxLikelyClasses, PgoInstrumentationSchema* schema, uint countSchemaItems, byte*pInstrumentationData, int ilOffset); + private extern static uint getLikelyClasses(LikelyClassMethodRecord* pLikelyClasses, uint maxLikelyClasses, PgoInstrumentationSchema* schema, uint countSchemaItems, byte*pInstrumentationData, int ilOffset); + + [DllImport(JitLibrary)] + private extern static uint getLikelyMethods(LikelyClassMethodRecord* pLikelyMethods, uint maxLikelyMethods, PgoInstrumentationSchema* schema, uint countSchemaItems, byte*pInstrumentationData, int ilOffset); [DllImport(JitSupportLibrary)] private extern static IntPtr GetJitHost(IntPtr configProvider); @@ -192,17 +195,18 @@ private Logger Logger public static IEnumerable ConvertTypeHandleHistogramsToCompactTypeHistogramFormat(PgoSchemaElem[] pgoData, CompilationModuleGroup compilationModuleGroup) { - bool hasTypeHistogram = false; + bool hasHistogram = false; foreach (var elem in pgoData) { - if (elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes) + if (elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes || + elem.InstrumentationKind == PgoInstrumentationKind.HandleHistogramMethods) { // found histogram - hasTypeHistogram = true; + hasHistogram = true; break; } } - if (!hasTypeHistogram) + if (!hasHistogram) { foreach (var elem in pgoData) { @@ -222,9 +226,10 @@ public static IEnumerable ConvertTypeHandleHistogramsToCompactTyp if ((i + 1 < pgoData.Length) && (pgoData[i].InstrumentationKind == PgoInstrumentationKind.HandleHistogramIntCount || pgoData[i].InstrumentationKind == PgoInstrumentationKind.HandleHistogramLongCount) && - (pgoData[i + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes)) + (pgoData[i + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes || + pgoData[i + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramMethods)) { - PgoSchemaElem? newElem = ComputeLikelyClass(i, handleToObject, nativeSchema, instrumentationData, compilationModuleGroup); + PgoSchemaElem? newElem = ComputeLikelyClassMethod(i, handleToObject, nativeSchema, instrumentationData, compilationModuleGroup); if (newElem.HasValue) { yield return newElem.Value; @@ -249,33 +254,63 @@ IntPtr LocalObjectToHandle(object input) } } - private static PgoSchemaElem? ComputeLikelyClass(int index, Dictionary handleToObject, PgoInstrumentationSchema[] nativeSchema, byte[] instrumentationData, CompilationModuleGroup compilationModuleGroup) + private static PgoSchemaElem? ComputeLikelyClassMethod(int index, Dictionary handleToObject, PgoInstrumentationSchema[] nativeSchema, byte[] instrumentationData, CompilationModuleGroup compilationModuleGroup) { // getLikelyClasses will use two entries from the native schema table. There must be at least two present to avoid overruning the buffer if (index > (nativeSchema.Length - 2)) return null; + bool isType = nativeSchema[index + 1].InstrumentationKind == PgoInstrumentationKind.HandleHistogramTypes; + fixed(PgoInstrumentationSchema* pSchema = &nativeSchema[index]) { fixed(byte* pInstrumentationData = &instrumentationData[0]) { - // We're going to store only the most popular type to reduce size of the profile - LikelyClassRecord* likelyClasses = stackalloc LikelyClassRecord[1]; - uint numberOfClasses = getLikelyClasses(likelyClasses, 1, pSchema, 2, pInstrumentationData, nativeSchema[index].ILOffset); + // We're going to store only the most popular type/method to reduce size of the profile + LikelyClassMethodRecord* likelyClassMethods = stackalloc LikelyClassMethodRecord[1]; + uint numberOfRecords; + if (isType) + { + numberOfRecords = getLikelyClasses(likelyClassMethods, 1, pSchema, 2, pInstrumentationData, nativeSchema[index].ILOffset); + } + else + { + numberOfRecords = getLikelyMethods(likelyClassMethods, 1, pSchema, 2, pInstrumentationData, nativeSchema[index].ILOffset); + } - if (numberOfClasses > 0) + if (numberOfRecords > 0) { - TypeDesc type = (TypeDesc)handleToObject[likelyClasses->clsHandle]; + TypeSystemEntityOrUnknown[] newData = null; + if (isType) + { + TypeDesc type = (TypeDesc)handleToObject[likelyClassMethods->handle]; +#if READYTORUN + if (compilationModuleGroup.VersionsWithType(type)) +#endif + { + newData = new[] { new TypeSystemEntityOrUnknown(type) }; + } + } + else + { + MethodDesc method = (MethodDesc)handleToObject[likelyClassMethods->handle]; + #if READYTORUN - if (compilationModuleGroup.VersionsWithType(type)) + if (compilationModuleGroup.VersionsWithMethodBody(method)) #endif + { + newData = new[] { new TypeSystemEntityOrUnknown(method) }; + } + } + + if (newData != null) { PgoSchemaElem likelyClassElem = new PgoSchemaElem(); - likelyClassElem.InstrumentationKind = PgoInstrumentationKind.GetLikelyClass; + likelyClassElem.InstrumentationKind = isType ? PgoInstrumentationKind.GetLikelyClass : PgoInstrumentationKind.GetLikelyMethod; likelyClassElem.ILOffset = nativeSchema[index].ILOffset; likelyClassElem.Count = 1; - likelyClassElem.Other = (int)(likelyClasses->likelihood | (numberOfClasses << 8)); - likelyClassElem.DataObject = new TypeSystemEntityOrUnknown[] { new TypeSystemEntityOrUnknown(type) }; + likelyClassElem.Other = (int)(likelyClassMethods->likelihood | (numberOfRecords << 8)); + likelyClassElem.DataObject = newData; return likelyClassElem; } } diff --git a/src/coreclr/tools/Common/Pgo/PgoFormat.cs b/src/coreclr/tools/Common/Pgo/PgoFormat.cs index a4e23286be4c67..5dc847b34810b0 100644 --- a/src/coreclr/tools/Common/Pgo/PgoFormat.cs +++ b/src/coreclr/tools/Common/Pgo/PgoFormat.cs @@ -49,6 +49,7 @@ public enum PgoInstrumentationKind EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int EdgeLongCount = (DescriptorMin * 6) | EightByte, // edge counter using unsigned 8 byte int GetLikelyClass = (DescriptorMin * 7) | TypeHandle, // Compressed get likely class data + GetLikelyMethod = (DescriptorMin * 7) | MethodHandle, // Compressed get likely method data } public interface IPgoSchemaDataLoader diff --git a/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp b/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp index 1cfbc598dbee16..2ff9b3cbe36908 100644 --- a/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp +++ b/src/coreclr/tools/superpmi/mcs/verbdumpmap.cpp @@ -93,8 +93,9 @@ void DumpMap(int index, MethodContext* mc) bool hasClassProfile = false; bool hasMethodProfile = false; bool hasLikelyClass = false; + bool hasLikelyMethod = false; ICorJitInfo::PgoSource pgoSource = ICorJitInfo::PgoSource::Unknown; - if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasMethodProfile, hasLikelyClass, pgoSource)) + if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasMethodProfile, hasLikelyClass, hasLikelyMethod, pgoSource)) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_PGO); @@ -118,6 +119,11 @@ void DumpMap(int index, MethodContext* mc) rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); } + if (hasLikelyMethod) + { + rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_LIKELY_METHOD); + } + if (pgoSource == ICorJitInfo::PgoSource::Static) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_STATIC_PROFILE); diff --git a/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp b/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp index a3e31a1f73062f..33190d3fbd9960 100644 --- a/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp +++ b/src/coreclr/tools/superpmi/mcs/verbjitflags.cpp @@ -31,8 +31,9 @@ int verbJitFlags::DoWork(const char* nameOfInput) bool hasClassProfile = false; bool hasMethodProfile = false; bool hasLikelyClass = false; + bool hasLikelyMethod = false; ICorJitInfo::PgoSource pgoSource = ICorJitInfo::PgoSource::Unknown; - if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasMethodProfile, hasLikelyClass, pgoSource)) + if (mc->hasPgoData(hasEdgeProfile, hasClassProfile, hasMethodProfile, hasLikelyClass, hasLikelyMethod, pgoSource)) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_PGO); @@ -56,6 +57,11 @@ int verbJitFlags::DoWork(const char* nameOfInput) rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); } + if (hasLikelyMethod) + { + rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_LIKELY_METHOD); + } + if (pgoSource == ICorJitInfo::PgoSource::Static) { rawFlags |= 1ULL << (EXTRA_JIT_FLAGS::HAS_STATIC_PROFILE); diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 97be9fbfc9fcac..df50466b476d31 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -5607,9 +5607,10 @@ void MethodContext::dmpGetPgoInstrumentationResults(DWORDLONG key, const Agnosti } break; case ICorJitInfo::PgoInstrumentationKind::GetLikelyClass: + case ICorJitInfo::PgoInstrumentationKind::GetLikelyMethod: { - // (N)umber, (L)ikelihood, (C)lass - printf("N %u L %u C %016llX", (unsigned)(pBuf[i].Other >> 8), (unsigned)(pBuf[i].Other && 0xFF), CastHandle(*(uintptr_t*)(pInstrumentationData + pBuf[i].Offset))); + // (N)umber, (L)ikelihood, (H)andle + printf("N %u L %u H %016llX", (unsigned)(pBuf[i].Other >> 8), (unsigned)(pBuf[i].Other && 0xFF), CastHandle(*(uintptr_t*)(pInstrumentationData + pBuf[i].Offset))); } break; default: @@ -7072,12 +7073,13 @@ int MethodContext::dumpMD5HashToBuffer(BYTE* pBuffer, int bufLen, char* hash, in return m_hash.HashBuffer(pBuffer, bufLen, hash, hashLen); } -bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasMethodProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource) +bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasMethodProfile, bool& hasLikelyClass, bool& hasLikelyMethod, ICorJitInfo::PgoSource& pgoSource) { hasEdgeProfile = false; hasClassProfile = false; hasMethodProfile = false; hasLikelyClass = false; + hasLikelyMethod = false; // Obtain the Method Info structure for this method CORINFO_METHOD_INFO info; @@ -7102,8 +7104,9 @@ bool MethodContext::hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool hasClassProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes); hasMethodProfile |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods); hasLikelyClass |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyClass); + hasLikelyMethod |= (schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::GetLikelyMethod); - if (hasEdgeProfile && hasClassProfile && hasLikelyClass) + if (hasEdgeProfile && hasClassProfile && hasLikelyClass && hasLikelyMethod) { break; } diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index 816682b1effe9e..5009d6b6a69c40 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -54,6 +54,7 @@ enum EXTRA_JIT_FLAGS HAS_STATIC_PROFILE = 59, HAS_DYNAMIC_PROFILE = 58, HAS_METHOD_PROFILE = 57, + HAS_LIKELY_METHOD = 56, }; // Asserts to catch changes in corjit flags definitions. @@ -64,6 +65,8 @@ static_assert((int)EXTRA_JIT_FLAGS::HAS_CLASS_PROFILE == (int)CORJIT_FLAGS::CorJ static_assert((int)EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS == (int)CORJIT_FLAGS::CorJitFlag::CORJIT_FLAG_UNUSED33, "Jit Flags Mismatch"); static_assert((int)EXTRA_JIT_FLAGS::HAS_STATIC_PROFILE == (int)CORJIT_FLAGS::CorJitFlag::CORJIT_FLAG_UNUSED32, "Jit Flags Mismatch"); static_assert((int)EXTRA_JIT_FLAGS::HAS_DYNAMIC_PROFILE == (int)CORJIT_FLAGS::CorJitFlag::CORJIT_FLAG_UNUSED31, "Jit Flags Mismatch"); +static_assert((int)EXTRA_JIT_FLAGS::HAS_METHOD_PROFILE == (int)CORJIT_FLAGS::CorJitFlag::CORJIT_FLAG_UNUSED30, "Jit Flags Mismatch"); +static_assert((int)EXTRA_JIT_FLAGS::HAS_LIKELY_METHOD == (int)CORJIT_FLAGS::CorJitFlag::CORJIT_FLAG_UNUSED29, "Jit Flags Mismatch"); class MethodContext { @@ -106,7 +109,7 @@ class MethodContext int dumpMethodIdentityInfoToBuffer(char* buff, int len, bool ignoreMethodName = false, CORINFO_METHOD_INFO* optInfo = nullptr, unsigned optFlags = 0); int dumpMethodMD5HashToBuffer(char* buff, int len, bool ignoreMethodName = false, CORINFO_METHOD_INFO* optInfo = nullptr, unsigned optFlags = 0); - bool hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasMethodProfile, bool& hasLikelyClass, ICorJitInfo::PgoSource& pgoSource); + bool hasPgoData(bool& hasEdgeProfile, bool& hasClassProfile, bool& hasMethodProfile, bool& hasLikelyClass, bool& hasLikelyMethod, ICorJitInfo::PgoSource& pgoSource); void recGlobalContext(const MethodContext& other); diff --git a/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp b/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp index 1d1d4d53b1a845..b51a54ba183851 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/spmidumphelper.cpp @@ -285,6 +285,7 @@ std::string SpmiDumpHelper::DumpJitFlags(unsigned long long flags) AddFlagNumeric(HAS_CLASS_PROFILE, EXTRA_JIT_FLAGS::HAS_CLASS_PROFILE); AddFlagNumeric(HAS_METHOD_PROFILE, EXTRA_JIT_FLAGS::HAS_METHOD_PROFILE); AddFlagNumeric(HAS_LIKELY_CLASS, EXTRA_JIT_FLAGS::HAS_LIKELY_CLASS); + AddFlagNumeric(HAS_LIKELY_METHOD, EXTRA_JIT_FLAGS::HAS_LIKELY_METHOD); AddFlagNumeric(HAS_STATIC_PROFILE, EXTRA_JIT_FLAGS::HAS_STATIC_PROFILE); AddFlagNumeric(HAS_DYNAMIC_PROFILE, EXTRA_JIT_FLAGS::HAS_DYNAMIC_PROFILE); From 1bfc4d8846df1c1ce91e7267a4622bc02c8ae0f6 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 15 Jun 2022 17:41:31 +0200 Subject: [PATCH 57/59] Update JIT-EE GUID --- src/coreclr/inc/jiteeversionguid.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 27f75a486cb53c..9a6cbc053e1ce6 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID; #define GUID_DEFINED #endif // !GUID_DEFINED -constexpr GUID JITEEVersionIdentifier = { /* af5b6632-6fbe-4a2e-82d6-24487a138e4a */ - 0xaf5b6632, - 0x6fbe, - 0x4a2e, - {0x82, 0xd6, 0x24, 0x48, 0x7a, 0x13, 0x8e, 0x4a} +constexpr GUID JITEEVersionIdentifier = { /* f2faa5fc-a1ec-4244-aebb-5597bfd7153a */ + 0xf2faa5fc, + 0xa1ec, + 0x4244, + {0xae, 0xbb, 0x55, 0x97, 0xbf, 0xd7, 0x15, 0x3a} }; ////////////////////////////////////////////////////////////////////////////////////////////////////////// From 55cdf81e1d064f381b43459325843f7de9c430a5 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 16 Jun 2022 10:23:12 +0200 Subject: [PATCH 58/59] Address some feedback, remove some unneeded stuff --- src/coreclr/jit/compiler.h | 5 ----- src/coreclr/jit/indirectcalltransformer.cpp | 1 - 2 files changed, 6 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 739dd65b57acd0..54abb9b8bd3e33 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6811,11 +6811,6 @@ class Compiler optMethodFlags |= OMF_HAS_GUARDEDDEVIRT; } - void clearMethodHasGuardedDevirtualization() - { - optMethodFlags &= ~OMF_HAS_GUARDEDDEVIRT; - } - void pickGDV(GenTreeCall* call, IL_OFFSET ilOffset, bool isInterface, diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 2385cc12fd5a7c..6d3f37d5936940 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -602,7 +602,6 @@ class IndirectCallTransformer if (reuseTarget) { m_targetLclNum = compiler->lvaGrabTemp(false DEBUGARG("guarded devirt call target temp")); - compiler->lvaGetDesc(m_targetLclNum)->lvSingleDef = 1; GenTree* asgTree = compiler->gtNewTempAssign(m_targetLclNum, tarTree); Statement* asgStmt = compiler->fgNewStmtFromTree(asgTree, stmt->GetDebugInfo()); From 55c9d5433d27d03022d41d3125bc79f5a2181309 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 16 Jun 2022 10:39:08 +0200 Subject: [PATCH 59/59] Address some more feedback --- src/coreclr/jit/block.h | 46 ++++++++++----------- src/coreclr/jit/fgprofile.cpp | 6 +-- src/coreclr/jit/indirectcalltransformer.cpp | 5 --- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index bfa4f856d73ad7..e9a539a2f35fa6 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -526,32 +526,32 @@ enum BasicBlockFlags : unsigned __int64 #endif // defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM) - BBF_BACKWARD_JUMP = MAKE_BBFLAG(24), // BB is surrounded by a backward jump/switch arc - BBF_RETLESS_CALL = MAKE_BBFLAG(25), // BBJ_CALLFINALLY that will never return (and therefore, won't need a paired - // BBJ_ALWAYS); see isBBCallAlwaysPair(). - BBF_LOOP_PREHEADER = MAKE_BBFLAG(26), // BB is a loop preheader block - BBF_COLD = MAKE_BBFLAG(27), // BB is cold - - BBF_PROF_WEIGHT = MAKE_BBFLAG(28), // BB weight is computed from profile data - BBF_IS_LIR = MAKE_BBFLAG(29), // Set if the basic block contains LIR (as opposed to HIR) - BBF_KEEP_BBJ_ALWAYS = MAKE_BBFLAG(30), // A special BBJ_ALWAYS block, used by EH code generation. Keep the jump kind - // as BBJ_ALWAYS. Used for the paired BBJ_ALWAYS block following the - // BBJ_CALLFINALLY block, as well as, on x86, the final step block out of a - // finally. - BBF_CLONED_FINALLY_BEGIN = MAKE_BBFLAG(31), // First block of a cloned finally region - - BBF_CLONED_FINALLY_END = MAKE_BBFLAG(32), // Last block of a cloned finally region - BBF_HAS_CALL = MAKE_BBFLAG(33), // BB contains a call + BBF_BACKWARD_JUMP = MAKE_BBFLAG(24), // BB is surrounded by a backward jump/switch arc + BBF_RETLESS_CALL = MAKE_BBFLAG(25), // BBJ_CALLFINALLY that will never return (and therefore, won't need a paired + // BBJ_ALWAYS); see isBBCallAlwaysPair(). + BBF_LOOP_PREHEADER = MAKE_BBFLAG(26), // BB is a loop preheader block + BBF_COLD = MAKE_BBFLAG(27), // BB is cold + + BBF_PROF_WEIGHT = MAKE_BBFLAG(28), // BB weight is computed from profile data + BBF_IS_LIR = MAKE_BBFLAG(29), // Set if the basic block contains LIR (as opposed to HIR) + BBF_KEEP_BBJ_ALWAYS = MAKE_BBFLAG(30), // A special BBJ_ALWAYS block, used by EH code generation. Keep the jump kind + // as BBJ_ALWAYS. Used for the paired BBJ_ALWAYS block following the + // BBJ_CALLFINALLY block, as well as, on x86, the final step block out of a + // finally. + BBF_CLONED_FINALLY_BEGIN = MAKE_BBFLAG(31), // First block of a cloned finally region + + BBF_CLONED_FINALLY_END = MAKE_BBFLAG(32), // Last block of a cloned finally region + BBF_HAS_CALL = MAKE_BBFLAG(33), // BB contains a call BBF_DOMINATED_BY_EXCEPTIONAL_ENTRY = MAKE_BBFLAG(34), // Block is dominated by exceptional entry. - BBF_BACKWARD_JUMP_TARGET = MAKE_BBFLAG(35), // Block is a target of a backward jump + BBF_BACKWARD_JUMP_TARGET = MAKE_BBFLAG(35), // Block is a target of a backward jump - BBF_PATCHPOINT = MAKE_BBFLAG(36), // Block is a patchpoint - BBF_HAS_HISTOGRAM_PROFILE = MAKE_BBFLAG(37), // BB contains a call needing a class profile - BBF_PARTIAL_COMPILATION_PATCHPOINT = MAKE_BBFLAG(38), // Block is a partial compilation patchpoint - BBF_HAS_ALIGN = MAKE_BBFLAG(39), // BB ends with 'align' instruction - BBF_TAILCALL_SUCCESSOR = MAKE_BBFLAG(40), // BB has pred that has potential tail call + BBF_PATCHPOINT = MAKE_BBFLAG(36), // Block is a patchpoint + BBF_HAS_HISTOGRAM_PROFILE = MAKE_BBFLAG(37), // BB contains a call needing a histogram profile + BBF_PARTIAL_COMPILATION_PATCHPOINT = MAKE_BBFLAG(38), // Block is a partial compilation patchpoint + BBF_HAS_ALIGN = MAKE_BBFLAG(39), // BB ends with 'align' instruction + BBF_TAILCALL_SUCCESSOR = MAKE_BBFLAG(40), // BB has pred that has potential tail call - BBF_BACKWARD_JUMP_SOURCE = MAKE_BBFLAG(41), // Block is a source of a backward jump + BBF_BACKWARD_JUMP_SOURCE = MAKE_BBFLAG(41), // Block is a source of a backward jump // The following are sets of flags. diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 3c00b547965c7d..4eb555a65ca94e 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1479,17 +1479,17 @@ class BuildHandleHistogramProbeSchemaGen if ((probeType == Compiler::GDVProbeType::ClassProfile) || (probeType == Compiler::GDVProbeType::MethodAndClassProfile)) { - CreateHistogramEntriesIfNecessary(compiler, call, true /* isTypeHistogram */); + CreateHistogramSchemaEntries(compiler, call, true /* isTypeHistogram */); } if ((probeType == Compiler::GDVProbeType::MethodProfile) || (probeType == Compiler::GDVProbeType::MethodAndClassProfile)) { - CreateHistogramEntriesIfNecessary(compiler, call, false /* isTypeHistogram */); + CreateHistogramSchemaEntries(compiler, call, false /* isTypeHistogram */); } } - void CreateHistogramEntriesIfNecessary(Compiler* compiler, GenTreeCall* call, bool isTypeHistogram) + void CreateHistogramSchemaEntries(Compiler* compiler, GenTreeCall* call, bool isTypeHistogram) { ICorJitInfo::PgoInstrumentationSchema schemaElem = {}; schemaElem.Count = 1; diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 6d3f37d5936940..e3d799f734b8ba 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -846,11 +846,6 @@ class IndirectCallTransformer } } - // Clear the inline candidate info (may be non-null since - // it's a union field used for other things by virtual - // stubs) - call->gtInlineCandidateInfo = nullptr; - context = MAKE_METHODCONTEXT(methodHnd); }