Support fast tailcalls in R2R (#56669)

* Support fast tailcalls in R2R

Partially addresses #5857

* Support ARM64

* Run jit-format

* Fix non-R2R ARM build

* Fix recursive-call-to-loop optimization with non-standard args

* Implement new delay load helper for fast tailcalls

* Minor changes and fix build break

* Switch to a define for tailcall register

* Fix x86

* Implement DefType.IsUnsafeValueType

* Emit rex.jmp for tailcall jumps on x64

* Refactor non standard args + refix recursive tailcall opt

* Set nonStandardArgKind for stack args also

* Regenerate JIT interface

* Improve recursion-to-loop decision about which args need to be reassigned

* Use INS_tail_i_jmp for func token indir

* More efficient arm64 VSD fast tailcalls, fix some bad merging

Also fix logic since we removed tailcall register. For ARM64 that means
we now use the single temp reg that is available for R2R/VSD calls, and
in LSRA we ensure this goes into a volatile register.

* Take R2R indirection into account for tail call profitability

On x64 an R2R indirected direct call normally disassembles the call site
to determine the indirection cell, so it is more expensive to do tail
calls in this scenario as the indirection cell needs to be passed in a
register. Take this into account: if there is no tail. prefix, do normal
calls, and otherwise use tail calls.

* Disallow tailcalls via JIT helper in R2R builds

* Revert "Take R2R indirection into account for tail call profitability"

This reverts commit 863ad4d.

Let's not divert on the behavior here. It is not clear that having a
smaller call site is better than tail calling albeit with a larger call
site.

* Add SPMI support, clean up mcPackets enum

* Take necessary conditions into account in canTailCall

Do not do implicit tailcalls when
* The caller is the entry point
* The caller is marked NoInline
* The callee requires security object

Author: jakobbotsch

src/coreclr/ToolBox/superpmi/superpmi-shared/lwmlist.h

@@ -128,6 +128,7 @@ LWM(GetSharedCCtorHelper, DWORDLONG, DWORD)
 LWM(GetStringConfigValue, DWORD, DWORD)
 LWM(GetSystemVAmd64PassStructInRegisterDescriptor, DWORDLONG, Agnostic_GetSystemVAmd64PassStructInRegisterDescriptor)
 LWM(GetTailCallHelpers, Agnostic_GetTailCallHelpers, Agnostic_CORINFO_TAILCALL_HELPERS)
+LWM(UpdateEntryPointForTailCall, Agnostic_CORINFO_CONST_LOOKUP, Agnostic_CORINFO_CONST_LOOKUP)
 LWM(GetThreadTLSIndex, DWORD, DLD)
 LWM(GetTokenTypeAsHandle, GetTokenTypeAsHandleValue, DWORDLONG)
 LWM(GetTypeForBox, DWORDLONG, DWORDLONG)

src/coreclr/ToolBox/superpmi/superpmi-shared/methodcontext.cpp

@@ -6608,6 +6608,41 @@ bool MethodContext::repGetTailCallHelpers(
     return true;
 }
 
+void MethodContext::recUpdateEntryPointForTailCall(
+    const CORINFO_CONST_LOOKUP& origEntryPoint,
+    const CORINFO_CONST_LOOKUP& newEntryPoint)
+{
+    if (UpdateEntryPointForTailCall == nullptr)
+        UpdateEntryPointForTailCall = new LightWeightMap<Agnostic_CORINFO_CONST_LOOKUP, Agnostic_CORINFO_CONST_LOOKUP>();
+
+    Agnostic_CORINFO_CONST_LOOKUP key = SpmiRecordsHelper::StoreAgnostic_CORINFO_CONST_LOOKUP(&origEntryPoint);
+    Agnostic_CORINFO_CONST_LOOKUP value = SpmiRecordsHelper::StoreAgnostic_CORINFO_CONST_LOOKUP(&newEntryPoint);
+    UpdateEntryPointForTailCall->Add(key, value);
+    DEBUG_REC(dmpUpdateEntryPointForTailCall(key, value));
+}
+
+void MethodContext::dmpUpdateEntryPointForTailCall(
+    const Agnostic_CORINFO_CONST_LOOKUP& origEntryPoint,
+    const Agnostic_CORINFO_CONST_LOOKUP& newEntryPoint)
+{
+    printf("UpdateEntryPointForTailcall orig=%s new=%s",
+        SpmiDumpHelper::DumpAgnostic_CORINFO_CONST_LOOKUP(origEntryPoint).c_str(),
+        SpmiDumpHelper::DumpAgnostic_CORINFO_CONST_LOOKUP(newEntryPoint).c_str());
+}
+
+void MethodContext::repUpdateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint)
+{
+    AssertMapExistsNoMessage(UpdateEntryPointForTailCall);
+
+    Agnostic_CORINFO_CONST_LOOKUP key = SpmiRecordsHelper::StoreAgnostic_CORINFO_CONST_LOOKUP(entryPoint);
+    AssertKeyExistsNoMessage(UpdateEntryPointForTailCall, key);
+
+    Agnostic_CORINFO_CONST_LOOKUP value = UpdateEntryPointForTailCall->Get(key);
+    DEBUG_REP(dmpUpdateEntryPointForTailCall(key, value));
+
+    *entryPoint = SpmiRecordsHelper::RestoreCORINFO_CONST_LOOKUP(value);
+}
+
 void MethodContext::recGetMethodDefFromMethod(CORINFO_METHOD_HANDLE hMethod, mdMethodDef result)
 {
     if (GetMethodDefFromMethod == nullptr)

src/coreclr/ToolBox/superpmi/superpmi-shared/methodcontext.h

@@ -97,9 +97,9 @@ class SpmiRecordsHelper
     static CORINFO_LOOKUP_KIND RestoreCORINFO_LOOKUP_KIND(Agnostic_CORINFO_LOOKUP_KIND& lookupKind);
 
     static Agnostic_CORINFO_CONST_LOOKUP StoreAgnostic_CORINFO_CONST_LOOKUP(
-        CORINFO_CONST_LOOKUP* pLookup);
+        const CORINFO_CONST_LOOKUP* pLookup);
 
-    static CORINFO_CONST_LOOKUP RestoreCORINFO_CONST_LOOKUP(Agnostic_CORINFO_CONST_LOOKUP& lookup);
+    static CORINFO_CONST_LOOKUP RestoreCORINFO_CONST_LOOKUP(const Agnostic_CORINFO_CONST_LOOKUP& lookup);
 
     static Agnostic_CORINFO_RUNTIME_LOOKUP StoreAgnostic_CORINFO_RUNTIME_LOOKUP(
         CORINFO_RUNTIME_LOOKUP* pLookup);
@@ -459,7 +459,7 @@ inline CORINFO_LOOKUP_KIND SpmiRecordsHelper::RestoreCORINFO_LOOKUP_KIND(
 }
 
 inline Agnostic_CORINFO_CONST_LOOKUP SpmiRecordsHelper::StoreAgnostic_CORINFO_CONST_LOOKUP(
-    CORINFO_CONST_LOOKUP* pLookup)
+    const CORINFO_CONST_LOOKUP* pLookup)
 {
     Agnostic_CORINFO_CONST_LOOKUP constLookup;
     ZeroMemory(&constLookup, sizeof(constLookup));
@@ -469,7 +469,7 @@ inline Agnostic_CORINFO_CONST_LOOKUP SpmiRecordsHelper::StoreAgnostic_CORINFO_CO
 }
 
 inline CORINFO_CONST_LOOKUP SpmiRecordsHelper::RestoreCORINFO_CONST_LOOKUP(
-    Agnostic_CORINFO_CONST_LOOKUP& lookup)
+    const Agnostic_CORINFO_CONST_LOOKUP& lookup)
 {
     CORINFO_CONST_LOOKUP constLookup;
     constLookup.accessType = (InfoAccessType)lookup.accessType;

src/coreclr/ToolBox/superpmi/superpmi-shared/spmirecordhelper.h

@@ -1776,6 +1776,14 @@ bool interceptor_ICJI::getTailCallHelpers(
     return result;
 }
 
+void interceptor_ICJI::updateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint)
+{
+    mc->cr->AddCall("updateEntryPointForTailCall");
+    CORINFO_CONST_LOOKUP origEntryPoint = *entryPoint;
+    original_ICorJitInfo->updateEntryPointForTailCall(entryPoint);
+    mc->recUpdateEntryPointForTailCall(origEntryPoint, *entryPoint);
+}
+
 // Stuff directly on ICorJitInfo
 
 // Returns extended flags for a particular compilation instance.

src/coreclr/ToolBox/superpmi/superpmi-shim-collector/icorjitinfo.cpp

@@ -1250,6 +1250,13 @@ bool interceptor_ICJI::notifyInstructionSetUsage(
     return original_ICorJitInfo->notifyInstructionSetUsage(instructionSet, supportEnabled);
 }
 
+void interceptor_ICJI::updateEntryPointForTailCall(
+          CORINFO_CONST_LOOKUP* entryPoint)
+{
+    mcs->AddCall("updateEntryPointForTailCall");
+    original_ICorJitInfo->updateEntryPointForTailCall(entryPoint);
+}
+
 void interceptor_ICJI::allocMem(
           AllocMemArgs* pArgs)
 {

src/coreclr/ToolBox/superpmi/superpmi-shim-counter/icorjitinfo.cpp

@@ -1094,6 +1094,12 @@ bool interceptor_ICJI::notifyInstructionSetUsage(
     return original_ICorJitInfo->notifyInstructionSetUsage(instructionSet, supportEnabled);
 }
 
+void interceptor_ICJI::updateEntryPointForTailCall(
+          CORINFO_CONST_LOOKUP* entryPoint)
+{
+    original_ICorJitInfo->updateEntryPointForTailCall(entryPoint);
+}
+
 void interceptor_ICJI::allocMem(
           AllocMemArgs* pArgs)
 {

src/coreclr/ToolBox/superpmi/superpmi-shim-simple/icorjitinfo.cpp

@@ -1556,6 +1556,12 @@ bool MyICJI::notifyInstructionSetUsage(CORINFO_InstructionSet instructionSet, bo
     return supported;
 }
 
+void MyICJI::updateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint)
+{
+    jitInstance->mc->cr->AddCall("updateEntryPointForTailCall");
+    jitInstance->mc->repUpdateEntryPointForTailCall(entryPoint);
+}
+
 // Stuff directly on ICorJitInfo
 
 // Returns extended flags for a particular compilation instance.

src/coreclr/ToolBox/superpmi/superpmi/icorjitinfo.cpp

@@ -3164,6 +3164,13 @@ class ICorDynamicInfo : public ICorStaticInfo
                 CORINFO_InstructionSet instructionSet,
                 bool supportEnabled
             ) = 0;
+
+    // Notify EE that JIT needs an entry-point that is tail-callable.
+    // This is used for AOT on x64 to support delay loaded fast tailcalls.
+    // Normally the indirection cell is retrieved from the return address,
+    // but for tailcalls, the contract is that JIT leaves the indirection cell in
+    // a register during tailcall.
+    virtual void updateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint) = 0;
 };
 
 /**********************************************************************************/

src/coreclr/inc/corinfo.h

@@ -635,6 +635,9 @@ bool notifyInstructionSetUsage(
           CORINFO_InstructionSet instructionSet,
           bool supportEnabled) override;
 
+void updateEntryPointForTailCall(
+          CORINFO_CONST_LOOKUP* entryPoint) override;
+
 void allocMem(
           AllocMemArgs* pArgs) override;