Fix deadlock when creating threads from ModuleInitializer in a shared library

src/coreclr/nativeaot/Bootstrap/main.cpp

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 #include <stdint.h>
+#include <stddef.h>
 #include <minipal/descriptorlimit.h>
 
 #if defined(DEBUG) && defined(_WIN32)
@@ -139,10 +140,16 @@ MANAGED_RUNTIME_EXPORT(ObjectiveCMarshalGetUnhandledExceptionPropagationHandler)
 
 typedef void (MANAGED_RUNTIME_EXPORT_CALLCONV *pfn)();
 
+#if defined(_WIN32)
+extern "C" int ThreadEntryPoint(void* pContext);
+#else
+extern "C" size_t ThreadEntryPoint(void* pContext);
+#endif
+
 static const pfn c_classlibFunctions[] = {
     &MANAGED_RUNTIME_EXPORT_NAME(GetRuntimeException),
     &MANAGED_RUNTIME_EXPORT_NAME(RuntimeFailFast),
-    nullptr, // &UnhandledExceptionHandler,
+    (pfn)&ThreadEntryPoint,
     &MANAGED_RUNTIME_EXPORT_NAME(AppendExceptionStackFrame),
     nullptr, // &CheckStaticClassConstruction,
     &MANAGED_RUNTIME_EXPORT_NAME(GetSystemArrayEEType),

src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs

@@ -38,7 +38,7 @@ internal enum ClassLibFunctionId
     {
         GetRuntimeException = 0,
         FailFast = 1,
-        // UnhandledExceptionHandler = 2, // unused
+        ThreadEntryPoint = 2,
         AppendExceptionStackFrame = 3,
         // unused = 4,
         GetSystemArrayEEType = 5,

src/coreclr/nativeaot/Runtime/ICodeManager.h

@@ -99,7 +99,7 @@ enum class ClasslibFunctionId
 {
     GetRuntimeException = 0,
     FailFast = 1,
-    UnhandledExceptionHandler = 2,
+    ThreadEntryPoint = 2,
     AppendExceptionStackFrame = 3,
     // unused = 4,
     GetSystemArrayEEType = 5,

src/coreclr/nativeaot/Runtime/thread.cpp

@@ -4,6 +4,7 @@
 #include "common.h"
 #include "gcenv.h"
 #include "gcheaputilities.h"
+#include "gchandleutilities.h"
 
 #include "CommonTypes.h"
 #include "CommonMacros.h"
@@ -1278,6 +1279,49 @@ FCIMPL0(Object**, RhGetThreadStaticStorage)
 }
 FCIMPLEND
 
+#ifdef TARGET_UNIX
+#define NEWTHREAD_RETURN_TYPE size_t
+#else
+#define NEWTHREAD_RETURN_TYPE uint32_t
+#endif
+
+static NEWTHREAD_RETURN_TYPE RhThreadEntryPoint(void* pContext)
+{
+    // We will attach the thread early so that when the managed thread entrypoint
+    // starts running and performs its reverse p/invoke transition, the thread is
+    // already attached.
+    //
+    // This avoids potential deadlocks with module initializers that may be running
+    // as part of runtime initialization (in non-EXE scenario) and creating new
+    // threads. When RhThreadEntryPoint runs, the runtime must already be initialized
+    // enough to be able to run managed code so we don't need to wait for it to
+    // finish.
+
+    ThreadStore::AttachCurrentThread();
+
+    Thread * pThread = ThreadStore::GetCurrentThread();
+    pThread->SetDeferredTransitionFrameForNativeHelperThread();
+    pThread->DisablePreemptiveMode();
+
+    Object * pThreadObject = ObjectFromHandle((OBJECTHANDLE)pContext);
+    MethodTable* pMT = pThreadObject->GetMethodTable();
+
+    pThread->EnablePreemptiveMode();
+
+    NEWTHREAD_RETURN_TYPE (*pFn)(void*) = (NEWTHREAD_RETURN_TYPE (*)(void*))
+        pMT->GetTypeManagerPtr()
+        ->AsTypeManager()
+        ->GetClasslibFunction(ClasslibFunctionId::ThreadEntryPoint);
+
+    return pFn(pContext);
+}
+
+FCIMPL0(void*, RhGetThreadEntryPointAddress)
+{
+    return (void*)&RhThreadEntryPoint;
+}
+FCIMPLEND
+
 InlinedThreadStaticRoot* Thread::GetInlinedThreadStaticList()
 {
     return m_pInlinedThreadLocalStatics;

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs

@@ -26,6 +26,14 @@ internal static partial class RuntimeImports
     {
         internal const string RuntimeLibrary = "*";
 
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        [RuntimeImport(RuntimeLibrary, "RhGetThreadEntryPointAddress")]
+#if TARGET_UNIX
+        internal static extern unsafe delegate* unmanaged<nint, nint> RhGetThreadEntryPointAddress();
+#else
+        internal static extern unsafe delegate* unmanaged<nint, uint> RhGetThreadEntryPointAddress();
+#endif
+
         [MethodImplAttribute(MethodImplOptions.InternalCall)]
         [RuntimeImport(RuntimeLibrary, "RhGetCrashInfoBuffer")]
         internal static extern unsafe byte* RhGetCrashInfoBuffer(out int cbMaxSize);

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Thread.NativeAot.Unix.cs

@@ -101,7 +101,7 @@ private unsafe bool CreateThread(GCHandle<Thread> thisThreadHandle)
                 stackSize = RuntimeImports.RhGetDefaultStackSize();
             }
 
-            if (!Interop.Sys.CreateThread(stackSize, &ThreadEntryPoint, GCHandle<Thread>.ToIntPtr(thisThreadHandle)))
+            if (!Interop.Sys.CreateThread(stackSize, RuntimeImports.RhGetThreadEntryPointAddress(), GCHandle<Thread>.ToIntPtr(thisThreadHandle)))
             {
                 return false;
             }
@@ -115,7 +115,7 @@ private unsafe bool CreateThread(GCHandle<Thread> thisThreadHandle)
         /// <summary>
         /// This is an entry point for managed threads created by application
         /// </summary>
-        [UnmanagedCallersOnly]
+        [UnmanagedCallersOnly(EntryPoint = "ThreadEntryPoint")]
         private static IntPtr ThreadEntryPoint(IntPtr parameter)
         {
             StartThread(parameter);

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Thread.NativeAot.Windows.cs

@@ -200,7 +200,7 @@ private unsafe bool CreateThread(GCHandle<Thread> thisThreadHandle)
             }
 
             _osHandle = Interop.Kernel32.CreateThread(IntPtr.Zero, (IntPtr)stackSize,
-                &ThreadEntryPoint, GCHandle<Thread>.ToIntPtr(thisThreadHandle),
+                RuntimeImports.RhGetThreadEntryPointAddress(), GCHandle<Thread>.ToIntPtr(thisThreadHandle),
                 Interop.Kernel32.CREATE_SUSPENDED | Interop.Kernel32.STACK_SIZE_PARAM_IS_A_RESERVATION,
                 out _);
 
@@ -219,7 +219,7 @@ private unsafe bool CreateThread(GCHandle<Thread> thisThreadHandle)
         /// <summary>
         /// This is an entry point for managed threads created by application
         /// </summary>
-        [UnmanagedCallersOnly]
+        [UnmanagedCallersOnly(EntryPoint = "ThreadEntryPoint")]
         private static uint ThreadEntryPoint(IntPtr parameter)
         {
             StartThread(parameter);

src/tests/nativeaot/SmokeTests/SharedLibrary/SharedLibrary.cs

@@ -4,15 +4,29 @@
 using System;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Threading;
 
 namespace SharedLibrary
 {
     public class ClassLibrary
     {
+        static Thread s_setterThread;
+        static int s_primitiveInt;
+
+        [ModuleInitializer]
+        public static void CreateThreadInModuleInitializer()
+        {
+            // Regression test for https://github.com/dotnet/runtime/issues/107699
+            // where creating threads in module initializer would lead to a deadlock.
+            s_setterThread = new Thread(() => { s_primitiveInt = 10; });
+            s_setterThread.Start();
+        }
+
         [UnmanagedCallersOnly(EntryPoint = "ReturnsPrimitiveInt", CallConvs = new Type[] { typeof(CallConvStdcall) })]
         public static int ReturnsPrimitiveInt()
         {
-            return 10;
+            s_setterThread.Join();
+            return s_primitiveInt;
         }
 
         [UnmanagedCallersOnly(EntryPoint = "ReturnsPrimitiveBool", CallConvs = new Type[] { typeof(CallConvStdcall) })]