Add reference assembly for System.Private.CoreLib.dll (#75311)

We now compile against the reference assembly in all places where we were compiling against the mono/coreclr System.Private.CoreLib.dll implementation assembly before.

The new reference assembly consumes sources from the existing contracts to avoid checking in a generated version of SPC.dll (this would add ~20k lines of .cs which is mostly duplicated with System.Runtime.cs)

Since a few contracts have only partially moved types to SPC we wrap contract types with `#if !BUILDING_CORELIB_REFERENCE` so we can hide them when compiling the SPC reference assembly.

Author: akoeplinger

Directory.Build.props

@@ -237,8 +237,10 @@
 
   <PropertyGroup>
     <CoreLibSharedDir>$([MSBuild]::NormalizeDirectory('$(LibrariesProjectRoot)', 'System.Private.CoreLib', 'src'))</CoreLibSharedDir>
+    <CoreLibRefDir>$([MSBuild]::NormalizeDirectory('$(LibrariesProjectRoot)', 'System.Private.CoreLib', 'ref'))</CoreLibRefDir>
     <CoreLibProject Condition="'$(RuntimeFlavor)' == 'CoreCLR'">$([MSBuild]::NormalizePath('$(CoreClrProjectRoot)', 'System.Private.CoreLib', 'System.Private.CoreLib.csproj'))</CoreLibProject>
     <CoreLibProject Condition="'$(RuntimeFlavor)' == 'Mono'">$([MSBuild]::NormalizePath('$(MonoProjectRoot)', 'System.Private.CoreLib', 'System.Private.CoreLib.csproj'))</CoreLibProject>
+    <UriProject>$([MSBuild]::NormalizePath('$(LibrariesProjectRoot)', 'System.Private.Uri', 'src', 'System.Private.Uri.csproj'))</UriProject>
 
     <!-- this property is used by the SDK to pull in mono-based runtime packs -->
     <UseMonoRuntime Condition="'$(UseMonoRuntime)' == '' and '$(RuntimeFlavor)' == 'Mono'">true</UseMonoRuntime>

docs/coding-guidelines/updating-ref-source.md

@@ -13,7 +13,7 @@ This document provides the steps you need to take to update the reference assemb
 
 These steps can also be applied to some unique assemblies which depend on changes in System.Private.Corelib. (partial facades like System.Memory, for example).
 
-1) Run `dotnet build --no-incremental /t:GenerateReferenceSource` from the System.Runtime/src directory.
+1) Run `dotnet build --no-incremental /t:GenerateReferenceAssemblySource` from the System.Runtime/src directory.
 2) Filter out all unrelated changes and extract the changes you care about (ignore certain attributes being removed). Generally, this step is not required for other reference assemblies.
 
 ## For Full Facade Assemblies implementation assemblies

eng/references.targets

@@ -59,4 +59,15 @@
     <Error Condition="'%(ReferencePath.ReferenceSourceTarget)' == 'ProjectReference' AND '%(ReferencePath.IsReferenceAssembly)' != 'true' AND '%(ReferencePath.ReferenceAssembly)' == ''"
            Text="Reference assemblies must only reference other reference assemblies and '%(ReferencePath.ProjectReferenceOriginalItemSpec)' is not a reference assembly project and does not set 'ProduceReferenceAssembly'." />
   </Target>
+
+  <Target Name="ReplaceCoreLibSrcWithRefAssemblyForCompilation"
+          AfterTargets="FindReferenceAssembliesForReferences"
+          Condition="'$(CompileUsingReferenceAssemblies)' != 'true' and '@(_coreLibProjectReference)' != ''">
+    <ItemGroup>
+      <_resolvedCoreLibProjectReference Include="@(_ResolvedProjectReferencePaths->WithMetadataValue('MSBuildSourceProjectFile','$(CoreLibProject)'))" />
+      <ReferencePathWithRefAssemblies Remove="@(_resolvedCoreLibProjectReference)" />
+      <ReferencePathWithRefAssemblies Include="@(_resolvedCoreLibProjectReference->Metadata('ReferenceAssembly'))" />
+    </ItemGroup>
+  </Target>
+
 </Project>

src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <EnableDefaultItems>false</EnableDefaultItems>
@@ -355,4 +355,10 @@
       <FileWrites Include="@(EventingSourceFile)" />
     </ItemGroup>
   </Target>
+
+  <!-- Import refererence assembly logic -->
+  <PropertyGroup>
+    <IsSourceProject>true</IsSourceProject>
+  </PropertyGroup>
+  <Import Project="$(RepositoryEngineeringDir)resolveContract.targets" />
 </Project>

src/libraries/System.Collections.Concurrent/ref/System.Collections.Concurrent.cs

@@ -6,6 +6,7 @@
 
 namespace System.Collections.Concurrent
 {
+#if !BUILDING_CORELIB_REFERENCE
     [System.Runtime.Versioning.UnsupportedOSPlatform("browser")]
     public partial class BlockingCollection<T> : System.Collections.Generic.IEnumerable<T>, System.Collections.Generic.IReadOnlyCollection<T>, System.Collections.ICollection, System.Collections.IEnumerable, System.IDisposable
     {
@@ -126,6 +127,7 @@ void System.Collections.IDictionary.Remove(object key) { }
         public bool TryRemove(TKey key, [System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute(false)] out TValue value) { throw null; }
         public bool TryUpdate(TKey key, TValue newValue, TValue comparisonValue) { throw null; }
     }
+#endif
     public partial class ConcurrentQueue<T> : System.Collections.Concurrent.IProducerConsumerCollection<T>, System.Collections.Generic.IEnumerable<T>, System.Collections.Generic.IReadOnlyCollection<T>, System.Collections.ICollection, System.Collections.IEnumerable
     {
         public ConcurrentQueue() { }
@@ -146,6 +148,7 @@ void System.Collections.ICollection.CopyTo(System.Array array, int index) { }
         public bool TryDequeue([System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute(false)] out T result) { throw null; }
         public bool TryPeek([System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute(false)] out T result) { throw null; }
     }
+#if !BUILDING_CORELIB_REFERENCE
     public partial class ConcurrentStack<T> : System.Collections.Concurrent.IProducerConsumerCollection<T>, System.Collections.Generic.IEnumerable<T>, System.Collections.Generic.IReadOnlyCollection<T>, System.Collections.ICollection, System.Collections.IEnumerable
     {
         public ConcurrentStack() { }
@@ -176,13 +179,15 @@ public enum EnumerablePartitionerOptions
         None = 0,
         NoBuffering = 1,
     }
+#endif
     public partial interface IProducerConsumerCollection<T> : System.Collections.Generic.IEnumerable<T>, System.Collections.ICollection, System.Collections.IEnumerable
     {
         void CopyTo(T[] array, int index);
         T[] ToArray();
         bool TryAdd(T item);
         bool TryTake([System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute(false)] out T item);
     }
+#if !BUILDING_CORELIB_REFERENCE
     public abstract partial class OrderablePartitioner<TSource> : System.Collections.Concurrent.Partitioner<TSource>
     {
         protected OrderablePartitioner(bool keysOrderedInEachPartition, bool keysOrderedAcrossPartitions, bool keysNormalized) { }
@@ -212,4 +217,5 @@ protected Partitioner() { }
         public virtual System.Collections.Generic.IEnumerable<TSource> GetDynamicPartitions() { throw null; }
         public abstract System.Collections.Generic.IList<System.Collections.Generic.IEnumerator<TSource>> GetPartitions(int partitionCount);
     }
+#endif
 }

src/libraries/System.Collections/ref/System.Collections.cs

@@ -6,6 +6,7 @@
 
 namespace System.Collections
 {
+#if !BUILDING_CORELIB_REFERENCE
     public sealed partial class BitArray : System.Collections.ICollection, System.Collections.IEnumerable, System.ICloneable
     {
         public BitArray(bool[] values) { }
@@ -38,16 +39,20 @@ public static partial class StructuralComparisons
         public static System.Collections.IComparer StructuralComparer { get { throw null; } }
         public static System.Collections.IEqualityComparer StructuralEqualityComparer { get { throw null; } }
     }
+#endif
 }
+
 namespace System.Collections.Generic
 {
+#if !BUILDING_CORELIB_REFERENCE
     public static partial class CollectionExtensions
     {
         public static TValue? GetValueOrDefault<TKey, TValue>(this System.Collections.Generic.IReadOnlyDictionary<TKey, TValue> dictionary, TKey key) { throw null; }
         public static TValue GetValueOrDefault<TKey, TValue>(this System.Collections.Generic.IReadOnlyDictionary<TKey, TValue> dictionary, TKey key, TValue defaultValue) { throw null; }
         public static bool Remove<TKey, TValue>(this System.Collections.Generic.IDictionary<TKey, TValue> dictionary, TKey key, [System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute(false)] out TValue value) { throw null; }
         public static bool TryAdd<TKey, TValue>(this System.Collections.Generic.IDictionary<TKey, TValue> dictionary, TKey key, TValue value) { throw null; }
     }
+#endif
     public abstract partial class Comparer<T> : System.Collections.Generic.IComparer<T>, System.Collections.IComparer
     {
         protected Comparer() { }
@@ -240,6 +245,7 @@ public void Dispose() { }
             void System.Collections.IEnumerator.Reset() { }
         }
     }
+#if !BUILDING_CORELIB_REFERENCE
     public sealed partial class LinkedListNode<T>
     {
         public LinkedListNode(T value) { }
@@ -298,6 +304,7 @@ void System.Runtime.Serialization.IDeserializationCallback.OnDeserialization(obj
             void System.Runtime.Serialization.ISerializable.GetObjectData(System.Runtime.Serialization.SerializationInfo info, System.Runtime.Serialization.StreamingContext context) { }
         }
     }
+#endif
     public partial class List<T> : System.Collections.Generic.ICollection<T>, System.Collections.Generic.IEnumerable<T>, System.Collections.Generic.IList<T>, System.Collections.Generic.IReadOnlyCollection<T>, System.Collections.Generic.IReadOnlyList<T>, System.Collections.ICollection, System.Collections.IEnumerable, System.Collections.IList
     {
         public List() { }
@@ -379,7 +386,7 @@ public void Dispose() { }
             void System.Collections.IEnumerator.Reset() { }
         }
     }
-
+#if !BUILDING_CORELIB_REFERENCE
     public partial class PriorityQueue<TElement, TPriority>
     {
         public PriorityQueue() { }
@@ -423,7 +430,7 @@ void System.Collections.IEnumerator.Reset() { }
             }
         }
     }
-
+#endif
     public partial class Queue<T> : System.Collections.Generic.IEnumerable<T>, System.Collections.Generic.IReadOnlyCollection<T>, System.Collections.ICollection, System.Collections.IEnumerable
     {
         public Queue() { }
@@ -466,6 +473,7 @@ private ReferenceEqualityComparer() { }
         public new bool Equals(object? x, object? y) { throw null; }
         public int GetHashCode(object? obj) { throw null; }
     }
+#if !BUILDING_CORELIB_REFERENCE
     public partial class SortedDictionary<TKey, TValue> : System.Collections.Generic.ICollection<System.Collections.Generic.KeyValuePair<TKey, TValue>>, System.Collections.Generic.IDictionary<TKey, TValue>, System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<TKey, TValue>>, System.Collections.Generic.IReadOnlyCollection<System.Collections.Generic.KeyValuePair<TKey, TValue>>, System.Collections.Generic.IReadOnlyDictionary<TKey, TValue>, System.Collections.ICollection, System.Collections.IDictionary, System.Collections.IEnumerable where TKey : notnull
     {
         public SortedDictionary() { }
@@ -718,4 +726,5 @@ public void Dispose() { }
             void System.Collections.IEnumerator.Reset() { }
         }
     }
+#endif
 }

src/libraries/System.Diagnostics.Debug/tests/System.Diagnostics.Debug.Tests.csproj

@@ -1,17 +1,18 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <RootNamespace>System.Diagnostics.Tests</RootNamespace>
-    <IgnoreArchitectureMismatches>true</IgnoreArchitectureMismatches>
-    <ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch>None</ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch>
     <TargetFrameworks>$(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent)-Unix</TargetFrameworks>
     <TestRuntime>true</TestRuntime>
+    <!-- Some tests need types like System.Diagnostics.DebugProvider which are only exposed from System.Private.CoreLib -->
+    <CompileUsingReferenceAssemblies>false</CompileUsingReferenceAssemblies>
   </PropertyGroup>
   <ItemGroup>
     <DefaultReferenceExclusion Include="System.Diagnostics.Debug" />
     <DefaultReferenceExclusion Include="System.Runtime.Extensions" />
     <ProjectReference Include="$(CoreLibProject)" />
     <ProjectReference Include="$(LibrariesProjectRoot)System.Runtime\src\System.Runtime.csproj" />
-    <ProjectReference Include="$(LibrariesProjectRoot)System.Threading\src\System.Threading.csproj" SkipUseReferenceAssembly="true" />
+    <ProjectReference Include="$(LibrariesProjectRoot)System.Threading\src\System.Threading.csproj" />
+    <ProjectReference Include="$(LibrariesProjectRoot)System.Collections\src\System.Collections.csproj" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="DebugTests.cs" />

src/libraries/System.Diagnostics.StackTrace/ref/System.Diagnostics.StackTrace.cs

@@ -52,6 +52,7 @@ public StackTrace(int skipFrames, bool fNeedFileInfo) { }
 }
 namespace System.Diagnostics.SymbolStore
 {
+#if !BUILDING_CORELIB_REFERENCE
     public partial interface ISymbolBinder
     {
         [System.ObsoleteAttribute("ISymbolBinder.GetReader has been deprecated because it is not 64-bit compatible. Use ISymbolBinder1.GetReader instead. ISymbolBinder1.GetReader accepts the importer interface pointer as an IntPtr instead of an Int32, and thus works on both 32-bit and 64-bit architectures.")]
@@ -74,11 +75,13 @@ public partial interface ISymbolDocument
         byte[] GetCheckSum();
         byte[] GetSourceRange(int startLine, int startColumn, int endLine, int endColumn);
     }
+#endif
     public partial interface ISymbolDocumentWriter
     {
         void SetCheckSum(System.Guid algorithmId, byte[] checkSum);
         void SetSource(byte[] source);
     }
+#if !BUILDING_CORELIB_REFERENCE
     public partial interface ISymbolMethod
     {
         System.Diagnostics.SymbolStore.ISymbolScope RootScope { get; }
@@ -205,4 +208,5 @@ public partial class SymLanguageVendor
         public static readonly System.Guid Microsoft;
         public SymLanguageVendor() { }
     }
+#endif
 }

src/libraries/System.Memory/ref/System.Memory.cs

@@ -165,6 +165,7 @@ public ref struct TryWriteInterpolatedStringHandler
             public bool AppendFormatted(string? value, int alignment = 0, string? format = null) { throw null; }
         }
     }
+#if !BUILDING_CORELIB_REFERENCE
     public readonly partial struct SequencePosition : System.IEquatable<System.SequencePosition>
     {
         private readonly object _dummy;
@@ -180,9 +181,11 @@ public ref struct TryWriteInterpolatedStringHandler
         [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)]
         public object? GetObject() { throw null; }
     }
+#endif
 }
 namespace System.Buffers
 {
+#if !BUILDING_CORELIB_REFERENCE
     public sealed partial class ArrayBufferWriter<T> : System.Buffers.IBufferWriter<T>
     {
         public ArrayBufferWriter() { }
@@ -315,6 +318,7 @@ public void Rewind(long count) { }
         public bool TryReadToAny(out System.Buffers.ReadOnlySequence<T> sequence, System.ReadOnlySpan<T> delimiters, bool advancePastDelimiter = true) { throw null; }
         public bool TryReadToAny(out System.ReadOnlySpan<T> span, System.ReadOnlySpan<T> delimiters, bool advancePastDelimiter = true) { throw null; }
     }
+#endif
     public readonly partial struct StandardFormat : System.IEquatable<System.Buffers.StandardFormat>
     {
         private readonly int _dummyPrimitive;
@@ -541,16 +545,19 @@ public static partial class MemoryMarshal
         public static bool TryWrite<T>(System.Span<byte> destination, ref T value) where T : struct { throw null; }
         public static void Write<T>(System.Span<byte> destination, ref T value) where T : struct { }
     }
+#if !BUILDING_CORELIB_REFERENCE
     public static partial class SequenceMarshal
     {
         public static bool TryGetArray<T>(System.Buffers.ReadOnlySequence<T> sequence, out System.ArraySegment<T> segment) { throw null; }
         public static bool TryGetReadOnlyMemory<T>(System.Buffers.ReadOnlySequence<T> sequence, out System.ReadOnlyMemory<T> memory) { throw null; }
         public static bool TryGetReadOnlySequenceSegment<T>(System.Buffers.ReadOnlySequence<T> sequence, [System.Diagnostics.CodeAnalysis.NotNullWhenAttribute(true)] out System.Buffers.ReadOnlySequenceSegment<T>? startSegment, out int startIndex, [System.Diagnostics.CodeAnalysis.NotNullWhenAttribute(true)] out System.Buffers.ReadOnlySequenceSegment<T>? endSegment, out int endIndex) { throw null; }
         public static bool TryRead<T>(ref System.Buffers.SequenceReader<byte> reader, out T value) where T : unmanaged { throw null; }
     }
+#endif
 }
 namespace System.Text
 {
+#if !BUILDING_CORELIB_REFERENCE
     public static partial class EncodingExtensions
     {
         public static void Convert(this System.Text.Decoder decoder, in System.Buffers.ReadOnlySequence<byte> bytes, System.Buffers.IBufferWriter<char> writer, bool flush, out long charsUsed, out bool completed) { throw null; }
@@ -566,6 +573,7 @@ public static partial class EncodingExtensions
         public static long GetChars(this System.Text.Encoding encoding, System.ReadOnlySpan<byte> bytes, System.Buffers.IBufferWriter<char> writer) { throw null; }
         public static string GetString(this System.Text.Encoding encoding, in System.Buffers.ReadOnlySequence<byte> bytes) { throw null; }
     }
+#endif
     public ref partial struct SpanLineEnumerator
     {
         private object _dummy;

src/libraries/System.Private.CoreLib/ref/System.Private.CoreLib.ExtraApis.cs

@@ -0,0 +1,41 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// NOTE: Types/members which are not publicly exposed in System.Runtime.dll but still used internally by libraries.
+//       Manually maintained, keep in sync with System.Private.CoreLib.ExtraApis.txt
+
+namespace System.Runtime.Serialization
+{
+    public readonly partial struct DeserializationToken : System.IDisposable
+    {
+#pragma warning disable CS0414
+        private readonly object _dummy = null;
+        private readonly int _dummyPrimitive = 0;
+#pragma warning restore CS0414
+        internal DeserializationToken(object tracker) { }
+        public void Dispose() { }
+    }
+    public sealed partial class SerializationInfo
+    {
+        public static System.Runtime.Serialization.DeserializationToken StartDeserialization() { throw null; }
+    }
+}
+namespace System.Diagnostics
+{
+    public partial class DebugProvider
+    {
+        public DebugProvider() { }
+        [System.Diagnostics.CodeAnalysis.DoesNotReturnAttribute]
+        public virtual void Fail(string? message, string? detailMessage) { throw null; }
+        public static void FailCore(string stackTrace, string? message, string? detailMessage, string errorSource) { }
+        public virtual void OnIndentLevelChanged(int indentLevel) { }
+        public virtual void OnIndentSizeChanged(int indentSize) { }
+        public virtual void Write(string? message) { }
+        public static void WriteCore(string message) { }
+        public virtual void WriteLine(string? message) { }
+    }
+    public static partial class Debug
+    {
+        public static System.Diagnostics.DebugProvider SetProvider(System.Diagnostics.DebugProvider provider) { throw null; }
+    }
+}