[release/8.0-staging] Remove reference / redistribution of externally provided packages in MSBuild tasks

eng/PackageDownloadAndReference.targets

@@ -0,0 +1,33 @@
+<Project>
+
+  <!-- This file supports using PackageDownloadAndReference items.
+
+  The PackageDownloadAndReference item is used to download a package and reference it in the project, without restoring the package's dependency closure.
+
+  When using PackageDownloadAndReference you are responsible for selecting the correct assets from the package and ensuring that the package and it's
+  dependencies are available at runtime.
+
+  The PackageDownloadAndReference item has the following metadata:
+    - Folder: The folder in the package where the assembly is located.
+    - AssemblyName: The name of the assembly to reference.
+    - Private: Whether the reference should be private (copied to the output directory) or not. Default is false.
+
+  A common use case for PackageDownloadAndReference is to reference assemblies provided by MSBuild or the .NET SDK.
+  -->
+
+  <ItemDefinitionGroup>
+    <PackageDownloadAndReference>
+      <Folder>lib/$(TargetFramework)</Folder>
+      <AssemblyName>%(Identity)</AssemblyName>
+      <Private>false</Private>
+    </PackageDownloadAndReference>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <PackageDownload Include="@(PackageDownloadAndReference)" />
+    <PackageDownload Update="@(PackageDownloadAndReference)" Version="[%(Version)]"/>
+    <PackageDownloadAndReference Update="@(PackageDownloadAndReference)" PackageFolder="$([System.String]::new(%(Identity)).ToLowerInvariant())" />
+    <Reference Include="@(PackageDownloadAndReference->'$(NuGetPackageRoot)%(PackageFolder)/%(Version)/%(Folder)/%(AssemblyName).dll')" />
+  </ItemGroup>
+
+</Project>

eng/Versions.props

@@ -136,8 +136,18 @@
     <SystemSecurityCryptographyOpenSslVersion>5.0.0</SystemSecurityCryptographyOpenSslVersion>
     <SystemSecurityPrincipalWindowsVersion>5.0.0</SystemSecurityPrincipalWindowsVersion>
     <SystemSecurityPermissionsVersion>7.0.0</SystemSecurityPermissionsVersion>
-    <!-- The JSON version that's present in minimum MSBuild / VS version that this release is supported on -->
+
+    <!-- The following package versions are present in minimum MSBuild / VS version that this release is supported on -->
+    <MicrosoftBclAsyncInterfacesToolsetVersion>7.0.0</MicrosoftBclAsyncInterfacesToolsetVersion>
+    <SystemBuffersToolsetVersion>4.5.1</SystemBuffersToolsetVersion>
+    <SystemCollectionsImmutableToolsetVersion>7.0.0</SystemCollectionsImmutableToolsetVersion>
+    <SystemMemoryToolsetVersion>4.5.5</SystemMemoryToolsetVersion>
+    <SystemReflectionMetadataLoadContextToolsetVersion>7.0.0</SystemReflectionMetadataLoadContextToolsetVersion> 
+    <SystemReflectionMetadataToolsetVersion>7.0.0</SystemReflectionMetadataToolsetVersion>
+    <SystemTextEncodingsWebToolsetVersion>7.0.0</SystemTextEncodingsWebToolsetVersion>
     <SystemTextJsonToolsetVersion>7.0.3</SystemTextJsonToolsetVersion>
+    <SystemThreadingTasksExtensionsToolsetVersion>4.5.4</SystemThreadingTasksExtensionsToolsetVersion>
+
     <SystemTextJsonVersion>8.0.0-rc.1.23406.6</SystemTextJsonVersion>
     <SystemRuntimeCompilerServicesUnsafeVersion>6.0.0</SystemRuntimeCompilerServicesUnsafeVersion>
     <SystemThreadingAccessControlVersion>7.0.0</SystemThreadingAccessControlVersion>
@@ -178,8 +188,10 @@
     <MicrosoftBuildTasksCoreVersion>$(MicrosoftBuildVersion)</MicrosoftBuildTasksCoreVersion>
     <MicrosoftBuildFrameworkVersion>$(MicrosoftBuildVersion)</MicrosoftBuildFrameworkVersion>
     <MicrosoftBuildUtilitiesCoreVersion>$(MicrosoftBuildVersion)</MicrosoftBuildUtilitiesCoreVersion>
+    <NugetFrameworksVersion>6.2.4</NugetFrameworksVersion>
     <NugetProjectModelVersion>6.2.4</NugetProjectModelVersion>
     <NugetPackagingVersion>6.2.4</NugetPackagingVersion>
+    <NugetVersioningVersion>6.2.4</NugetVersioningVersion>
     <DotnetSosVersion>7.0.412701</DotnetSosVersion>
     <DotnetSosTargetFrameworkVersion>6.0</DotnetSosTargetFrameworkVersion>
     <!-- Testing -->

src/tasks/AndroidAppBuilder/AndroidAppBuilder.csproj

@@ -10,9 +10,6 @@
   <ItemGroup>
     <EmbeddedResource Include="Templates\*.*" />
   </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
-  </ItemGroup>
   <ItemGroup>
       <ProjectReference Include="$(RepoRoot)src\tasks\MobileBuildTasks\MobileBuildTasks.csproj" />
   </ItemGroup>

src/tasks/AotCompilerTask/MonoAOTCompiler.csproj

@@ -11,11 +11,6 @@
     <NoWarn Condition="$(TargetFramework.StartsWith('net4'))">$(NoWarn),CS8604,CS8602</NoWarn>
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
   </PropertyGroup>
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="$(MicrosoftBuildVersion)" />
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
-    <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
-  </ItemGroup>
   <ItemGroup>
     <Compile Include="MonoAOTCompiler.cs" />
     <Compile Include="..\Common\CompilerCache.cs" />

src/tasks/AppleAppBuilder/AppleAppBuilder.csproj

@@ -10,9 +10,6 @@
   <ItemGroup>
     <EmbeddedResource Include="Templates\*.*" />
   </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
-  </ItemGroup>
   <ItemGroup>
     <Compile Include="AppleAppBuilder.cs" />
     <Compile Include="..\Common\Utils.cs" />

src/tasks/Crossgen2Tasks/Crossgen2Tasks.csproj

@@ -9,8 +9,13 @@
     <NoWarn Condition="$(TargetFramework.StartsWith('net4'))">$(NoWarn),CS8604,CS8602</NoWarn>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
-    <PackageReference Include="NuGet.ProjectModel" Version="$(NugetProjectModelVersion)" />
+    <!-- Bring in references for these assemblies which are provided by the SDK.
+         We do this to avoid bringing the package closure for assemblies we don't use here. -->
+    <PackageDownloadAndReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" Folder="lib/netstandard2.0" />
+    <PackageDownloadAndReference Include="NuGet.Frameworks" Version="$(NugetFrameworksVersion)" Folder="lib/netstandard2.0" />
+    <PackageDownloadAndReference Include="NuGet.Packaging" Version="$(NugetPackagingVersion)" Folder="lib/netstandard2.0" />
+    <PackageDownloadAndReference Include="NuGet.ProjectModel" Version="$(NugetProjectModelVersion)" Folder="lib/netstandard2.0" />
+    <PackageDownloadAndReference Include="NuGet.Versioning" Version="$(NugetVersioningVersion)" Folder="lib/netstandard2.0" />
   </ItemGroup>
   <ItemGroup>
     <Content Include="Microsoft.NET.CrossGen.targets">

src/tasks/Directory.Build.targets

@@ -0,0 +1,28 @@
+<Project>
+  <ItemGroup>
+    <!-- reference MSBuild directly to avoid bringing in it's package closure.  These all represent assemblies available to tasks and provided by MSBuild -->
+    <PackageDownloadAndReference Include="Microsoft.Build" Version="$(MicrosoftBuildVersion)" Folder="ref/net472" Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net472'))" />
+    <PackageDownloadAndReference Include="Microsoft.Build" Version="$(MicrosoftBuildVersion)" Folder="ref/net8.0" Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))" />
+    <PackageDownloadAndReference Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildFrameworkVersion)" Folder="ref/net472" Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net472'))" />
+    <PackageDownloadAndReference Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildFrameworkVersion)" Folder="ref/net8.0" Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))" />
+    <PackageDownloadAndReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildUtilitiesCoreVersion)" Folder="ref/net472" Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net472'))" />
+    <PackageDownloadAndReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildUtilitiesCoreVersion)" Folder="ref/net8.0" Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))" />
+    <PackageDownloadAndReference Include="System.Reflection.MetadataLoadContext" Version="$(SystemReflectionMetadataLoadContextToolsetVersion)" Folder="lib/netstandard2.0" />
+  </ItemGroup>
+
+  <ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net462'))">
+    <!-- These assemblies are made available by MSBuild on .NET Framework -->
+    <PackageDownloadAndReference Include="Microsoft.Bcl.AsyncInterfaces" Version="$(MicrosoftBclAsyncInterfacesToolsetVersion)" Folder="lib/net462" />
+    <PackageDownloadAndReference Include="System.Buffers" Version="$(SystemBuffersToolsetVersion)" Folder="lib/net461" />
+    <PackageDownloadAndReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableToolsetVersion)" Folder="lib/net462" />
+    <PackageDownloadAndReference Include="System.Memory" Version="$(SystemMemoryToolsetVersion)" Folder="lib/net461" />
+    <PackageDownloadAndReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataToolsetVersion)" Folder="lib/net462" />
+    <PackageDownloadAndReference Include="System.Text.Encodings.Web" Version="$(SystemTextEncodingsWebToolsetVersion)" Folder="lib/net462" />
+    <PackageDownloadAndReference Include="System.Text.Json" Version="$(SystemTextJsonToolsetVersion)" Folder="lib/net462" />
+    <PackageDownloadAndReference Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsToolsetVersion)" Folder="lib/net461" />
+  </ItemGroup>
+
+  <Import Project="$(RepositoryEngineeringDir)PackageDownloadAndReference.targets" />
+
+  <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.targets, $(MSBuildThisFileDirectory)..))" />
+</Project>

src/tasks/LibraryBuilder/LibraryBuilder.csproj

@@ -10,9 +10,6 @@
   <ItemGroup>
     <EmbeddedResource Include="Templates\*.*" />
   </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
-  </ItemGroup>
   <ItemGroup>
       <ProjectReference Include="$(RepoRoot)src\tasks\MobileBuildTasks\MobileBuildTasks.csproj" />
   </ItemGroup>

src/tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks/Microsoft.NET.Sdk.WebAssembly.Pack.Tasks.csproj

@@ -19,11 +19,7 @@
   <ItemGroup>
     <Compile Include="..\Common\Utils.cs" />
     <Compile Include="..\WasmAppBuilder\WebcilConverter.cs" />
-
-    <PackageReference Include="Microsoft.Build" Version="$(MicrosoftBuildVersion)" />
-    <PackageReference Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildFrameworkVersion)" ExcludeAssets="runtime" />
-    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildUtilitiesCoreVersion)" ExcludeAssets="runtime" />
-
+
     <ProjectReference Include="..\Microsoft.NET.WebAssembly.Webcil\Microsoft.NET.WebAssembly.Webcil.csproj" />
   </ItemGroup>
 

src/tasks/Microsoft.NET.WebAssembly.Webcil/Microsoft.NET.WebAssembly.Webcil.csproj

@@ -14,13 +14,6 @@
     <IsShippingPackage>false</IsShippingPackage>
   </PropertyGroup>
 
-  <ItemGroup>
-    <!-- we need to keep the version of System.Reflection.Metadata in sync with dotnet/msbuild and dotnet/sdk -->
-    <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" /> 
-    <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
-    <PackageReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
-  </ItemGroup>
-
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">
     <Compile Include="$(CoreLibSharedDir)System\Diagnostics\CodeAnalysis\NullableAttributes.cs" Link="System\Diagnostics\CodeAnalysis\NullableAttributes.cs" />
     <Compile Include="..\Common\IsExternalInit.cs" />

src/tasks/MobileBuildTasks/MobileBuildTasks.csproj

@@ -13,11 +13,6 @@
     <Compile Include="..\Common\Utils.cs" />
   </ItemGroup>
 
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="$(MicrosoftBuildVersion)" />
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
-  </ItemGroup>
-
   <Target Name="GetFilesToPackage" Returns="@(FilesToPackage)">
     <ItemGroup>
       <!-- non-net4* -->

src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj

@@ -5,21 +5,6 @@
     <Nullable>enable</Nullable>
     <NoWarn>$(NoWarn),CA1050,CA1850</NoWarn>
   </PropertyGroup>
-  <ItemGroup Condition="'$(TargetFramework)' == '$(TargetFrameworkForNETCoreTasks)'">
-    <PackageReference Include="Microsoft.Build" Version="$(MicrosoftBuildVersion)" />
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
-    <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" />
-  </ItemGroup>
-  <ItemGroup Condition="'$(TargetFramework)' == '$(TargetFrameworkForNETFrameworkTasks)'">
-    <!-- On .NET Framework, make sure we don't include a copy of the MSBuild assemblies with the task. The NETCore version doesn't do it already. -->
-    <PackageReference Include="Microsoft.Build" Version="$(MicrosoftBuildVersion)" IncludeAssets="compile" />
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" IncludeAssets="compile" />
-    <PackageReference Include="System.Reflection.Metadata" Version="$(SystemReflectionMetadataVersion)" PrivateAssets="All" />
-    <!-- These versions should not be newer than what Visual Studio MSBuild uses -->
-    <PackageReference Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsVersion)" PrivateAssets="all" />
-    <PackageReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
-  </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="ILStrip\AssemblyStripper\AssemblyStripper.csproj" />
   </ItemGroup>

src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj

@@ -13,9 +13,6 @@
     <Compile Include="../AndroidAppBuilder/ApkBuilder.cs" />
     <Compile Include="../Common/Utils.cs" />
   </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
-  </ItemGroup>
   <ItemGroup>
       <ProjectReference Include="$(RepoRoot)src\tasks\MobileBuildTasks\MobileBuildTasks.csproj" />
   </ItemGroup>

src/tasks/WasmAppBuilder/PInvokeCollector.cs

@@ -167,13 +167,7 @@ static bool MethodHasCallbackAttributes(MethodInfo method)
         }
     }
 
-    public static bool IsBlittable(Type type)
-    {
-        if (type.IsPrimitive || type.IsByRef || type.IsPointer || type.IsEnum)
-            return true;
-        else
-            return false;
-    }
+    public static bool IsBlittable(Type type) => type.IsPrimitive || type.IsByRef || type.IsPointer || type.IsEnum;
 
     private static void Error(string msg) => throw new LogAsErrorException(msg);
 
@@ -200,11 +194,24 @@ private static bool HasAttribute(MemberInfo element, params string[] attributeNa
         return false;
     }
 
-    private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotNullWhen(true)] out string? reason)
+    public static bool IsFunctionPointer(Type type)
+    {
+        object? bIsFunctionPointer = type.GetType().GetProperty("IsFunctionPointer")?.GetValue(type);
+        return (bIsFunctionPointer is bool b) && b;
+    }
+
+    internal static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotNullWhen(true)] out string? reason)
     {
         try
         {
             method.GetParameters();
+            foreach (var p in method.GetParameters())
+            {
+                if (IsFunctionPointer(p.ParameterType))
+                {
+                    throw new NotSupportedException("Parsing function pointer types in signatures is not supported.");
+                }
+            }
         }
         catch (NotSupportedException nse)
         {

src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs

@@ -13,8 +13,6 @@
 
 internal sealed class PInvokeTableGenerator
 {
-    private readonly Dictionary<Assembly, bool> _assemblyDisableRuntimeMarshallingAttributeCache = new();
-
     private TaskLoggingHelper Log { get; set; }
     private readonly Func<string, string> _fixupSymbolName;
     private readonly HashSet<string> signatures = new();
@@ -168,13 +166,13 @@ static bool ShouldTreatAsVariadic(PInvoke[] candidates)
                 return false;
 
             PInvoke first = candidates[0];
-            if (TryIsMethodGetParametersUnsupported(first.Method, out _))
+            if (PInvokeCollector.TryIsMethodGetParametersUnsupported(first.Method, out _))
                 return false;
 
             int firstNumArgs = first.Method.GetParameters().Length;
             return candidates
                         .Skip(1)
-                        .Any(c => !TryIsMethodGetParametersUnsupported(c.Method, out _) &&
+                        .Any(c => !PInvokeCollector.TryIsMethodGetParametersUnsupported(c.Method, out _) &&
                                     c.Method.GetParameters().Length != firstNumArgs);
         }
     }
@@ -200,28 +198,6 @@ private string SymbolNameForMethod(MethodInfo method)
         _ => "int"
     };
 
-    // FIXME: System.Reflection.MetadataLoadContext can't decode function pointer types
-    // https://github.com/dotnet/runtime/issues/43791
-    private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotNullWhen(true)] out string? reason)
-    {
-        try
-        {
-            method.GetParameters();
-        }
-        catch (NotSupportedException nse)
-        {
-            reason = nse.Message;
-            return true;
-        }
-        catch
-        {
-            // not concerned with other exceptions
-        }
-
-        reason = null;
-        return false;
-    }
-
     private string? GenPInvokeDecl(PInvoke pinvoke)
     {
         var sb = new StringBuilder();
@@ -234,7 +210,7 @@ private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotN
             return sb.ToString();
         }
 
-        if (TryIsMethodGetParametersUnsupported(pinvoke.Method, out string? reason))
+        if (PInvokeCollector.TryIsMethodGetParametersUnsupported(pinvoke.Method, out string? reason))
         {
             // Don't use method.ToString() or any of it's parameters, or return type
             // because at least one of those are unsupported, and will throw
@@ -381,25 +357,5 @@ private void EmitNativeToInterp(StreamWriter w, List<PInvokeCallback> callbacks)
         w.WriteLine("};");
     }
 
-    private bool HasAssemblyDisableRuntimeMarshallingAttribute(Assembly assembly)
-    {
-        if (!_assemblyDisableRuntimeMarshallingAttributeCache.TryGetValue(assembly, out var value))
-        {
-            _assemblyDisableRuntimeMarshallingAttributeCache[assembly] = value = assembly
-                .GetCustomAttributesData()
-                .Any(d => d.AttributeType.Name == "DisableRuntimeMarshallingAttribute");
-        }
-
-        return value;
-    }
-
-    private static bool IsBlittable(Type type)
-    {
-        if (type.IsPrimitive || type.IsByRef || type.IsPointer || type.IsEnum)
-            return true;
-        else
-            return false;
-    }
-
     private static void Error(string msg) => throw new LogAsErrorException(msg);
 }

src/tasks/WasmAppBuilder/SignatureMapper.cs

@@ -59,6 +59,10 @@ internal static class SignatureMapper
 
         foreach (var parameter in method.GetParameters())
         {
+            if (PInvokeCollector.IsFunctionPointer(parameter.ParameterType))
+            {
+                throw new NotSupportedException("Parsing function pointer types in signatures is not supported.");
+            }
             char? parameterChar = TypeToChar(parameter.ParameterType);
             if (parameterChar == null)
             {