From 8f0974178f7eb4f6674267fac98e9606a4410399 Mon Sep 17 00:00:00 2001 From: martincostello Date: Thu, 27 Feb 2025 15:56:31 +0000 Subject: [PATCH 01/15] Update to ASP.NET Core 10 preview 1 Update to preview 1 of ASP.NET Core 10. --- Directory.Build.props | 4 ++-- Directory.Packages.props | 9 +++------ eng/Versions.props | 8 ++++---- global.json | 4 ++-- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 816bd61de..89bd090bd 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -3,7 +3,7 @@ - net9.0 + net10.0 latest $(NoWarn);CA1515;CS1591 true @@ -77,7 +77,7 @@ $(IsPackable) - + false diff --git a/Directory.Packages.props b/Directory.Packages.props index 79ebe9d8e..8d37474fb 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,16 +4,13 @@ - - - + + + - - - diff --git a/eng/Versions.props b/eng/Versions.props index 0b8e81be7..b9ac5b9b3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,13 +1,13 @@ - 9 - 1 + 10 + 0 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) - 9.0.0 + 10.0.0 preview - + 1 Preview $(PreReleaseVersionIteration) false release diff --git a/global.json b/global.json index 9513c3772..be91326aa 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "9.0.200", + "version": "10.0.100-preview.1.25120.13", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "9.0.200" + "dotnet": "10.0.100-preview.1.25120.13" }, "msbuild-sdks": { From 468415f62d5b228cdcd41e8cd17601db37056f9a Mon Sep 17 00:00:00 2001 From: martincostello Date: Tue, 18 Mar 2025 19:17:34 +0000 Subject: [PATCH 02/15] Update to ASP.NET Core 10 preview 2 Update to preview 2 of ASP.NET Core 10. --- Directory.Packages.props | 6 +++--- eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 8d37474fb..d282750a0 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,9 +4,9 @@ - - - + + + diff --git a/eng/Versions.props b/eng/Versions.props index b9ac5b9b3..89cf6153f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,7 +7,7 @@ $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 preview - 1 + 2 Preview $(PreReleaseVersionIteration) false release diff --git a/global.json b/global.json index be91326aa..0a095bd6a 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-preview.1.25120.13", + "version": "10.0.100-preview.2.25164.34", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-preview.1.25120.13" + "dotnet": "10.0.100-preview.2.25164.34" }, "msbuild-sdks": { From 59a8c4fe3a9f77b432339647fb595e8f2bfa4e97 Mon Sep 17 00:00:00 2001 From: martincostello Date: Sun, 23 Mar 2025 08:27:37 +0000 Subject: [PATCH 03/15] Update package baselines Remove custom baselines for 9.x packages. --- .../AspNet.Security.OAuth.Atlassian.csproj | 1 - .../AspNet.Security.OAuth.GitCode.csproj | 1 - .../AspNet.Security.OAuth.Linear.csproj | 1 - src/AspNet.Security.OAuth.Miro/AspNet.Security.OAuth.Miro.csproj | 1 - .../AspNet.Security.OAuth.Webflow.csproj | 1 - 5 files changed, 5 deletions(-) diff --git a/src/AspNet.Security.OAuth.Atlassian/AspNet.Security.OAuth.Atlassian.csproj b/src/AspNet.Security.OAuth.Atlassian/AspNet.Security.OAuth.Atlassian.csproj index e59faea19..9a9ec1954 100644 --- a/src/AspNet.Security.OAuth.Atlassian/AspNet.Security.OAuth.Atlassian.csproj +++ b/src/AspNet.Security.OAuth.Atlassian/AspNet.Security.OAuth.Atlassian.csproj @@ -1,7 +1,6 @@  - 9.1.0 $(DefaultNetCoreTargetFramework) diff --git a/src/AspNet.Security.OAuth.GitCode/AspNet.Security.OAuth.GitCode.csproj b/src/AspNet.Security.OAuth.GitCode/AspNet.Security.OAuth.GitCode.csproj index 2927d25ec..b8a085588 100644 --- a/src/AspNet.Security.OAuth.GitCode/AspNet.Security.OAuth.GitCode.csproj +++ b/src/AspNet.Security.OAuth.GitCode/AspNet.Security.OAuth.GitCode.csproj @@ -1,7 +1,6 @@ - 9.1.0 $(DefaultNetCoreTargetFramework) diff --git a/src/AspNet.Security.OAuth.Linear/AspNet.Security.OAuth.Linear.csproj b/src/AspNet.Security.OAuth.Linear/AspNet.Security.OAuth.Linear.csproj index 7f10d0cf3..f9f0a9cb0 100644 --- a/src/AspNet.Security.OAuth.Linear/AspNet.Security.OAuth.Linear.csproj +++ b/src/AspNet.Security.OAuth.Linear/AspNet.Security.OAuth.Linear.csproj @@ -1,7 +1,6 @@  - 9.2.0 $(DefaultNetCoreTargetFramework) diff --git a/src/AspNet.Security.OAuth.Miro/AspNet.Security.OAuth.Miro.csproj b/src/AspNet.Security.OAuth.Miro/AspNet.Security.OAuth.Miro.csproj index 5b21ac6f8..dd6c15d7b 100644 --- a/src/AspNet.Security.OAuth.Miro/AspNet.Security.OAuth.Miro.csproj +++ b/src/AspNet.Security.OAuth.Miro/AspNet.Security.OAuth.Miro.csproj @@ -1,7 +1,6 @@  - 9.2.0 $(DefaultNetCoreTargetFramework) diff --git a/src/AspNet.Security.OAuth.Webflow/AspNet.Security.OAuth.Webflow.csproj b/src/AspNet.Security.OAuth.Webflow/AspNet.Security.OAuth.Webflow.csproj index 41e47c4fa..60a8b4e8f 100644 --- a/src/AspNet.Security.OAuth.Webflow/AspNet.Security.OAuth.Webflow.csproj +++ b/src/AspNet.Security.OAuth.Webflow/AspNet.Security.OAuth.Webflow.csproj @@ -1,7 +1,6 @@  - 9.2.0 $(DefaultNetCoreTargetFramework) From fe9caa0e652d8d91fae6be06878360209db84665 Mon Sep 17 00:00:00 2001 From: martincostello Date: Thu, 10 Apr 2025 16:57:35 +0100 Subject: [PATCH 04/15] Update to ASP.NET Core 10 preview 3 Update to preview 3 of ASP.NET Core 10. --- Directory.Packages.props | 6 +++--- eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d282750a0..a9c6c3c1f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,9 +4,9 @@ - - - + + + diff --git a/eng/Versions.props b/eng/Versions.props index 89cf6153f..6f64fb31c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,7 +7,7 @@ $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 preview - 2 + 3 Preview $(PreReleaseVersionIteration) false release diff --git a/global.json b/global.json index 0a095bd6a..54729b210 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-preview.2.25164.34", + "version": "10.0.100-preview.3.25201.16", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-preview.2.25164.34" + "dotnet": "10.0.100-preview.3.25201.16" }, "msbuild-sdks": { From 311218eb3e7f2cb3f3fc1d0577a308bb0e5e44db Mon Sep 17 00:00:00 2001 From: martincostello Date: Thu, 1 May 2025 13:01:06 +0100 Subject: [PATCH 05/15] Remove PackageValidationBaselineVersion Remove PackageValidationBaselineVersion for 10.0. --- .../AspNet.Security.OAuth.Contentful.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/AspNet.Security.OAuth.Contentful/AspNet.Security.OAuth.Contentful.csproj b/src/AspNet.Security.OAuth.Contentful/AspNet.Security.OAuth.Contentful.csproj index c2fee2bf3..6b4056a2a 100644 --- a/src/AspNet.Security.OAuth.Contentful/AspNet.Security.OAuth.Contentful.csproj +++ b/src/AspNet.Security.OAuth.Contentful/AspNet.Security.OAuth.Contentful.csproj @@ -1,7 +1,6 @@  - 9.3.0 $(DefaultNetCoreTargetFramework) From 7ce0ee5af29e0fc8ef1c1fcdc9c0d641603250d8 Mon Sep 17 00:00:00 2001 From: martincostello Date: Wed, 14 May 2025 08:32:36 -0400 Subject: [PATCH 06/15] Update to ASP.NET Core 10 preview 4 Update to preview 4 of ASP.NET Core 10. --- Directory.Packages.props | 6 +++--- eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index a9c6c3c1f..639bc5d05 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,9 +4,9 @@ - - - + + + diff --git a/eng/Versions.props b/eng/Versions.props index 6f64fb31c..9e2b807ba 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,7 +7,7 @@ $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 preview - 3 + 4 Preview $(PreReleaseVersionIteration) false release diff --git a/global.json b/global.json index 54729b210..7b70c3922 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-preview.3.25201.16", + "version": "10.0.100-preview.4.25258.110", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-preview.3.25201.16" + "dotnet": "10.0.100-preview.4.25258.110" }, "msbuild-sdks": { From f4307b11e865ff2c88eccaa995e279c3fcc9b59b Mon Sep 17 00:00:00 2001 From: martincostello Date: Wed, 21 May 2025 09:52:44 +0100 Subject: [PATCH 07/15] Remove PackageValidationBaselineVersion Remove PackageValidationBaselineVersion for Bilibili. --- .../AspNet.Security.OAuth.Bilibili.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/AspNet.Security.OAuth.Bilibili/AspNet.Security.OAuth.Bilibili.csproj b/src/AspNet.Security.OAuth.Bilibili/AspNet.Security.OAuth.Bilibili.csproj index 7741884d7..c015f7b8a 100644 --- a/src/AspNet.Security.OAuth.Bilibili/AspNet.Security.OAuth.Bilibili.csproj +++ b/src/AspNet.Security.OAuth.Bilibili/AspNet.Security.OAuth.Bilibili.csproj @@ -1,7 +1,6 @@  - 9.4.0 $(DefaultNetCoreTargetFramework) From 2a25245f0bede7644fa2d036077f8be2e3700e39 Mon Sep 17 00:00:00 2001 From: martincostello Date: Tue, 10 Jun 2025 21:12:13 +0100 Subject: [PATCH 08/15] Update to ASP.NET Core 10 preview 5 Update to preview 5 of ASP.NET Core 10. --- Directory.Packages.props | 10 +++++----- eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 639bc5d05..3efd1a635 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,11 +3,11 @@ - - - - - + + + + + diff --git a/eng/Versions.props b/eng/Versions.props index 9e2b807ba..888deaa39 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,7 +7,7 @@ $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 preview - 4 + 5 Preview $(PreReleaseVersionIteration) false release diff --git a/global.json b/global.json index 7b70c3922..b3483215a 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-preview.4.25258.110", + "version": "10.0.100-preview.5.25277.114", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-preview.4.25258.110" + "dotnet": "10.0.100-preview.5.25277.114" }, "msbuild-sdks": { From 2e16e993b63da9762f7e19a44ccdacaf8a0942cc Mon Sep 17 00:00:00 2001 From: martincostello Date: Mon, 14 Jul 2025 22:37:05 +0100 Subject: [PATCH 09/15] Update to ASP.NET Core 10 preview 6 Update to preview 6 of ASP.NET Core 10. --- Directory.Packages.props | 6 +++--- eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 9560fa2ce..3eb7b2105 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,9 +4,9 @@ - - - + + + diff --git a/eng/Versions.props b/eng/Versions.props index 888deaa39..1e8c2626d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,7 +7,7 @@ $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 preview - 5 + 6 Preview $(PreReleaseVersionIteration) false release diff --git a/global.json b/global.json index b3483215a..438b4d9fb 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-preview.5.25277.114", + "version": "10.0.100-preview.6.25358.103", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-preview.5.25277.114" + "dotnet": "10.0.100-preview.6.25358.103" }, "msbuild-sdks": { From b9e036943f3bcb510e3d100a2ae6f3d719fceb8e Mon Sep 17 00:00:00 2001 From: Martin Costello Date: Tue, 12 Aug 2025 17:40:26 +0100 Subject: [PATCH 10/15] Update to ASP.NET Core 10 preview 7 (#1092) Update to preview 7 of ASP.NET Core 10. --- Directory.Packages.props | 6 +++--- eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 3eb7b2105..c013b1852 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,9 +4,9 @@ - - - + + + diff --git a/eng/Versions.props b/eng/Versions.props index 1e8c2626d..551b99fb6 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,7 +7,7 @@ $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 preview - 6 + 7 Preview $(PreReleaseVersionIteration) false release diff --git a/global.json b/global.json index 438b4d9fb..5ada88321 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-preview.6.25358.103", + "version": "10.0.100-preview.7.25380.108", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-preview.6.25358.103" + "dotnet": "10.0.100-preview.7.25380.108" }, "msbuild-sdks": { From 5fec0128dde45fcc95298fd6d9cc61215fbaa550 Mon Sep 17 00:00:00 2001 From: Martin Costello Date: Tue, 9 Sep 2025 14:36:34 +0100 Subject: [PATCH 11/15] Update to ASP.NET Core 10 RC1 (#1104) - Update to release candidate 1 of ASP.NET Core 10. - Fix obsolete usage of `WebHostBuilder`. - Update `JetBrains.Annotations` and `Microsoft.IdentityModel.Protocols.OpenIdConnect` to their latest versions. - Disable `IDE0058`. --- .editorconfig | 2 ++ Directory.Packages.props | 10 ++++---- eng/Versions.props | 6 ++--- global.json | 4 +-- .../Apple/AppleClientSecretGeneratorTests.cs | 25 ++++++++----------- .../Infrastructure/ApplicationFactory.cs | 12 ++++++--- 6 files changed, 32 insertions(+), 27 deletions(-) diff --git a/.editorconfig b/.editorconfig index 582a12360..61b517326 100644 --- a/.editorconfig +++ b/.editorconfig @@ -152,3 +152,5 @@ csharp_space_between_method_call_empty_parameter_list_parentheses = false # Wrapping preferences csharp_preserve_single_line_statements = true csharp_preserve_single_line_blocks = true + +dotnet_diagnostic.IDE0058.severity = none diff --git a/Directory.Packages.props b/Directory.Packages.props index c013b1852..2a157c0af 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,13 +1,13 @@ - + - - - - + + + + diff --git a/eng/Versions.props b/eng/Versions.props index 551b99fb6..9efa09cd0 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,9 +6,9 @@ 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 - preview - 7 - Preview $(PreReleaseVersionIteration) + rc + 1 + Release Candidate $(PreReleaseVersionIteration) false release true diff --git a/global.json b/global.json index 5ada88321..5eeb905df 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-preview.7.25380.108", + "version": "10.0.100-rc.1.25451.107", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-preview.7.25380.108" + "dotnet": "10.0.100-rc.1.25451.107" }, "msbuild-sdks": { diff --git a/test/AspNet.Security.OAuth.Providers.Tests/Apple/AppleClientSecretGeneratorTests.cs b/test/AspNet.Security.OAuth.Providers.Tests/Apple/AppleClientSecretGeneratorTests.cs index 9e4905ee3..ced6354b7 100644 --- a/test/AspNet.Security.OAuth.Providers.Tests/Apple/AppleClientSecretGeneratorTests.cs +++ b/test/AspNet.Security.OAuth.Providers.Tests/Apple/AppleClientSecretGeneratorTests.cs @@ -8,21 +8,15 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.IdentityModel.JsonWebTokens; namespace AspNet.Security.OAuth.Apple; -public class AppleClientSecretGeneratorTests +public class AppleClientSecretGeneratorTests(ITestOutputHelper outputHelper) { - private readonly ITestOutputHelper _outputHelper; - - public AppleClientSecretGeneratorTests(ITestOutputHelper outputHelper) - { - _outputHelper = outputHelper; - } - [Fact] public async Task GenerateAsync_Generates_Valid_Signed_Jwt() { @@ -186,13 +180,16 @@ private async Task GenerateTokenAsync( Func actAndAssert) { // Arrange - var builder = new WebHostBuilder() - .ConfigureLogging((p) => p.AddXUnit(_outputHelper).SetMinimumLevel(LogLevel.Debug)) - .Configure((app) => app.UseAuthentication()) - .ConfigureServices((services) => + var builder = new HostBuilder() + .ConfigureWebHost((builder) => { - services.AddAuthentication() - .AddApple(); + builder.ConfigureLogging((p) => p.AddXUnit(outputHelper).SetMinimumLevel(LogLevel.Debug)) + .Configure((app) => app.UseAuthentication()) + .ConfigureServices((services) => + { + services.AddAuthentication() + .AddApple(); + }); }); using var host = builder.Build(); diff --git a/test/AspNet.Security.OAuth.Providers.Tests/Infrastructure/ApplicationFactory.cs b/test/AspNet.Security.OAuth.Providers.Tests/Infrastructure/ApplicationFactory.cs index 9e7dd0424..6b49444a1 100644 --- a/test/AspNet.Security.OAuth.Providers.Tests/Infrastructure/ApplicationFactory.cs +++ b/test/AspNet.Security.OAuth.Providers.Tests/Infrastructure/ApplicationFactory.cs @@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; @@ -137,10 +138,15 @@ private static string IdentityToXmlString(ClaimsPrincipal user) private sealed class TestApplicationFactory : WebApplicationFactory { - protected override IWebHostBuilder CreateWebHostBuilder() + protected override void ConfigureWebHost(IWebHostBuilder builder) { - return new WebHostBuilder() - .UseSetting("TEST_CONTENTROOT_ASPNET_SECURITY_OAUTH_PROVIDERS_TESTS", "."); // Use a dummy content root + base.ConfigureWebHost(builder); + builder.UseContentRoot("."); // Use a dummy content root + } + + protected override IHostBuilder? CreateHostBuilder() + { + return new HostBuilder(); } } } From 40d146435192af5ae868d5b44213cc2c46026c0b Mon Sep 17 00:00:00 2001 From: martincostello Date: Tue, 14 Oct 2025 18:36:04 +0200 Subject: [PATCH 12/15] Update to ASP.NET Core 10 RC2 Update to Release Candidate 2 of ASP.NET Core 10. --- Directory.Packages.props | 6 +++--- eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 2a157c0af..3965cc84d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,9 +4,9 @@ - - - + + + diff --git a/eng/Versions.props b/eng/Versions.props index 9efa09cd0..a1ec564a7 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,7 +7,7 @@ $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 rc - 1 + 2 Release Candidate $(PreReleaseVersionIteration) false release diff --git a/global.json b/global.json index 5eeb905df..19bec01a5 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-rc.1.25451.107", + "version": "10.0.100-rc.2.25502.107", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-rc.1.25451.107" + "dotnet": "10.0.100-rc.2.25502.107" }, "msbuild-sdks": { From a9d8170733874f916ef9b6b94fd476607e15c8cc Mon Sep 17 00:00:00 2001 From: martincostello Date: Fri, 24 Oct 2025 10:15:36 +0100 Subject: [PATCH 13/15] Update Arcade for .NET 10 Apply Arcade updates for .NET 10. --- .editorconfig | 3 + .gitattributes | 9 +- eng/common/CIBuild.cmd | 2 +- eng/common/SetupNugetSources.ps1 | 77 +++-- eng/common/SetupNugetSources.sh | 179 +++++----- eng/common/build.ps1 | 11 +- eng/common/build.sh | 33 +- eng/common/cibuild.sh | 4 +- eng/common/cross/arm64/tizen/tizen.patch | 2 +- eng/common/cross/armel/armel.jessie.patch | 43 --- eng/common/cross/build-android-rootfs.sh | 49 ++- eng/common/cross/build-rootfs.sh | 237 +++++++------ eng/common/cross/install-debs.py | 334 +++++++++++++++++++ eng/common/cross/tizen-fetch.sh | 9 +- eng/common/cross/toolchain.cmake | 149 ++++----- eng/common/darc-init.sh | 2 +- eng/common/dotnet.cmd | 7 + eng/common/dotnet.ps1 | 11 + eng/common/dotnet.sh | 26 ++ eng/common/generate-locproject.ps1 | 49 ++- eng/common/generate-sbom-prep.ps1 | 20 +- eng/common/generate-sbom-prep.sh | 17 +- eng/common/internal/NuGet.config | 3 + eng/common/internal/Tools.csproj | 10 - eng/common/native/install-dependencies.sh | 62 ++++ eng/common/post-build/nuget-verification.ps1 | 2 +- eng/common/post-build/publish-using-darc.ps1 | 12 +- eng/common/sdk-task.ps1 | 12 +- eng/common/sdk-task.sh | 121 +++++++ eng/common/sdl/packages.config | 2 +- eng/common/tools.ps1 | 74 ++-- eng/common/tools.sh | 77 +++-- global.json | 4 +- 33 files changed, 1168 insertions(+), 484 deletions(-) delete mode 100644 eng/common/cross/armel/armel.jessie.patch create mode 100644 eng/common/cross/install-debs.py create mode 100644 eng/common/dotnet.cmd create mode 100644 eng/common/dotnet.ps1 create mode 100644 eng/common/dotnet.sh create mode 100644 eng/common/native/install-dependencies.sh create mode 100644 eng/common/sdk-task.sh diff --git a/.editorconfig b/.editorconfig index 61b517326..bbb9ca9a5 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,6 +13,9 @@ indent_style = space insert_final_newline = true trim_trailing_whitespace = true +[*.{sh}] +end_of_line = lf + [*.{csproj,json,props,ruleset,targets}] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index 10b136d73..0d50e1e1b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,11 +1,12 @@ ############################################################################### # Set default behavior to automatically normalize line endings. ############################################################################### -* text=auto +* text=auto *.cer binary *.p8 binary *.pfx binary +*.sh eol=lf ############################################################################### # Set default behavior for command prompt diff. @@ -21,7 +22,7 @@ # # Merging from the command prompt will add diff markers to the files if there # are conflicts (Merging from VS is not affected by the settings below, in VS -# the diff markers are never inserted). Diff markers may cause the following +# the diff markers are never inserted). Diff markers may cause the following # file extensions to fail to load in VS. An alternative would be to treat # these files as binary and thus will always conflict and require user # intervention with every merge. To do so, just uncomment the entries below @@ -50,9 +51,9 @@ ############################################################################### # diff behavior for common document formats -# +# # Convert binary document formats to text before diffing them. This feature -# is only available from the command line. Turn it on by uncommenting the +# is only available from the command line. Turn it on by uncommenting the # entries below. ############################################################################### #*.doc diff=astextplain diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd index 56c2f25ac..ac1f72bf9 100644 --- a/eng/common/CIBuild.cmd +++ b/eng/common/CIBuild.cmd @@ -1,2 +1,2 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" \ No newline at end of file +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 5db4ad71e..fc8d61801 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -7,11 +7,11 @@ # See example call for this script below. # # - task: PowerShell@2 -# displayName: Setup Private Feeds Credentials +# displayName: Setup internal Feeds Credentials # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: -# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token +# filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 +# arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token # env: # Token: $(dn-bot-dnceng-artifact-feeds-rw) # @@ -34,19 +34,28 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 +# Adds or enables the package source with the given name +function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { + if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName)) { + AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $userName -pwd $Password + } +} + # Add source entry to PackageSources function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) { + Write-Host "Adding package source $SourceName" + $packageSource = $doc.CreateElement("add") $packageSource.SetAttribute("key", $SourceName) $packageSource.SetAttribute("value", $SourceEndPoint) $sources.AppendChild($packageSource) | Out-Null } else { - Write-Host "Package source $SourceName already present." + Write-Host "Package source $SourceName already present and enabled." } AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd @@ -59,6 +68,8 @@ function AddCredential($creds, $source, $username, $pwd) { return; } + Write-Host "Inserting credential for feed: " $source + # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -91,24 +102,27 @@ function AddCredential($creds, $source, $username, $pwd) { $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { - $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") - - Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." - - ForEach ($PackageSource in $maestroPrivateSources) { - Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd +# Enable all darc-int package sources. +function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds) { + $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroInternalSources) { + EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key } } -function EnablePrivatePackageSources($DisabledPackageSources) { - $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($DisabledPackageSource in $maestroPrivateSources) { - Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" +# Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise. +function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName) { + $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']") + if ($DisabledPackageSource) { + Write-Host "Enabling internal source '$($DisabledPackageSource.key)'." + # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries $DisabledPackageSources.RemoveChild($DisabledPackageSource) + + AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -pwd $Password + return $true } + return $false } if (!(Test-Path $ConfigFile -PathType Leaf)) { @@ -121,15 +135,17 @@ $doc = New-Object System.Xml.XmlDocument $filename = (Get-Item $ConfigFile).FullName $doc.Load($filename) -# Get reference to or create one if none exist already +# Get reference to - fail if none exist $sources = $doc.DocumentElement.SelectSingleNode("packageSources") if ($sources -eq $null) { - $sources = $doc.CreateElement("packageSources") - $doc.DocumentElement.AppendChild($sources) | Out-Null + Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile" + ExitWithExitCode 1 } $creds = $null +$feedSuffix = "v3/index.json" if ($Password) { + $feedSuffix = "v2" # Looks for a node. Create it if none is found. $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") if ($creds -eq $null) { @@ -138,33 +154,22 @@ if ($Password) { } } +$userName = "dn-bot" + # Check for disabledPackageSources; we'll enable any darc-int ones we find there $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") if ($disabledSources -ne $null) { Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnablePrivatePackageSources -DisabledPackageSources $disabledSources -} - -$userName = "dn-bot" - -# Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password - -# 3.1 uses a different feed url format so it's handled differently here -$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") -if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds } - -$dotnetVersions = @('5','6','7','8','9') +$dotnetVersions = @('5','6','7','8','9','10') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password } } diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 4604b61b0..b97cc5363 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -11,8 +11,8 @@ # - task: Bash@3 # displayName: Setup Internal Feeds # inputs: -# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh -# arguments: $(Build.SourcesDirectory)/NuGet.config +# filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh +# arguments: $(System.DefaultWorkingDirectory)/NuGet.config # condition: ne(variables['Agent.OS'], 'Windows_NT') # - task: NuGetAuthenticate@1 # @@ -52,78 +52,124 @@ if [[ `uname -s` == "Darwin" ]]; then TB='' fi -# Ensure there is a ... section. -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding ... section." - ConfigNodeHeader="" - PackageSourcesTemplate="${TB}${NL}${TB}" +# Enables an internal package source by name, if found. Returns 0 if found and enabled, 1 if not found. +EnableInternalPackageSource() { + local PackageSourceName="$1" + + # Check if disabledPackageSources section exists + grep -i "" "$ConfigFile" > /dev/null + if [ "$?" != "0" ]; then + return 1 # No disabled sources section + fi + + # Check if this source name is disabled + grep -i " /dev/null + if [ "$?" == "0" ]; then + echo "Enabling internal source '$PackageSourceName'." + # Remove the disabled entry (including any surrounding comments or whitespace on the same line) + sed -i.bak "//d" "$ConfigFile" + + # Add the source name to PackageSources for credential handling + PackageSources+=("$PackageSourceName") + return 0 # Found and enabled + fi + + return 1 # Not found in disabled sources +} + +# Add source entry to PackageSources +AddPackageSource() { + local SourceName="$1" + local SourceEndPoint="$2" + + # Check if source already exists + grep -i " /dev/null + if [ "$?" == "0" ]; then + echo "Package source $SourceName already present and enabled." + PackageSources+=("$SourceName") + return + fi + + echo "Adding package source $SourceName" + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" "$ConfigFile" + PackageSources+=("$SourceName") +} + +# Adds or enables the package source with the given name +AddOrEnablePackageSource() { + local SourceName="$1" + local SourceEndPoint="$2" + + # Try to enable if disabled, if not found then add new source + EnableInternalPackageSource "$SourceName" + if [ "$?" != "0" ]; then + AddPackageSource "$SourceName" "$SourceEndPoint" + fi +} - sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile -fi +# Enable all darc-int package sources +EnableMaestroInternalPackageSources() { + # Check if disabledPackageSources section exists + grep -i "" "$ConfigFile" > /dev/null + if [ "$?" != "0" ]; then + return # No disabled sources section + fi + + # Find all darc-int disabled sources + local DisabledDarcIntSources=() + DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' "$ConfigFile" | tr -d '"') + + for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do + if [[ $DisabledSourceName == darc-int* ]]; then + EnableInternalPackageSource "$DisabledSourceName" + fi + done +} -# Ensure there is a ... section. -grep -i "" $ConfigFile +# Ensure there is a ... section. +grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding ... section." - - PackageSourcesNodeFooter="" - PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile + Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile" + ExitWithExitCode 1 fi PackageSources=() -# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present -grep -i "... section. + grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + echo "Adding ... section." - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet3.1-internal') - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal-transport to the packageSources." PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile fi - PackageSources+=('dotnet3.1-internal-transport') fi -DotNetVersions=('5' '6' '7' '8' '9') +# Check for disabledPackageSources; we'll enable any darc-int ones we find there +grep -i "" $ConfigFile > /dev/null +if [ "$?" == "0" ]; then + echo "Checking for any darc-int disabled package sources in the disabledPackageSources node" + EnableMaestroInternalPackageSources +fi + +DotNetVersions=('5' '6' '7' '8' '9' '10') for DotNetVersion in ${DotNetVersions[@]} ; do FeedPrefix="dotnet${DotNetVersion}"; - grep -i " /dev/null if [ "$?" == "0" ]; then - grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal") - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding $FeedPrefix-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal-transport") + AddOrEnablePackageSource "$FeedPrefix-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/$FeedSuffix" + AddOrEnablePackageSource "$FeedPrefix-internal-transport" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/$FeedSuffix" fi done @@ -139,29 +185,12 @@ if [ "$CredToken" ]; then # Check if there is no existing credential for this FeedName grep -i "<$FeedName>" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." + echo " Inserting credential for feed: $FeedName" PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + NewCredential="${TB}${TB}<$FeedName>${NL}${TB}${NL}${TB}${TB}${NL}${TB}${TB}" sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile fi done fi - -# Re-enable any entries in disabledPackageSources where the feed name contains darc-int -grep -i "" $ConfigFile -if [ "$?" == "0" ]; then - DisabledDarcIntSources=() - echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" - DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') - for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do - if [[ $DisabledSourceName == darc-int* ]] - then - OldDisableValue="" - NewDisableValue="" - sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile - echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" - fi - done -fi diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 438f9920c..8cfee107e 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -7,6 +7,7 @@ Param( [string] $msbuildEngine = $null, [bool] $warnAsError = $true, [bool] $nodeReuse = $true, + [switch] $buildCheck = $false, [switch][Alias('r')]$restore, [switch] $deployDeps, [switch][Alias('b')]$build, @@ -20,6 +21,7 @@ Param( [switch] $publish, [switch] $clean, [switch][Alias('pb')]$productBuild, + [switch]$fromVMR, [switch][Alias('bl')]$binaryLog, [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, @@ -71,6 +73,9 @@ function Print-Usage() { Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" + Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + Write-Host " -buildCheck Sets /check msbuild parameter" + Write-Host " -fromVMR Set when building from within the VMR" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -97,6 +102,7 @@ function Build { $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' } $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' } + $check = if ($buildCheck) { '/check' } else { '' } if ($projects) { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. @@ -113,6 +119,7 @@ function Build { MSBuild $toolsetBuildProj ` $bl ` $platformArg ` + $check ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` /p:Restore=$restore ` @@ -122,11 +129,13 @@ function Build { /p:Deploy=$deploy ` /p:Test=$test ` /p:Pack=$pack ` - /p:DotNetBuildRepo=$productBuild ` + /p:DotNetBuild=$productBuild ` + /p:DotNetBuildFromVMR=$fromVMR ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` /p:Publish=$publish ` + /p:RestoreStaticGraphEnableBinaryLogger=$binaryLog ` @properties } diff --git a/eng/common/build.sh b/eng/common/build.sh index ac1ee8620..9767bb411 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -42,6 +42,8 @@ usage() echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo " --buildCheck Sets /check msbuild parameter" + echo " --fromVMR Set when building from within the VMR" echo "" echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." @@ -63,6 +65,7 @@ restore=false build=false source_build=false product_build=false +from_vmr=false rebuild=false test=false integration_test=false @@ -76,6 +79,7 @@ clean=false warn_as_error=true node_reuse=true +build_check=false binary_log=false exclude_ci_binary_log=false pipelines_log=false @@ -87,7 +91,7 @@ verbosity='minimal' runtime_source_feed='' runtime_source_feed_key='' -properties='' +properties=() while [[ $# > 0 ]]; do opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in @@ -127,19 +131,22 @@ while [[ $# > 0 ]]; do -pack) pack=true ;; - -sourcebuild|-sb) + -sourcebuild|-source-build|-sb) build=true source_build=true product_build=true restore=true pack=true ;; - -productBuild|-pb) + -productbuild|-product-build|-pb) build=true product_build=true restore=true pack=true ;; + -fromvmr|-from-vmr) + from_vmr=true + ;; -test|-t) test=true ;; @@ -173,6 +180,9 @@ while [[ $# > 0 ]]; do node_reuse=$2 shift ;; + -buildcheck) + build_check=true + ;; -runtimesourcefeed) runtime_source_feed=$2 shift @@ -182,7 +192,7 @@ while [[ $# > 0 ]]; do shift ;; *) - properties="$properties $1" + properties+=("$1") ;; esac @@ -216,7 +226,7 @@ function Build { InitializeCustomToolset if [[ ! -z "$projects" ]]; then - properties="$properties /p:Projects=$projects" + properties+=("/p:Projects=$projects") fi local bl="" @@ -224,15 +234,21 @@ function Build { bl="/bl:\"$log_dir/Build.binlog\"" fi + local check="" + if [[ "$build_check" == true ]]; then + check="/check" + fi + MSBuild $_InitializeToolset \ $bl \ + $check \ /p:Configuration=$configuration \ /p:RepoRoot="$repo_root" \ /p:Restore=$restore \ /p:Build=$build \ - /p:DotNetBuildRepo=$product_build \ - /p:ArcadeBuildFromSource=$source_build \ + /p:DotNetBuild=$product_build \ /p:DotNetBuildSourceOnly=$source_build \ + /p:DotNetBuildFromVMR=$from_vmr \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ @@ -240,7 +256,8 @@ function Build { /p:PerformanceTest=$performance_test \ /p:Sign=$sign \ /p:Publish=$publish \ - $properties + /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \ + ${properties[@]+"${properties[@]}"} ExitWithExitCode 0 } diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh index 1a02c0dec..9d9001df5 100755 --- a/eng/common/cibuild.sh +++ b/eng/common/cibuild.sh @@ -7,10 +7,10 @@ while [[ -h $source ]]; do scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where + # if $source was a relative symlink, we need to resolve it relative to the path where # the symlink file was located [[ $source != /* ]] && source="$scriptroot/$source" done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@ \ No newline at end of file +. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@ diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch index af7c8be05..2cebc5473 100644 --- a/eng/common/cross/arm64/tizen/tizen.patch +++ b/eng/common/cross/arm64/tizen/tizen.patch @@ -5,5 +5,5 @@ diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so Use the shared library, but some functions are only in the static library, so try that secondarily. */ OUTPUT_FORMAT(elf64-littleaarch64) --GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) ) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-aarch64.so.1 ) ) +GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) ) diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch deleted file mode 100644 index 2d2615619..000000000 --- a/eng/common/cross/armel/armel.jessie.patch +++ /dev/null @@ -1,43 +0,0 @@ -diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h ---- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700 -+++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700 -@@ -69,10 +69,10 @@ - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_val_compare_and_swap_2(addr, old, _new); -+ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new); - #endif - case 4: -- return __sync_val_compare_and_swap_4(addr, old, _new); -+ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new); - #if (CAA_BITS_PER_LONG == 64) - case 8: - return __sync_val_compare_and_swap_8(addr, old, _new); -@@ -109,7 +109,7 @@ - return; - #endif - case 4: -- __sync_and_and_fetch_4(addr, val); -+ __sync_and_and_fetch_4((uint32_t*) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -@@ -148,7 +148,7 @@ - return; - #endif - case 4: -- __sync_or_and_fetch_4(addr, val); -+ __sync_or_and_fetch_4((uint32_t*) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -@@ -187,7 +187,7 @@ - return __sync_add_and_fetch_2(addr, val); - #endif - case 4: -- return __sync_add_and_fetch_4(addr, val); -+ return __sync_add_and_fetch_4((uint32_t*) addr, val); - #if (CAA_BITS_PER_LONG == 64) - case 8: - return __sync_add_and_fetch_8(addr, val); diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index 7e9ba2b75..fbd8d8084 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -6,10 +6,11 @@ usage() { echo "Creates a toolchain and sysroot used for cross-compiling for Android." echo - echo "Usage: $0 [BuildArch] [ApiLevel]" + echo "Usage: $0 [BuildArch] [ApiLevel] [--ndk NDKVersion]" echo echo "BuildArch is the target architecture of Android. Currently only arm64 is supported." echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html" + echo "NDKVersion is the version of Android NDK. The default is r21. See https://developer.android.com/ndk/downloads/revision_history" echo echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior" echo "by setting the TOOLCHAIN_DIR environment variable" @@ -25,10 +26,15 @@ __BuildArch=arm64 __AndroidArch=aarch64 __AndroidToolchain=aarch64-linux-android -for i in "$@" - do - lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")" - case $lowerI in +while :; do + if [[ "$#" -le 0 ]]; then + break + fi + + i=$1 + + lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")" + case $lowerI in -?|-h|--help) usage exit 1 @@ -43,6 +49,10 @@ for i in "$@" __AndroidArch=arm __AndroidToolchain=arm-linux-androideabi ;; + --ndk) + shift + __NDK_Version=$1 + ;; *[0-9]) __ApiLevel=$i ;; @@ -50,8 +60,17 @@ for i in "$@" __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" ;; esac + shift done +if [[ "$__NDK_Version" == "r21" ]] || [[ "$__NDK_Version" == "r22" ]]; then + __NDK_File_Arch_Spec=-x86_64 + __SysRoot=sysroot +else + __NDK_File_Arch_Spec= + __SysRoot=toolchains/llvm/prebuilt/linux-x86_64/sysroot +fi + # Obtain the location of the bash script to figure out where the root of the repo is. __ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -78,6 +97,7 @@ fi echo "Target API level: $__ApiLevel" echo "Target architecture: $__BuildArch" +echo "NDK version: $__NDK_Version" echo "NDK location: $__NDK_Dir" echo "Target Toolchain location: $__ToolchainDir" @@ -85,8 +105,8 @@ echo "Target Toolchain location: $__ToolchainDir" if [ ! -d $__NDK_Dir ]; then echo Downloading the NDK into $__NDK_Dir mkdir -p $__NDK_Dir - wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip - unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux$__NDK_File_Arch_Spec.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux.zip + unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux.zip -d $__CrossDir fi if [ ! -d $__lldb_Dir ]; then @@ -116,16 +136,11 @@ for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/ fi done -cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" +cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/$__SysRoot/usr/" # Generate platform file for build.sh script to assign to __DistroRid echo "Generating platform file..." -echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform - -echo "Now to build coreclr, libraries and installers; run:" -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory coreclr -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory libraries -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory installer +echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/$__SysRoot/android_platform + +echo "Now to build coreclr, libraries and host; run:" +echo ROOTFS_DIR=$(realpath $__ToolchainDir/$__SysRoot) ./build.sh clr+libs+host --cross --arch $__BuildArch diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 4b5e8d716..8abfb71f7 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -5,7 +5,7 @@ set -e usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" - echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86" + echo "BuildArch can be: arm(default), arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" echo " for alpine can be specified with version: alpineX.YY or alpineedge" echo " for FreeBSD can be: freebsd13, freebsd14" @@ -15,6 +15,7 @@ usage() echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)." + echo "--skipemulation - optional, will skip qemu and debootstrap requirement when building environment for debian based systems." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." echo "--jobs N - optional, restrict to N jobs." exit 1 @@ -52,28 +53,27 @@ __UbuntuPackages+=" symlinks" __UbuntuPackages+=" libicu-dev" __UbuntuPackages+=" liblttng-ust-dev" __UbuntuPackages+=" libunwind8-dev" -__UbuntuPackages+=" libnuma-dev" __AlpinePackages+=" gettext-dev" __AlpinePackages+=" icu-dev" __AlpinePackages+=" libunwind-dev" __AlpinePackages+=" lttng-ust-dev" __AlpinePackages+=" compiler-rt" -__AlpinePackages+=" numactl-dev" # runtime libraries' dependencies __UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libssl-dev" __UbuntuPackages+=" zlib1g-dev" +__UbuntuPackages+=" libbrotli-dev" __AlpinePackages+=" curl-dev" __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="13.3-RELEASE" -__FreeBSDPkg="1.17.0" +__FreeBSDBase="13.4-RELEASE" +__FreeBSDPkg="1.21.3" __FreeBSDABI="13" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" @@ -91,18 +91,18 @@ __HaikuPackages="gcc_syslibs" __HaikuPackages+=" gcc_syslibs_devel" __HaikuPackages+=" gmp" __HaikuPackages+=" gmp_devel" -__HaikuPackages+=" icu66" -__HaikuPackages+=" icu66_devel" +__HaikuPackages+=" icu[0-9]+" +__HaikuPackages+=" icu[0-9]*_devel" __HaikuPackages+=" krb5" __HaikuPackages+=" krb5_devel" __HaikuPackages+=" libiconv" __HaikuPackages+=" libiconv_devel" -__HaikuPackages+=" llvm12_libunwind" -__HaikuPackages+=" llvm12_libunwind_devel" +__HaikuPackages+=" llvm[0-9]*_libunwind" +__HaikuPackages+=" llvm[0-9]*_libunwind_devel" __HaikuPackages+=" mpfr" __HaikuPackages+=" mpfr_devel" -__HaikuPackages+=" openssl" -__HaikuPackages+=" openssl_devel" +__HaikuPackages+=" openssl3" +__HaikuPackages+=" openssl3_devel" __HaikuPackages+=" zlib" __HaikuPackages+=" zlib_devel" @@ -128,10 +128,12 @@ __AlpineKeys=' 616adfeb:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU\npNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X\n0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6\nADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ\nraQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy\nKLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj\n/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H\nKuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs\nmFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1\n/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq\nXIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4\nVORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ== 616ae350:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0\nXkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ\npiM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46\nxLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP\nODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM\nEyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr\nDzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+\nf53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF\nHwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk\nOk9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6\n9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT\nJZDr++FjKrnnijbyNF8b98UCAwEAAQ== 616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ== +66ba20fe:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtfB12w4ZgqsXWZDfUAV/\n6Y4aHUKIu3q4SXrNZ7CXF9nXoAVYrS7NAxJdAodsY3vPCN0g5O8DFXR+390LdOuQ\n+HsGKCc1k5tX5ZXld37EZNTNSbR0k+NKhd9h6X3u6wqPOx7SIKxwAQR8qeeFq4pP\nrt9GAGlxtuYgzIIcKJPwE0dZlcBCg+GnptCUZXp/38BP1eYC+xTXSL6Muq1etYfg\nodXdb7Yl+2h1IHuOwo5rjgY5kpY7GcAs8AjGk3lDD/av60OTYccknH0NCVSmPoXK\nvrxDBOn0LQRNBLcAfnTKgHrzy0Q5h4TNkkyTgxkoQw5ObDk9nnabTxql732yy9BY\ns+hM9+dSFO1HKeVXreYSA2n1ndF18YAvAumzgyqzB7I4pMHXq1kC/8bONMJxwSkS\nYm6CoXKyavp7RqGMyeVpRC7tV+blkrrUml0BwNkxE+XnwDRB3xDV6hqgWe0XrifD\nYTfvd9ScZQP83ip0r4IKlq4GMv/R5shcCRJSkSZ6QSGshH40JYSoiwJf5FHbj9ND\n7do0UAqebWo4yNx63j/wb2ULorW3AClv0BCFSdPsIrCStiGdpgJDBR2P2NZOCob3\nG9uMj+wJD6JJg2nWqNJxkANXX37Qf8plgzssrhrgOvB0fjjS7GYhfkfmZTJ0wPOw\nA8+KzFseBh4UFGgue78KwgkCAwEAAQ== ' __Keyring= __KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg" __SkipSigCheck=0 +__SkipEmulation=0 __UseMirror=0 __UnprocessedBuildArgs= @@ -162,9 +164,13 @@ while :; do armel) __BuildArch=armel __UbuntuArch=armel - __UbuntuRepo="http://ftp.debian.org/debian/" - __CodeName=jessie + __UbuntuRepo="http://archive.debian.org/debian/" + __CodeName=buster __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" + __LLDB_Package="liblldb-6.0-dev" + __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp5/}" + __UbuntuSuites= ;; armv6) __BuildArch=armv6 @@ -180,6 +186,18 @@ while :; do __Keyring="--keyring $__KeyringFile" fi ;; + loongarch64) + __BuildArch=loongarch64 + __AlpineArch=loongarch64 + __QEMUArch=loongarch64 + __UbuntuArch=loong64 + __UbuntuSuites=unreleased + __LLDB_Package="liblldb-19-dev" + + if [[ "$__CodeName" == "sid" ]]; then + __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/" + fi + ;; riscv64) __BuildArch=riscv64 __AlpineArch=riscv64 @@ -264,44 +282,21 @@ while :; do ;; xenial) # Ubuntu 16.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=xenial - fi - ;; - zesty) # Ubuntu 17.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=zesty - fi + __CodeName=xenial ;; bionic) # Ubuntu 18.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=bionic - fi + __CodeName=bionic ;; focal) # Ubuntu 20.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=focal - fi + __CodeName=focal ;; jammy) # Ubuntu 22.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=jammy - fi + __CodeName=jammy ;; noble) # Ubuntu 24.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=noble - fi - if [[ -n "$__LLDB_Package" ]]; then - __LLDB_Package="liblldb-18-dev" - fi - ;; - jessie) # Debian 8 - __CodeName=jessie - __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" - - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __CodeName=noble + if [[ -z "$__LLDB_Package" ]]; then + __LLDB_Package="liblldb-19-dev" fi ;; stretch) # Debian 9 @@ -319,7 +314,7 @@ while :; do __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + __UbuntuRepo="http://archive.debian.org/debian/" fi ;; bullseye) # Debian 11 @@ -340,10 +335,28 @@ while :; do ;; sid) # Debian sid __CodeName=sid - __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" + __UbuntuSuites= - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" + # Debian-Ports architectures need different values + case "$__UbuntuArch" in + amd64|arm64|armel|armhf|i386|mips64el|ppc64el|riscv64|s390x) + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.debian.org/debian/" + fi + ;; + *) + __KeyringFile="/usr/share/keyrings/debian-ports-archive-keyring.gpg" + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/" + fi + ;; + esac + + if [[ -e "$__KeyringFile" ]]; then + __Keyring="--keyring $__KeyringFile" fi ;; tizen) @@ -370,7 +383,7 @@ while :; do ;; freebsd14) __CodeName=freebsd - __FreeBSDBase="14.0-RELEASE" + __FreeBSDBase="14.2-RELEASE" __FreeBSDABI="14" __SkipUnmount=1 ;; @@ -388,6 +401,9 @@ while :; do --skipsigcheck) __SkipSigCheck=1 ;; + --skipemulation) + __SkipEmulation=1 + ;; --rootfsdir|-rootfsdir) shift __RootfsDir="$1" @@ -420,16 +436,15 @@ case "$__AlpineVersion" in elif [[ "$__AlpineArch" == "x86" ]]; then __AlpineVersion=3.17 # minimum version that supports lldb-dev __AlpinePackages+=" llvm15-libs" - elif [[ "$__AlpineArch" == "riscv64" ]]; then + elif [[ "$__AlpineArch" == "riscv64" || "$__AlpineArch" == "loongarch64" ]]; then + __AlpineVersion=3.21 # minimum version that supports lldb-dev + __AlpinePackages+=" llvm19-libs" + elif [[ -n "$__AlpineMajorVersion" ]]; then + # use whichever alpine version is provided and select the latest toolchain libs __AlpineLlvmLibsLookup=1 - __AlpineVersion=edge # minimum version with APKINDEX.tar.gz (packages archive) else __AlpineVersion=3.13 # 3.13 to maximize compatibility __AlpinePackages+=" llvm10-libs" - - if [[ "$__AlpineArch" == "armv7" ]]; then - __AlpinePackages="${__AlpinePackages//numactl-dev/}" - fi fi esac @@ -439,15 +454,6 @@ if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then __AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}" fi -if [[ "$__BuildArch" == "armel" ]]; then - __LLDB_Package="lldb-3.5-dev" -fi - -if [[ "$__CodeName" == "xenial" && "$__UbuntuArch" == "armhf" ]]; then - # libnuma-dev is not available on armhf for xenial - __UbuntuPackages="${__UbuntuPackages//libnuma-dev/}" -fi - __UbuntuPackages+=" ${__LLDB_Package:-}" if [[ -z "$__UbuntuRepo" ]]; then @@ -496,7 +502,7 @@ if [[ "$__CodeName" == "alpine" ]]; then arch="$(uname -m)" ensureDownloadTool - + if [[ "$__hasWget" == 1 ]]; then wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" else @@ -512,11 +518,6 @@ if [[ "$__CodeName" == "alpine" ]]; then echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c chmod +x "$__ApkToolsDir/apk.static" - if [[ -f "/usr/bin/qemu-$__QEMUArch-static" ]]; then - mkdir -p "$__RootfsDir"/usr/bin - cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin" - fi - if [[ "$__AlpineVersion" == "edge" ]]; then version=edge else @@ -536,6 +537,10 @@ if [[ "$__CodeName" == "alpine" ]]; then __ApkSignatureArg="--keys-dir $__ApkKeysDir" fi + if [[ "$__SkipEmulation" == "1" ]]; then + __NoEmulationArg="--no-scripts" + fi + # initialize DB # shellcheck disable=SC2086 "$__ApkToolsDir/apk.static" \ @@ -557,7 +562,7 @@ if [[ "$__CodeName" == "alpine" ]]; then "$__ApkToolsDir/apk.static" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ + -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \ add $__AlpinePackages rm -r "$__ApkToolsDir" @@ -573,7 +578,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version fi echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf - echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf + echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf mkdir -p "$__RootfsDir"/tmp # get and build package manager if [[ "$__hasWget" == 1 ]]; then @@ -681,7 +686,7 @@ elif [[ "$__CodeName" == "haiku" ]]; then ensureDownloadTool - echo "Downloading Haiku package tool" + echo "Downloading Haiku package tools" git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script" if [[ "$__hasWget" == 1 ]]; then wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)" @@ -691,34 +696,42 @@ elif [[ "$__CodeName" == "haiku" ]]; then unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" - DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" - HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" + HaikuBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" + HaikuPortsBaseUrl="https://eu.hpkg.haiku-os.org/haikuports/master/$__HaikuArch/current" + + echo "Downloading HaikuPorts package repository index..." + if [[ "$__hasWget" == 1 ]]; then + wget -P "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo" + else + curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo" + fi - # Download Haiku packages echo "Downloading Haiku packages" read -ra array <<<"$__HaikuPackages" for package in "${array[@]}"; do echo "Downloading $package..." - # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 - # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 + hpkgFilename="$(LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package_repo" list -f "$__RootfsDir/tmp/download/repo" | + grep -E "${package}-" | sort -V | tail -n 1 | xargs)" + if [ -z "$hpkgFilename" ]; then + >&2 echo "ERROR: package $package missing." + exit 1 + fi + echo "Resolved filename: $hpkgFilename..." + hpkgDownloadUrl="$HaikuPortsBaseUrl/packages/$hpkgFilename" if [[ "$__hasWget" == 1 ]]; then - hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" else - hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" fi done for package in haiku haiku_devel; do echo "Downloading $package..." if [[ "$__hasWget" == 1 ]]; then - hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" - wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + hpkgVersion="$(wget -qO- "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" else - hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" - curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" + hpkgVersion="$(curl -sSL "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" fi done @@ -744,25 +757,67 @@ elif [[ "$__CodeName" == "haiku" ]]; then popd rm -rf "$__RootfsDir/tmp" elif [[ -n "$__CodeName" ]]; then + __Suites="$__CodeName $(for suite in $__UbuntuSuites; do echo -n "$__CodeName-$suite "; done)" + + if [[ "$__SkipEmulation" == "1" ]]; then + if [[ -z "$AR" ]]; then + if command -v ar &>/dev/null; then + AR="$(command -v ar)" + elif command -v llvm-ar &>/dev/null; then + AR="$(command -v llvm-ar)" + else + echo "Unable to find ar or llvm-ar on PATH, add them to PATH or set AR environment variable pointing to the available AR tool" + exit 1 + fi + fi + + PYTHON=${PYTHON_EXECUTABLE:-python3} + + # shellcheck disable=SC2086,SC2046 + echo running "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ + $(for suite in $__Suites; do echo -n "--suite $suite "; done) \ + $__UbuntuPackages + + # shellcheck disable=SC2086,SC2046 + "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \ + $(for suite in $__Suites; do echo -n "--suite $suite "; done) \ + $__UbuntuPackages + exit 0 + fi + + __UpdateOptions= if [[ "$__SkipSigCheck" == "0" ]]; then __Keyring="$__Keyring --force-check-gpg" + else + __Keyring= + __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories" fi # shellcheck disable=SC2086 echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" - debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + # shellcheck disable=SC2086 + if ! debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then + echo "debootstrap failed! dumping debootstrap.log" + cat "$__RootfsDir/debootstrap/debootstrap.log" + exit 1 + fi + + rm -rf "$__RootfsDir"/etc/apt/*.{sources,list} "$__RootfsDir"/etc/apt/sources.list.d mkdir -p "$__RootfsDir/etc/apt/sources.list.d/" + + # shellcheck disable=SC2086 cat > "$__RootfsDir/etc/apt/sources.list.d/$__CodeName.sources" < token2) - (token1 < token2) + else: + return -1 if isinstance(token1, str) else 1 + + return len(tokens1) - len(tokens2) + +def compare_debian_versions(version1, version2): + """Compare two Debian package versions.""" + epoch1, upstream1, revision1 = parse_debian_version(version1) + epoch2, upstream2, revision2 = parse_debian_version(version2) + + if epoch1 != epoch2: + return epoch1 - epoch2 + + result = compare_upstream_version(upstream1, upstream2) + if result != 0: + return result + + return compare_upstream_version(revision1, revision2) + +def resolve_dependencies(packages, aliases, desired_packages): + """Recursively resolves dependencies for the desired packages.""" + resolved = [] + to_process = deque(desired_packages) + + while to_process: + current = to_process.popleft() + resolved_package = current if current in packages else aliases.get(current, [None])[0] + + if not resolved_package: + print(f"Error: Package '{current}' was not found in the available packages.") + sys.exit(1) + + if resolved_package not in resolved: + resolved.append(resolved_package) + + deps = packages.get(resolved_package, {}).get("Depends", "") + if deps: + deps = [dep.split(' ')[0] for dep in deps.split(', ') if dep] + for dep in deps: + if dep not in resolved and dep not in to_process and dep in packages: + to_process.append(dep) + + return resolved + +def parse_package_index(content): + """Parses the Packages.gz file and returns package information.""" + packages = {} + aliases = {} + entries = re.split(r'\n\n+', content) + + for entry in entries: + fields = dict(re.findall(r'^(\S+): (.+)$', entry, re.MULTILINE)) + if "Package" in fields: + package_name = fields["Package"] + version = fields.get("Version") + filename = fields.get("Filename") + depends = fields.get("Depends") + provides = fields.get("Provides", None) + + # Only update if package_name is not in packages or if the new version is higher + if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0: + packages[package_name] = { + "Version": version, + "Filename": filename, + "Depends": depends + } + + # Update aliases if package provides any alternatives + if provides: + provides_list = [x.strip() for x in provides.split(",")] + for alias in provides_list: + # Strip version specifiers + alias_name = re.sub(r'\s*\(=.*\)', '', alias) + if alias_name not in aliases: + aliases[alias_name] = [] + if package_name not in aliases[alias_name]: + aliases[alias_name].append(package_name) + + return packages, aliases + +def install_packages(mirror, packages_info, aliases, tmp_dir, extract_dir, ar_tool, desired_packages): + """Downloads .deb files and extracts them.""" + resolved_packages = resolve_dependencies(packages_info, aliases, desired_packages) + print(f"Resolved packages (including dependencies): {resolved_packages}") + + packages_to_download = {} + + for pkg in resolved_packages: + if pkg in packages_info: + packages_to_download[pkg] = packages_info[pkg] + + if pkg in aliases: + for alias in aliases[pkg]: + if alias in packages_info: + packages_to_download[alias] = packages_info[alias] + + asyncio.run(download_deb_files_parallel(mirror, packages_to_download, tmp_dir)) + + package_to_deb_file_map = {} + for pkg in resolved_packages: + pkg_info = packages_info.get(pkg) + if pkg_info: + deb_filename = pkg_info.get("Filename") + if deb_filename: + deb_file_path = os.path.join(tmp_dir, os.path.basename(deb_filename)) + package_to_deb_file_map[pkg] = deb_file_path + + for pkg in reversed(resolved_packages): + deb_file = package_to_deb_file_map.get(pkg) + if deb_file and os.path.exists(deb_file): + extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool) + + print("All done!") + +def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): + """Extract .deb file contents""" + + os.makedirs(extract_dir, exist_ok=True) + + with tempfile.TemporaryDirectory(dir=tmp_dir) as tmp_subdir: + result = subprocess.run(f"{ar_tool} t {os.path.abspath(deb_file)}", cwd=tmp_subdir, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + tar_filename = None + for line in result.stdout.decode().splitlines(): + if line.startswith("data.tar"): + tar_filename = line.strip() + break + + if not tar_filename: + raise FileNotFoundError(f"Could not find 'data.tar.*' in {deb_file}.") + + tar_file_path = os.path.join(tmp_subdir, tar_filename) + print(f"Extracting {tar_filename} from {deb_file}..") + + subprocess.run(f"{ar_tool} p {os.path.abspath(deb_file)} {tar_filename} > {tar_file_path}", check=True, shell=True) + + file_extension = os.path.splitext(tar_file_path)[1].lower() + + if file_extension == ".xz": + mode = "r:xz" + elif file_extension == ".gz": + mode = "r:gz" + elif file_extension == ".zst": + # zstd is not supported by standard library yet + decompressed_tar_path = tar_file_path.replace(".zst", "") + with open(tar_file_path, "rb") as zst_file, open(decompressed_tar_path, "wb") as decompressed_file: + dctx = zstandard.ZstdDecompressor() + dctx.copy_stream(zst_file, decompressed_file) + + tar_file_path = decompressed_tar_path + mode = "r" + else: + raise ValueError(f"Unsupported compression format: {file_extension}") + + with tarfile.open(tar_file_path, mode) as tar: + tar.extractall(path=extract_dir, filter='fully_trusted') + +def finalize_setup(rootfsdir): + lib_dir = os.path.join(rootfsdir, 'lib') + usr_lib_dir = os.path.join(rootfsdir, 'usr', 'lib') + + if os.path.exists(lib_dir): + if os.path.islink(lib_dir): + os.remove(lib_dir) + else: + os.makedirs(usr_lib_dir, exist_ok=True) + + for item in os.listdir(lib_dir): + src = os.path.join(lib_dir, item) + dest = os.path.join(usr_lib_dir, item) + + if os.path.isdir(src): + shutil.copytree(src, dest, dirs_exist_ok=True) + else: + shutil.copy2(src, dest) + + shutil.rmtree(lib_dir) + + os.symlink(usr_lib_dir, lib_dir) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate rootfs for .NET runtime on Debian-like OS") + parser.add_argument("--distro", required=False, help="Distro name (e.g., debian, ubuntu, etc.)") + parser.add_argument("--arch", required=True, help="Architecture (e.g., amd64, loong64, etc.)") + parser.add_argument("--rootfsdir", required=True, help="Destination directory.") + parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.') + parser.add_argument("--mirror", required=False, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)") + parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)") + parser.add_argument("packages", nargs="+", help="List of package names to be installed.") + + args = parser.parse_args() + + if args.mirror is None: + if args.distro == "ubuntu": + args.mirror = "http://archive.ubuntu.com/ubuntu" if args.arch in ["amd64", "i386"] else "http://ports.ubuntu.com/ubuntu-ports" + elif args.distro == "debian": + args.mirror = "http://ftp.debian.org/debian-ports" + else: + raise Exception("Unsupported distro") + + DESIRED_PACKAGES = args.packages + [ # base packages + "dpkg", + "busybox", + "libc-bin", + "base-files", + "base-passwd", + "debianutils" + ] + + print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, distro: {args.distro}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}") + + package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite)) + + packages_info, aliases = parse_package_index(package_index_content) + + with tempfile.TemporaryDirectory() as tmp_dir: + install_packages(args.mirror, packages_info, aliases, tmp_dir, args.rootfsdir, args.artool, DESIRED_PACKAGES) + + finalize_setup(args.rootfsdir) diff --git a/eng/common/cross/tizen-fetch.sh b/eng/common/cross/tizen-fetch.sh index 28936ceef..37c3a61f1 100755 --- a/eng/common/cross/tizen-fetch.sh +++ b/eng/common/cross/tizen-fetch.sh @@ -156,13 +156,8 @@ fetch_tizen_pkgs() done } -if [ "$TIZEN_ARCH" == "riscv64" ]; then - BASE="Tizen-Base-RISCV" - UNIFIED="Tizen-Unified-RISCV" -else - BASE="Tizen-Base" - UNIFIED="Tizen-Unified" -fi +BASE="Tizen-Base" +UNIFIED="Tizen-Unified" Inform "Initialize ${TIZEN_ARCH} base" fetch_tizen_pkgs_init standard $BASE diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 9a4e285a5..0ff85cf03 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -40,7 +40,7 @@ if(TARGET_ARCH_NAME STREQUAL "arm") set(TOOLCHAIN "arm-linux-gnueabihf") endif() if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) @@ -49,7 +49,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(LINUX) set(TOOLCHAIN "aarch64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu") endif() elseif(FREEBSD) set(triple "aarch64-unknown-freebsd12") @@ -58,7 +58,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) set(TOOLCHAIN "arm-linux-gnueabi") if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi") endif() elseif(TARGET_ARCH_NAME STREQUAL "armv6") set(CMAKE_SYSTEM_PROCESSOR armv6l) @@ -67,6 +67,13 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6") else() set(TOOLCHAIN "arm-linux-gnueabihf") endif() +elseif(TARGET_ARCH_NAME STREQUAL "loongarch64") + set(CMAKE_SYSTEM_PROCESSOR "loongarch64") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/loongarch64-alpine-linux-musl) + set(TOOLCHAIN "loongarch64-alpine-linux-musl") + else() + set(TOOLCHAIN "loongarch64-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") set(CMAKE_SYSTEM_PROCESSOR ppc64le) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl) @@ -81,7 +88,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "riscv64") else() set(TOOLCHAIN "riscv64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "riscv64-tizen-linux-gnu/13.1.0") + set(TIZEN_TOOLCHAIN "riscv64-tizen-linux-gnu") endif() endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") @@ -98,7 +105,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(LINUX) set(TOOLCHAIN "x86_64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu") endif() elseif(FREEBSD) set(triple "x86_64-unknown-freebsd12") @@ -115,10 +122,10 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") set(TOOLCHAIN "i686-linux-gnu") endif() if(TIZEN) - set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu") endif() else() - message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!") + message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64 and x86 are supported!") endif() if(DEFINED ENV{TOOLCHAIN}) @@ -127,32 +134,46 @@ endif() # Specify include paths if(TIZEN) - if(TARGET_ARCH_NAME STREQUAL "arm") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf) - endif() - if(TARGET_ARCH_NAME STREQUAL "armel") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) - endif() - if(TARGET_ARCH_NAME STREQUAL "arm64") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) - endif() - if(TARGET_ARCH_NAME STREQUAL "x86") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu) - endif() - if(TARGET_ARCH_NAME STREQUAL "x64") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu) - endif() - if(TARGET_ARCH_NAME STREQUAL "riscv64") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/riscv64-tizen-linux-gnu) + function(find_toolchain_dir prefix) + # Dynamically find the version subdirectory + file(GLOB DIRECTORIES "${prefix}/*") + list(GET DIRECTORIES 0 FIRST_MATCH) + get_filename_component(TOOLCHAIN_VERSION ${FIRST_MATCH} NAME) + + set(TIZEN_TOOLCHAIN_PATH "${prefix}/${TOOLCHAIN_VERSION}" PARENT_SCOPE) + endfunction() + + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") + find_toolchain_dir("${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + else() + find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() + + message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}") + + include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++) + include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN}) endif() +function(locate_toolchain_exec exec var) + set(TOOLSET_PREFIX ${TOOLCHAIN}-) + string(TOUPPER ${exec} EXEC_UPPERCASE) + if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") + set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) + return() + endif() + + find_program(EXEC_LOCATION_${exec} + NAMES + "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" + "${TOOLSET_PREFIX}${exec}") + + if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") + message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") + endif() + set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) +endfunction() + if(ANDROID) if(TARGET_ARCH_NAME STREQUAL "arm") set(ANDROID_ABI armeabi-v7a) @@ -183,66 +204,24 @@ elseif(FREEBSD) set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") elseif(ILLUMOS) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") include_directories(SYSTEM ${CROSS_ROOTFS}/include) - set(TOOLSET_PREFIX ${TOOLCHAIN}-) - function(locate_toolchain_exec exec var) - string(TOUPPER ${exec} EXEC_UPPERCASE) - if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") - set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) - return() - endif() - - find_program(EXEC_LOCATION_${exec} - NAMES - "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" - "${TOOLSET_PREFIX}${exec}") - - if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") - message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") - endif() - set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) - endfunction() - - set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") - locate_toolchain_exec(gcc CMAKE_C_COMPILER) locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) - - set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") - set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") elseif(HAIKU) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") - - set(TOOLSET_PREFIX ${TOOLCHAIN}-) - function(locate_toolchain_exec exec var) - string(TOUPPER ${exec} EXEC_UPPERCASE) - if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") - set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) - return() - endif() - - find_program(EXEC_LOCATION_${exec} - NAMES - "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" - "${TOOLSET_PREFIX}${exec}") - - if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") - message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") - endif() - set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) - endfunction() - set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") locate_toolchain_exec(gcc CMAKE_C_COMPILER) locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) - set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") - set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") - # let CMake set up the correct search paths include(Platform/Haiku) else() @@ -272,21 +251,21 @@ endif() if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") endif() elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64|riscv64)$") if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") - add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${TIZEN_TOOLCHAIN_PATH}") endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") add_toolchain_linker_flag("--target=${TOOLCHAIN}") @@ -297,10 +276,10 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") endif() add_toolchain_linker_flag(-m32) if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") endif() elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") @@ -312,7 +291,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 36dbd45e1..e889f439b 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -68,7 +68,7 @@ function InstallDarcCli { fi fi - local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" + local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." diff --git a/eng/common/dotnet.cmd b/eng/common/dotnet.cmd new file mode 100644 index 000000000..527fa4bb3 --- /dev/null +++ b/eng/common/dotnet.cmd @@ -0,0 +1,7 @@ +@echo off + +:: This script is used to install the .NET SDK. +:: It will also invoke the SDK with any provided arguments. + +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet.ps1""" %*" +exit /b %ErrorLevel% diff --git a/eng/common/dotnet.ps1 b/eng/common/dotnet.ps1 new file mode 100644 index 000000000..45e5676c9 --- /dev/null +++ b/eng/common/dotnet.ps1 @@ -0,0 +1,11 @@ +# This script is used to install the .NET SDK. +# It will also invoke the SDK with any provided arguments. + +. $PSScriptRoot\tools.ps1 +$dotnetRoot = InitializeDotNetCli -install:$true + +# Invoke acquired SDK with args if they are provided +if ($args.count -gt 0) { + $env:DOTNET_NOLOGO=1 + & "$dotnetRoot\dotnet.exe" $args +} diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh new file mode 100644 index 000000000..2ef682356 --- /dev/null +++ b/eng/common/dotnet.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# This script is used to install the .NET SDK. +# It will also invoke the SDK with any provided arguments. + +source="${BASH_SOURCE[0]}" +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +source $scriptroot/tools.sh +InitializeDotNetCli true # install + +# Invoke acquired SDK with args if they are provided +if [[ $# > 0 ]]; then + __dotnetDir=${_InitializeDotNetCli} + dotnetPath=${__dotnetDir}/dotnet + ${dotnetPath} "$@" +fi diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 524aaa57f..fa1cdc2b3 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -33,15 +33,27 @@ $jsonTemplateFiles | ForEach-Object { $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$wxlFilesV3 = @() +$wxlFilesV5 = @() $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them if (-not $wxlFiles) { $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files if ($wxlEnFiles) { - $wxlFiles = @() - $wxlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $content = Get-Content $_.FullName -Raw + + # Split files on schema to select different parser settings in the generated project. + if ($content -like "*http://wixtoolset.org/schemas/v4/wxl*") + { + $wxlFilesV5 += Copy-Item $_.FullName -Destination $destinationFile -PassThru + } + elseif ($content -like "*http://schemas.microsoft.com/wix/2006/localization*") + { + $wxlFilesV3 += Copy-Item $_.FullName -Destination $destinationFile -PassThru + } + } } } @@ -114,7 +126,32 @@ $locJson = @{ CloneLanguageSet = "WiX_CloneLanguages" LssFiles = @( "wxl_loc.lss" ) LocItems = @( - $wxlFiles | ForEach-Object { + $wxlFilesV3 | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "WiX_CloneLanguages" + LssFiles = @( "P210WxlSchemaV4.lss" ) + LocItems = @( + $wxlFilesV5 | ForEach-Object { $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 index 3e5c1c74a..a0c7d792a 100644 --- a/eng/common/generate-sbom-prep.ps1 +++ b/eng/common/generate-sbom-prep.ps1 @@ -4,18 +4,26 @@ Param( . $PSScriptRoot\pipeline-logging-functions.ps1 +# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly +# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. +$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" +$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' +$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName + +Write-Host "Artifact name before : $ArtifactName" +Write-Host "Artifact name after : $SafeArtifactName" + Write-Host "Creating dir $ManifestDirPath" + # create directory for sbom manifest to be placed -if (!(Test-Path -path $ManifestDirPath)) +if (!(Test-Path -path $SbomGenerationDir)) { - New-Item -ItemType Directory -path $ManifestDirPath - Write-Host "Successfully created directory $ManifestDirPath" + New-Item -ItemType Directory -path $SbomGenerationDir + Write-Host "Successfully created directory $SbomGenerationDir" } else{ Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." } Write-Host "Updating artifact name" -$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_' -Write-Host "Artifact name $artifact_name" -Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name" +Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh index d5c76dc82..b8ecca72b 100755 --- a/eng/common/generate-sbom-prep.sh +++ b/eng/common/generate-sbom-prep.sh @@ -14,19 +14,24 @@ done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" . $scriptroot/pipeline-logging-functions.sh + +# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. +artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" +safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" manifest_dir=$1 -if [ ! -d "$manifest_dir" ] ; then - mkdir -p "$manifest_dir" - echo "Sbom directory created." $manifest_dir +# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly +# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. +sbom_generation_dir="$manifest_dir/$safe_artifact_name" + +if [ ! -d "$sbom_generation_dir" ] ; then + mkdir -p "$sbom_generation_dir" + echo "Sbom directory created." $sbom_generation_dir else Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." fi -artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" echo "Artifact name before : "$artifact_name -# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. -safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" echo "Artifact name after : "$safe_artifact_name export ARTIFACT_NAME=$safe_artifact_name echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config index 19d3d311b..f70261ed6 100644 --- a/eng/common/internal/NuGet.config +++ b/eng/common/internal/NuGet.config @@ -4,4 +4,7 @@ + + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 32f79dfb3..feaa6d208 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -15,16 +15,6 @@ - - - - https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json; - - - $(RestoreSources); - https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; - - diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh new file mode 100644 index 000000000..477a44f33 --- /dev/null +++ b/eng/common/native/install-dependencies.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +set -e + +# This is a simple script primarily used for CI to install necessary dependencies +# +# Usage: +# +# ./install-dependencies.sh + +os="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + +if [ -z "$os" ]; then + . "$(dirname "$0")"/init-os-and-arch.sh +fi + +case "$os" in + linux) + if [ -e /etc/os-release ]; then + . /etc/os-release + fi + + if [ "$ID" = "debian" ] || [ "$ID_LIKE" = "debian" ]; then + apt update + + apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \ + libssl-dev libkrb5-dev pigz cpio + + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 + elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then + pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)" + $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio + elif [ "$ID" = "alpine" ]; then + apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio + else + echo "Unsupported distro. distro: $ID" + exit 1 + fi + ;; + + osx|maccatalyst|ios|iossimulator|tvos|tvossimulator) + echo "Installed xcode version: $(xcode-select -p)" + + export HOMEBREW_NO_INSTALL_CLEANUP=1 + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + # Skip brew update for now, see https://github.com/actions/setup-python/issues/577 + # brew update --preinstall + brew bundle --no-upgrade --file=- < Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -excludeCIBinaryLog When running on CI, allow no binary log (short: -nobl)" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." } @@ -34,10 +37,11 @@ function Print-Usage() { function Build([string]$target) { $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" } $log = Join-Path $LogDir "$task$logSuffix.binlog" + $binaryLogArg = if ($binaryLog) { "/bl:$log" } else { "" } $outputPath = Join-Path $ToolsetDir "$task\" MSBuild $taskProject ` - /bl:$log ` + $binaryLogArg ` /t:$target ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` @@ -64,7 +68,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.10.0-pre.4.0" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.13.0" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh new file mode 100644 index 000000000..3270f83fa --- /dev/null +++ b/eng/common/sdk-task.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +show_usage() { + echo "Common settings:" + echo " --task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + echo " --restore Restore dependencies" + echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" + echo " --help Print help and exit" + echo "" + + echo "Advanced settings:" + echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" + echo " --noWarnAsError Do not warn as error" + echo "" + echo "Command line arguments not listed above are passed thru to msbuild." +} + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +Build() { + local target=$1 + local log_suffix="" + [[ "$target" != "Execute" ]] && log_suffix=".$target" + local log="$log_dir/$task$log_suffix.binlog" + local binaryLogArg="" + [[ $binary_log == true ]] && binaryLogArg="/bl:$log" + local output_path="$toolset_dir/$task/" + + MSBuild "$taskProject" \ + $binaryLogArg \ + /t:"$target" \ + /p:Configuration="$configuration" \ + /p:RepoRoot="$repo_root" \ + /p:BaseIntermediateOutputPath="$output_path" \ + /v:"$verbosity" \ + $properties +} + +binary_log=true +configuration="Debug" +verbosity="minimal" +exclude_ci_binary_log=false +restore=false +help=false +properties='' +warnAsError=true + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --task) + task=$2 + shift 2 + ;; + --restore) + restore=true + shift 1 + ;; + --verbosity) + verbosity=$2 + shift 2 + ;; + --excludecibinarylog|--nobl) + binary_log=false + exclude_ci_binary_log=true + shift 1 + ;; + --noWarnAsError) + warnAsError=false + shift 1 + ;; + --help) + help=true + shift 1 + ;; + *) + properties="$properties $1" + shift 1 + ;; + esac +done + +ci=true + +if $help; then + show_usage + exit 0 +fi + +. "$scriptroot/tools.sh" +InitializeToolset + +if [[ -z "$task" ]]; then + Write-PipelineTelemetryError -Category 'Task' -Name 'MissingTask' -Message "Missing required parameter '-task '" + ExitWithExitCode 1 +fi + +taskProject=$(GetSdkTaskProject "$task") +if [[ ! -e "$taskProject" ]]; then + Write-PipelineTelemetryError -Category 'Task' -Name 'UnknownTask' -Message "Unknown task: $task" + ExitWithExitCode 1 +fi + +if $restore; then + Build "Restore" +fi + +Build "Execute" + + +ExitWithExitCode 0 diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 4585cfd6b..e5f543ea6 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 800364742..06b44de78 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -42,7 +42,7 @@ [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } # Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1 +# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.ps1 [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' } # True to use global NuGet cache instead of restoring packages to repository-local directory. @@ -65,10 +65,8 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } -# True if the build is a product build -[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } - -[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() } +# True when the build is running within the VMR. +[bool]$fromVMR = if (Test-Path variable:fromVMR) { $fromVMR } else { $false } function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null @@ -259,10 +257,23 @@ function Retry($downloadBlock, $maxRetries = 5) { function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' + $shouldDownload = $false + if (!(Test-Path $installScript)) { + $shouldDownload = $true + } else { + # Check if the script is older than 30 days + $fileAge = (Get-Date) - (Get-Item $installScript).LastWriteTime + if ($fileAge.Days -gt 30) { + Write-Host "Existing install script is too old, re-downloading..." + $shouldDownload = $true + } + } + + if ($shouldDownload) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" + $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" Retry({ Write-Host "GET $uri" @@ -383,8 +394,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.10.0-pre.4.0 - $defaultXCopyMSBuildVersion = '17.10.0-pre.4.0' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.13.0 + $defaultXCopyMSBuildVersion = '17.13.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -416,7 +427,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # Locate Visual Studio installation or download x-copy msbuild. $vsInfo = LocateVisualStudio $vsRequirements - if ($vsInfo -ne $null) { + if ($vsInfo -ne $null -and $env:ForceUseXCopyMSBuild -eq $null) { # Ensure vsInstallDir has a trailing slash $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] @@ -533,7 +544,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { - $vswhereVersion = '2.5.2' + # keep this in sync with the VSWhereVersion in DefaultVersions.props + $vswhereVersion = '3.1.7' } $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion" @@ -541,7 +553,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir - Write-Host 'Downloading vswhere' + Write-Host "Downloading vswhere $vswhereVersion" + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Retry({ Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe }) @@ -604,14 +617,7 @@ function InitializeBuildTool() { } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - # Use override if it exists - commonly set by source-build - if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) { - $initializeBuildToolFramework="net9.0" - } else { - $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework - } - - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework } + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -620,7 +626,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS } + $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "netframework"; ExcludePrereleaseVS = $excludePrereleaseVS } } else { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 @@ -653,7 +659,6 @@ function GetNuGetPackageCachePath() { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOHTTPCACHE = $true } } @@ -775,26 +780,13 @@ function MSBuild() { $toolsetBuildProject = InitializeToolset $basePath = Split-Path -parent $toolsetBuildProject - $possiblePaths = @( - # new scripts need to work with old packages, so we need to look for the old names/versions - (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), - (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')), - (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll')) - ) - $selectedPath = $null - foreach ($path in $possiblePaths) { - if (Test-Path $path -PathType Leaf) { - $selectedPath = $path - break - } - } + $selectedPath = Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll') + if (-not $selectedPath) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.' + Write-PipelineTelemetryError -Category 'Build' -Message "Unable to find arcade sdk logger assembly: $selectedPath" ExitWithExitCode 1 } + $args += "/logger:$selectedPath" } @@ -857,8 +849,8 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) { + # Skip this when the build is a child of the VMR build. + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error @@ -900,7 +892,7 @@ function IsWindowsPlatform() { } function Get-Darc($version) { - $darcPath = "$TempDir\darc\$(New-Guid)" + $darcPath = "$TempDir\darc\$([guid]::NewGuid())" if ($version -ne $null) { & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host } else { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 1159726a1..c1841c9df 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -5,6 +5,9 @@ # CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} +# Build mode +source_build=${source_build:-false} + # Set to true to use the pipelines logger which will enable Azure logging output. # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md # This flag is meant as a temporary opt-opt for the feature while validate it across @@ -54,11 +57,12 @@ warn_as_error=${warn_as_error:-true} use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} # Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh +# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'} # True to use global NuGet cache instead of restoring packages to repository-local directory. -if [[ "$ci" == true ]]; then +# Keep in sync with NuGetPackageroot in Arcade SDK's RepositoryLayout.props. +if [[ "$ci" == true || "$source_build" == true ]]; then use_global_nuget_cache=${use_global_nuget_cache:-false} else use_global_nuget_cache=${use_global_nuget_cache:-true} @@ -68,8 +72,8 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} -# True if the build is a product build -product_build=${product_build:-false} +# True when the build is running within the VMR. +from_vmr=${from_vmr:-false} # Resolve any symlinks in the given path. function ResolvePath { @@ -295,9 +299,30 @@ function with_retries { function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" - local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" + local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" + local timestamp_file="$root/.dotnet-install.timestamp" + local should_download=false if [[ ! -a "$install_script" ]]; then + should_download=true + elif [[ -f "$timestamp_file" ]]; then + # Check if the script is older than 30 days using timestamp file + local download_time=$(cat "$timestamp_file" 2>/dev/null || echo "0") + local current_time=$(date +%s) + local age_seconds=$((current_time - download_time)) + + # 30 days = 30 * 24 * 60 * 60 = 2592000 seconds + if [[ $age_seconds -gt 2592000 ]]; then + echo "Existing install script is too old, re-downloading..." + should_download=true + fi + else + # No timestamp file exists, assume script is old and re-download + echo "No timestamp found for existing install script, re-downloading..." + should_download=true + fi + + if [[ "$should_download" == true ]]; then mkdir -p "$root" echo "Downloading '$install_script_url'" @@ -324,6 +349,9 @@ function GetDotNetInstallScript { ExitWithExitCode $exit_code } fi + + # Create timestamp file to track download time in seconds from epoch + date +%s > "$timestamp_file" fi # return value _GetDotNetInstallScript="$install_script" @@ -339,22 +367,14 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - # use override if it exists - commonly set by source-build - if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then - _InitializeBuildToolFramework="net9.0" - else - _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}" - fi } -# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" else export NUGET_PACKAGES="$repo_root/.packages/" - export RESTORENOHTTPCACHE=true fi fi @@ -451,25 +471,13 @@ function MSBuild { fi local toolset_dir="${_InitializeToolset%/*}" - # new scripts need to work with old packages, so we need to look for the old names/versions - local selectedPath= - local possiblePaths=() - possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" ) - for path in "${possiblePaths[@]}"; do - if [[ -f $path ]]; then - selectedPath=$path - break - fi - done + local selectedPath="$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll" + if [[ -z "$selectedPath" ]]; then - Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly." + Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly: $selectedPath" ExitWithExitCode 1 fi + args+=( "-logger:$selectedPath" ) fi @@ -506,8 +514,8 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then + # Skip this when the build is a child of the VMR build. + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error @@ -530,6 +538,13 @@ function GetDarc { fi "$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version + darc_tool="$darc_path/darc" +} + +# Returns a full path to an Arcade SDK task project file. +function GetSdkTaskProject { + taskName=$1 + echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj" } ResolvePath "${BASH_SOURCE[0]}" diff --git a/global.json b/global.json index 19bec01a5..b54a37d32 100644 --- a/global.json +++ b/global.json @@ -10,7 +10,7 @@ }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24453.1", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24453.1" + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25517.104", + "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.25517.104" } } From 2cad1e9ad750b7836f6506f06f84100c740ac1a5 Mon Sep 17 00:00:00 2001 From: martincostello Date: Fri, 24 Oct 2025 10:16:21 +0100 Subject: [PATCH 14/15] Set executable bit Set the executable bit for Linux and macOS. --- eng/common/dotnet.sh | 0 eng/common/native/install-dependencies.sh | 0 eng/common/sdk-task.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 eng/common/dotnet.sh mode change 100644 => 100755 eng/common/native/install-dependencies.sh mode change 100644 => 100755 eng/common/sdk-task.sh diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh old mode 100644 new mode 100755 diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh old mode 100644 new mode 100755 diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh old mode 100644 new mode 100755 From 16f2dee1b66561c00df29be62e3bdfbea30a2847 Mon Sep 17 00:00:00 2001 From: martincostello Date: Sun, 19 Oct 2025 14:48:24 +0100 Subject: [PATCH 15/15] Update to .NET 10 RTM Update to stable version of .NET 10. --- Directory.Packages.props | 6 +++--- eng/Versions.props | 6 +++--- global.json | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 3965cc84d..d11733e4d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,9 +4,9 @@ - - - + + + diff --git a/eng/Versions.props b/eng/Versions.props index a1ec564a7..eb8c82735 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,9 +6,9 @@ 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) 10.0.0 - rc - 2 - Release Candidate $(PreReleaseVersionIteration) + preview + + Preview $(PreReleaseVersionIteration) false release true diff --git a/global.json b/global.json index b54a37d32..3dcc5ba85 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,12 @@ { "sdk": { - "version": "10.0.100-rc.2.25502.107", + "version": "10.0.100", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "10.0.100-rc.2.25502.107" + "dotnet": "10.0.100" }, "msbuild-sdks": {