Update Arcade & opt into automated pool provider usage functionality

azure-pipelines.yml

@@ -68,6 +68,8 @@ variables:
   - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}:
     - name: RunningAsPullRequest
       value: true
+  # Pick up pool provider name behavior from shared yaml template
+  - template: /eng/common/templates/variables/pool-providers.yml
 
 # Variables defined in yml cannot be overridden at queue time; instead overridable variables must be defined in the web UI.
 # Commenting out until something like this is supported: https://github.com/Microsoft/azure-pipelines-yaml/pull/129
@@ -104,7 +106,7 @@ stages:
         jobs:
         - job: Full_Signed
           pool:
-            name: NetCore1ESPool-Svc-Internal
+            name: $(DncEngInternalBuildPool)
             demands: ImageOverride -equals windows.vs2022.amd64
           timeoutInMinutes: 300
           variables:
@@ -210,7 +212,7 @@ stages:
           - name: _SignType
             value: Test
           pool:
-            name: NetCore-Public
+            name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           timeoutInMinutes: 90
           steps:
@@ -283,7 +285,7 @@ stages:
             #   WindowsMachineQueueName=Windows.vs2022.amd64.open
             # and there is an alternate build definition that sets this to a queue that is always scouting the
             # next preview of Visual Studio.
-            name: NetCore-Public
+            name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           timeoutInMinutes: 120
           strategy:
@@ -341,7 +343,7 @@ stages:
             #   WindowsMachineQueueName=Windows.vs2022.amd64.open
             # and there is an alternate build definition that sets this to a queue that is always scouting the
             # next preview of Visual Studio.
-            name: NetCore-Public
+            name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           timeoutInMinutes: 120
           strategy:
@@ -395,7 +397,7 @@ stages:
         # Mock official build
         - job: MockOfficial
           pool:
-            name: NetCore-Public
+            name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           steps:
           - checkout: self
@@ -486,7 +488,7 @@ stages:
         # End to end build
         - job: EndToEndBuildTests
           pool:
-            name: NetCore-Public
+            name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           steps:
           - checkout: self
@@ -515,7 +517,7 @@ stages:
             - name: _SignType
               value: Test
           pool:
-            name: NetCore-Public
+            name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           timeoutInMinutes: 90
           steps:
@@ -541,7 +543,7 @@ stages:
         # Plain build Windows
         - job: Plain_Build_Windows
           pool:
-            name: NetCore-Public
+            name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           variables:
           - name: _BuildConfig
@@ -638,7 +640,7 @@ stages:
         # Test trimming on Windows
         - job: Build_And_Test_Trimming_Windows
           pool:
-            name: NetCore-Public
+            name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           strategy:
             maxParallel: 2

eng/Version.Details.xml

@@ -8,14 +8,14 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.22473.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.22524.5">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ba4d2568dd2e3e7538feeaba60215f7bcb99e89c</Sha>
+      <Sha>c5dd6a1da2e6d9b3423ab809fcda8af2927a408b</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.22473.1">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.22524.5">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ba4d2568dd2e3e7538feeaba60215f7bcb99e89c</Sha>
+      <Sha>c5dd6a1da2e6d9b3423ab809fcda8af2927a408b</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -56,9 +56,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <VSMajorVersion>17</VSMajorVersion>
-
     <VSMinorVersion>4</VSMinorVersion>
-
     <VSGeneralVersion>$(VSMajorVersion).0</VSGeneralVersion>
     <VSAssemblyVersionPrefix>$(VSMajorVersion).$(VSMinorVersion).0</VSAssemblyVersionPrefix>
     <VSAssemblyVersion>$(VSAssemblyVersionPrefix).0</VSAssemblyVersion>

eng/common/cross/arm/tizen-fetch.sh

@@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then
 	mkdir -p $TMPDIR 
 fi
 
-TIZEN_URL=http://download.tizen.org/snapshots/tizen
+TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen
 BUILD_XML=build.xml
 REPOMD_XML=repomd.xml
 PRIMARY_XML=primary.xml
@@ -155,7 +155,7 @@ fetch_tizen_pkgs()
 }
 
 Inform "Initialize arm base"
-fetch_tizen_pkgs_init standard base
+fetch_tizen_pkgs_init standard Tizen-Base
 Inform "fetch common packages"
 fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
 Inform "fetch coreclr packages"
@@ -164,7 +164,7 @@ Inform "fetch corefx packages"
 fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
 
 Inform "Initialize standard unified"
-fetch_tizen_pkgs_init standard unified
+fetch_tizen_pkgs_init standard Tizen-Unified
 Inform "fetch corefx packages"
 fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release
 

eng/common/cross/arm64/tizen-fetch.sh

@@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then
 	mkdir -p $TMPDIR
 fi
 
-TIZEN_URL=http://download.tizen.org/snapshots/tizen/
+TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen
 BUILD_XML=build.xml
 REPOMD_XML=repomd.xml
 PRIMARY_XML=primary.xml
@@ -154,8 +154,8 @@ fetch_tizen_pkgs()
 	done
 }
 
-Inform "Initialize arm base"
-fetch_tizen_pkgs_init standard base
+Inform "Initialize arm64 base"
+fetch_tizen_pkgs_init standard Tizen-Base
 Inform "fetch common packages"
 fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
 Inform "fetch coreclr packages"
@@ -164,7 +164,7 @@ Inform "fetch corefx packages"
 fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
 
 Inform "Initialize standard unified"
-fetch_tizen_pkgs_init standard unified
+fetch_tizen_pkgs_init standard Tizen-Unified
 Inform "fetch corefx packages"
 fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release
 

eng/common/cross/armel/tizen-fetch.sh

@@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then
 	mkdir -p $TMPDIR 
 fi
 
-TIZEN_URL=http://download.tizen.org/snapshots/tizen
+TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen
 BUILD_XML=build.xml
 REPOMD_XML=repomd.xml
 PRIMARY_XML=primary.xml
@@ -154,8 +154,8 @@ fetch_tizen_pkgs()
 	done
 }
 
-Inform "Initialize arm base"
-fetch_tizen_pkgs_init standard base
+Inform "Initialize armel base"
+fetch_tizen_pkgs_init standard Tizen-Base
 Inform "fetch common packages"
 fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
 Inform "fetch coreclr packages"
@@ -164,7 +164,7 @@ Inform "fetch corefx packages"
 fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
 
 Inform "Initialize standard unified"
-fetch_tizen_pkgs_init standard unified
+fetch_tizen_pkgs_init standard Tizen-Unified
 Inform "fetch corefx packages"
 fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release
 

eng/common/cross/build-android-rootfs.sh

@@ -107,12 +107,12 @@ __AndroidPackages+=" liblzma"
 __AndroidPackages+=" krb5"
 __AndroidPackages+=" openssl"
 
-for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\
+for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/main/binary-$__AndroidArch/Packages |\
     grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do
 
     if [[ "$path" != "Filename:" ]]; then
         echo "Working on: $path"
-        wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir"
+        wget -qO- https://packages.termux.dev/termux-main-21/$path | dpkg -x - "$__TmpDir"
     fi
 done
 

eng/common/cross/build-rootfs.sh

@@ -391,9 +391,9 @@ elif [[ "$__CodeName" == "illumos" ]]; then
         --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \
         --disable-libquadmath-support --disable-shared --enable-tls
     make -j "$JOBS" && make install && cd ..
-    BaseUrl=https://pkgsrc.joyent.com
+    BaseUrl=https://pkgsrc.smartos.org
     if [[ "$__UseMirror" == 1 ]]; then
-        BaseUrl=http://pkgsrc.smartos.skylime.net
+        BaseUrl=https://pkgsrc.smartos.skylime.net
     fi
     BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
     echo "Downloading manifest"
@@ -402,7 +402,8 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     read -ra array <<<"$__IllumosPackages"
     for package in "${array[@]}"; do
         echo "Installing '$package'"
-        package="$(grep ">$package-[0-9]" All | sed -En 's/.*href="(.*)\.tgz".*/\1/p')"
+        # find last occurrence of package in listing and extract its name
+        package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)"
         echo "Resolved name '$package'"
         wget "$BaseUrl"/"$package".tgz
         ar -x "$package".tgz

eng/common/cross/x86/tizen-fetch.sh

@@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then
 	mkdir -p $TMPDIR 
 fi
 
-TIZEN_URL=http://download.tizen.org/snapshots/tizen
+TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen
 BUILD_XML=build.xml
 REPOMD_XML=repomd.xml
 PRIMARY_XML=primary.xml
@@ -155,7 +155,7 @@ fetch_tizen_pkgs()
 }
 
 Inform "Initialize i686 base"
-fetch_tizen_pkgs_init standard base
+fetch_tizen_pkgs_init standard Tizen-Base
 Inform "fetch common packages"
 fetch_tizen_pkgs i686 gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
 Inform "fetch coreclr packages"
@@ -164,7 +164,7 @@ Inform "fetch corefx packages"
 fetch_tizen_pkgs i686 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
 
 Inform "Initialize standard unified"
-fetch_tizen_pkgs_init standard unified
+fetch_tizen_pkgs_init standard Tizen-Unified
 Inform "fetch corefx packages"
 fetch_tizen_pkgs i686 gssdp gssdp-devel tizen-release
 

eng/common/init-tools-native.ps1

@@ -113,6 +113,7 @@ try {
             $ToolPath = Convert-Path -Path $BinPath
             Write-Host "Adding $ToolName to the path ($ToolPath)..."
             Write-Host "##vso[task.prependpath]$ToolPath"
+            $env:PATH = "$ToolPath;$env:PATH"
             $InstalledTools += @{ $ToolName = $ToolDirectory.FullName }
           }
         }

eng/common/templates/job/execute-sdl.yml

@@ -34,7 +34,7 @@ jobs:
 - job: Run_SDL
   dependsOn: ${{ parameters.dependsOn }}
   displayName: Run SDL tool
-  condition: eq( ${{ parameters.enable }}, 'true')
+  condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true'))
   variables:
     - group: DotNet-VSTS-Bot
     - name: AzDOProjectName
@@ -46,14 +46,15 @@ jobs:
     - template: /eng/common/templates/variables/sdl-variables.yml
     - name: GuardianVersion
       value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+    - template: /eng/common/templates/variables/pool-providers.yml
   pool:
     # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
     ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
       name: VSEngSS-MicroBuild2022-1ES
       demands: Cmd
     # If it's not devdiv, it's dnceng
     ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-      name: NetCore1ESPool-Internal
+      name: $(DncEngInternalBuildPool)
       demands: ImageOverride -equals windows.vs2019.amd64
   steps:
   - checkout: self

eng/common/templates/job/job.yml

@@ -25,6 +25,7 @@ parameters:
   enablePublishTestResults: false
   enablePublishUsingPipelines: false
   disableComponentGovernance: false
+  componentGovernanceIgnoreDirectories: ''
   mergeTestResults: false
   testRunTitle: ''
   testResultsFormat: ''
@@ -146,6 +147,8 @@ jobs:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}:
       - task: ComponentGovernanceComponentDetection@0
         continueOnError: true
+        inputs:
+          ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
 
   - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
     - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -223,4 +226,5 @@ jobs:
       parameters:
         PackageVersion: ${{ parameters.packageVersion}}
         BuildDropPath: ${{ parameters.buildDropPath }}
+        IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}