Update SetupNugetSource.ps1/sh with new functionality and model

Arcade.slnx

@@ -29,6 +29,7 @@
     <Project Path="src/Microsoft.DotNet.NuGetRepack/tests/Microsoft.DotNet.NuGetRepack.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.PackageTesting.Tests/Microsoft.DotNet.PackageTesting.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.RemoteExecutor/tests/Microsoft.DotNet.RemoteExecutor.Tests.csproj" />
+    <Project Path="src/Microsoft.DotNet.SetupNugetSources.Tests/Microsoft.DotNet.SetupNugetSources.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.SignTool.Tests/Microsoft.DotNet.SignTool.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.SourceBuild/tests/Microsoft.DotNet.SourceBuild.Tasks.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.XliffTasks.Tests/Microsoft.DotNet.XliffTasks.Tests.csproj" />

eng/common/SetupNugetSources.ps1

@@ -7,7 +7,7 @@
 # See example call for this script below.
 #
 #  - task: PowerShell@2
-#    displayName: Setup Private Feeds Credentials
+#    displayName: Setup internal Feeds Credentials
 #    condition: eq(variables['Agent.OS'], 'Windows_NT')
 #    inputs:
 #      filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1
@@ -34,19 +34,28 @@ Set-StrictMode -Version 2.0
 
 . $PSScriptRoot\tools.ps1
 
+# Adds or enables the package source with the given name
+function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
+    if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName)) {
+        AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $userName -pwd $Password
+    }
+}
+
 # Add source entry to PackageSources
 function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
     $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
 
     if ($packageSource -eq $null)
     {
+        Write-Host "Adding package source $SourceName"
+
         $packageSource = $doc.CreateElement("add")
         $packageSource.SetAttribute("key", $SourceName)
         $packageSource.SetAttribute("value", $SourceEndPoint)
         $sources.AppendChild($packageSource) | Out-Null
     }
     else {
-        Write-Host "Package source $SourceName already present."
+        Write-Host "Package source $SourceName already present and enabled."
     }
 
     AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
@@ -59,6 +68,8 @@ function AddCredential($creds, $source, $username, $pwd) {
         return;
     }
 
+    Write-Host "Inserting credential for feed: " $source
+
     # Looks for credential configuration for the given SourceName. Create it if none is found.
     $sourceElement = $creds.SelectSingleNode($Source)
     if ($sourceElement -eq $null)
@@ -91,24 +102,27 @@ function AddCredential($creds, $source, $username, $pwd) {
     $passwordElement.SetAttribute("value", $pwd)
 }
 
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
-    $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
-
-    Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
-
-    ForEach ($PackageSource in $maestroPrivateSources) {
-        Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
-        AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
+# Enable all darc-int package sources.
+function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds) {
+    $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
+    ForEach ($DisabledPackageSource in $maestroInternalSources) {
+        EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key
     }
 }
 
-function EnablePrivatePackageSources($DisabledPackageSources) {
-    $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
-    ForEach ($DisabledPackageSource in $maestroPrivateSources) {
-        Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
+# Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise.
+function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName) {
+    $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']")
+    if ($DisabledPackageSource) {
+        Write-Host "Enabling internal source '$($DisabledPackageSource.key)'."
+
         # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
         $DisabledPackageSources.RemoveChild($DisabledPackageSource)
+
+        AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -pwd $Password
+        return $true
     }
+    return $false
 }
 
 if (!(Test-Path $ConfigFile -PathType Leaf)) {
@@ -121,15 +135,17 @@ $doc = New-Object System.Xml.XmlDocument
 $filename = (Get-Item $ConfigFile).FullName
 $doc.Load($filename)
 
-# Get reference to <PackageSources> or create one if none exist already
+# Get reference to <PackageSources> - fail if none exist
 $sources = $doc.DocumentElement.SelectSingleNode("packageSources")
 if ($sources -eq $null) {
-    $sources = $doc.CreateElement("packageSources")
-    $doc.DocumentElement.AppendChild($sources) | Out-Null
+    Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile"
+    ExitWithExitCode 1
 }
 
 $creds = $null
+$feedSuffix = "v3/index.json"
 if ($Password) {
+    $feedSuffix = "v2"
     # Looks for a <PackageSourceCredentials> node. Create it if none is found.
     $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
     if ($creds -eq $null) {
@@ -138,33 +154,22 @@ if ($Password) {
     }
 }
 
+$userName = "dn-bot"
+
 # Check for disabledPackageSources; we'll enable any darc-int ones we find there
 $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources")
 if ($disabledSources -ne $null) {
     Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node"
-    EnablePrivatePackageSources -DisabledPackageSources $disabledSources
-}
-
-$userName = "dn-bot"
-
-# Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password
-
-# 3.1 uses a different feed url format so it's handled differently here
-$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
-if ($dotnet31Source -ne $null) {
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+    EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds
 }
-
 $dotnetVersions = @('5','6','7','8','9','10')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;
     $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
     if ($dotnetSource -ne $null) {
-        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
-        AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+        AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password
+        AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password
     }
 }
 

eng/common/SetupNugetSources.sh

@@ -52,78 +52,126 @@ if [[ `uname -s` == "Darwin" ]]; then
     TB=''
 fi
 
-# Ensure there is a <packageSources>...</packageSources> section.
-grep -i "<packageSources>" $ConfigFile
-if [ "$?" != "0" ]; then
-    echo "Adding <packageSources>...</packageSources> section."
-    ConfigNodeHeader="<configuration>"
-    PackageSourcesTemplate="${TB}<packageSources>${NL}${TB}</packageSources>"
+# Enables an internal package source by name, if found. Returns 0 if found and enabled, 1 if not found.
+EnableInternalPackageSource() {
+    local PackageSourceName="$1"
+
+    # Check if disabledPackageSources section exists
+    grep -i "<disabledPackageSources>" "$ConfigFile" > /dev/null
+    if [ "$?" != "0" ]; then
+        return 1  # No disabled sources section
+    fi
+
+    # Check if this source name is disabled
+    grep -i "<add key=\"$PackageSourceName\" value=\"true\"" "$ConfigFile" > /dev/null
+    if [ "$?" == "0" ]; then
+        echo "Enabling internal source '$PackageSourceName'."
+        # Remove the disabled entry
+        local OldDisableValue="<add key=\"$PackageSourceName\" value=\"true\" />"
+        local NewDisableValue="<!-- Reenabled for build : $PackageSourceName -->"
+        sed -i.bak "s|$OldDisableValue|$NewDisableValue|" "$ConfigFile"
+
+        # Add the source name to PackageSources for credential handling
+        PackageSources+=("$PackageSourceName")
+        return 0  # Found and enabled
+    fi
+
+    return 1  # Not found in disabled sources
+}
+
+# Add source entry to PackageSources
+AddPackageSource() {
+    local SourceName="$1"
+    local SourceEndPoint="$2"
+
+    # Check if source already exists
+    grep -i "<add key=\"$SourceName\"" "$ConfigFile" > /dev/null
+    if [ "$?" == "0" ]; then
+        echo "Package source $SourceName already present and enabled."
+        PackageSources+=("$SourceName")
+        return
+    fi
+
+    echo "Adding package source $SourceName"
+    PackageSourcesNodeFooter="</packageSources>"
+    PackageSourceTemplate="${TB}<add key=\"$SourceName\" value=\"$SourceEndPoint\" />"
+
+    sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" "$ConfigFile"
+    PackageSources+=("$SourceName")
+}
+
+# Adds or enables the package source with the given name
+AddOrEnablePackageSource() {
+    local SourceName="$1"
+    local SourceEndPoint="$2"
+
+    # Try to enable if disabled, if not found then add new source
+    EnableInternalPackageSource "$SourceName"
+    if [ "$?" != "0" ]; then
+        AddPackageSource "$SourceName" "$SourceEndPoint"
+    fi
+}
 
-    sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile
-fi
+# Enable all darc-int package sources
+EnableMaestroInternalPackageSources() {
+    # Check if disabledPackageSources section exists
+    grep -i "<disabledPackageSources>" "$ConfigFile" > /dev/null
+    if [ "$?" != "0" ]; then
+        return  # No disabled sources section
+    fi
+
+    # Find all darc-int disabled sources
+    local DisabledDarcIntSources=()
+    DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' "$ConfigFile" | tr -d '"')
+
+    for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
+        if [[ $DisabledSourceName == darc-int* ]]; then
+            EnableInternalPackageSource "$DisabledSourceName"
+        fi
+    done
+}
 
-# Ensure there is a <packageSourceCredentials>...</packageSourceCredentials> section. 
-grep -i "<packageSourceCredentials>" $ConfigFile
+# Ensure there is a <packageSources>...</packageSources> section.
+grep -i "<packageSources>" $ConfigFile
 if [ "$?" != "0" ]; then
-    echo "Adding <packageSourceCredentials>...</packageSourceCredentials> section."
-
-    PackageSourcesNodeFooter="</packageSources>"
-    PackageSourceCredentialsTemplate="${TB}<packageSourceCredentials>${NL}${TB}</packageSourceCredentials>"
-
-    sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
+    Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile"
+    ExitWithExitCode 1
 fi
 
 PackageSources=()
 
-# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present
-grep -i "<add key=\"dotnet3.1\"" $ConfigFile
-if [ "$?" == "0" ]; then
-    grep -i "<add key=\"dotnet3.1-internal\"" $ConfigFile
+# Set feed suffix based on whether credentials are provided
+FeedSuffix="v3/index.json"
+if [ -n "$CredToken" ]; then
+    FeedSuffix="v2"
+
+    # Ensure there is a <packageSourceCredentials>...</packageSourceCredentials> section.
+    grep -i "<packageSourceCredentials>" $ConfigFile
     if [ "$?" != "0" ]; then
-        echo "Adding dotnet3.1-internal to the packageSources."
-        PackageSourcesNodeFooter="</packageSources>"
-        PackageSourceTemplate="${TB}<add key=\"dotnet3.1-internal\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2\" />"
+        echo "Adding <packageSourceCredentials>...</packageSourceCredentials> section."
 
-        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
-    fi
-    PackageSources+=('dotnet3.1-internal')
-
-    grep -i "<add key=\"dotnet3.1-internal-transport\">" $ConfigFile
-    if [ "$?" != "0" ]; then
-        echo "Adding dotnet3.1-internal-transport to the packageSources."
         PackageSourcesNodeFooter="</packageSources>"
-        PackageSourceTemplate="${TB}<add key=\"dotnet3.1-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2\" />"
+        PackageSourceCredentialsTemplate="${TB}<packageSourceCredentials>${NL}${TB}</packageSourceCredentials>"
 
-        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile
     fi
-    PackageSources+=('dotnet3.1-internal-transport')
+fi
+
+# Check for disabledPackageSources; we'll enable any darc-int ones we find there
+grep -i "<disabledPackageSources>" $ConfigFile > /dev/null
+if [ "$?" == "0" ]; then
+    echo "Checking for any darc-int disabled package sources in the disabledPackageSources node"
+    EnableMaestroInternalPackageSources
 fi
 
 DotNetVersions=('5' '6' '7' '8' '9' '10')
 
 for DotNetVersion in ${DotNetVersions[@]} ; do
     FeedPrefix="dotnet${DotNetVersion}";
-    grep -i "<add key=\"$FeedPrefix\"" $ConfigFile
+    grep -i "<add key=\"$FeedPrefix\"" $ConfigFile > /dev/null
     if [ "$?" == "0" ]; then
-        grep -i "<add key=\"$FeedPrefix-internal\"" $ConfigFile
-        if [ "$?" != "0" ]; then
-            echo "Adding $FeedPrefix-internal to the packageSources."
-            PackageSourcesNodeFooter="</packageSources>"
-            PackageSourceTemplate="${TB}<add key=\"$FeedPrefix-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/v2\" />"
-
-            sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
-        fi
-        PackageSources+=("$FeedPrefix-internal")
-
-        grep -i "<add key=\"$FeedPrefix-internal-transport\">" $ConfigFile
-        if [ "$?" != "0" ]; then
-            echo "Adding $FeedPrefix-internal-transport to the packageSources."
-            PackageSourcesNodeFooter="</packageSources>"
-            PackageSourceTemplate="${TB}<add key=\"$FeedPrefix-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/v2\" />"
-
-            sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
-        fi
-        PackageSources+=("$FeedPrefix-internal-transport")
+        AddOrEnablePackageSource "$FeedPrefix-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/$FeedSuffix"
+        AddOrEnablePackageSource "$FeedPrefix-internal-transport" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/$FeedSuffix"
     fi
 done
 
@@ -139,29 +187,12 @@ if [ "$CredToken" ]; then
         # Check if there is no existing credential for this FeedName
         grep -i "<$FeedName>" $ConfigFile 
         if [ "$?" != "0" ]; then
-            echo "Adding credentials for $FeedName."
+            echo "	Inserting credential for feed: $FeedName"
 
             PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
-            NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
+            NewCredential="${TB}${TB}<$FeedName>${NL}${TB}<add key=\"Username\" value=\"dn-bot\" />${NL}${TB}${TB}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}${TB}${TB}</$FeedName>"
 
             sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
         fi
     done
 fi
-
-# Re-enable any entries in disabledPackageSources where the feed name contains darc-int
-grep -i "<disabledPackageSources>" $ConfigFile
-if [ "$?" == "0" ]; then
-    DisabledDarcIntSources=()
-    echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile"
-    DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile  | tr -d '"')
-    for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
-        if [[ $DisabledSourceName == darc-int* ]]
-            then
-                OldDisableValue="<add key=\"$DisabledSourceName\" value=\"true\" />"
-                NewDisableValue="<!-- Reenabled for build : $DisabledSourceName -->"
-                sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
-                echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
-        fi
-    done
-fi