diff --git a/NuGet.config b/NuGet.config index b3c69e7ae99..c0346aafa45 100644 --- a/NuGet.config +++ b/NuGet.config @@ -1,15 +1,15 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a5fc67de65d..2746d859a53 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,111 +1,111 @@ - - - - - https://github.com/dotnet/winforms - 9b0ec7bb6e82818a4ac80702a9cd5c45f5d2ffde - - - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/core-setup - e1d780539e85f4d8de263957715f9d08db2ceef4 - - - https://github.com/dotnet/arcade - 12508aec923685c972842c646349faf0f8991e86 - - - https://github.com/dotnet/arcade - 12508aec923685c972842c646349faf0f8991e86 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://dev.azure.com/dnceng/internal/_git/dotnet-wpf-int - 914f5f71a797720a04c0808e4dd3e50b1fb2f4b6 - - - https://github.com/dotnet/corefx - 468355e66daf9eaeb1978703fa6365cd1e37e644 - - - https://github.com/dotnet/coreclr - 8974a699899bdc2cc5687504e1ada606ac803e9b - - - https://github.com/dotnet/coreclr - 8974a699899bdc2cc5687504e1ada606ac803e9b - - - https://github.com/dotnet/coreclr - 8974a699899bdc2cc5687504e1ada606ac803e9b - - - https://github.com/dotnet/arcade - 12508aec923685c972842c646349faf0f8991e86 - - - https://github.com/dotnet/arcade - 12508aec923685c972842c646349faf0f8991e86 - - - https://github.com/dotnet/arcade - 12508aec923685c972842c646349faf0f8991e86 - - - + + + + + https://github.com/dotnet/winforms + 9b0ec7bb6e82818a4ac80702a9cd5c45f5d2ffde + + + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/core-setup + e1d780539e85f4d8de263957715f9d08db2ceef4 + + + https://github.com/dotnet/arcade + 4b3d46cc75969c4e2de5786ec2b10a430b26dd9f + + + https://github.com/dotnet/arcade + 4b3d46cc75969c4e2de5786ec2b10a430b26dd9f + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf-int + 914f5f71a797720a04c0808e4dd3e50b1fb2f4b6 + + + https://github.com/dotnet/corefx + 468355e66daf9eaeb1978703fa6365cd1e37e644 + + + https://github.com/dotnet/coreclr + 8974a699899bdc2cc5687504e1ada606ac803e9b + + + https://github.com/dotnet/coreclr + 8974a699899bdc2cc5687504e1ada606ac803e9b + + + https://github.com/dotnet/coreclr + 8974a699899bdc2cc5687504e1ada606ac803e9b + + + https://github.com/dotnet/arcade + 4b3d46cc75969c4e2de5786ec2b10a430b26dd9f + + + https://github.com/dotnet/arcade + 4b3d46cc75969c4e2de5786ec2b10a430b26dd9f + + + https://github.com/dotnet/arcade + 4b3d46cc75969c4e2de5786ec2b10a430b26dd9f + + + diff --git a/eng/Versions.props b/eng/Versions.props index d11b66f0414..8f4ed2dc96c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,112 +1,112 @@ - - - - 4.8.0 - preview7 - 4.0.0.0 - 4.6.0-preview7.19320.1 - - - - 4.8.0-preview7.19325.3 - - - - 3.0.0-preview7.19323.2 - 3.0.0-preview7.19323.2 - 3.0.0-preview7.19323.2 - - - - 3.0.0-preview7-27824-03 - 3.0.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - - - - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview4.19176.11 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - 4.6.0-preview7.19320.1 - - - - 1.0.0-beta.19324.7 - 1.0.0-beta.19324.7 - 1.0.0-beta.19324.7 - - - - 0.1.0-preview2-181205-2 - - - - 4.4.0 - - - - 15.9.20 - 15.9.20 - 2.4.0 - $(XUnitVersion) - $(XUnitVersion) - - 1.0.0-beta.19263.1 - - 5.28.1.1-1 - 0.0.0.1 - - - - $(RestoreSources); - https://dotnet.myget.org/F/dotnet-corefxlab/api/v3/index.json; - https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json; - https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json; - https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json; - - - - - System.Reflection.MetadataLoadContext - Microsoft.Private.Winforms - System.Drawing.Common - System.DirectoryServices - Microsoft.DotNet.Wpf.DncEng - System.IO.Packaging - System.Reflection.MetadataLoadContext - - - - 4.8.0-preview7.19325.4 - - + + + + 4.8.0 + preview7 + 4.0.0.0 + 4.6.0-preview7.19320.1 + + + + 4.8.0-preview7.19325.3 + + + + 3.0.0-preview7.19323.2 + 3.0.0-preview7.19323.2 + 3.0.0-preview7.19323.2 + + + + 3.0.0-preview7-27824-03 + 3.0.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + + + + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview4.19176.11 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + 4.6.0-preview7.19320.1 + + + + 1.0.0-beta.19326.2 + 1.0.0-beta.19326.2 + 1.0.0-beta.19326.2 + + + + 0.1.0-preview2-181205-2 + + + + 4.4.0 + + + + 15.9.20 + 15.9.20 + 2.4.0 + $(XUnitVersion) + $(XUnitVersion) + + 1.0.0-beta.19263.1 + + 5.28.1.1-1 + 0.0.0.1 + + + + $(RestoreSources); + https://dotnet.myget.org/F/dotnet-corefxlab/api/v3/index.json; + https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json; + https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json; + https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json; + + + + + System.Reflection.MetadataLoadContext + Microsoft.Private.Winforms + System.Drawing.Common + System.DirectoryServices + Microsoft.DotNet.Wpf.DncEng + System.IO.Packaging + System.Reflection.MetadataLoadContext + + + + 4.8.0-preview7.19325.4 + + diff --git a/eng/WpfArcadeSdk/tools/Packaging.targets b/eng/WpfArcadeSdk/tools/Packaging.targets index 9200a1c939d..7f18c8b434a 100644 --- a/eng/WpfArcadeSdk/tools/Packaging.targets +++ b/eng/WpfArcadeSdk/tools/Packaging.targets @@ -372,12 +372,9 @@ $(PreparePackageAssetsDependsOn): Ensure that SourceRevisionId is obtained if it has not already been read in Reuse tasks from Microsoft.Build.Tasks.Git.targets --> - - - - + - + diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd index 56c2f25ac22..7fd3912c9b6 100644 --- a/eng/common/CIBuild.cmd +++ b/eng/common/CIBuild.cmd @@ -1,2 +1,2 @@ -@echo off +@echo off powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" \ No newline at end of file diff --git a/eng/common/CheckSymbols.ps1 b/eng/common/CheckSymbols.ps1 index b8d84607b89..a7f6a8627fb 100644 --- a/eng/common/CheckSymbols.ps1 +++ b/eng/common/CheckSymbols.ps1 @@ -1,158 +1,158 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $SymbolToolPath # Full path to directory where dotnet symbol-tool was installed -) - -Add-Type -AssemblyName System.IO.Compression.FileSystem - -function FirstMatchingSymbolDescriptionOrDefault { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + "\" + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, ".pdb") - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") - - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") - - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") - - .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null - - if (Test-Path $PdbPath) { - return "PDB" - } - elseif (Test-Path $NGenPdb) { - return "NGen PDB" - } - elseif (Test-Path $SODbg) { - return "DBG for SO" - } - elseif (Test-Path $DylibDwarf) { - return "Dwarf for Dylib" - } - elseif (Test-Path $SymbolPath) { - return "Module" - } - else { - return $null - } -} - -function CountMissingSymbols { - param( - [string] $PackagePath # Path to a NuGet package - ) - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - throw "Input file does not exist: $PackagePath" - } - - # Extensions for which we'll look for symbols - $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") - - # How many files are missing symbol information - $MissingSymbols = 0 - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid - $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" - - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) - - # Makes easier to reference `symbol tool` - Push-Location $SymbolToolPath - - Get-ChildItem -Recurse $ExtractPath | - Where-Object {$RelevantExtensions -contains $_.Extension} | - ForEach-Object { - if ($_.FullName -Match "\\ref\\") { - Write-Host "`t Ignoring reference assembly file" $_.FullName - return - } - - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath - - Write-Host -NoNewLine "`t Checking file" $_.FullName "... " - - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" - } - else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host "No symbols found on MSDL or SymWeb!" - } - else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host "No symbols found on MSDL!" - } - else { - Write-Host "No symbols found on SymWeb!" - } - } - } - } - - Pop-Location - - return $MissingSymbols -} - -function CheckSymbolsAvailable { - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $FileName = $_.Name - - # These packages from Arcade-Services include some native libraries that - # our current symbol uploader can't handle. Below is a workaround until - # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. - if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - - Write-Host "Validating $FileName " - $Status = CountMissingSymbols "$InputPath\$FileName" - - if ($Status -ne 0) { - Write-Error "Missing symbols for $Status modules in the package $FileName" - } - - Write-Host - } -} - -CheckSymbolsAvailable +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $SymbolToolPath # Full path to directory where dotnet symbol-tool was installed +) + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function FirstMatchingSymbolDescriptionOrDefault { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + "\" + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") + + .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return "PDB" + } + elseif (Test-Path $NGenPdb) { + return "NGen PDB" + } + elseif (Test-Path $SODbg) { + return "DBG for SO" + } + elseif (Test-Path $DylibDwarf) { + return "Dwarf for Dylib" + } + elseif (Test-Path $SymbolPath) { + return "Module" + } + else { + return $null + } +} + +function CountMissingSymbols { + param( + [string] $PackagePath # Path to a NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + throw "Input file does not exist: $PackagePath" + } + + # Extensions for which we'll look for symbols + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") + + # How many files are missing symbol information + $MissingSymbols = 0 + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $PackageGuid = New-Guid + $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" + + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + + # Makes easier to reference `symbol tool` + Push-Location $SymbolToolPath + + Get-ChildItem -Recurse $ExtractPath | + Where-Object {$RelevantExtensions -contains $_.Extension} | + ForEach-Object { + if ($_.FullName -Match "\\ref\\") { + Write-Host "`t Ignoring reference assembly file" $_.FullName + return + } + + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath + + Write-Host -NoNewLine "`t Checking file" $_.FullName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host "No symbols found on MSDL or SymWeb!" + } + else { + if ($SymbolsOnMSDL -eq $null) { + Write-Host "No symbols found on MSDL!" + } + else { + Write-Host "No symbols found on SymWeb!" + } + } + } + } + + Pop-Location + + return $MissingSymbols +} + +function CheckSymbolsAvailable { + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $FileName = $_.Name + + # These packages from Arcade-Services include some native libraries that + # our current symbol uploader can't handle. Below is a workaround until + # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + + Write-Host "Validating $FileName " + $Status = CountMissingSymbols "$InputPath\$FileName" + + if ($Status -ne 0) { + Write-Error "Missing symbols for $Status modules in the package $FileName" + } + + Write-Host + } +} + +CheckSymbolsAvailable diff --git a/eng/common/PSScriptAnalyzerSettings.psd1 b/eng/common/PSScriptAnalyzerSettings.psd1 index 4c1ea7c98ea..7c0f6a411bf 100644 --- a/eng/common/PSScriptAnalyzerSettings.psd1 +++ b/eng/common/PSScriptAnalyzerSettings.psd1 @@ -1,11 +1,11 @@ -@{ - IncludeRules=@('PSAvoidUsingCmdletAliases', - 'PSAvoidUsingWMICmdlet', - 'PSAvoidUsingPositionalParameters', - 'PSAvoidUsingInvokeExpression', - 'PSUseDeclaredVarsMoreThanAssignments', - 'PSUseCmdletCorrectly', - 'PSStandardDSCFunctionsInResource', - 'PSUseIdenticalMandatoryParametersForDSC', - 'PSUseIdenticalParametersForDSC') +@{ + IncludeRules=@('PSAvoidUsingCmdletAliases', + 'PSAvoidUsingWMICmdlet', + 'PSAvoidUsingPositionalParameters', + 'PSAvoidUsingInvokeExpression', + 'PSUseDeclaredVarsMoreThanAssignments', + 'PSUseCmdletCorrectly', + 'PSStandardDSCFunctionsInResource', + 'PSUseIdenticalMandatoryParametersForDSC', + 'PSUseIdenticalParametersForDSC') } \ No newline at end of file diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index a1b1333723e..2730713f307 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -1,83 +1,83 @@ - - - - - - netcoreapp2.1 - - - - - - - - - - - - - - - - - - - - - - - - - https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json - https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json - https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json - - - - - - - - - - - - + + + + + + netcoreapp2.1 + + + + + + + + + + + + + + + + + + + + + + + + + https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json + https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json + https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json + + + + + + + + + + + + diff --git a/eng/common/PublishToSymbolServers.proj b/eng/common/PublishToSymbolServers.proj index 5d55e312b01..56c22ff63f3 100644 --- a/eng/common/PublishToSymbolServers.proj +++ b/eng/common/PublishToSymbolServers.proj @@ -1,82 +1,82 @@ - - - - - - netcoreapp2.1 - - - - - - - - - - - - - - - - 3650 - true - false - - - - - - - - - - - - - - - - - + + + + + + netcoreapp2.1 + + + + + + + + + + + + + + + + 3650 + true + false + + + + + + + + + + + + + + + + + diff --git a/eng/common/README.md b/eng/common/README.md index ff49c371527..61eb96ec87a 100644 --- a/eng/common/README.md +++ b/eng/common/README.md @@ -1,28 +1,28 @@ -# Don't touch this folder - - uuuuuuuuuuuuuuuuuuuu - u" uuuuuuuuuuuuuuuuuu "u - u" u$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ - $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ - $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ - $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ - $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$" u" - "u """""""""""""""""" u" - """""""""""""""""""" - -!!! Changes made in this directory are subject to being overwritten by automation !!! - -The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. +# Don't touch this folder + + uuuuuuuuuuuuuuuuuuuu + u" uuuuuuuuuuuuuuuuuu "u + u" u$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ + $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ + $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ + $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ + $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$" u" + "u """""""""""""""""" u" + """""""""""""""""""" + +!!! Changes made in this directory are subject to being overwritten by automation !!! + +The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. diff --git a/eng/common/SigningValidation.proj b/eng/common/SigningValidation.proj index 7045fb6fb9d..8791acee336 100644 --- a/eng/common/SigningValidation.proj +++ b/eng/common/SigningValidation.proj @@ -1,83 +1,83 @@ - - - - - - netcoreapp2.1 - - - - - - - - $(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(SignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe - - $(PackageBasePath) - signcheck.log - signcheck.errors.log - signcheck.exclusions.txt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + netcoreapp2.1 + + + + + + + + $(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(SignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe + + $(PackageBasePath) + signcheck.log + signcheck.errors.log + signcheck.exclusions.txt + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/eng/common/SourceLinkValidation.ps1 b/eng/common/SourceLinkValidation.ps1 index cb2d28cb99e..8ba975e6915 100644 --- a/eng/common/SourceLinkValidation.ps1 +++ b/eng/common/SourceLinkValidation.ps1 @@ -1,184 +1,184 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $SourceLinkToolPath, # Full path to directory where dotnet SourceLink CLI was installed - [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages -) - -# Cache/HashMap (File -> Exist flag) used to consult whether a file exist -# in the repository at a specific commit point. This is populated by inserting -# all files present in the repo at a specific commit point. -$global:RepoFiles = @{} - -$ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - throw "Input file does not exist: $PackagePath" - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @(".dll", ".exe", ".pdb") - - Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith(".resources.dll")) { - return - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - # Makes easier to reference `sourcelink cli` - Push-Location $using:SourceLinkToolPath - - $SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/") - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - $Status = 0 - } - } - catch { - $Status = 0 - } - } - - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - - Pop-Location - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - - $zip.Dispose() - - if ($FailedFiles -eq 0) { - Write-Host "Passed." - } -} - -function ValidateSourceLinkLinks { - if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { - Write-Host "GHRepoName should be in the format /" - $global:LASTEXITCODE = 1 - return - } - - if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { - Write-Host "GHCommit should be a 40 chars hexadecimal string" - $global:LASTEXITCODE = 1 - return - } - - $RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") - $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" - $global:LASTEXITCODE = 1 - return - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - # Process each NuGet package in parallel - $Jobs = @() - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -Measure-Command { ValidateSourceLinkLinks } +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $SourceLinkToolPath, # Full path to directory where dotnet SourceLink CLI was installed + [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages +) + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + throw "Input file does not exist: $PackagePath" + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @(".dll", ".exe", ".pdb") + + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith(".resources.dll")) { + return + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + # Makes easier to reference `sourcelink cli` + Push-Location $using:SourceLinkToolPath + + $SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/") + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + $Status = 0 + } + } + catch { + $Status = 0 + } + } + + if ($Status -ne 200) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + + Pop-Location + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + + $zip.Dispose() + + if ($FailedFiles -eq 0) { + Write-Host "Passed." + } +} + +function ValidateSourceLinkLinks { + if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + Write-Host "GHRepoName should be in the format /" + $global:LASTEXITCODE = 1 + return + } + + if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-Host "GHCommit should be a 40 chars hexadecimal string" + $global:LASTEXITCODE = 1 + return + } + + $RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" + $global:LASTEXITCODE = 1 + return + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + # Process each NuGet package in parallel + $Jobs = @() + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +Measure-Command { ValidateSourceLinkLinks } diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index feb58d14191..75a7871a610 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -1,141 +1,141 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string][Alias('c')]$configuration = "Debug", - [string]$platform = $null, - [string] $projects, - [string][Alias('v')]$verbosity = "minimal", - [string] $msbuildEngine = $null, - [bool] $warnAsError = $true, - [bool] $nodeReuse = $true, - [switch][Alias('r')]$restore, - [switch] $deployDeps, - [switch][Alias('b')]$build, - [switch] $rebuild, - [switch] $deploy, - [switch][Alias('t')]$test, - [switch] $integrationTest, - [switch] $performanceTest, - [switch] $sign, - [switch] $pack, - [switch] $publish, - [switch][Alias('bl')]$binaryLog, - [switch] $ci, - [switch] $prepareMachine, - [switch] $help, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties -) - -. $PSScriptRoot\tools.ps1 - -function Print-Usage() { - Write-Host "Common settings:" - Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" - Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" - Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" - Write-Host " -binaryLog Output binary log (short: -bl)" - Write-Host " -help Print help and exit" - Write-Host "" - - Write-Host "Actions:" - Write-Host " -restore Restore dependencies (short: -r)" - Write-Host " -build Build solution (short: -b)" - Write-Host " -rebuild Rebuild solution" - Write-Host " -deploy Deploy built VSIXes" - Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)" - Write-Host " -test Run all unit tests in the solution (short: -t)" - Write-Host " -integrationTest Run all integration tests in the solution" - Write-Host " -performanceTest Run all performance tests in the solution" - Write-Host " -pack Package build outputs into NuGet packages and Willow components" - Write-Host " -sign Sign build outputs" - Write-Host " -publish Publish artifacts (e.g. symbols)" - Write-Host "" - - Write-Host "Advanced settings:" - Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" - Write-Host " -ci Set when running on CI server" - Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" - Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" - Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host "" - - Write-Host "Command line arguments not listed above are passed thru to msbuild." - Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." -} - -function InitializeCustomToolset { - if (-not $restore) { - return - } - - $script = Join-Path $EngRoot "restore-toolset.ps1" - - if (Test-Path $script) { - . $script - } -} - -function Build { - $toolsetBuildProj = InitializeToolset - InitializeCustomToolset - - $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } - $platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } - - if ($projects) { - # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. - # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty. - [string[]] $msbuildArgs = $properties - $msbuildArgs += "/p:Projects=$projects" - $properties = $msbuildArgs - } - - MSBuild $toolsetBuildProj ` - $bl ` - $platformArg ` - /p:Configuration=$configuration ` - /p:RepoRoot=$RepoRoot ` - /p:Restore=$restore ` - /p:DeployDeps=$deployDeps ` - /p:Build=$build ` - /p:Rebuild=$rebuild ` - /p:Deploy=$deploy ` - /p:Test=$test ` - /p:Pack=$pack ` - /p:IntegrationTest=$integrationTest ` - /p:PerformanceTest=$performanceTest ` - /p:Sign=$sign ` - /p:Publish=$publish ` - @properties -} - -try { - if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { - Print-Usage - exit 0 - } - - if ($ci) { - $binaryLog = $true - $nodeReuse = $false - } - - # Import custom tools configuration, if present in the repo. - # Note: Import in global scope so that the script set top-level variables without qualification. - $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1" - if (Test-Path $configureToolsetScript) { - . $configureToolsetScript - } - - if (($restore) -and ($null -eq $env:DisableNativeToolsetInstalls)) { - InitializeNativeTools - } - - Build -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 +[CmdletBinding(PositionalBinding=$false)] +Param( + [string][Alias('c')]$configuration = "Debug", + [string]$platform = $null, + [string] $projects, + [string][Alias('v')]$verbosity = "minimal", + [string] $msbuildEngine = $null, + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, + [switch][Alias('r')]$restore, + [switch] $deployDeps, + [switch][Alias('b')]$build, + [switch] $rebuild, + [switch] $deploy, + [switch][Alias('t')]$test, + [switch] $integrationTest, + [switch] $performanceTest, + [switch] $sign, + [switch] $pack, + [switch] $publish, + [switch][Alias('bl')]$binaryLog, + [switch] $ci, + [switch] $prepareMachine, + [switch] $help, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties +) + +. $PSScriptRoot\tools.ps1 + +function Print-Usage() { + Write-Host "Common settings:" + Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" + Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + Write-Host " -binaryLog Output binary log (short: -bl)" + Write-Host " -help Print help and exit" + Write-Host "" + + Write-Host "Actions:" + Write-Host " -restore Restore dependencies (short: -r)" + Write-Host " -build Build solution (short: -b)" + Write-Host " -rebuild Rebuild solution" + Write-Host " -deploy Deploy built VSIXes" + Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)" + Write-Host " -test Run all unit tests in the solution (short: -t)" + Write-Host " -integrationTest Run all integration tests in the solution" + Write-Host " -performanceTest Run all performance tests in the solution" + Write-Host " -pack Package build outputs into NuGet packages and Willow components" + Write-Host " -sign Sign build outputs" + Write-Host " -publish Publish artifacts (e.g. symbols)" + Write-Host "" + + Write-Host "Advanced settings:" + Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" + Write-Host " -ci Set when running on CI server" + Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" + Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host "" + + Write-Host "Command line arguments not listed above are passed thru to msbuild." + Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." +} + +function InitializeCustomToolset { + if (-not $restore) { + return + } + + $script = Join-Path $EngRoot "restore-toolset.ps1" + + if (Test-Path $script) { + . $script + } +} + +function Build { + $toolsetBuildProj = InitializeToolset + InitializeCustomToolset + + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + $platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } + + if ($projects) { + # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. + # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty. + [string[]] $msbuildArgs = $properties + $msbuildArgs += "/p:Projects=$projects" + $properties = $msbuildArgs + } + + MSBuild $toolsetBuildProj ` + $bl ` + $platformArg ` + /p:Configuration=$configuration ` + /p:RepoRoot=$RepoRoot ` + /p:Restore=$restore ` + /p:DeployDeps=$deployDeps ` + /p:Build=$build ` + /p:Rebuild=$rebuild ` + /p:Deploy=$deploy ` + /p:Test=$test ` + /p:Pack=$pack ` + /p:IntegrationTest=$integrationTest ` + /p:PerformanceTest=$performanceTest ` + /p:Sign=$sign ` + /p:Publish=$publish ` + @properties +} + +try { + if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { + Print-Usage + exit 0 + } + + if ($ci) { + $binaryLog = $true + $nodeReuse = $false + } + + # Import custom tools configuration, if present in the repo. + # Note: Import in global scope so that the script set top-level variables without qualification. + $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1" + if (Test-Path $configureToolsetScript) { + . $configureToolsetScript + } + + if (($restore) -and ($null -eq $env:DisableNativeToolsetInstalls)) { + InitializeNativeTools + } + + Build +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/eng/common/build.sh b/eng/common/build.sh index 6236fc4d38c..dd6e7898d1a 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -1,216 +1,216 @@ -#!/usr/bin/env bash - -# Stop script if unbound variable found (use ${var:-} if intentional) -set -u - -# Stop script if command returns non-zero exit code. -# Prevents hidden errors caused by missing error code propagation. -set -e - -usage() -{ - echo "Common settings:" - echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)" - echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" - echo " --binaryLog Create MSBuild binary log (short: -bl)" - echo " --help Print help and exit (short: -h)" - echo "" - - echo "Actions:" - echo " --restore Restore dependencies (short: -r)" - echo " --build Build solution (short: -b)" - echo " --rebuild Rebuild solution" - echo " --test Run all unit tests in the solution (short: -t)" - echo " --integrationTest Run all integration tests in the solution" - echo " --performanceTest Run all performance tests in the solution" - echo " --pack Package build outputs into NuGet packages and Willow components" - echo " --sign Sign build outputs" - echo " --publish Publish artifacts (e.g. symbols)" - echo "" - - echo "Advanced settings:" - echo " --projects Project or solution file(s) to build" - echo " --ci Set when running on CI server" - echo " --prepareMachine Prepare machine for CI run, clean up processes after build" - echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" - echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" - echo "" - echo "Command line arguments not listed above are passed thru to msbuild." - echo "Arguments can also be passed in with a single hyphen." -} - -source="${BASH_SOURCE[0]}" - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -restore=false -build=false -rebuild=false -test=false -integration_test=false -performance_test=false -pack=false -publish=false -sign=false -public=false -ci=false - -warn_as_error=true -node_reuse=true -binary_log=false -pipelines_log=false - -projects='' -configuration='Debug' -prepare_machine=false -verbosity='minimal' - -properties='' - -while [[ $# > 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" - case "$opt" in - -help|-h) - usage - exit 0 - ;; - -configuration|-c) - configuration=$2 - shift - ;; - -verbosity|-v) - verbosity=$2 - shift - ;; - -binarylog|-bl) - binary_log=true - ;; - -pipelineslog|-pl) - pipelines_log=true - ;; - -restore|-r) - restore=true - ;; - -build|-b) - build=true - ;; - -rebuild) - rebuild=true - ;; - -pack) - pack=true - ;; - -test|-t) - test=true - ;; - -integrationtest) - integration_test=true - ;; - -performancetest) - performance_test=true - ;; - -sign) - sign=true - ;; - -publish) - publish=true - ;; - -preparemachine) - prepare_machine=true - ;; - -projects) - projects=$2 - shift - ;; - -ci) - ci=true - ;; - -warnaserror) - warn_as_error=$2 - shift - ;; - -nodereuse) - node_reuse=$2 - shift - ;; - *) - properties="$properties $1" - ;; - esac - - shift -done - -if [[ "$ci" == true ]]; then - pipelines_log=true - binary_log=true - node_reuse=false -fi - -. "$scriptroot/tools.sh" - -function InitializeCustomToolset { - local script="$eng_root/restore-toolset.sh" - - if [[ -a "$script" ]]; then - . "$script" - fi -} - -function Build { - InitializeToolset - InitializeCustomToolset - - if [[ ! -z "$projects" ]]; then - properties="$properties /p:Projects=$projects" - fi - - local bl="" - if [[ "$binary_log" == true ]]; then - bl="/bl:\"$log_dir/Build.binlog\"" - fi - - MSBuild $_InitializeToolset \ - $bl \ - /p:Configuration=$configuration \ - /p:RepoRoot="$repo_root" \ - /p:Restore=$restore \ - /p:Build=$build \ - /p:Rebuild=$rebuild \ - /p:Test=$test \ - /p:Pack=$pack \ - /p:IntegrationTest=$integration_test \ - /p:PerformanceTest=$performance_test \ - /p:Sign=$sign \ - /p:Publish=$publish \ - $properties - - ExitWithExitCode 0 -} - -# Import custom tools configuration, if present in the repo. -configure_toolset_script="$eng_root/configure-toolset.sh" -if [[ -a "$configure_toolset_script" ]]; then - . "$configure_toolset_script" -fi - -# TODO: https://github.com/dotnet/arcade/issues/1468 -# Temporary workaround to avoid breaking change. -# Remove once repos are updated. -if [[ -n "${useInstalledDotNetCli:-}" ]]; then - use_installed_dotnet_cli="$useInstalledDotNetCli" -fi - -if [[ "$restore" == true && -z ${DisableNativeToolsetInstalls:-} ]]; then - InitializeNativeTools -fi - -Build +#!/usr/bin/env bash + +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u + +# Stop script if command returns non-zero exit code. +# Prevents hidden errors caused by missing error code propagation. +set -e + +usage() +{ + echo "Common settings:" + echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)" + echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + echo " --binaryLog Create MSBuild binary log (short: -bl)" + echo " --help Print help and exit (short: -h)" + echo "" + + echo "Actions:" + echo " --restore Restore dependencies (short: -r)" + echo " --build Build solution (short: -b)" + echo " --rebuild Rebuild solution" + echo " --test Run all unit tests in the solution (short: -t)" + echo " --integrationTest Run all integration tests in the solution" + echo " --performanceTest Run all performance tests in the solution" + echo " --pack Package build outputs into NuGet packages and Willow components" + echo " --sign Sign build outputs" + echo " --publish Publish artifacts (e.g. symbols)" + echo "" + + echo "Advanced settings:" + echo " --projects Project or solution file(s) to build" + echo " --ci Set when running on CI server" + echo " --prepareMachine Prepare machine for CI run, clean up processes after build" + echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo "" + echo "Command line arguments not listed above are passed thru to msbuild." + echo "Arguments can also be passed in with a single hyphen." +} + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +restore=false +build=false +rebuild=false +test=false +integration_test=false +performance_test=false +pack=false +publish=false +sign=false +public=false +ci=false + +warn_as_error=true +node_reuse=true +binary_log=false +pipelines_log=false + +projects='' +configuration='Debug' +prepare_machine=false +verbosity='minimal' + +properties='' + +while [[ $# > 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -help|-h) + usage + exit 0 + ;; + -configuration|-c) + configuration=$2 + shift + ;; + -verbosity|-v) + verbosity=$2 + shift + ;; + -binarylog|-bl) + binary_log=true + ;; + -pipelineslog|-pl) + pipelines_log=true + ;; + -restore|-r) + restore=true + ;; + -build|-b) + build=true + ;; + -rebuild) + rebuild=true + ;; + -pack) + pack=true + ;; + -test|-t) + test=true + ;; + -integrationtest) + integration_test=true + ;; + -performancetest) + performance_test=true + ;; + -sign) + sign=true + ;; + -publish) + publish=true + ;; + -preparemachine) + prepare_machine=true + ;; + -projects) + projects=$2 + shift + ;; + -ci) + ci=true + ;; + -warnaserror) + warn_as_error=$2 + shift + ;; + -nodereuse) + node_reuse=$2 + shift + ;; + *) + properties="$properties $1" + ;; + esac + + shift +done + +if [[ "$ci" == true ]]; then + pipelines_log=true + binary_log=true + node_reuse=false +fi + +. "$scriptroot/tools.sh" + +function InitializeCustomToolset { + local script="$eng_root/restore-toolset.sh" + + if [[ -a "$script" ]]; then + . "$script" + fi +} + +function Build { + InitializeToolset + InitializeCustomToolset + + if [[ ! -z "$projects" ]]; then + properties="$properties /p:Projects=$projects" + fi + + local bl="" + if [[ "$binary_log" == true ]]; then + bl="/bl:\"$log_dir/Build.binlog\"" + fi + + MSBuild $_InitializeToolset \ + $bl \ + /p:Configuration=$configuration \ + /p:RepoRoot="$repo_root" \ + /p:Restore=$restore \ + /p:Build=$build \ + /p:Rebuild=$rebuild \ + /p:Test=$test \ + /p:Pack=$pack \ + /p:IntegrationTest=$integration_test \ + /p:PerformanceTest=$performance_test \ + /p:Sign=$sign \ + /p:Publish=$publish \ + $properties + + ExitWithExitCode 0 +} + +# Import custom tools configuration, if present in the repo. +configure_toolset_script="$eng_root/configure-toolset.sh" +if [[ -a "$configure_toolset_script" ]]; then + . "$configure_toolset_script" +fi + +# TODO: https://github.com/dotnet/arcade/issues/1468 +# Temporary workaround to avoid breaking change. +# Remove once repos are updated. +if [[ -n "${useInstalledDotNetCli:-}" ]]; then + use_installed_dotnet_cli="$useInstalledDotNetCli" +fi + +if [[ "$restore" == true && -z ${DisableNativeToolsetInstalls:-} ]]; then + InitializeNativeTools +fi + +Build diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh index 1a02c0dec8f..01746f84a0e 100755 --- a/eng/common/cibuild.sh +++ b/eng/common/cibuild.sh @@ -1,16 +1,16 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where - # the symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where + # the symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + . "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@ \ No newline at end of file diff --git a/eng/common/cross/android/arm/toolchain.cmake b/eng/common/cross/android/arm/toolchain.cmake index a7e1c73501b..1cbc48958dc 100644 --- a/eng/common/cross/android/arm/toolchain.cmake +++ b/eng/common/cross/android/arm/toolchain.cmake @@ -1,41 +1,41 @@ -set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../) -set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot) -set(CLR_CMAKE_PLATFORM_ANDROID "Android") - -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR arm) - -## Specify the toolchain -set(TOOLCHAIN "arm-linux-androideabi") -set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN}) -set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-) - -find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++) -find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy) -find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump) - -add_compile_options(--sysroot=${CROSS_ROOTFS}) -add_compile_options(-fPIE) -add_compile_options(-mfloat-abi=soft) -include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/) -include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/arm-linux-androideabi/) - -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie") - -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) - -set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../) +set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot) +set(CLR_CMAKE_PLATFORM_ANDROID "Android") + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR arm) + +## Specify the toolchain +set(TOOLCHAIN "arm-linux-androideabi") +set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN}) +set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-) + +find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang) +find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++) +find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang) +find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar) +find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar) +find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy) +find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump) + +add_compile_options(--sysroot=${CROSS_ROOTFS}) +add_compile_options(-fPIE) +add_compile_options(-mfloat-abi=soft) +include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/) +include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/arm-linux-androideabi/) + +set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") +set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}") +set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}") +set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie") + +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) + +set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/android/arm64/toolchain.cmake b/eng/common/cross/android/arm64/toolchain.cmake index 29415899c1c..7c3d0d06e88 100644 --- a/eng/common/cross/android/arm64/toolchain.cmake +++ b/eng/common/cross/android/arm64/toolchain.cmake @@ -1,42 +1,42 @@ -set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../) -set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot) -set(CLR_CMAKE_PLATFORM_ANDROID "Android") - -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR aarch64) - -## Specify the toolchain -set(TOOLCHAIN "aarch64-linux-android") -set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN}) -set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-) - -find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++) -find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy) -find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump) - -add_compile_options(--sysroot=${CROSS_ROOTFS}) -add_compile_options(-fPIE) - -## Needed for Android or bionic specific conditionals -add_compile_options(-D__ANDROID__) -add_compile_options(-D__BIONIC__) - -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie") - -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) - -set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../) +set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot) +set(CLR_CMAKE_PLATFORM_ANDROID "Android") + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR aarch64) + +## Specify the toolchain +set(TOOLCHAIN "aarch64-linux-android") +set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN}) +set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-) + +find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang) +find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++) +find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang) +find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar) +find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar) +find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy) +find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump) + +add_compile_options(--sysroot=${CROSS_ROOTFS}) +add_compile_options(-fPIE) + +## Needed for Android or bionic specific conditionals +add_compile_options(-D__ANDROID__) +add_compile_options(-D__BIONIC__) + +set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") +set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}") +set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}") +set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie") + +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) + +set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/arm/sources.list.bionic b/eng/common/cross/arm/sources.list.bionic index 21095574095..a447075be91 100644 --- a/eng/common/cross/arm/sources.list.bionic +++ b/eng/common/cross/arm/sources.list.bionic @@ -1,11 +1,11 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.jessie b/eng/common/cross/arm/sources.list.jessie index 4d142ac9b10..b209ca88ceb 100644 --- a/eng/common/cross/arm/sources.list.jessie +++ b/eng/common/cross/arm/sources.list.jessie @@ -1,3 +1,3 @@ -# Debian (sid) # UNSTABLE -deb http://ftp.debian.org/debian/ sid main contrib non-free -deb-src http://ftp.debian.org/debian/ sid main contrib non-free +# Debian (sid) # UNSTABLE +deb http://ftp.debian.org/debian/ sid main contrib non-free +deb-src http://ftp.debian.org/debian/ sid main contrib non-free diff --git a/eng/common/cross/arm/sources.list.trusty b/eng/common/cross/arm/sources.list.trusty index 07d8f88d82e..b79622576b4 100644 --- a/eng/common/cross/arm/sources.list.trusty +++ b/eng/common/cross/arm/sources.list.trusty @@ -1,11 +1,11 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse +deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm/sources.list.xenial b/eng/common/cross/arm/sources.list.xenial index eacd86b7df3..1e043483fe4 100644 --- a/eng/common/cross/arm/sources.list.xenial +++ b/eng/common/cross/arm/sources.list.xenial @@ -1,11 +1,11 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse +deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm/sources.list.zesty b/eng/common/cross/arm/sources.list.zesty index ea2c14a7874..103d793a356 100644 --- a/eng/common/cross/arm/sources.list.zesty +++ b/eng/common/cross/arm/sources.list.zesty @@ -1,11 +1,11 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse +deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse diff --git a/eng/common/cross/arm/trusty-lttng-2.4.patch b/eng/common/cross/arm/trusty-lttng-2.4.patch index 8e4dd7ae719..f906212bb29 100644 --- a/eng/common/cross/arm/trusty-lttng-2.4.patch +++ b/eng/common/cross/arm/trusty-lttng-2.4.patch @@ -1,71 +1,71 @@ -From e72c9d7ead60e3317bd6d1fade995c07021c947b Mon Sep 17 00:00:00 2001 -From: Mathieu Desnoyers -Date: Thu, 7 May 2015 13:25:04 -0400 -Subject: [PATCH] Fix: building probe providers with C++ compiler - -Robert Daniels wrote: -> > I'm attempting to use lttng userspace tracing with a C++ application -> > on an ARM platform. I'm using GCC 4.8.4 on Linux 3.14 with the 2.6 -> > release of lttng. I've compiled lttng-modules, lttng-ust, and -> > lttng-tools and have been able to get a simple test working with C -> > code. When I attempt to run the hello.cxx test on my target it will -> > segfault. -> -> -> I spent a little time digging into this issue and finally discovered the -> cause of my segfault with ARM C++ tracepoints. -> -> There is a struct called 'lttng_event' in ust-events.h which contains an -> empty union 'u'. This was the cause of my issue. Under C, this empty union -> compiles to a zero byte member while under C++ it compiles to a one byte -> member, and in my case was four-byte aligned which caused my C++ code to -> have the 'cds_list_head node' offset incorrectly by four bytes. This lead -> to an incorrect linked list structure which caused my issue. -> -> Since this union is empty, I simply removed it from the struct and everything -> worked correctly. -> -> I don't know the history or purpose behind this empty union so I'd like to -> know if this is a safe fix. If it is I can submit a patch with the union -> removed. - -That's a very nice catch! - -We do not support building tracepoint probe provider with -g++ yet, as stated in lttng-ust(3): - -"- Note for C++ support: although an application instrumented with - tracepoints can be compiled with g++, tracepoint probes should be - compiled with gcc (only tested with gcc so far)." - -However, if it works fine with this fix, then I'm tempted to take it, -especially because removing the empty union does not appear to affect -the layout of struct lttng_event as seen from liblttng-ust, which must -be compiled with a C compiler, and from probe providers compiled with -a C compiler. So all we are changing is the layout of a probe provider -compiled with a C++ compiler, which is anyway buggy at the moment, -because it is not compatible with the layout expected by liblttng-ust -compiled with a C compiler. - -Reported-by: Robert Daniels -Signed-off-by: Mathieu Desnoyers ---- - include/lttng/ust-events.h | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/usr/include/lttng/ust-events.h b/usr/include/lttng/ust-events.h -index 328a875..3d7a274 100644 ---- a/usr/include/lttng/ust-events.h -+++ b/usr/include/lttng/ust-events.h -@@ -407,8 +407,6 @@ struct lttng_event { - void *_deprecated1; - struct lttng_ctx *ctx; - enum lttng_ust_instrumentation instrumentation; -- union { -- } u; - struct cds_list_head node; /* Event list in session */ - struct cds_list_head _deprecated2; - void *_deprecated3; --- -2.7.4 - +From e72c9d7ead60e3317bd6d1fade995c07021c947b Mon Sep 17 00:00:00 2001 +From: Mathieu Desnoyers +Date: Thu, 7 May 2015 13:25:04 -0400 +Subject: [PATCH] Fix: building probe providers with C++ compiler + +Robert Daniels wrote: +> > I'm attempting to use lttng userspace tracing with a C++ application +> > on an ARM platform. I'm using GCC 4.8.4 on Linux 3.14 with the 2.6 +> > release of lttng. I've compiled lttng-modules, lttng-ust, and +> > lttng-tools and have been able to get a simple test working with C +> > code. When I attempt to run the hello.cxx test on my target it will +> > segfault. +> +> +> I spent a little time digging into this issue and finally discovered the +> cause of my segfault with ARM C++ tracepoints. +> +> There is a struct called 'lttng_event' in ust-events.h which contains an +> empty union 'u'. This was the cause of my issue. Under C, this empty union +> compiles to a zero byte member while under C++ it compiles to a one byte +> member, and in my case was four-byte aligned which caused my C++ code to +> have the 'cds_list_head node' offset incorrectly by four bytes. This lead +> to an incorrect linked list structure which caused my issue. +> +> Since this union is empty, I simply removed it from the struct and everything +> worked correctly. +> +> I don't know the history or purpose behind this empty union so I'd like to +> know if this is a safe fix. If it is I can submit a patch with the union +> removed. + +That's a very nice catch! + +We do not support building tracepoint probe provider with +g++ yet, as stated in lttng-ust(3): + +"- Note for C++ support: although an application instrumented with + tracepoints can be compiled with g++, tracepoint probes should be + compiled with gcc (only tested with gcc so far)." + +However, if it works fine with this fix, then I'm tempted to take it, +especially because removing the empty union does not appear to affect +the layout of struct lttng_event as seen from liblttng-ust, which must +be compiled with a C compiler, and from probe providers compiled with +a C compiler. So all we are changing is the layout of a probe provider +compiled with a C++ compiler, which is anyway buggy at the moment, +because it is not compatible with the layout expected by liblttng-ust +compiled with a C compiler. + +Reported-by: Robert Daniels +Signed-off-by: Mathieu Desnoyers +--- + include/lttng/ust-events.h | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/usr/include/lttng/ust-events.h b/usr/include/lttng/ust-events.h +index 328a875..3d7a274 100644 +--- a/usr/include/lttng/ust-events.h ++++ b/usr/include/lttng/ust-events.h +@@ -407,8 +407,6 @@ struct lttng_event { + void *_deprecated1; + struct lttng_ctx *ctx; + enum lttng_ust_instrumentation instrumentation; +- union { +- } u; + struct cds_list_head node; /* Event list in session */ + struct cds_list_head _deprecated2; + void *_deprecated3; +-- +2.7.4 + diff --git a/eng/common/cross/arm/trusty.patch b/eng/common/cross/arm/trusty.patch index 2f2972f8eb5..d578de062a8 100644 --- a/eng/common/cross/arm/trusty.patch +++ b/eng/common/cross/arm/trusty.patch @@ -1,97 +1,97 @@ -diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h ---- a/usr/include/urcu/uatomic/generic.h 2014-03-28 06:04:42.000000000 +0900 -+++ b/usr/include/urcu/uatomic/generic.h 2017-02-13 10:35:21.189927116 +0900 -@@ -65,17 +65,17 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- return __sync_val_compare_and_swap_1(addr, old, _new); -+ return __sync_val_compare_and_swap_1((uint8_t *) addr, old, _new); - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_val_compare_and_swap_2(addr, old, _new); -+ return __sync_val_compare_and_swap_2((uint16_t *) addr, old, _new); - #endif - case 4: -- return __sync_val_compare_and_swap_4(addr, old, _new); -+ return __sync_val_compare_and_swap_4((uint32_t *) addr, old, _new); - #if (CAA_BITS_PER_LONG == 64) - case 8: -- return __sync_val_compare_and_swap_8(addr, old, _new); -+ return __sync_val_compare_and_swap_8((uint64_t *) addr, old, _new); - #endif - } - _uatomic_link_error(); -@@ -100,20 +100,20 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- __sync_and_and_fetch_1(addr, val); -+ __sync_and_and_fetch_1((uint8_t *) addr, val); - return; - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- __sync_and_and_fetch_2(addr, val); -+ __sync_and_and_fetch_2((uint16_t *) addr, val); - return; - #endif - case 4: -- __sync_and_and_fetch_4(addr, val); -+ __sync_and_and_fetch_4((uint32_t *) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -- __sync_and_and_fetch_8(addr, val); -+ __sync_and_and_fetch_8((uint64_t *) addr, val); - return; - #endif - } -@@ -139,20 +139,20 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- __sync_or_and_fetch_1(addr, val); -+ __sync_or_and_fetch_1((uint8_t *) addr, val); - return; - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- __sync_or_and_fetch_2(addr, val); -+ __sync_or_and_fetch_2((uint16_t *) addr, val); - return; - #endif - case 4: -- __sync_or_and_fetch_4(addr, val); -+ __sync_or_and_fetch_4((uint32_t *) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -- __sync_or_and_fetch_8(addr, val); -+ __sync_or_and_fetch_8((uint64_t *) addr, val); - return; - #endif - } -@@ -180,17 +180,17 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- return __sync_add_and_fetch_1(addr, val); -+ return __sync_add_and_fetch_1((uint8_t *) addr, val); - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_add_and_fetch_2(addr, val); -+ return __sync_add_and_fetch_2((uint16_t *) addr, val); - #endif - case 4: -- return __sync_add_and_fetch_4(addr, val); -+ return __sync_add_and_fetch_4((uint32_t *) addr, val); - #if (CAA_BITS_PER_LONG == 64) - case 8: -- return __sync_add_and_fetch_8(addr, val); -+ return __sync_add_and_fetch_8((uint64_t *) addr, val); - #endif - } - _uatomic_link_error(); +diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h +--- a/usr/include/urcu/uatomic/generic.h 2014-03-28 06:04:42.000000000 +0900 ++++ b/usr/include/urcu/uatomic/generic.h 2017-02-13 10:35:21.189927116 +0900 +@@ -65,17 +65,17 @@ + switch (len) { + #ifdef UATOMIC_HAS_ATOMIC_BYTE + case 1: +- return __sync_val_compare_and_swap_1(addr, old, _new); ++ return __sync_val_compare_and_swap_1((uint8_t *) addr, old, _new); + #endif + #ifdef UATOMIC_HAS_ATOMIC_SHORT + case 2: +- return __sync_val_compare_and_swap_2(addr, old, _new); ++ return __sync_val_compare_and_swap_2((uint16_t *) addr, old, _new); + #endif + case 4: +- return __sync_val_compare_and_swap_4(addr, old, _new); ++ return __sync_val_compare_and_swap_4((uint32_t *) addr, old, _new); + #if (CAA_BITS_PER_LONG == 64) + case 8: +- return __sync_val_compare_and_swap_8(addr, old, _new); ++ return __sync_val_compare_and_swap_8((uint64_t *) addr, old, _new); + #endif + } + _uatomic_link_error(); +@@ -100,20 +100,20 @@ + switch (len) { + #ifdef UATOMIC_HAS_ATOMIC_BYTE + case 1: +- __sync_and_and_fetch_1(addr, val); ++ __sync_and_and_fetch_1((uint8_t *) addr, val); + return; + #endif + #ifdef UATOMIC_HAS_ATOMIC_SHORT + case 2: +- __sync_and_and_fetch_2(addr, val); ++ __sync_and_and_fetch_2((uint16_t *) addr, val); + return; + #endif + case 4: +- __sync_and_and_fetch_4(addr, val); ++ __sync_and_and_fetch_4((uint32_t *) addr, val); + return; + #if (CAA_BITS_PER_LONG == 64) + case 8: +- __sync_and_and_fetch_8(addr, val); ++ __sync_and_and_fetch_8((uint64_t *) addr, val); + return; + #endif + } +@@ -139,20 +139,20 @@ + switch (len) { + #ifdef UATOMIC_HAS_ATOMIC_BYTE + case 1: +- __sync_or_and_fetch_1(addr, val); ++ __sync_or_and_fetch_1((uint8_t *) addr, val); + return; + #endif + #ifdef UATOMIC_HAS_ATOMIC_SHORT + case 2: +- __sync_or_and_fetch_2(addr, val); ++ __sync_or_and_fetch_2((uint16_t *) addr, val); + return; + #endif + case 4: +- __sync_or_and_fetch_4(addr, val); ++ __sync_or_and_fetch_4((uint32_t *) addr, val); + return; + #if (CAA_BITS_PER_LONG == 64) + case 8: +- __sync_or_and_fetch_8(addr, val); ++ __sync_or_and_fetch_8((uint64_t *) addr, val); + return; + #endif + } +@@ -180,17 +180,17 @@ + switch (len) { + #ifdef UATOMIC_HAS_ATOMIC_BYTE + case 1: +- return __sync_add_and_fetch_1(addr, val); ++ return __sync_add_and_fetch_1((uint8_t *) addr, val); + #endif + #ifdef UATOMIC_HAS_ATOMIC_SHORT + case 2: +- return __sync_add_and_fetch_2(addr, val); ++ return __sync_add_and_fetch_2((uint16_t *) addr, val); + #endif + case 4: +- return __sync_add_and_fetch_4(addr, val); ++ return __sync_add_and_fetch_4((uint32_t *) addr, val); + #if (CAA_BITS_PER_LONG == 64) + case 8: +- return __sync_add_and_fetch_8(addr, val); ++ return __sync_add_and_fetch_8((uint64_t *) addr, val); + #endif + } + _uatomic_link_error(); diff --git a/eng/common/cross/arm64/sources.list.bionic b/eng/common/cross/arm64/sources.list.bionic index 21095574095..a447075be91 100644 --- a/eng/common/cross/arm64/sources.list.bionic +++ b/eng/common/cross/arm64/sources.list.bionic @@ -1,11 +1,11 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.buster b/eng/common/cross/arm64/sources.list.buster index 7194ac64a96..96a02b81c04 100644 --- a/eng/common/cross/arm64/sources.list.buster +++ b/eng/common/cross/arm64/sources.list.buster @@ -1,11 +1,11 @@ -deb http://deb.debian.org/debian buster main -deb-src http://deb.debian.org/debian buster main - -deb http://deb.debian.org/debian-security/ buster/updates main -deb-src http://deb.debian.org/debian-security/ buster/updates main - -deb http://deb.debian.org/debian buster-updates main -deb-src http://deb.debian.org/debian buster-updates main - -deb http://deb.debian.org/debian buster-backports main contrib non-free -deb-src http://deb.debian.org/debian buster-backports main contrib non-free +deb http://deb.debian.org/debian buster main +deb-src http://deb.debian.org/debian buster main + +deb http://deb.debian.org/debian-security/ buster/updates main +deb-src http://deb.debian.org/debian-security/ buster/updates main + +deb http://deb.debian.org/debian buster-updates main +deb-src http://deb.debian.org/debian buster-updates main + +deb http://deb.debian.org/debian buster-backports main contrib non-free +deb-src http://deb.debian.org/debian buster-backports main contrib non-free diff --git a/eng/common/cross/arm64/sources.list.stretch b/eng/common/cross/arm64/sources.list.stretch index 0e121577436..658d0721c71 100644 --- a/eng/common/cross/arm64/sources.list.stretch +++ b/eng/common/cross/arm64/sources.list.stretch @@ -1,12 +1,12 @@ -deb http://deb.debian.org/debian stretch main -deb-src http://deb.debian.org/debian stretch main - -deb http://deb.debian.org/debian-security/ stretch/updates main -deb-src http://deb.debian.org/debian-security/ stretch/updates main - -deb http://deb.debian.org/debian stretch-updates main -deb-src http://deb.debian.org/debian stretch-updates main - -deb http://deb.debian.org/debian stretch-backports main contrib non-free -deb-src http://deb.debian.org/debian stretch-backports main contrib non-free - +deb http://deb.debian.org/debian stretch main +deb-src http://deb.debian.org/debian stretch main + +deb http://deb.debian.org/debian-security/ stretch/updates main +deb-src http://deb.debian.org/debian-security/ stretch/updates main + +deb http://deb.debian.org/debian stretch-updates main +deb-src http://deb.debian.org/debian stretch-updates main + +deb http://deb.debian.org/debian stretch-backports main contrib non-free +deb-src http://deb.debian.org/debian stretch-backports main contrib non-free + diff --git a/eng/common/cross/arm64/sources.list.trusty b/eng/common/cross/arm64/sources.list.trusty index 07d8f88d82e..b79622576b4 100644 --- a/eng/common/cross/arm64/sources.list.trusty +++ b/eng/common/cross/arm64/sources.list.trusty @@ -1,11 +1,11 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse +deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm64/sources.list.xenial b/eng/common/cross/arm64/sources.list.xenial index eacd86b7df3..1e043483fe4 100644 --- a/eng/common/cross/arm64/sources.list.xenial +++ b/eng/common/cross/arm64/sources.list.xenial @@ -1,11 +1,11 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse +deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm64/sources.list.zesty b/eng/common/cross/arm64/sources.list.zesty index ea2c14a7874..103d793a356 100644 --- a/eng/common/cross/arm64/sources.list.zesty +++ b/eng/common/cross/arm64/sources.list.zesty @@ -1,11 +1,11 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse +deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse diff --git a/eng/common/cross/armel/sources.list.jessie b/eng/common/cross/armel/sources.list.jessie index 3d9c3059d89..5afd7b191ea 100644 --- a/eng/common/cross/armel/sources.list.jessie +++ b/eng/common/cross/armel/sources.list.jessie @@ -1,3 +1,3 @@ -# Debian (jessie) # Stable -deb http://ftp.debian.org/debian/ jessie main contrib non-free -deb-src http://ftp.debian.org/debian/ jessie main contrib non-free +# Debian (jessie) # Stable +deb http://ftp.debian.org/debian/ jessie main contrib non-free +deb-src http://ftp.debian.org/debian/ jessie main contrib non-free diff --git a/eng/common/cross/armel/tizen-build-rootfs.sh b/eng/common/cross/armel/tizen-build-rootfs.sh index 87c48e78fbb..590e0af0cd9 100644 --- a/eng/common/cross/armel/tizen-build-rootfs.sh +++ b/eng/common/cross/armel/tizen-build-rootfs.sh @@ -1,44 +1,44 @@ -#!/usr/bin/env bash -set -e - -__ARM_SOFTFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -__TIZEN_CROSSDIR="$__ARM_SOFTFP_CrossDir/tizen" - -if [[ -z "$ROOTFS_DIR" ]]; then - echo "ROOTFS_DIR is not defined." - exit 1; -fi - -# Clean-up (TODO-Cleanup: We may already delete $ROOTFS_DIR at ./cross/build-rootfs.sh.) -# hk0110 -if [ -d "$ROOTFS_DIR" ]; then - umount $ROOTFS_DIR/* - rm -rf $ROOTFS_DIR -fi - -TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp -mkdir -p $TIZEN_TMP_DIR - -# Download files -echo ">>Start downloading files" -VERBOSE=1 $__ARM_SOFTFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR -echo "<>Start constructing Tizen rootfs" -TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` -cd $ROOTFS_DIR -for f in $TIZEN_RPM_FILES; do - rpm2cpio $f | cpio -idm --quiet -done -echo "<>Start configuring Tizen rootfs" -rm ./usr/lib/libunwind.so -ln -s libunwind.so.8 ./usr/lib/libunwind.so -ln -sfn asm-arm ./usr/include/asm -patch -p1 < $__TIZEN_CROSSDIR/tizen.patch -echo "<>Start downloading files" +VERBOSE=1 $__ARM_SOFTFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +rm ./usr/lib/libunwind.so +ln -s libunwind.so.8 ./usr/lib/libunwind.so +ln -sfn asm-arm ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then - VERBOSE=0 -fi - -Log() -{ - if [ $VERBOSE -ge $1 ]; then - echo ${@:2} - fi -} - -Inform() -{ - Log 1 -e "\x1B[0;34m$@\x1B[m" -} - -Debug() -{ - Log 2 -e "\x1B[0;32m$@\x1B[m" -} - -Error() -{ - >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" -} - -Fetch() -{ - URL=$1 - FILE=$2 - PROGRESS=$3 - if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then - CURL_OPT="--progress-bar" - else - CURL_OPT="--silent" - fi - curl $CURL_OPT $URL > $FILE -} - -hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } -hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } -hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } - -TMPDIR=$1 -if [ ! -d $TMPDIR ]; then - TMPDIR=./tizen_tmp - Debug "Create temporary directory : $TMPDIR" - mkdir -p $TMPDIR -fi - -TIZEN_URL=http://download.tizen.org/releases/milestone/tizen -BUILD_XML=build.xml -REPOMD_XML=repomd.xml -PRIMARY_XML=primary.xml -TARGET_URL="http://__not_initialized" - -Xpath_get() -{ - XPATH_RESULT='' - XPATH=$1 - XML_FILE=$2 - RESULT=$(xmllint --xpath $XPATH $XML_FILE) - if [[ -z ${RESULT// } ]]; then - Error "Can not find target from $XML_FILE" - Debug "Xpath = $XPATH" - exit 1 - fi - XPATH_RESULT=$RESULT -} - -fetch_tizen_pkgs_init() -{ - TARGET=$1 - PROFILE=$2 - Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" - - TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs - if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi - mkdir -p $TMP_PKG_DIR - - PKG_URL=$TIZEN_URL/$PROFILE/latest - - BUILD_XML_URL=$PKG_URL/$BUILD_XML - TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML - TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML - TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML - TMP_PRIMARYGZ=${TMP_PRIMARY}.gz - - Fetch $BUILD_XML_URL $TMP_BUILD - - Debug "fetch $BUILD_XML_URL to $TMP_BUILD" - - TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" - Xpath_get $TARGET_XPATH $TMP_BUILD - TARGET_PATH=$XPATH_RESULT - TARGET_URL=$PKG_URL/$TARGET_PATH - - REPOMD_URL=$TARGET_URL/repodata/repomd.xml - PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' - - Fetch $REPOMD_URL $TMP_REPOMD - - Debug "fetch $REPOMD_URL to $TMP_REPOMD" - - Xpath_get $PRIMARY_XPATH $TMP_REPOMD - PRIMARY_XML_PATH=$XPATH_RESULT - PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH - - Fetch $PRIMARY_URL $TMP_PRIMARYGZ - - Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" - - gunzip $TMP_PRIMARYGZ - - Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" -} - -fetch_tizen_pkgs() -{ - ARCH=$1 - PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' - - PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' - - for pkg in ${@:2} - do - Inform "Fetching... $pkg" - XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} - XPATH=${XPATH/_ARCH_/$ARCH} - Xpath_get $XPATH $TMP_PRIMARY - PKG_PATH=$XPATH_RESULT - - XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} - XPATH=${XPATH/_ARCH_/$ARCH} - Xpath_get $XPATH $TMP_PRIMARY - CHECKSUM=$XPATH_RESULT - - PKG_URL=$TARGET_URL/$PKG_PATH - PKG_FILE=$(basename $PKG_PATH) - PKG_PATH=$TMPDIR/$PKG_FILE - - Debug "Download $PKG_URL to $PKG_PATH" - Fetch $PKG_URL $PKG_PATH true - - echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null - if [ $? -ne 0 ]; then - Error "Fail to fetch $PKG_URL to $PKG_PATH" - Debug "Checksum = $CHECKSUM" - exit 1 - fi - done -} - -Inform "Initialize arm base" -fetch_tizen_pkgs_init standard base -Inform "fetch common packages" -fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic -fetch_tizen_pkgs noarch linux-glibc-devel -Inform "fetch coreclr packages" -fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu -Inform "fetch corefx packages" -fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl-devel krb5 krb5-devel libcurl libcurl-devel - -Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard unified -Inform "fetch corefx packages" -fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release - +#!/usr/bin/env bash +set -e + +if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/releases/milestone/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic +fetch_tizen_pkgs noarch linux-glibc-devel +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl-devel krb5 krb5-devel libcurl libcurl-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/armel/tizen/tizen-dotnet.ks b/eng/common/cross/armel/tizen/tizen-dotnet.ks index 506d455bd4f..c258134a79b 100644 --- a/eng/common/cross/armel/tizen/tizen-dotnet.ks +++ b/eng/common/cross/armel/tizen/tizen-dotnet.ks @@ -1,50 +1,50 @@ -lang en_US.UTF-8 -keyboard us -timezone --utc Asia/Seoul - -part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime - -rootpw tizen -desktop --autologinuser=root -user --name root --groups audio,video --password 'tizen' - -repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no -repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no - -%packages -tar -gzip - -sed -grep -gawk -perl - -binutils -findutils -util-linux -lttng-ust -userspace-rcu -procps-ng -tzdata -ca-certificates - - -### Core FX -libicu -libunwind -iputils -zlib -krb5 -libcurl -libopenssl - -%end - -%post - -### Update /tmp privilege -chmod 777 /tmp -#################################### - -%end +lang en_US.UTF-8 +keyboard us +timezone --utc Asia/Seoul + +part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime + +rootpw tizen +desktop --autologinuser=root +user --name root --groups audio,video --password 'tizen' + +repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no +repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no + +%packages +tar +gzip + +sed +grep +gawk +perl + +binutils +findutils +util-linux +lttng-ust +userspace-rcu +procps-ng +tzdata +ca-certificates + + +### Core FX +libicu +libunwind +iputils +zlib +krb5 +libcurl +libopenssl + +%end + +%post + +### Update /tmp privilege +chmod 777 /tmp +#################################### + +%end diff --git a/eng/common/cross/armel/tizen/tizen.patch b/eng/common/cross/armel/tizen/tizen.patch index d223427c978..47648b0305f 100644 --- a/eng/common/cross/armel/tizen/tizen.patch +++ b/eng/common/cross/armel/tizen/tizen.patch @@ -1,18 +1,18 @@ -diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so ---- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 -+++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 -@@ -2,4 +2,4 @@ - Use the shared library, but some functions are only in - the static library, so try that secondarily. */ - OUTPUT_FORMAT(elf32-littlearm) --GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) ) -+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) ) -diff -u -r a/usr/lib/libpthread.so b/usr/lib/libpthread.so ---- a/usr/lib/libpthread.so 2016-12-30 23:00:19.408951841 +0900 -+++ b/usr/lib/libpthread.so 2016-12-30 23:00:39.068951801 +0900 -@@ -2,4 +2,4 @@ - Use the shared library, but some functions are only in - the static library, so try that secondarily. */ - OUTPUT_FORMAT(elf32-littlearm) --GROUP ( /lib/libpthread.so.0 /usr/lib/libpthread_nonshared.a ) -+GROUP ( libpthread.so.0 libpthread_nonshared.a ) +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) ) +diff -u -r a/usr/lib/libpthread.so b/usr/lib/libpthread.so +--- a/usr/lib/libpthread.so 2016-12-30 23:00:19.408951841 +0900 ++++ b/usr/lib/libpthread.so 2016-12-30 23:00:39.068951801 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libpthread.so.0 /usr/lib/libpthread_nonshared.a ) ++GROUP ( libpthread.so.0 libpthread_nonshared.a ) diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index adceda877ad..c23fb58f806 100644 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -1,137 +1,137 @@ -#!/usr/bin/env bash -set -e -__NDK_Version=r14 - -usage() -{ - echo "Creates a toolchain and sysroot used for cross-compiling for Android." - echo. - echo "Usage: $0 [BuildArch] [ApiLevel]" - echo. - echo "BuildArch is the target architecture of Android. Currently only arm64 is supported." - echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html" - echo. - echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior" - echo "by setting the TOOLCHAIN_DIR environment variable" - echo. - echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," - echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." - echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.21-arm64. This file is to replace '/etc/os-release', which is not available for Android." - exit 1 -} - -__ApiLevel=21 # The minimum platform for arm64 is API level 21 -__BuildArch=arm64 -__AndroidArch=aarch64 -__AndroidToolchain=aarch64-linux-android - -for i in "$@" - do - lowerI="$(echo $i | awk '{print tolower($0)}')" - case $lowerI in - -?|-h|--help) - usage - exit 1 - ;; - arm64) - __BuildArch=arm64 - __AndroidArch=aarch64 - __AndroidToolchain=aarch64-linux-android - ;; - arm) - __BuildArch=arm - __AndroidArch=arm - __AndroidToolchain=arm-linux-androideabi - ;; - *[0-9]) - __ApiLevel=$i - ;; - *) - __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" - ;; - esac -done - -# Obtain the location of the bash script to figure out where the root of the repo is. -__CrossDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -__Android_Cross_Dir="$__CrossDir/android-rootfs" -__NDK_Dir="$__Android_Cross_Dir/android-ndk-$__NDK_Version" -__libunwind_Dir="$__Android_Cross_Dir/libunwind" -__lldb_Dir="$__Android_Cross_Dir/lldb" -__ToolchainDir="$__Android_Cross_Dir/toolchain/$__BuildArch" - -if [[ -n "$TOOLCHAIN_DIR" ]]; then - __ToolchainDir=$TOOLCHAIN_DIR -fi - -if [[ -n "$NDK_DIR" ]]; then - __NDK_Dir=$NDK_DIR -fi - -echo "Target API level: $__ApiLevel" -echo "Target architecture: $__BuildArch" -echo "NDK location: $__NDK_Dir" -echo "Target Toolchain location: $__ToolchainDir" - -# Download the NDK if required -if [ ! -d $__NDK_Dir ]; then - echo Downloading the NDK into $__NDK_Dir - mkdir -p $__NDK_Dir - wget -nv -nc --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip - unzip -q $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__Android_Cross_Dir -fi - -if [ ! -d $__lldb_Dir ]; then - mkdir -p $__lldb_Dir - echo Downloading LLDB into $__lldb_Dir - wget -nv -nc --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip - unzip -q $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir -fi - -# Create the RootFS for both arm64 as well as aarch -rm -rf $__Android_Cross_Dir/toolchain - -echo Generating the $__BuildArch toolchain -$__NDK_Dir/build/tools/make_standalone_toolchain.py --arch $__BuildArch --api $__ApiLevel --install-dir $__ToolchainDir - -# Install the required packages into the toolchain -# TODO: Add logic to get latest pkg version instead of specific version number -rm -rf $__Android_Cross_Dir/deb/ -rm -rf $__Android_Cross_Dir/tmp - -mkdir -p $__Android_Cross_Dir/deb/ -mkdir -p $__Android_Cross_Dir/tmp/$arch/ -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu-dev_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb - -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob-dev_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support-dev_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma-dev_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind-dev_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb - -echo Unpacking Termux packages -dpkg -x $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ - -cp -R $__Android_Cross_Dir/tmp/$__AndroidArch/data/data/com.termux/files/usr/* $__ToolchainDir/sysroot/usr/ - -# Generate platform file for build.sh script to assign to __DistroRid -echo "Generating platform file..." - -echo "RID=android.21-arm64" > $__ToolchainDir/sysroot/android_platform -echo Now run: -echo CONFIG_DIR=\`realpath cross/android/$__BuildArch\` ROOTFS_DIR=\`realpath $__ToolchainDir/sysroot\` ./build.sh cross $__BuildArch skipgenerateversion skipnuget cmakeargs -DENABLE_LLDBPLUGIN=0 - +#!/usr/bin/env bash +set -e +__NDK_Version=r14 + +usage() +{ + echo "Creates a toolchain and sysroot used for cross-compiling for Android." + echo. + echo "Usage: $0 [BuildArch] [ApiLevel]" + echo. + echo "BuildArch is the target architecture of Android. Currently only arm64 is supported." + echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html" + echo. + echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior" + echo "by setting the TOOLCHAIN_DIR environment variable" + echo. + echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," + echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." + echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.21-arm64. This file is to replace '/etc/os-release', which is not available for Android." + exit 1 +} + +__ApiLevel=21 # The minimum platform for arm64 is API level 21 +__BuildArch=arm64 +__AndroidArch=aarch64 +__AndroidToolchain=aarch64-linux-android + +for i in "$@" + do + lowerI="$(echo $i | awk '{print tolower($0)}')" + case $lowerI in + -?|-h|--help) + usage + exit 1 + ;; + arm64) + __BuildArch=arm64 + __AndroidArch=aarch64 + __AndroidToolchain=aarch64-linux-android + ;; + arm) + __BuildArch=arm + __AndroidArch=arm + __AndroidToolchain=arm-linux-androideabi + ;; + *[0-9]) + __ApiLevel=$i + ;; + *) + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + ;; + esac +done + +# Obtain the location of the bash script to figure out where the root of the repo is. +__CrossDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +__Android_Cross_Dir="$__CrossDir/android-rootfs" +__NDK_Dir="$__Android_Cross_Dir/android-ndk-$__NDK_Version" +__libunwind_Dir="$__Android_Cross_Dir/libunwind" +__lldb_Dir="$__Android_Cross_Dir/lldb" +__ToolchainDir="$__Android_Cross_Dir/toolchain/$__BuildArch" + +if [[ -n "$TOOLCHAIN_DIR" ]]; then + __ToolchainDir=$TOOLCHAIN_DIR +fi + +if [[ -n "$NDK_DIR" ]]; then + __NDK_Dir=$NDK_DIR +fi + +echo "Target API level: $__ApiLevel" +echo "Target architecture: $__BuildArch" +echo "NDK location: $__NDK_Dir" +echo "Target Toolchain location: $__ToolchainDir" + +# Download the NDK if required +if [ ! -d $__NDK_Dir ]; then + echo Downloading the NDK into $__NDK_Dir + mkdir -p $__NDK_Dir + wget -nv -nc --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip + unzip -q $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__Android_Cross_Dir +fi + +if [ ! -d $__lldb_Dir ]; then + mkdir -p $__lldb_Dir + echo Downloading LLDB into $__lldb_Dir + wget -nv -nc --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip + unzip -q $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir +fi + +# Create the RootFS for both arm64 as well as aarch +rm -rf $__Android_Cross_Dir/toolchain + +echo Generating the $__BuildArch toolchain +$__NDK_Dir/build/tools/make_standalone_toolchain.py --arch $__BuildArch --api $__ApiLevel --install-dir $__ToolchainDir + +# Install the required packages into the toolchain +# TODO: Add logic to get latest pkg version instead of specific version number +rm -rf $__Android_Cross_Dir/deb/ +rm -rf $__Android_Cross_Dir/tmp + +mkdir -p $__Android_Cross_Dir/deb/ +mkdir -p $__Android_Cross_Dir/tmp/$arch/ +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu-dev_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb + +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob-dev_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support-dev_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma-dev_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind-dev_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb +wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb + +echo Unpacking Termux packages +dpkg -x $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ +dpkg -x $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ + +cp -R $__Android_Cross_Dir/tmp/$__AndroidArch/data/data/com.termux/files/usr/* $__ToolchainDir/sysroot/usr/ + +# Generate platform file for build.sh script to assign to __DistroRid +echo "Generating platform file..." + +echo "RID=android.21-arm64" > $__ToolchainDir/sysroot/android_platform +echo Now run: +echo CONFIG_DIR=\`realpath cross/android/$__BuildArch\` ROOTFS_DIR=\`realpath $__ToolchainDir/sysroot\` ./build.sh cross $__BuildArch skipgenerateversion skipnuget cmakeargs -DENABLE_LLDBPLUGIN=0 + diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index d7d5d7d5f44..1ccb34a42e0 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -1,234 +1,234 @@ -#!/usr/bin/env bash - -usage() -{ - echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" - echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" - echo "--skipunmount - optional, will skip the unmount of rootfs folder." - exit 1 -} - -__LinuxCodeName=xenial -__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -__InitialDir=$PWD -__BuildArch=arm -__UbuntuArch=armhf -__UbuntuRepo="http://ports.ubuntu.com/" -__LLDB_Package="liblldb-3.9-dev" -__SkipUnmount=0 - -# base development support -__UbuntuPackages="build-essential" - -__AlpinePackages="alpine-base" -__AlpinePackages+=" build-base" -__AlpinePackages+=" linux-headers" -__AlpinePackages+=" lldb-dev" -__AlpinePackages+=" llvm-dev" - -# symlinks fixer -__UbuntuPackages+=" symlinks" - -# CoreCLR and CoreFX dependencies -__UbuntuPackages+=" libicu-dev" -__UbuntuPackages+=" liblttng-ust-dev" -__UbuntuPackages+=" libunwind8-dev" - -__AlpinePackages+=" gettext-dev" -__AlpinePackages+=" icu-dev" -__AlpinePackages+=" libunwind-dev" -__AlpinePackages+=" lttng-ust-dev" - -# CoreFX dependencies -__UbuntuPackages+=" libcurl4-openssl-dev" -__UbuntuPackages+=" libkrb5-dev" -__UbuntuPackages+=" libssl-dev" -__UbuntuPackages+=" zlib1g-dev" - -__AlpinePackages+=" curl-dev" -__AlpinePackages+=" krb5-dev" -__AlpinePackages+=" openssl-dev" -__AlpinePackages+=" zlib-dev" - -__UnprocessedBuildArgs= -while :; do - if [ $# -le 0 ]; then - break - fi - - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - -?|-h|--help) - usage - exit 1 - ;; - arm) - __BuildArch=arm - __UbuntuArch=armhf - __AlpineArch=armhf - __QEMUArch=arm - ;; - arm64) - __BuildArch=arm64 - __UbuntuArch=arm64 - __AlpineArch=aarch64 - __QEMUArch=aarch64 - ;; - armel) - __BuildArch=armel - __UbuntuArch=armel - __UbuntuRepo="http://ftp.debian.org/debian/" - __LinuxCodeName=jessie - ;; - x86) - __BuildArch=x86 - __UbuntuArch=i386 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" - ;; - lldb3.6) - __LLDB_Package="lldb-3.6-dev" - ;; - lldb3.8) - __LLDB_Package="lldb-3.8-dev" - ;; - lldb3.9) - __LLDB_Package="liblldb-3.9-dev" - ;; - lldb4.0) - __LLDB_Package="liblldb-4.0-dev" - ;; - lldb5.0) - __LLDB_Package="liblldb-5.0-dev" - ;; - lldb6.0) - __LLDB_Package="liblldb-6.0-dev" - ;; - no-lldb) - unset __LLDB_Package - ;; - trusty) # Ubuntu 14.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=trusty - fi - ;; - xenial) # Ubuntu 16.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=xenial - fi - ;; - zesty) # Ubuntu 17.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=zesty - fi - ;; - bionic) # Ubuntu 18.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=bionic - fi - ;; - jessie) # Debian 8 - __LinuxCodeName=jessie - __UbuntuRepo="http://ftp.debian.org/debian/" - ;; - stretch) # Debian 9 - __LinuxCodeName=stretch - __UbuntuRepo="http://ftp.debian.org/debian/" - __LLDB_Package="liblldb-6.0-dev" - ;; - buster) # Debian 10 - __LinuxCodeName=buster - __UbuntuRepo="http://ftp.debian.org/debian/" - __LLDB_Package="liblldb-6.0-dev" - ;; - tizen) - if [ "$__BuildArch" != "armel" ]; then - echo "Tizen is available only for armel." - usage; - exit 1; - fi - __LinuxCodeName= - __UbuntuRepo= - __Tizen=tizen - ;; - alpine) - __LinuxCodeName=alpine - __UbuntuRepo= - ;; - --skipunmount) - __SkipUnmount=1 - ;; - --rootfsdir|-rootfsdir) - shift - __RootfsDir=$1 - ;; - *) - __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" - ;; - esac - - shift -done - -if [ "$__BuildArch" == "armel" ]; then - __LLDB_Package="lldb-3.5-dev" -fi -__UbuntuPackages+=" ${__LLDB_Package:-}" - -if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then - __RootfsDir=$ROOTFS_DIR -fi - -if [ -z "$__RootfsDir" ]; then - __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" -fi - -if [ -d "$__RootfsDir" ]; then - if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* - fi - rm -rf $__RootfsDir -fi - -if [[ "$__LinuxCodeName" == "alpine" ]]; then - __ApkToolsVersion=2.9.1 - __AlpineVersion=3.7 - __ApkToolsDir=$(mktemp -d) - wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir - tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir - mkdir -p $__RootfsDir/usr/bin - cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ - -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackages - rm -r $__ApkToolsDir -elif [[ -n $__LinuxCodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__LinuxCodeName $__RootfsDir $__UbuntuRepo - cp $__CrossDir/$__BuildArch/sources.list.$__LinuxCodeName $__RootfsDir/etc/apt/sources.list - chroot $__RootfsDir apt-get update - chroot $__RootfsDir apt-get -f -y install - chroot $__RootfsDir apt-get -y install $__UbuntuPackages - chroot $__RootfsDir symlinks -cr /usr - - if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* - fi - - if [[ "$__BuildArch" == "arm" && "$__LinuxCodeName" == "trusty" ]]; then - pushd $__RootfsDir - patch -p1 < $__CrossDir/$__BuildArch/trusty.patch - patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch - popd - fi -elif [ "$__Tizen" == "tizen" ]; then - ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh -else - echo "Unsupported target platform." - usage; - exit 1 -fi +#!/usr/bin/env bash + +usage() +{ + echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" + echo "BuildArch can be: arm(default), armel, arm64, x86" + echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" + echo "--skipunmount - optional, will skip the unmount of rootfs folder." + exit 1 +} + +__LinuxCodeName=xenial +__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__InitialDir=$PWD +__BuildArch=arm +__UbuntuArch=armhf +__UbuntuRepo="http://ports.ubuntu.com/" +__LLDB_Package="liblldb-3.9-dev" +__SkipUnmount=0 + +# base development support +__UbuntuPackages="build-essential" + +__AlpinePackages="alpine-base" +__AlpinePackages+=" build-base" +__AlpinePackages+=" linux-headers" +__AlpinePackages+=" lldb-dev" +__AlpinePackages+=" llvm-dev" + +# symlinks fixer +__UbuntuPackages+=" symlinks" + +# CoreCLR and CoreFX dependencies +__UbuntuPackages+=" libicu-dev" +__UbuntuPackages+=" liblttng-ust-dev" +__UbuntuPackages+=" libunwind8-dev" + +__AlpinePackages+=" gettext-dev" +__AlpinePackages+=" icu-dev" +__AlpinePackages+=" libunwind-dev" +__AlpinePackages+=" lttng-ust-dev" + +# CoreFX dependencies +__UbuntuPackages+=" libcurl4-openssl-dev" +__UbuntuPackages+=" libkrb5-dev" +__UbuntuPackages+=" libssl-dev" +__UbuntuPackages+=" zlib1g-dev" + +__AlpinePackages+=" curl-dev" +__AlpinePackages+=" krb5-dev" +__AlpinePackages+=" openssl-dev" +__AlpinePackages+=" zlib-dev" + +__UnprocessedBuildArgs= +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + -?|-h|--help) + usage + exit 1 + ;; + arm) + __BuildArch=arm + __UbuntuArch=armhf + __AlpineArch=armhf + __QEMUArch=arm + ;; + arm64) + __BuildArch=arm64 + __UbuntuArch=arm64 + __AlpineArch=aarch64 + __QEMUArch=aarch64 + ;; + armel) + __BuildArch=armel + __UbuntuArch=armel + __UbuntuRepo="http://ftp.debian.org/debian/" + __LinuxCodeName=jessie + ;; + x86) + __BuildArch=x86 + __UbuntuArch=i386 + __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" + ;; + lldb3.6) + __LLDB_Package="lldb-3.6-dev" + ;; + lldb3.8) + __LLDB_Package="lldb-3.8-dev" + ;; + lldb3.9) + __LLDB_Package="liblldb-3.9-dev" + ;; + lldb4.0) + __LLDB_Package="liblldb-4.0-dev" + ;; + lldb5.0) + __LLDB_Package="liblldb-5.0-dev" + ;; + lldb6.0) + __LLDB_Package="liblldb-6.0-dev" + ;; + no-lldb) + unset __LLDB_Package + ;; + trusty) # Ubuntu 14.04 + if [ "$__LinuxCodeName" != "jessie" ]; then + __LinuxCodeName=trusty + fi + ;; + xenial) # Ubuntu 16.04 + if [ "$__LinuxCodeName" != "jessie" ]; then + __LinuxCodeName=xenial + fi + ;; + zesty) # Ubuntu 17.04 + if [ "$__LinuxCodeName" != "jessie" ]; then + __LinuxCodeName=zesty + fi + ;; + bionic) # Ubuntu 18.04 + if [ "$__LinuxCodeName" != "jessie" ]; then + __LinuxCodeName=bionic + fi + ;; + jessie) # Debian 8 + __LinuxCodeName=jessie + __UbuntuRepo="http://ftp.debian.org/debian/" + ;; + stretch) # Debian 9 + __LinuxCodeName=stretch + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; + buster) # Debian 10 + __LinuxCodeName=buster + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; + tizen) + if [ "$__BuildArch" != "armel" ]; then + echo "Tizen is available only for armel." + usage; + exit 1; + fi + __LinuxCodeName= + __UbuntuRepo= + __Tizen=tizen + ;; + alpine) + __LinuxCodeName=alpine + __UbuntuRepo= + ;; + --skipunmount) + __SkipUnmount=1 + ;; + --rootfsdir|-rootfsdir) + shift + __RootfsDir=$1 + ;; + *) + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" + ;; + esac + + shift +done + +if [ "$__BuildArch" == "armel" ]; then + __LLDB_Package="lldb-3.5-dev" +fi +__UbuntuPackages+=" ${__LLDB_Package:-}" + +if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then + __RootfsDir=$ROOTFS_DIR +fi + +if [ -z "$__RootfsDir" ]; then + __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" +fi + +if [ -d "$__RootfsDir" ]; then + if [ $__SkipUnmount == 0 ]; then + umount $__RootfsDir/* + fi + rm -rf $__RootfsDir +fi + +if [[ "$__LinuxCodeName" == "alpine" ]]; then + __ApkToolsVersion=2.9.1 + __AlpineVersion=3.7 + __ApkToolsDir=$(mktemp -d) + wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir + tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir + mkdir -p $__RootfsDir/usr/bin + cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ + -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackages + rm -r $__ApkToolsDir +elif [[ -n $__LinuxCodeName ]]; then + qemu-debootstrap --arch $__UbuntuArch $__LinuxCodeName $__RootfsDir $__UbuntuRepo + cp $__CrossDir/$__BuildArch/sources.list.$__LinuxCodeName $__RootfsDir/etc/apt/sources.list + chroot $__RootfsDir apt-get update + chroot $__RootfsDir apt-get -f -y install + chroot $__RootfsDir apt-get -y install $__UbuntuPackages + chroot $__RootfsDir symlinks -cr /usr + + if [ $__SkipUnmount == 0 ]; then + umount $__RootfsDir/* + fi + + if [[ "$__BuildArch" == "arm" && "$__LinuxCodeName" == "trusty" ]]; then + pushd $__RootfsDir + patch -p1 < $__CrossDir/$__BuildArch/trusty.patch + patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch + popd + fi +elif [ "$__Tizen" == "tizen" ]; then + ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh +else + echo "Unsupported target platform." + usage; + exit 1 +fi diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 071d4112419..46bda586040 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -1,138 +1,138 @@ -set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) - -set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) - -if(TARGET_ARCH_NAME STREQUAL "armel") - set(CMAKE_SYSTEM_PROCESSOR armv7l) - set(TOOLCHAIN "arm-linux-gnueabi") - if("$ENV{__DistroRid}" MATCHES "tizen.*") - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/6.2.1") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "arm") - set(CMAKE_SYSTEM_PROCESSOR armv7l) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) - set(TOOLCHAIN "armv6-alpine-linux-musleabihf") - else() - set(TOOLCHAIN "arm-linux-gnueabihf") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "arm64") - set(CMAKE_SYSTEM_PROCESSOR aarch64) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) - set(TOOLCHAIN "aarch64-alpine-linux-musl") - else() - set(TOOLCHAIN "aarch64-linux-gnu") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "x86") - set(CMAKE_SYSTEM_PROCESSOR i686) - set(TOOLCHAIN "i686-linux-gnu") -else() - message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") -endif() - -# Specify include paths -if(TARGET_ARCH_NAME STREQUAL "armel") - if(DEFINED TIZEN_TOOLCHAIN) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) - endif() -endif() - -# add_compile_param - adds only new options without duplicates. -# arg0 - list with result options, arg1 - list with new options. -# arg2 - optional argument, quick summary string for optional using CACHE FORCE mode. -macro(add_compile_param) - if(NOT ${ARGC} MATCHES "^(2|3)$") - message(FATAL_ERROR "Wrong using add_compile_param! Two or three parameters must be given! See add_compile_param description.") - endif() - foreach(OPTION ${ARGV1}) - if(NOT ${ARGV0} MATCHES "${OPTION}($| )") - set(${ARGV0} "${${ARGV0}} ${OPTION}") - if(${ARGC} EQUAL "3") # CACHE FORCE mode - set(${ARGV0} "${${ARGV0}}" CACHE STRING "${ARGV2}" FORCE) - endif() - endif() - endforeach() -endmacro() - -# Specify link flags -add_compile_param(CROSS_LINK_FLAGS "--sysroot=${CROSS_ROOTFS}") -add_compile_param(CROSS_LINK_FLAGS "--gcc-toolchain=${CROSS_ROOTFS}/usr") -add_compile_param(CROSS_LINK_FLAGS "--target=${TOOLCHAIN}") -add_compile_param(CROSS_LINK_FLAGS "-fuse-ld=gold") - -if(TARGET_ARCH_NAME STREQUAL "armel") - if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only - add_compile_param(CROSS_LINK_FLAGS "-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") - add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/lib") - add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib") - add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "x86") - add_compile_param(CROSS_LINK_FLAGS "-m32") -endif() - -add_compile_param(CMAKE_EXE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS") -add_compile_param(CMAKE_SHARED_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS") -add_compile_param(CMAKE_MODULE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS") - -# Specify compile options -add_compile_options("--sysroot=${CROSS_ROOTFS}") -add_compile_options("--target=${TOOLCHAIN}") -add_compile_options("--gcc-toolchain=${CROSS_ROOTFS}/usr") - -if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") - set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) - set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) - set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) -endif() - -if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") - add_compile_options(-mthumb) - add_compile_options(-mfpu=vfpv3) - if(TARGET_ARCH_NAME STREQUAL "armel") - add_compile_options(-mfloat-abi=softfp) - if(DEFINED TIZEN_TOOLCHAIN) - add_compile_options(-Wno-deprecated-declarations) # compile-time option - add_compile_options(-D__extern_always_inline=inline) # compile-time option - endif() - endif() -elseif(TARGET_ARCH_NAME STREQUAL "x86") - add_compile_options(-m32) - add_compile_options(-Wno-error=unused-command-line-argument) -endif() - -# Set LLDB include and library paths -if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") - if(TARGET_ARCH_NAME STREQUAL "x86") - set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}") - else() # arm/armel case - set(LLVM_CROSS_DIR "$ENV{LLVM_ARM_HOME}") - endif() - if(LLVM_CROSS_DIR) - set(WITH_LLDB_LIBS "${LLVM_CROSS_DIR}/lib/" CACHE STRING "") - set(WITH_LLDB_INCLUDES "${LLVM_CROSS_DIR}/include" CACHE STRING "") - set(LLDB_H "${WITH_LLDB_INCLUDES}" CACHE STRING "") - set(LLDB "${LLVM_CROSS_DIR}/lib/liblldb.so" CACHE STRING "") - else() - if(TARGET_ARCH_NAME STREQUAL "x86") - set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/i386-linux-gnu" CACHE STRING "") - set(CHECK_LLVM_DIR "${CROSS_ROOTFS}/usr/lib/llvm-3.8/include") - if(EXISTS "${CHECK_LLVM_DIR}" AND IS_DIRECTORY "${CHECK_LLVM_DIR}") - set(WITH_LLDB_INCLUDES "${CHECK_LLVM_DIR}") - else() - set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include") - endif() - else() # arm/armel case - set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}" CACHE STRING "") - set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include" CACHE STRING "") - endif() - endif() -endif() - -set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) + +set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) + +if(TARGET_ARCH_NAME STREQUAL "armel") + set(CMAKE_SYSTEM_PROCESSOR armv7l) + set(TOOLCHAIN "arm-linux-gnueabi") + if("$ENV{__DistroRid}" MATCHES "tizen.*") + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/6.2.1") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "arm") + set(CMAKE_SYSTEM_PROCESSOR armv7l) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) + set(TOOLCHAIN "armv6-alpine-linux-musleabihf") + else() + set(TOOLCHAIN "arm-linux-gnueabihf") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "arm64") + set(CMAKE_SYSTEM_PROCESSOR aarch64) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) + set(TOOLCHAIN "aarch64-alpine-linux-musl") + else() + set(TOOLCHAIN "aarch64-linux-gnu") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "x86") + set(CMAKE_SYSTEM_PROCESSOR i686) + set(TOOLCHAIN "i686-linux-gnu") +else() + message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") +endif() + +# Specify include paths +if(TARGET_ARCH_NAME STREQUAL "armel") + if(DEFINED TIZEN_TOOLCHAIN) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) + endif() +endif() + +# add_compile_param - adds only new options without duplicates. +# arg0 - list with result options, arg1 - list with new options. +# arg2 - optional argument, quick summary string for optional using CACHE FORCE mode. +macro(add_compile_param) + if(NOT ${ARGC} MATCHES "^(2|3)$") + message(FATAL_ERROR "Wrong using add_compile_param! Two or three parameters must be given! See add_compile_param description.") + endif() + foreach(OPTION ${ARGV1}) + if(NOT ${ARGV0} MATCHES "${OPTION}($| )") + set(${ARGV0} "${${ARGV0}} ${OPTION}") + if(${ARGC} EQUAL "3") # CACHE FORCE mode + set(${ARGV0} "${${ARGV0}}" CACHE STRING "${ARGV2}" FORCE) + endif() + endif() + endforeach() +endmacro() + +# Specify link flags +add_compile_param(CROSS_LINK_FLAGS "--sysroot=${CROSS_ROOTFS}") +add_compile_param(CROSS_LINK_FLAGS "--gcc-toolchain=${CROSS_ROOTFS}/usr") +add_compile_param(CROSS_LINK_FLAGS "--target=${TOOLCHAIN}") +add_compile_param(CROSS_LINK_FLAGS "-fuse-ld=gold") + +if(TARGET_ARCH_NAME STREQUAL "armel") + if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only + add_compile_param(CROSS_LINK_FLAGS "-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/lib") + add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib") + add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "x86") + add_compile_param(CROSS_LINK_FLAGS "-m32") +endif() + +add_compile_param(CMAKE_EXE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS") +add_compile_param(CMAKE_SHARED_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS") +add_compile_param(CMAKE_MODULE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS") + +# Specify compile options +add_compile_options("--sysroot=${CROSS_ROOTFS}") +add_compile_options("--target=${TOOLCHAIN}") +add_compile_options("--gcc-toolchain=${CROSS_ROOTFS}/usr") + +if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") + set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) + set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) + set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) +endif() + +if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") + add_compile_options(-mthumb) + add_compile_options(-mfpu=vfpv3) + if(TARGET_ARCH_NAME STREQUAL "armel") + add_compile_options(-mfloat-abi=softfp) + if(DEFINED TIZEN_TOOLCHAIN) + add_compile_options(-Wno-deprecated-declarations) # compile-time option + add_compile_options(-D__extern_always_inline=inline) # compile-time option + endif() + endif() +elseif(TARGET_ARCH_NAME STREQUAL "x86") + add_compile_options(-m32) + add_compile_options(-Wno-error=unused-command-line-argument) +endif() + +# Set LLDB include and library paths +if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") + if(TARGET_ARCH_NAME STREQUAL "x86") + set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}") + else() # arm/armel case + set(LLVM_CROSS_DIR "$ENV{LLVM_ARM_HOME}") + endif() + if(LLVM_CROSS_DIR) + set(WITH_LLDB_LIBS "${LLVM_CROSS_DIR}/lib/" CACHE STRING "") + set(WITH_LLDB_INCLUDES "${LLVM_CROSS_DIR}/include" CACHE STRING "") + set(LLDB_H "${WITH_LLDB_INCLUDES}" CACHE STRING "") + set(LLDB "${LLVM_CROSS_DIR}/lib/liblldb.so" CACHE STRING "") + else() + if(TARGET_ARCH_NAME STREQUAL "x86") + set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/i386-linux-gnu" CACHE STRING "") + set(CHECK_LLVM_DIR "${CROSS_ROOTFS}/usr/lib/llvm-3.8/include") + if(EXISTS "${CHECK_LLVM_DIR}" AND IS_DIRECTORY "${CHECK_LLVM_DIR}") + set(WITH_LLDB_INCLUDES "${CHECK_LLVM_DIR}") + else() + set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include") + endif() + else() # arm/armel case + set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}" CACHE STRING "") + set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include" CACHE STRING "") + endif() + endif() +endif() + +set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/x86/sources.list.bionic b/eng/common/cross/x86/sources.list.bionic index a71ccadcffa..e3038256950 100644 --- a/eng/common/cross/x86/sources.list.bionic +++ b/eng/common/cross/x86/sources.list.bionic @@ -1,11 +1,11 @@ -deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/x86/sources.list.trusty b/eng/common/cross/x86/sources.list.trusty index 9b3085436e9..fd6902adfe9 100644 --- a/eng/common/cross/x86/sources.list.trusty +++ b/eng/common/cross/x86/sources.list.trusty @@ -1,11 +1,11 @@ -deb http://archive.ubuntu.com/ubuntu/ trusty main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ trusty main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu/ trusty main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ trusty main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse diff --git a/eng/common/cross/x86/sources.list.xenial b/eng/common/cross/x86/sources.list.xenial index ad9c5a0144e..360816a43ea 100644 --- a/eng/common/cross/x86/sources.list.xenial +++ b/eng/common/cross/x86/sources.list.xenial @@ -1,11 +1,11 @@ -deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 8854d979f37..0fd5d411144 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,33 +1,33 @@ -param ( - $darcVersion = $null, - $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" -) - -$verbosity = "m" -. $PSScriptRoot\tools.ps1 - -function InstallDarcCli ($darcVersion) { - $darcCliPackageName = "microsoft.dotnet.darc" - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list -g - - if ($toolList -like "*$darcCliPackageName*") { - & "$dotnet" tool uninstall $darcCliPackageName -g - } - - # If the user didn't explicitly specify the darc version, - # query the Maestro API for the correct version of darc to install. - if (-not $darcVersion) { - $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content - } - - $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' - - Write-Host "Installing Darc CLI version $darcVersion..." - Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g -} - -InstallDarcCli $darcVersion +param ( + $darcVersion = $null, + $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" +) + +$verbosity = "m" +. $PSScriptRoot\tools.ps1 + +function InstallDarcCli ($darcVersion) { + $darcCliPackageName = "microsoft.dotnet.darc" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list -g + + if ($toolList -like "*$darcCliPackageName*") { + & "$dotnet" tool uninstall $darcCliPackageName -g + } + + # If the user didn't explicitly specify the darc version, + # query the Maestro API for the correct version of darc to install. + if (-not $darcVersion) { + $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content + } + + $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' + + Write-Host "Installing Darc CLI version $darcVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g +} + +InstallDarcCli $darcVersion diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index abdd0bc05a1..845a38b16da 100644 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,64 +1,64 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -darcVersion='' -versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" - -while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" - case "$opt" in - --darcversion) - darcVersion=$2 - shift - ;; - --versionendpoint) - versionEndpoint=$2 - shift - ;; - *) - echo "Invalid argument: $1" - usage - exit 1 - ;; - esac - - shift -done - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -verbosity=m - -. "$scriptroot/tools.sh" - -if [ -z "$darcVersion" ]; then - darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") -fi - -function InstallDarcCli { - local darc_cli_package_name="microsoft.dotnet.darc" - - InitializeDotNetCli - local dotnet_root=$_InitializeDotNetCli - - local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g` - local tool_list=$($dotnet_root/dotnet tool list -g) - if [[ $tool_list = *$darc_cli_package_name* ]]; then - echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) - fi - - local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" - - echo "Installing Darc CLI version $darcVersion..." - echo "You may need to restart your command shell if this is the first dotnet tool you have installed." - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) -} - -InstallDarcCli +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +darcVersion='' +versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --darcversion) + darcVersion=$2 + shift + ;; + --versionendpoint) + versionEndpoint=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +verbosity=m + +. "$scriptroot/tools.sh" + +if [ -z "$darcVersion" ]; then + darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") +fi + +function InstallDarcCli { + local darc_cli_package_name="microsoft.dotnet.darc" + + InitializeDotNetCli + local dotnet_root=$_InitializeDotNetCli + + local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g` + local tool_list=$($dotnet_root/dotnet tool list -g) + if [[ $tool_list = *$darc_cli_package_name* ]]; then + echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) + fi + + local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" + + echo "Installing Darc CLI version $darcVersion..." + echo "You may need to restart your command shell if this is the first dotnet tool you have installed." + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) +} + +InstallDarcCli diff --git a/eng/common/dotnet-install.cmd b/eng/common/dotnet-install.cmd index b1c2642e76f..ac4fb3af05f 100644 --- a/eng/common/dotnet-install.cmd +++ b/eng/common/dotnet-install.cmd @@ -1,2 +1,2 @@ -@echo off +@echo off powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 0b629b8301a..a2b0fb18afa 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -1,27 +1,27 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $verbosity = "minimal", - [string] $architecture = "", - [string] $version = "Latest", - [string] $runtime = "dotnet" -) - -. $PSScriptRoot\tools.ps1 - -$dotnetRoot = Join-Path $RepoRoot ".dotnet" - -$installdir = $dotnetRoot -try { - if ($architecture -and $architecture.Trim() -eq "x86") { - $installdir = Join-Path $installdir "x86" - } - InstallDotNet $installdir $version $architecture $runtime $true -} -catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} - -ExitWithExitCode 0 +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = "minimal", + [string] $architecture = "", + [string] $version = "Latest", + [string] $runtime = "dotnet" +) + +. $PSScriptRoot\tools.ps1 + +$dotnetRoot = Join-Path $RepoRoot ".dotnet" + +$installdir = $dotnetRoot +try { + if ($architecture -and $architecture.Trim() -eq "x86") { + $installdir = Join-Path $installdir "x86" + } + InstallDotNet $installdir $version $architecture $runtime $true +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index c3072c958af..3c483d1c762 100644 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -1,49 +1,49 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -version='Latest' -architecture='' -runtime='dotnet' -while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" - case "$opt" in - -version|-v) - shift - version="$1" - ;; - -architecture|-a) - shift - architecture="$1" - ;; - -runtime|-r) - shift - runtime="$1" - ;; - *) - echo "Invalid argument: $1" - usage - exit 1 - ;; - esac - shift -done - -. "$scriptroot/tools.sh" -dotnetRoot="$repo_root/.dotnet" -InstallDotNet $dotnetRoot $version "$architecture" $runtime true || { - local exit_code=$? - echo "dotnet-install.sh failed (exit code '$exit_code')." >&2 - ExitWithExitCode $exit_code -} - -ExitWithExitCode 0 +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +version='Latest' +architecture='' +runtime='dotnet' +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + -version|-v) + shift + version="$1" + ;; + -architecture|-a) + shift + architecture="$1" + ;; + -runtime|-r) + shift + runtime="$1" + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + shift +done + +. "$scriptroot/tools.sh" +dotnetRoot="$repo_root/.dotnet" +InstallDotNet $dotnetRoot $version "$architecture" $runtime true || { + local exit_code=$? + echo "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code +} + +ExitWithExitCode 0 diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index b056e4c1ac2..fc388eb20f1 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -1,87 +1,87 @@ -Param( - [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens - [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) - [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) - [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created - [string] $darcVersion = '1.1.0-beta.19175.6', # darc's version - [string] $graphvizVersion = '2.38', # GraphViz version - [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about - # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies -) - -$ErrorActionPreference = "Stop" -. $PSScriptRoot\tools.ps1 - -Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1") - -function CheckExitCode ([string]$stage) -{ - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } -} - -try { - Push-Location $PSScriptRoot - - Write-Host "Installing darc..." - . .\darc-init.ps1 -darcVersion $darcVersion - CheckExitCode "Running darc-init" - - $engCommonBaseDir = Join-Path $PSScriptRoot "native\" - $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory - $nativeToolBaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external" - $installBin = Join-Path $graphvizInstallDir "bin" - - Write-Host "Installing dot..." - .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose - - $darcExe = "$env:USERPROFILE\.dotnet\tools" - $darcExe = Resolve-Path "$darcExe\darc.exe" - - Create-Directory $outputFolder - - # Generate 3 graph descriptions: - # 1. Flat with coherency information - # 2. Graphviz (dot) file - # 3. Standard dependency graph - $graphVizFilePath = "$outputFolder\graphviz.txt" - $graphVizImageFilePath = "$outputFolder\graph.png" - $normalGraphFilePath = "$outputFolder\graph-full.txt" - $flatGraphFilePath = "$outputFolder\graph-flat.txt" - $baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" ) - - if ($includeToolset) { - Write-Host "Toolsets will be included in the graph..." - $baseOptions += @( "--include-toolset" ) - } - - Write-Host "Generating standard dependency graph..." - & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath - CheckExitCode "Generating normal dependency graph" - - Write-Host "Generating flat dependency graph and graphviz file..." - & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath - CheckExitCode "Generating flat and graphviz dependency graph" - - Write-Host "Generating graph image $graphVizFilePath" - $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" - & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" - CheckExitCode "Generating graphviz image" - - Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" -} -catch { - if (!$includeToolset) { - Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow - Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow - } - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} finally { - Pop-Location +Param( + [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens + [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) + [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) + [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created + [string] $darcVersion = '1.1.0-beta.19175.6', # darc's version + [string] $graphvizVersion = '2.38', # GraphViz version + [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about + # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies +) + +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1") + +function CheckExitCode ([string]$stage) +{ + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Push-Location $PSScriptRoot + + Write-Host "Installing darc..." + . .\darc-init.ps1 -darcVersion $darcVersion + CheckExitCode "Running darc-init" + + $engCommonBaseDir = Join-Path $PSScriptRoot "native\" + $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory + $nativeToolBaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external" + $installBin = Join-Path $graphvizInstallDir "bin" + + Write-Host "Installing dot..." + .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose + + $darcExe = "$env:USERPROFILE\.dotnet\tools" + $darcExe = Resolve-Path "$darcExe\darc.exe" + + Create-Directory $outputFolder + + # Generate 3 graph descriptions: + # 1. Flat with coherency information + # 2. Graphviz (dot) file + # 3. Standard dependency graph + $graphVizFilePath = "$outputFolder\graphviz.txt" + $graphVizImageFilePath = "$outputFolder\graph.png" + $normalGraphFilePath = "$outputFolder\graph-full.txt" + $flatGraphFilePath = "$outputFolder\graph-flat.txt" + $baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" ) + + if ($includeToolset) { + Write-Host "Toolsets will be included in the graph..." + $baseOptions += @( "--include-toolset" ) + } + + Write-Host "Generating standard dependency graph..." + & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath + CheckExitCode "Generating normal dependency graph" + + Write-Host "Generating flat dependency graph and graphviz file..." + & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath + CheckExitCode "Generating flat and graphviz dependency graph" + + Write-Host "Generating graph image $graphVizFilePath" + $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" + & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" + CheckExitCode "Generating graphviz image" + + Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" +} +catch { + if (!$includeToolset) { + Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow + Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow + } + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} finally { + Pop-Location } \ No newline at end of file diff --git a/eng/common/helixpublish.proj b/eng/common/helixpublish.proj index d7f185856e7..f58bbf3f879 100644 --- a/eng/common/helixpublish.proj +++ b/eng/common/helixpublish.proj @@ -1,26 +1,26 @@ - - - - msbuild - - - - - %(Identity) - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) - $(WorkItemTimeout) - - - - - - - - - + + + + msbuild + + + + + %(Identity) + + + + + + $(WorkItemDirectory) + $(WorkItemCommand) + $(WorkItemTimeout) + + + + + + + + + diff --git a/eng/common/init-tools-native.cmd b/eng/common/init-tools-native.cmd index 438cd548c45..fca2e139b41 100644 --- a/eng/common/init-tools-native.cmd +++ b/eng/common/init-tools-native.cmd @@ -1,3 +1,3 @@ -@echo off -powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*" +@echo off +powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*" exit /b %ErrorLevel% \ No newline at end of file diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index 9d18645f455..389394ee945 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -1,141 +1,141 @@ -<# -.SYNOPSIS -Entry point script for installing native tools - -.DESCRIPTION -Reads $RepoRoot\global.json file to determine native assets to install -and executes installers for those tools - -.PARAMETER BaseUri -Base file directory or Url from which to acquire tool archives - -.PARAMETER InstallDirectory -Directory to install native toolset. This is a command-line override for the default -Install directory precedence order: -- InstallDirectory command-line override -- NETCOREENG_INSTALL_DIRECTORY environment variable -- (default) %USERPROFILE%/.netcoreeng/native - -.PARAMETER Clean -Switch specifying to not install anything, but cleanup native asset folders - -.PARAMETER Force -Clean and then install tools - -.PARAMETER DownloadRetries -Total number of retry attempts - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds - -.PARAMETER GlobalJsonFile -File path to global.json file - -.NOTES -#> -[CmdletBinding(PositionalBinding=$false)] -Param ( - [string] $BaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external", - [string] $InstallDirectory, - [switch] $Clean = $False, - [switch] $Force = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile -) - -if (!$GlobalJsonFile) { - $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName "global.json" -} - -Set-StrictMode -version 2.0 -$ErrorActionPreference="Stop" - -Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1") - -try { - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq "Continue" - - $EngCommonBaseDir = Join-Path $PSScriptRoot "native\" - $NativeBaseDir = $InstallDirectory - if (!$NativeBaseDir) { - $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory - } - $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir - $InstallBin = Join-Path $NativeBaseDir "bin" - $InstallerPath = Join-Path $EngCommonBaseDir "install-tool.ps1" - - # Process tools list - Write-Host "Processing $GlobalJsonFile" - If (-Not (Test-Path $GlobalJsonFile)) { - Write-Host "Unable to find '$GlobalJsonFile'" - exit 0 - } - $NativeTools = Get-Content($GlobalJsonFile) -Raw | - ConvertFrom-Json | - Select-Object -Expand "native-tools" -ErrorAction SilentlyContinue - if ($NativeTools) { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - Write-Warning $errMsg - $toolInstallationFailure = $true - } else { - Write-Error $errMsg - exit 1 - } - } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - exit 1 - } - } - else { - Write-Host "No native tools defined in global.json" - exit 0 - } - - if ($Clean) { - exit 0 - } - if (Test-Path $InstallBin) { - Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin) - Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" - } - else { - Write-Error "Native tools install directory does not exist, installation failed" - exit 1 - } - exit 0 -} -catch { - Write-Host $_ - Write-Host $_.Exception - exit 1 -} +<# +.SYNOPSIS +Entry point script for installing native tools + +.DESCRIPTION +Reads $RepoRoot\global.json file to determine native assets to install +and executes installers for those tools + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER InstallDirectory +Directory to install native toolset. This is a command-line override for the default +Install directory precedence order: +- InstallDirectory command-line override +- NETCOREENG_INSTALL_DIRECTORY environment variable +- (default) %USERPROFILE%/.netcoreeng/native + +.PARAMETER Clean +Switch specifying to not install anything, but cleanup native asset folders + +.PARAMETER Force +Clean and then install tools + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.PARAMETER GlobalJsonFile +File path to global.json file + +.NOTES +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [string] $BaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external", + [string] $InstallDirectory, + [switch] $Clean = $False, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [string] $GlobalJsonFile +) + +if (!$GlobalJsonFile) { + $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName "global.json" +} + +Set-StrictMode -version 2.0 +$ErrorActionPreference="Stop" + +Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1") + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $EngCommonBaseDir = Join-Path $PSScriptRoot "native\" + $NativeBaseDir = $InstallDirectory + if (!$NativeBaseDir) { + $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory + } + $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir + $InstallBin = Join-Path $NativeBaseDir "bin" + $InstallerPath = Join-Path $EngCommonBaseDir "install-tool.ps1" + + # Process tools list + Write-Host "Processing $GlobalJsonFile" + If (-Not (Test-Path $GlobalJsonFile)) { + Write-Host "Unable to find '$GlobalJsonFile'" + exit 0 + } + $NativeTools = Get-Content($GlobalJsonFile) -Raw | + ConvertFrom-Json | + Select-Object -Expand "native-tools" -ErrorAction SilentlyContinue + if ($NativeTools) { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + Write-Warning $errMsg + $toolInstallationFailure = $true + } else { + Write-Error $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + exit 1 + } + } + else { + Write-Host "No native tools defined in global.json" + exit 0 + } + + if ($Clean) { + exit 0 + } + if (Test-Path $InstallBin) { + Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin) + Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" + } + else { + Write-Error "Native tools install directory does not exist, installation failed" + exit 1 + } + exit 0 +} +catch { + Write-Host $_ + Write-Host $_.Exception + exit 1 +} diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index fc72d13948e..1d3df860d5f 100644 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -1,141 +1,141 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' -install_directory='' -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 -global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" -declare -A native_assets - -. $scriptroot/native/common-library.sh - -while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installdirectory) - install_directory=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --installdirectory Directory to install native toolset." - echo " This is a command-line override for the default" - echo " Install directory precedence order:" - echo " - InstallDirectory command-line override" - echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" - echo " - (default) %USERPROFILE%/.netcoreeng/native" - echo "" - echo " --clean Switch specifying not to install anything, but cleanup native asset folders" - echo " --force Clean and then install tools" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --baseuri Base URI for where to download native tools from" - echo " --downloadretries Number of times a download should be attempted" - echo " --retrywaittimeseconds Wait time between download attempts" - echo "" - exit 0 - ;; - esac -done - -function ReadGlobalJsonNativeTools { - # Get the native-tools section from the global.json. - local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/') - # Only extract the contents of the object. - local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}') - native_tools_list=${native_tools_list//[\" ]/} - native_tools_list=${native_tools_list//,/$'\n'} - native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[:space:]')" - - local old_IFS=$IFS - while read -r line; do - # Lines are of the form: 'tool:version' - IFS=: - while read -r key value; do - native_assets[$key]=$value - done <<< "$line" - done <<< "$native_tools_list" - IFS=$old_IFS - - return 0; -} - -native_base_dir=$install_directory -if [[ -z $install_directory ]]; then - native_base_dir=$(GetNativeInstallDirectory) -fi - -install_bin="${native_base_dir}/bin" - -ReadGlobalJsonNativeTools - -if [[ ${#native_assets[@]} -eq 0 ]]; then - echo "No native tools defined in global.json" - exit 0; -else - native_installer_dir="$scriptroot/native" - for tool in "${!native_assets[@]}" - do - tool_version=${native_assets[$tool]} - installer_name="install-$tool.sh" - installer_command="$native_installer_dir/$installer_name" - installer_command+=" --baseuri $base_uri" - installer_command+=" --installpath $install_bin" - installer_command+=" --version $tool_version" - - if [[ $force = true ]]; then - installer_command+=" --force" - fi - - if [[ $clean = true ]]; then - installer_command+=" --clean" - fi - - $installer_command - - if [[ $? != 0 ]]; then - echo "Execution Failed" >&2 - exit 1 - fi - done -fi - -if [[ $clean = true ]]; then - exit 0 -fi - -if [[ -d $install_bin ]]; then - echo "Native tools are available from $install_bin" - echo "##vso[task.prependpath]$install_bin" -else - echo "Native tools install directory does not exist, installation failed" >&2 - exit 1 -fi - -exit 0 +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' +install_directory='' +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 +global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" +declare -A native_assets + +. $scriptroot/native/common-library.sh + +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installdirectory) + install_directory=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --installdirectory Directory to install native toolset." + echo " This is a command-line override for the default" + echo " Install directory precedence order:" + echo " - InstallDirectory command-line override" + echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" + echo " - (default) %USERPROFILE%/.netcoreeng/native" + echo "" + echo " --clean Switch specifying not to install anything, but cleanup native asset folders" + echo " --force Clean and then install tools" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --baseuri Base URI for where to download native tools from" + echo " --downloadretries Number of times a download should be attempted" + echo " --retrywaittimeseconds Wait time between download attempts" + echo "" + exit 0 + ;; + esac +done + +function ReadGlobalJsonNativeTools { + # Get the native-tools section from the global.json. + local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/') + # Only extract the contents of the object. + local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}') + native_tools_list=${native_tools_list//[\" ]/} + native_tools_list=${native_tools_list//,/$'\n'} + native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[:space:]')" + + local old_IFS=$IFS + while read -r line; do + # Lines are of the form: 'tool:version' + IFS=: + while read -r key value; do + native_assets[$key]=$value + done <<< "$line" + done <<< "$native_tools_list" + IFS=$old_IFS + + return 0; +} + +native_base_dir=$install_directory +if [[ -z $install_directory ]]; then + native_base_dir=$(GetNativeInstallDirectory) +fi + +install_bin="${native_base_dir}/bin" + +ReadGlobalJsonNativeTools + +if [[ ${#native_assets[@]} -eq 0 ]]; then + echo "No native tools defined in global.json" + exit 0; +else + native_installer_dir="$scriptroot/native" + for tool in "${!native_assets[@]}" + do + tool_version=${native_assets[$tool]} + installer_name="install-$tool.sh" + installer_command="$native_installer_dir/$installer_name" + installer_command+=" --baseuri $base_uri" + installer_command+=" --installpath $install_bin" + installer_command+=" --version $tool_version" + + if [[ $force = true ]]; then + installer_command+=" --force" + fi + + if [[ $clean = true ]]; then + installer_command+=" --clean" + fi + + $installer_command + + if [[ $? != 0 ]]; then + echo "Execution Failed" >&2 + exit 1 + fi + done +fi + +if [[ $clean = true ]]; then + exit 0 +fi + +if [[ -d $install_bin ]]; then + echo "Native tools are available from $install_bin" + echo "##vso[task.prependpath]$install_bin" +else + echo "Native tools install directory does not exist, installation failed" >&2 + exit 1 +fi + +exit 0 diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index 8b8bafd6a89..f1a55ac19b9 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -1,135 +1,135 @@ -param( - [Parameter(Mandatory=$true)][string] $Operation, - [string] $AuthToken, - [string] $CommitSha, - [string] $RepoName, - [switch] $IsFeedPrivate -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\tools.ps1 - -# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed -# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in -# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified -# internal builds -function SetupCredProvider { - param( - [string] $AuthToken - ) - - # Install the Cred Provider NuGet plugin - Write-Host "Setting up Cred Provider NuGet plugin in the agent..." - Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." - - $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' - - Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." - Invoke-WebRequest $url -OutFile installcredprovider.ps1 - - Write-Host "Installing plugin..." - .\installcredprovider.ps1 -Force - - Write-Host "Deleting local copy of 'installcredprovider.ps1'..." - Remove-Item .\installcredprovider.ps1 - - if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { - Write-Host "CredProvider plugin was not installed correctly!" - ExitWithExitCode 1 - } - else { - Write-Host "CredProvider plugin was installed correctly!" - } - - # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable - # feeds successfully - - $nugetConfigPath = "$RepoRoot\NuGet.config" - - if (-Not (Test-Path -Path $nugetConfigPath)) { - Write-Host "NuGet.config file not found in repo's root!" - ExitWithExitCode 1 - } - - $endpoints = New-Object System.Collections.ArrayList - $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} - - if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { - foreach ($stableRestoreResource in $nugetConfigPackageSources) { - $trimmedResource = ([string]$stableRestoreResource).Trim() - [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) - } - } - - if (($endpoints | Measure-Object).Count -gt 0) { - # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' - $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress - - # Create the environment variables the AzDo way - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ - 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' - 'issecret' = 'false' - } - - # We don't want sessions cached since we will be updating the endpoints quite frequently - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ - 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' - 'issecret' = 'false' - } - } - else - { - Write-Host "No internal endpoints found in NuGet.config" - } -} - -#Workaround for https://github.com/microsoft/msbuild/issues/4430 -function InstallDotNetSdkAndRestoreArcade { - $dotnetTempDir = "$RepoRoot\dotnet" - $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) - $dotnet = "$dotnetTempDir\dotnet.exe" - $restoreProjPath = "$PSScriptRoot\restore.proj" - - Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." - InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" - - '' | Out-File "$restoreProjPath" - - & $dotnet restore $restoreProjPath - - Write-Host "Arcade SDK restored!" - - if (Test-Path -Path $restoreProjPath) { - Remove-Item $restoreProjPath - } - - if (Test-Path -Path $dotnetTempDir) { - Remove-Item $dotnetTempDir -Recurse - } -} - -try { - Push-Location $PSScriptRoot - - if ($Operation -like "setup") { - SetupCredProvider $AuthToken - } - elseif ($Operation -like "install-restore") { - InstallDotNetSdkAndRestoreArcade - } - else { - Write-Host "Unknown operation '$Operation'!" - ExitWithExitCode 1 - } -} -catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} -finally { - Pop-Location -} +param( + [Parameter(Mandatory=$true)][string] $Operation, + [string] $AuthToken, + [string] $CommitSha, + [string] $RepoName, + [switch] $IsFeedPrivate +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\tools.ps1 + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified +# internal builds +function SetupCredProvider { + param( + [string] $AuthToken + ) + + # Install the Cred Provider NuGet plugin + Write-Host "Setting up Cred Provider NuGet plugin in the agent..." + Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' + + Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." + Invoke-WebRequest $url -OutFile installcredprovider.ps1 + + Write-Host "Installing plugin..." + .\installcredprovider.ps1 -Force + + Write-Host "Deleting local copy of 'installcredprovider.ps1'..." + Remove-Item .\installcredprovider.ps1 + + if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { + Write-Host "CredProvider plugin was not installed correctly!" + ExitWithExitCode 1 + } + else { + Write-Host "CredProvider plugin was installed correctly!" + } + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + $nugetConfigPath = "$RepoRoot\NuGet.config" + + if (-Not (Test-Path -Path $nugetConfigPath)) { + Write-Host "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + } + + $endpoints = New-Object System.Collections.ArrayList + $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} + + if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { + foreach ($stableRestoreResource in $nugetConfigPackageSources) { + $trimmedResource = ([string]$stableRestoreResource).Trim() + [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) + } + } + + if (($endpoints | Measure-Object).Count -gt 0) { + # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' + $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress + + # Create the environment variables the AzDo way + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ + 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' + 'issecret' = 'false' + } + + # We don't want sessions cached since we will be updating the endpoints quite frequently + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ + 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' + 'issecret' = 'false' + } + } + else + { + Write-Host "No internal endpoints found in NuGet.config" + } +} + +#Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + $dotnetTempDir = "$RepoRoot\dotnet" + $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + $dotnet = "$dotnetTempDir\dotnet.exe" + $restoreProjPath = "$PSScriptRoot\restore.proj" + + Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + '' | Out-File "$restoreProjPath" + + & $dotnet restore $restoreProjPath + + Write-Host "Arcade SDK restored!" + + if (Test-Path -Path $restoreProjPath) { + Remove-Item $restoreProjPath + } + + if (Test-Path -Path $dotnetTempDir) { + Remove-Item $dotnetTempDir -Recurse + } +} + +try { + Push-Location $PSScriptRoot + + if ($Operation -like "setup") { + SetupCredProvider $AuthToken + } + elseif ($Operation -like "install-restore") { + InstallDotNetSdkAndRestoreArcade + } + else { + Write-Host "Unknown operation '$Operation'!" + ExitWithExitCode 1 + } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} +finally { + Pop-Location +} diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 1ff654d2ffc..cdf8c62edb5 100644 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -1,142 +1,142 @@ -#!/usr/bin/env bash - -set -e - -# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed -# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in -# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. -# This should ONLY be called from identified internal builds -function SetupCredProvider { - local authToken=$1 - - # Install the Cred Provider NuGet plugin - echo "Setting up Cred Provider NuGet plugin in the agent..."... - echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." - - local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" - - echo "Writing the contents of 'installcredprovider.ps1' locally..." - local installcredproviderPath="installcredprovider.sh" - if command -v curl > /dev/null; then - curl $url > "$installcredproviderPath" - else - wget -q -O "$installcredproviderPath" "$url" - fi - - echo "Installing plugin..." - . "$installcredproviderPath" - - echo "Deleting local copy of 'installcredprovider.sh'..." - rm installcredprovider.sh - - if [ ! -d "$HOME/.nuget/plugins" ]; then - echo "CredProvider plugin was not installed correctly!" - ExitWithExitCode 1 - else - echo "CredProvider plugin was installed correctly!" - fi - - # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable - # feeds successfully - - local nugetConfigPath="$repo_root/NuGet.config" - - if [ ! "$nugetConfigPath" ]; then - echo "NuGet.config file not found in repo's root!" - ExitWithExitCode 1 - fi - - local endpoints='[' - local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` - local pattern="value=\"(.*)\"" - - for value in $nugetConfigPackageValues - do - if [[ $value =~ $pattern ]]; then - local endpoint="${BASH_REMATCH[1]}" - endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," - fi - done - - endpoints=${endpoints%?} - endpoints+=']' - - if [ ${#endpoints} -gt 2 ]; then - # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' - local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" - - echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" - echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" - else - echo "No internal endpoints found in NuGet.config" - fi -} - -# Workaround for https://github.com/microsoft/msbuild/issues/4430 -function InstallDotNetSdkAndRestoreArcade { - local dotnetTempDir="$repo_root/dotnet" - local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) - local restoreProjPath="$repo_root/eng/common/restore.proj" - - echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." - echo "" > "$restoreProjPath" - - InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" - - local res=`$dotnetTempDir/dotnet restore $restoreProjPath` - echo "Arcade SDK restored!" - - # Cleanup - if [ "$restoreProjPath" ]; then - rm "$restoreProjPath" - fi - - if [ "$dotnetTempDir" ]; then - rm -r $dotnetTempDir - fi -} - -source="${BASH_SOURCE[0]}" -operation='' -authToken='' -repoName='' - -while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" - case "$opt" in - --operation) - operation=$2 - shift - ;; - --authtoken) - authToken=$2 - shift - ;; - *) - echo "Invalid argument: $1" - usage - exit 1 - ;; - esac - - shift -done - -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -if [ "$operation" = "setup" ]; then - SetupCredProvider $authToken -elif [ "$operation" = "install-restore" ]; then - InstallDotNetSdkAndRestoreArcade -else - echo "Unknown operation '$operation'!" -fi +#!/usr/bin/env bash + +set -e + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. +# This should ONLY be called from identified internal builds +function SetupCredProvider { + local authToken=$1 + + # Install the Cred Provider NuGet plugin + echo "Setting up Cred Provider NuGet plugin in the agent..."... + echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" + + echo "Writing the contents of 'installcredprovider.ps1' locally..." + local installcredproviderPath="installcredprovider.sh" + if command -v curl > /dev/null; then + curl $url > "$installcredproviderPath" + else + wget -q -O "$installcredproviderPath" "$url" + fi + + echo "Installing plugin..." + . "$installcredproviderPath" + + echo "Deleting local copy of 'installcredprovider.sh'..." + rm installcredprovider.sh + + if [ ! -d "$HOME/.nuget/plugins" ]; then + echo "CredProvider plugin was not installed correctly!" + ExitWithExitCode 1 + else + echo "CredProvider plugin was installed correctly!" + fi + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + local nugetConfigPath="$repo_root/NuGet.config" + + if [ ! "$nugetConfigPath" ]; then + echo "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + fi + + local endpoints='[' + local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` + local pattern="value=\"(.*)\"" + + for value in $nugetConfigPackageValues + do + if [[ $value =~ $pattern ]]; then + local endpoint="${BASH_REMATCH[1]}" + endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," + fi + done + + endpoints=${endpoints%?} + endpoints+=']' + + if [ ${#endpoints} -gt 2 ]; then + # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' + local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" + + echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" + echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" + else + echo "No internal endpoints found in NuGet.config" + fi +} + +# Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + local dotnetTempDir="$repo_root/dotnet" + local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + local restoreProjPath="$repo_root/eng/common/restore.proj" + + echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + echo "" > "$restoreProjPath" + + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + local res=`$dotnetTempDir/dotnet restore $restoreProjPath` + echo "Arcade SDK restored!" + + # Cleanup + if [ "$restoreProjPath" ]; then + rm "$restoreProjPath" + fi + + if [ "$dotnetTempDir" ]; then + rm -r $dotnetTempDir + fi +} + +source="${BASH_SOURCE[0]}" +operation='' +authToken='' +repoName='' + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --operation) + operation=$2 + shift + ;; + --authtoken) + authToken=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ "$operation" = "setup" ]; then + SetupCredProvider $authToken +elif [ "$operation" = "install-restore" ]; then + InstallDotNetSdkAndRestoreArcade +else + echo "Unknown operation '$operation'!" +fi diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props index e33179ef373..d554b8a63d0 100644 --- a/eng/common/internal/Directory.Build.props +++ b/eng/common/internal/Directory.Build.props @@ -1,4 +1,4 @@ - - - - + + + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 1a39a7ef3f6..a70213f3071 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -1,27 +1,27 @@ - - - - - net472 - false - - - - - - - - - - - https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json; - - - $(RestoreSources); - https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; - - - - - - + + + + + net472 + false + + + + + + + + + + + https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json; + + + $(RestoreSources); + https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; + + + + + + diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index b37fd3d5e97..7b585ad9ec2 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -1,27 +1,27 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $verbosity = "minimal", - [bool] $warnAsError = $true, - [bool] $nodeReuse = $true, - [switch] $ci, - [switch] $prepareMachine, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs -) - -. $PSScriptRoot\tools.ps1 - -try { - if ($ci) { - $nodeReuse = $false - } - - MSBuild @extraArgs -} -catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} - +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = "minimal", + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, + [switch] $ci, + [switch] $prepareMachine, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs +) + +. $PSScriptRoot\tools.ps1 + +try { + if ($ci) { + $nodeReuse = $false + } + + MSBuild @extraArgs +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} + ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh index 8160cd5a59d..396fbe76498 100755 --- a/eng/common/msbuild.sh +++ b/eng/common/msbuild.sh @@ -1,58 +1,58 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -verbosity='minimal' -warn_as_error=true -node_reuse=true -prepare_machine=false -extra_args='' - -while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - --verbosity) - verbosity=$2 - shift 2 - ;; - --warnaserror) - warn_as_error=$2 - shift 2 - ;; - --nodereuse) - node_reuse=$2 - shift 2 - ;; - --ci) - ci=true - shift 1 - ;; - --preparemachine) - prepare_machine=true - shift 1 - ;; - *) - extra_args="$extra_args $1" - shift 1 - ;; - esac -done - -. "$scriptroot/tools.sh" - -if [[ "$ci" == true ]]; then - node_reuse=false -fi - -MSBuild $extra_args -ExitWithExitCode 0 +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +verbosity='minimal' +warn_as_error=true +node_reuse=true +prepare_machine=false +extra_args='' + +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --verbosity) + verbosity=$2 + shift 2 + ;; + --warnaserror) + warn_as_error=$2 + shift 2 + ;; + --nodereuse) + node_reuse=$2 + shift 2 + ;; + --ci) + ci=true + shift 1 + ;; + --preparemachine) + prepare_machine=true + shift 1 + ;; + *) + extra_args="$extra_args $1" + shift 1 + ;; + esac +done + +. "$scriptroot/tools.sh" + +if [[ "$ci" == true ]]; then + node_reuse=false +fi + +MSBuild $extra_args +ExitWithExitCode 0 diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 7a34c7e8a42..bbae72055e2 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -1,358 +1,358 @@ -<# -.SYNOPSIS -Helper module to install an archive to a directory - -.DESCRIPTION -Helper module to download and extract an archive to a specified directory - -.PARAMETER Uri -Uri of artifact to download - -.PARAMETER InstallDirectory -Directory to extract artifact contents to - -.PARAMETER Force -Force download / extraction if file or contents already exist. Default = False - -.PARAMETER DownloadRetries -Total number of retry attempts. Default = 5 - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds. Default = 30 - -.NOTES -Returns False if download or extraction fail, True otherwise -#> -function DownloadAndExtract { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Uri, - [Parameter(Mandatory=$True)] - [string] $InstallDirectory, - [switch] $Force = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30 - ) - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq "Continue" - - $TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri - - # Download native tool - $DownloadStatus = CommonLibrary\Get-File -Uri $Uri ` - -Path $TempToolPath ` - -DownloadRetries $DownloadRetries ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Force:$Force ` - -Verbose:$Verbose - - if ($DownloadStatus -Eq $False) { - Write-Error "Download failed" - return $False - } - - # Extract native tool - $UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` - -OutputDirectory $InstallDirectory ` - -Force:$Force ` - -Verbose:$Verbose - - if ($UnzipStatus -Eq $False) { - Write-Error "Unzip failed" - return $False - } - return $True -} - -<# -.SYNOPSIS -Download a file, retry on failure - -.DESCRIPTION -Download specified file and retry if attempt fails - -.PARAMETER Uri -Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded - -.PARAMETER Path -Path to download or copy uri file to - -.PARAMETER Force -Overwrite existing file if present. Default = False - -.PARAMETER DownloadRetries -Total number of retry attempts. Default = 5 - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds Default = 30 - -#> -function Get-File { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Uri, - [Parameter(Mandatory=$True)] - [string] $Path, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30, - [switch] $Force = $False - ) - $Attempt = 0 - - if ($Force) { - if (Test-Path $Path) { - Remove-Item $Path -Force - } - } - if (Test-Path $Path) { - Write-Host "File '$Path' already exists, skipping download" - return $True - } - - $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent - if (-Not (Test-Path $DownloadDirectory)) { - New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null - } - - if (Test-Path -IsValid -Path $Uri) { - Write-Verbose "'$Uri' is a file path, copying file to '$Path'" - Copy-Item -Path $Uri -Destination $Path - return $? - } - else { - Write-Verbose "Downloading $Uri" - while($Attempt -Lt $DownloadRetries) - { - try { - Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path - Write-Verbose "Downloaded to '$Path'" - return $True - } - catch { - $Attempt++ - if ($Attempt -Lt $DownloadRetries) { - $AttemptsLeft = $DownloadRetries - $Attempt - Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds" - Start-Sleep -Seconds $RetryWaitTimeInSeconds - } - else { - Write-Error $_ - Write-Error $_.Exception - } - } - } - } - - return $False -} - -<# -.SYNOPSIS -Generate a shim for a native tool - -.DESCRIPTION -Creates a wrapper script (shim) that passes arguments forward to native tool assembly - -.PARAMETER ShimName -The name of the shim - -.PARAMETER ShimDirectory -The directory where shims are stored - -.PARAMETER ToolFilePath -Path to file that shim forwards to - -.PARAMETER Force -Replace shim if already present. Default = False - -.NOTES -Returns $True if generating shim succeeds, $False otherwise -#> -function New-ScriptShim { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $ShimName, - [Parameter(Mandatory=$True)] - [string] $ShimDirectory, - [Parameter(Mandatory=$True)] - [string] $ToolFilePath, - [Parameter(Mandatory=$True)] - [string] $BaseUri, - [switch] $Force - ) - try { - Write-Verbose "Generating '$ShimName' shim" - - if (-Not (Test-Path $ToolFilePath)){ - Write-Error "Specified tool file path '$ToolFilePath' does not exist" - return $False - } - - # WinShimmer is a small .NET Framework program that creates .exe shims to bootstrapped programs - # Many of the checks for installed programs expect a .exe extension for Windows tools, rather - # than a .bat or .cmd file. - # Source: https://github.com/dotnet/arcade/tree/master/src/WinShimmer - if (-Not (Test-Path "$ShimDirectory\WinShimmer\winshimmer.exe")) { - $InstallStatus = DownloadAndExtract -Uri "$BaseUri/windows/winshimmer/WinShimmer.zip" ` - -InstallDirectory $ShimDirectory\WinShimmer ` - -Force:$Force ` - -DownloadRetries 2 ` - -RetryWaitTimeInSeconds 5 ` - -Verbose:$Verbose - } - - if ((Test-Path (Join-Path $ShimDirectory "$ShimName.exe"))) { - Write-Host "$ShimName.exe already exists; replacing..." - Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") - } - - & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory - return $True - } - catch { - Write-Host $_ - Write-Host $_.Exception - return $False - } -} - -<# -.SYNOPSIS -Returns the machine architecture of the host machine - -.NOTES -Returns 'x64' on 64 bit machines - Returns 'x86' on 32 bit machines -#> -function Get-MachineArchitecture { - $ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE - $ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432 - if($ProcessorArchitecture -Eq "X86") - { - if(($ProcessorArchitectureW6432 -Eq "") -Or - ($ProcessorArchitectureW6432 -Eq "X86")) { - return "x86" - } - $ProcessorArchitecture = $ProcessorArchitectureW6432 - } - if (($ProcessorArchitecture -Eq "AMD64") -Or - ($ProcessorArchitecture -Eq "IA64") -Or - ($ProcessorArchitecture -Eq "ARM64")) { - return "x64" - } - return "x86" -} - -<# -.SYNOPSIS -Get the name of a temporary folder under the native install directory -#> -function Get-TempDirectory { - return Join-Path (Get-NativeInstallDirectory) "temp/" -} - -function Get-TempPathFilename { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Path - ) - $TempDir = CommonLibrary\Get-TempDirectory - $TempFilename = Split-Path $Path -leaf - $TempPath = Join-Path $TempDir $TempFilename - return $TempPath -} - -<# -.SYNOPSIS -Returns the base directory to use for native tool installation - -.NOTES -Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable -is set, or otherwise returns an install directory under the %USERPROFILE% -#> -function Get-NativeInstallDirectory { - $InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY - if (!$InstallDir) { - $InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/" - } - return $InstallDir -} - -<# -.SYNOPSIS -Unzip an archive - -.DESCRIPTION -Powershell module to unzip an archive to a specified directory - -.PARAMETER ZipPath (Required) -Path to archive to unzip - -.PARAMETER OutputDirectory (Required) -Output directory for archive contents - -.PARAMETER Force -Overwrite output directory contents if they already exist - -.NOTES -- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True. -- Returns True if unzip operation is successful -- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory -- Returns False if unable to extract zip archive -#> -function Expand-Zip { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $ZipPath, - [Parameter(Mandatory=$True)] - [string] $OutputDirectory, - [switch] $Force - ) - - Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'" - try { - if ((Test-Path $OutputDirectory) -And (-Not $Force)) { - Write-Host "Directory '$OutputDirectory' already exists, skipping extract" - return $True - } - if (Test-Path $OutputDirectory) { - Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory" - Remove-Item $OutputDirectory -Force -Recurse - if ($? -Eq $False) { - Write-Error "Unable to remove '$OutputDirectory'" - return $False - } - } - if (-Not (Test-Path $OutputDirectory)) { - New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null - } - - Add-Type -assembly "system.io.compression.filesystem" - [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory") - if ($? -Eq $False) { - Write-Error "Unable to extract '$ZipPath'" - return $False - } - } - catch { - Write-Host $_ - Write-Host $_.Exception - - return $False - } - return $True -} - -export-modulemember -function DownloadAndExtract -export-modulemember -function Expand-Zip -export-modulemember -function Get-File -export-modulemember -function Get-MachineArchitecture -export-modulemember -function Get-NativeInstallDirectory -export-modulemember -function Get-TempDirectory -export-modulemember -function Get-TempPathFilename -export-modulemember -function New-ScriptShim +<# +.SYNOPSIS +Helper module to install an archive to a directory + +.DESCRIPTION +Helper module to download and extract an archive to a specified directory + +.PARAMETER Uri +Uri of artifact to download + +.PARAMETER InstallDirectory +Directory to extract artifact contents to + +.PARAMETER Force +Force download / extraction if file or contents already exist. Default = False + +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds. Default = 30 + +.NOTES +Returns False if download or extraction fail, True otherwise +#> +function DownloadAndExtract { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $InstallDirectory, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 + ) + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri + + # Download native tool + $DownloadStatus = CommonLibrary\Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$Force ` + -Verbose:$Verbose + + if ($DownloadStatus -Eq $False) { + Write-Error "Download failed" + return $False + } + + # Extract native tool + $UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$Force ` + -Verbose:$Verbose + + if ($UnzipStatus -Eq $False) { + Write-Error "Unzip failed" + return $False + } + return $True +} + +<# +.SYNOPSIS +Download a file, retry on failure + +.DESCRIPTION +Download specified file and retry if attempt fails + +.PARAMETER Uri +Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded + +.PARAMETER Path +Path to download or copy uri file to + +.PARAMETER Force +Overwrite existing file if present. Default = False + +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds Default = 30 + +#> +function Get-File { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $Path, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [switch] $Force = $False + ) + $Attempt = 0 + + if ($Force) { + if (Test-Path $Path) { + Remove-Item $Path -Force + } + } + if (Test-Path $Path) { + Write-Host "File '$Path' already exists, skipping download" + return $True + } + + $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent + if (-Not (Test-Path $DownloadDirectory)) { + New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null + } + + if (Test-Path -IsValid -Path $Uri) { + Write-Verbose "'$Uri' is a file path, copying file to '$Path'" + Copy-Item -Path $Uri -Destination $Path + return $? + } + else { + Write-Verbose "Downloading $Uri" + while($Attempt -Lt $DownloadRetries) + { + try { + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path + Write-Verbose "Downloaded to '$Path'" + return $True + } + catch { + $Attempt++ + if ($Attempt -Lt $DownloadRetries) { + $AttemptsLeft = $DownloadRetries - $Attempt + Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds" + Start-Sleep -Seconds $RetryWaitTimeInSeconds + } + else { + Write-Error $_ + Write-Error $_.Exception + } + } + } + } + + return $False +} + +<# +.SYNOPSIS +Generate a shim for a native tool + +.DESCRIPTION +Creates a wrapper script (shim) that passes arguments forward to native tool assembly + +.PARAMETER ShimName +The name of the shim + +.PARAMETER ShimDirectory +The directory where shims are stored + +.PARAMETER ToolFilePath +Path to file that shim forwards to + +.PARAMETER Force +Replace shim if already present. Default = False + +.NOTES +Returns $True if generating shim succeeds, $False otherwise +#> +function New-ScriptShim { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ShimName, + [Parameter(Mandatory=$True)] + [string] $ShimDirectory, + [Parameter(Mandatory=$True)] + [string] $ToolFilePath, + [Parameter(Mandatory=$True)] + [string] $BaseUri, + [switch] $Force + ) + try { + Write-Verbose "Generating '$ShimName' shim" + + if (-Not (Test-Path $ToolFilePath)){ + Write-Error "Specified tool file path '$ToolFilePath' does not exist" + return $False + } + + # WinShimmer is a small .NET Framework program that creates .exe shims to bootstrapped programs + # Many of the checks for installed programs expect a .exe extension for Windows tools, rather + # than a .bat or .cmd file. + # Source: https://github.com/dotnet/arcade/tree/master/src/WinShimmer + if (-Not (Test-Path "$ShimDirectory\WinShimmer\winshimmer.exe")) { + $InstallStatus = DownloadAndExtract -Uri "$BaseUri/windows/winshimmer/WinShimmer.zip" ` + -InstallDirectory $ShimDirectory\WinShimmer ` + -Force:$Force ` + -DownloadRetries 2 ` + -RetryWaitTimeInSeconds 5 ` + -Verbose:$Verbose + } + + if ((Test-Path (Join-Path $ShimDirectory "$ShimName.exe"))) { + Write-Host "$ShimName.exe already exists; replacing..." + Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") + } + + & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory + return $True + } + catch { + Write-Host $_ + Write-Host $_.Exception + return $False + } +} + +<# +.SYNOPSIS +Returns the machine architecture of the host machine + +.NOTES +Returns 'x64' on 64 bit machines + Returns 'x86' on 32 bit machines +#> +function Get-MachineArchitecture { + $ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE + $ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432 + if($ProcessorArchitecture -Eq "X86") + { + if(($ProcessorArchitectureW6432 -Eq "") -Or + ($ProcessorArchitectureW6432 -Eq "X86")) { + return "x86" + } + $ProcessorArchitecture = $ProcessorArchitectureW6432 + } + if (($ProcessorArchitecture -Eq "AMD64") -Or + ($ProcessorArchitecture -Eq "IA64") -Or + ($ProcessorArchitecture -Eq "ARM64")) { + return "x64" + } + return "x86" +} + +<# +.SYNOPSIS +Get the name of a temporary folder under the native install directory +#> +function Get-TempDirectory { + return Join-Path (Get-NativeInstallDirectory) "temp/" +} + +function Get-TempPathFilename { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Path + ) + $TempDir = CommonLibrary\Get-TempDirectory + $TempFilename = Split-Path $Path -leaf + $TempPath = Join-Path $TempDir $TempFilename + return $TempPath +} + +<# +.SYNOPSIS +Returns the base directory to use for native tool installation + +.NOTES +Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable +is set, or otherwise returns an install directory under the %USERPROFILE% +#> +function Get-NativeInstallDirectory { + $InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY + if (!$InstallDir) { + $InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/" + } + return $InstallDir +} + +<# +.SYNOPSIS +Unzip an archive + +.DESCRIPTION +Powershell module to unzip an archive to a specified directory + +.PARAMETER ZipPath (Required) +Path to archive to unzip + +.PARAMETER OutputDirectory (Required) +Output directory for archive contents + +.PARAMETER Force +Overwrite output directory contents if they already exist + +.NOTES +- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True. +- Returns True if unzip operation is successful +- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory +- Returns False if unable to extract zip archive +#> +function Expand-Zip { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ZipPath, + [Parameter(Mandatory=$True)] + [string] $OutputDirectory, + [switch] $Force + ) + + Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'" + try { + if ((Test-Path $OutputDirectory) -And (-Not $Force)) { + Write-Host "Directory '$OutputDirectory' already exists, skipping extract" + return $True + } + if (Test-Path $OutputDirectory) { + Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory" + Remove-Item $OutputDirectory -Force -Recurse + if ($? -Eq $False) { + Write-Error "Unable to remove '$OutputDirectory'" + return $False + } + } + if (-Not (Test-Path $OutputDirectory)) { + New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null + } + + Add-Type -assembly "system.io.compression.filesystem" + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory") + if ($? -Eq $False) { + Write-Error "Unable to extract '$ZipPath'" + return $False + } + } + catch { + Write-Host $_ + Write-Host $_.Exception + + return $False + } + return $True +} + +export-modulemember -function DownloadAndExtract +export-modulemember -function Expand-Zip +export-modulemember -function Get-File +export-modulemember -function Get-MachineArchitecture +export-modulemember -function Get-NativeInstallDirectory +export-modulemember -function Get-TempDirectory +export-modulemember -function Get-TempPathFilename +export-modulemember -function New-ScriptShim diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh index 271bddfac5a..fcbddad9ce2 100644 --- a/eng/common/native/common-library.sh +++ b/eng/common/native/common-library.sh @@ -1,168 +1,168 @@ -#!/usr/bin/env bash - -function GetNativeInstallDirectory { - local install_dir - - if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then - install_dir=$HOME/.netcoreeng/native/ - else - install_dir=$NETCOREENG_INSTALL_DIRECTORY - fi - - echo $install_dir - return 0 -} - -function GetTempDirectory { - - echo $(GetNativeInstallDirectory)temp/ - return 0 -} - -function ExpandZip { - local zip_path=$1 - local output_directory=$2 - local force=${3:-false} - - echo "Extracting $zip_path to $output_directory" - if [[ -d $output_directory ]] && [[ $force = false ]]; then - echo "Directory '$output_directory' already exists, skipping extract" - return 0 - fi - - if [[ -d $output_directory ]]; then - echo "'Force flag enabled, but '$output_directory' exists. Removing directory" - rm -rf $output_directory - if [[ $? != 0 ]]; then - echo Unable to remove '$output_directory'>&2 - return 1 - fi - fi - - echo "Creating directory: '$output_directory'" - mkdir -p $output_directory - - echo "Extracting archive" - tar -xf $zip_path -C $output_directory - if [[ $? != 0 ]]; then - echo "Unable to extract '$zip_path'" >&2 - return 1 - fi - - return 0 -} - -function GetCurrentOS { - local unameOut="$(uname -s)" - case $unameOut in - Linux*) echo "Linux";; - Darwin*) echo "MacOS";; - esac - return 0 -} - -function GetFile { - local uri=$1 - local path=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - if [[ -f $path ]]; then - if [[ $force = false ]]; then - echo "File '$path' already exists. Skipping download" - return 0 - else - rm -rf $path - fi - fi - - if [[ -f $uri ]]; then - echo "'$uri' is a file path, copying file to '$path'" - cp $uri $path - return $? - fi - - echo "Downloading $uri" - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail - else - wget -q -O "$path" "$uri" --tries="$download_retries" - fi - - return $? -} - -function GetTempPathFileName { - local path=$1 - - local temp_dir=$(GetTempDirectory) - local temp_file_name=$(basename $path) - echo $temp_dir$temp_file_name - return 0 -} - -function DownloadAndExtract { - local uri=$1 - local installDir=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - local temp_tool_path=$(GetTempPathFileName $uri) - - echo "downloading to: $temp_tool_path" - - # Download file - GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds - if [[ $? != 0 ]]; then - echo "Failed to download '$uri' to '$temp_tool_path'." >&2 - return 1 - fi - - # Extract File - echo "extracting from $temp_tool_path to $installDir" - ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds - if [[ $? != 0 ]]; then - echo "Failed to extract '$temp_tool_path' to '$installDir'." >&2 - return 1 - fi - - return 0 -} - -function NewScriptShim { - local shimpath=$1 - local tool_file_path=$2 - local force=${3:-false} - - echo "Generating '$shimpath' shim" - if [[ -f $shimpath ]]; then - if [[ $force = false ]]; then - echo "File '$shimpath' already exists." >&2 - return 1 - else - rm -rf $shimpath - fi - fi - - if [[ ! -f $tool_file_path ]]; then - echo "Specified tool file path:'$tool_file_path' does not exist" >&2 - return 1 - fi - - local shim_contents=$'#!/usr/bin/env bash\n' - shim_contents+="SHIMARGS="$'$1\n' - shim_contents+="$tool_file_path"$' $SHIMARGS\n' - - # Write shim file - echo "$shim_contents" > $shimpath - - chmod +x $shimpath - - echo "Finished generating shim '$shimpath'" - - return $? -} - +#!/usr/bin/env bash + +function GetNativeInstallDirectory { + local install_dir + + if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then + install_dir=$HOME/.netcoreeng/native/ + else + install_dir=$NETCOREENG_INSTALL_DIRECTORY + fi + + echo $install_dir + return 0 +} + +function GetTempDirectory { + + echo $(GetNativeInstallDirectory)temp/ + return 0 +} + +function ExpandZip { + local zip_path=$1 + local output_directory=$2 + local force=${3:-false} + + echo "Extracting $zip_path to $output_directory" + if [[ -d $output_directory ]] && [[ $force = false ]]; then + echo "Directory '$output_directory' already exists, skipping extract" + return 0 + fi + + if [[ -d $output_directory ]]; then + echo "'Force flag enabled, but '$output_directory' exists. Removing directory" + rm -rf $output_directory + if [[ $? != 0 ]]; then + echo Unable to remove '$output_directory'>&2 + return 1 + fi + fi + + echo "Creating directory: '$output_directory'" + mkdir -p $output_directory + + echo "Extracting archive" + tar -xf $zip_path -C $output_directory + if [[ $? != 0 ]]; then + echo "Unable to extract '$zip_path'" >&2 + return 1 + fi + + return 0 +} + +function GetCurrentOS { + local unameOut="$(uname -s)" + case $unameOut in + Linux*) echo "Linux";; + Darwin*) echo "MacOS";; + esac + return 0 +} + +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + else + wget -q -O "$path" "$uri" --tries="$download_retries" + fi + + return $? +} + +function GetTempPathFileName { + local path=$1 + + local temp_dir=$(GetTempDirectory) + local temp_file_name=$(basename $path) + echo $temp_dir$temp_file_name + return 0 +} + +function DownloadAndExtract { + local uri=$1 + local installDir=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + local temp_tool_path=$(GetTempPathFileName $uri) + + echo "downloading to: $temp_tool_path" + + # Download file + GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + echo "Failed to download '$uri' to '$temp_tool_path'." >&2 + return 1 + fi + + # Extract File + echo "extracting from $temp_tool_path to $installDir" + ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + echo "Failed to extract '$temp_tool_path' to '$installDir'." >&2 + return 1 + fi + + return 0 +} + +function NewScriptShim { + local shimpath=$1 + local tool_file_path=$2 + local force=${3:-false} + + echo "Generating '$shimpath' shim" + if [[ -f $shimpath ]]; then + if [[ $force = false ]]; then + echo "File '$shimpath' already exists." >&2 + return 1 + else + rm -rf $shimpath + fi + fi + + if [[ ! -f $tool_file_path ]]; then + echo "Specified tool file path:'$tool_file_path' does not exist" >&2 + return 1 + fi + + local shim_contents=$'#!/usr/bin/env bash\n' + shim_contents+="SHIMARGS="$'$1\n' + shim_contents+="$tool_file_path"$' $SHIMARGS\n' + + # Write shim file + echo "$shim_contents" > $shimpath + + chmod +x $shimpath + + echo "Finished generating shim '$shimpath'" + + return $? +} + diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh index 293af6017df..7074f3b0f18 100644 --- a/eng/common/native/install-cmake.sh +++ b/eng/common/native/install-cmake.sh @@ -1,117 +1,117 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. $scriptroot/common-library.sh - -base_uri= -install_path= -version= -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 - -while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installpath) - install_path=$2 - shift 2 - ;; - --version) - version=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --baseuri Base file directory or Url wrom which to acquire tool archives" - echo " --installpath Base directory to install native tool to" - echo " --clean Don't install the tool, just clean up the current install of the tool" - echo " --force Force install of tools even if they previously exist" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --downloadretries Total number of retry attempts" - echo " --retrywaittimeseconds Wait time between retry attempts in seconds" - echo "" - exit 0 - ;; - esac -done - -tool_name="cmake" -tool_os=$(GetCurrentOS) -tool_folder=$(echo $tool_os | awk '{print tolower($0)}') -tool_arch="x86_64" -tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" -tool_install_directory="$install_path/$tool_name/$version" -tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" -shim_path="$install_path/$tool_name.sh" -uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz" - -# Clean up tool and installers -if [[ $clean = true ]]; then - echo "Cleaning $tool_install_directory" - if [[ -d $tool_install_directory ]]; then - rm -rf $tool_install_directory - fi - - echo "Cleaning $shim_path" - if [[ -f $shim_path ]]; then - rm -rf $shim_path - fi - - tool_temp_path=$(GetTempPathFileName $uri) - echo "Cleaning $tool_temp_path" - if [[ -f $tool_temp_path ]]; then - rm -rf $tool_temp_path - fi - - exit 0 -fi - -# Install tool -if [[ -f $tool_file_path ]] && [[ $force = false ]]; then - echo "$tool_name ($version) already exists, skipping install" - exit 0 -fi - -DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds - -if [[ $? != 0 ]]; then - echo "Installation failed" >&2 - exit 1 -fi - -# Generate Shim -# Always rewrite shims so that we are referencing the expected version -NewScriptShim $shim_path $tool_file_path true - -if [[ $? != 0 ]]; then - echo "Shim generation failed" >&2 - exit 1 -fi - +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. $scriptroot/common-library.sh + +base_uri= +install_path= +version= +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 + +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installpath) + install_path=$2 + shift 2 + ;; + --version) + version=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --baseuri Base file directory or Url wrom which to acquire tool archives" + echo " --installpath Base directory to install native tool to" + echo " --clean Don't install the tool, just clean up the current install of the tool" + echo " --force Force install of tools even if they previously exist" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --downloadretries Total number of retry attempts" + echo " --retrywaittimeseconds Wait time between retry attempts in seconds" + echo "" + exit 0 + ;; + esac +done + +tool_name="cmake" +tool_os=$(GetCurrentOS) +tool_folder=$(echo $tool_os | awk '{print tolower($0)}') +tool_arch="x86_64" +tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" +tool_install_directory="$install_path/$tool_name/$version" +tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" +shim_path="$install_path/$tool_name.sh" +uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz" + +# Clean up tool and installers +if [[ $clean = true ]]; then + echo "Cleaning $tool_install_directory" + if [[ -d $tool_install_directory ]]; then + rm -rf $tool_install_directory + fi + + echo "Cleaning $shim_path" + if [[ -f $shim_path ]]; then + rm -rf $shim_path + fi + + tool_temp_path=$(GetTempPathFileName $uri) + echo "Cleaning $tool_temp_path" + if [[ -f $tool_temp_path ]]; then + rm -rf $tool_temp_path + fi + + exit 0 +fi + +# Install tool +if [[ -f $tool_file_path ]] && [[ $force = false ]]; then + echo "$tool_name ($version) already exists, skipping install" + exit 0 +fi + +DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds + +if [[ $? != 0 ]]; then + echo "Installation failed" >&2 + exit 1 +fi + +# Generate Shim +# Always rewrite shims so that we are referencing the expected version +NewScriptShim $shim_path $tool_file_path true + +if [[ $? != 0 ]]; then + echo "Shim generation failed" >&2 + exit 1 +fi + exit 0 \ No newline at end of file diff --git a/eng/common/native/install-tool.ps1 b/eng/common/native/install-tool.ps1 index 635ab3fd414..d295745f050 100644 --- a/eng/common/native/install-tool.ps1 +++ b/eng/common/native/install-tool.ps1 @@ -1,130 +1,130 @@ -<# -.SYNOPSIS -Install native tool - -.DESCRIPTION -Install cmake native tool from Azure blob storage - -.PARAMETER InstallPath -Base directory to install native tool to - -.PARAMETER BaseUri -Base file directory or Url from which to acquire tool archives - -.PARAMETER CommonLibraryDirectory -Path to folder containing common library modules - -.PARAMETER Force -Force install of tools even if they previously exist - -.PARAMETER Clean -Don't install the tool, just clean up the current install of the tool - -.PARAMETER DownloadRetries -Total number of retry attempts - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds - -.NOTES -Returns 0 if install succeeds, 1 otherwise -#> -[CmdletBinding(PositionalBinding=$false)] -Param ( - [Parameter(Mandatory=$True)] - [string] $ToolName, - [Parameter(Mandatory=$True)] - [string] $InstallPath, - [Parameter(Mandatory=$True)] - [string] $BaseUri, - [Parameter(Mandatory=$True)] - [string] $Version, - [string] $CommonLibraryDirectory = $PSScriptRoot, - [switch] $Force = $False, - [switch] $Clean = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30 -) - -# Import common library modules -Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") - -try { - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq "Continue" - - $Arch = CommonLibrary\Get-MachineArchitecture - $ToolOs = "win64" - if($Arch -Eq "x32") { - $ToolOs = "win32" - } - $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch" - $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\" - $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip" - $ShimPath = Join-Path $InstallPath "$ToolName.exe" - - if ($Clean) { - Write-Host "Cleaning $ToolInstallDirectory" - if (Test-Path $ToolInstallDirectory) { - Remove-Item $ToolInstallDirectory -Force -Recurse - } - Write-Host "Cleaning $ShimPath" - if (Test-Path $ShimPath) { - Remove-Item $ShimPath -Force - } - $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri - Write-Host "Cleaning $ToolTempPath" - if (Test-Path $ToolTempPath) { - Remove-Item $ToolTempPath -Force - } - exit 0 - } - - # Install tool - if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) { - Write-Verbose "$ToolName ($Version) already exists, skipping install" - } - else { - $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri ` - -InstallDirectory $ToolInstallDirectory ` - -Force:$Force ` - -DownloadRetries $DownloadRetries ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Verbose:$Verbose - - if ($InstallStatus -Eq $False) { - Write-Error "Installation failed" - exit 1 - } - } - - $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName } - if (@($ToolFilePath).Length -Gt 1) { - Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" - exit 1 - } elseif (@($ToolFilePath).Length -Lt 1) { - Write-Error "$ToolName was not found in $ToolFilePath." - exit 1 - } - - # Generate shim - # Always rewrite shims so that we are referencing the expected version - $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName ` - -ShimDirectory $InstallPath ` - -ToolFilePath "$ToolFilePath" ` - -BaseUri $BaseUri ` - -Force:$Force ` - -Verbose:$Verbose - - if ($GenerateShimStatus -Eq $False) { - Write-Error "Generate shim failed" - return 1 - } - - exit 0 -} -catch { - Write-Host $_ - Write-Host $_.Exception - exit 1 -} +<# +.SYNOPSIS +Install native tool + +.DESCRIPTION +Install cmake native tool from Azure blob storage + +.PARAMETER InstallPath +Base directory to install native tool to + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER CommonLibraryDirectory +Path to folder containing common library modules + +.PARAMETER Force +Force install of tools even if they previously exist + +.PARAMETER Clean +Don't install the tool, just clean up the current install of the tool + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.NOTES +Returns 0 if install succeeds, 1 otherwise +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [Parameter(Mandatory=$True)] + [string] $ToolName, + [Parameter(Mandatory=$True)] + [string] $InstallPath, + [Parameter(Mandatory=$True)] + [string] $BaseUri, + [Parameter(Mandatory=$True)] + [string] $Version, + [string] $CommonLibraryDirectory = $PSScriptRoot, + [switch] $Force = $False, + [switch] $Clean = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 +) + +# Import common library modules +Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $Arch = CommonLibrary\Get-MachineArchitecture + $ToolOs = "win64" + if($Arch -Eq "x32") { + $ToolOs = "win32" + } + $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch" + $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\" + $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip" + $ShimPath = Join-Path $InstallPath "$ToolName.exe" + + if ($Clean) { + Write-Host "Cleaning $ToolInstallDirectory" + if (Test-Path $ToolInstallDirectory) { + Remove-Item $ToolInstallDirectory -Force -Recurse + } + Write-Host "Cleaning $ShimPath" + if (Test-Path $ShimPath) { + Remove-Item $ShimPath -Force + } + $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri + Write-Host "Cleaning $ToolTempPath" + if (Test-Path $ToolTempPath) { + Remove-Item $ToolTempPath -Force + } + exit 0 + } + + # Install tool + if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) { + Write-Verbose "$ToolName ($Version) already exists, skipping install" + } + else { + $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri ` + -InstallDirectory $ToolInstallDirectory ` + -Force:$Force ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Verbose:$Verbose + + if ($InstallStatus -Eq $False) { + Write-Error "Installation failed" + exit 1 + } + } + + $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName } + if (@($ToolFilePath).Length -Gt 1) { + Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" + exit 1 + } elseif (@($ToolFilePath).Length -Lt 1) { + Write-Error "$ToolName was not found in $ToolFilePath." + exit 1 + } + + # Generate shim + # Always rewrite shims so that we are referencing the expected version + $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName ` + -ShimDirectory $InstallPath ` + -ToolFilePath "$ToolFilePath" ` + -BaseUri $BaseUri ` + -Force:$Force ` + -Verbose:$Verbose + + if ($GenerateShimStatus -Eq $False) { + Write-Error "Generate shim failed" + return 1 + } + + exit 0 +} +catch { + Write-Host $_ + Write-Host $_.Exception + exit 1 +} diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 7b61376f8aa..5d993591203 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -1,233 +1,233 @@ -# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. - -# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 - -$script:loggingCommandPrefix = '##vso[' -$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? - New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } - New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } - New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } - New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } -) -# TODO: BUG: Escape % ??? -# TODO: Add test to verify don't need to escape "=". - -function Write-PipelineTelemetryError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Category, - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - $PSBoundParameters.Remove("Category") | Out-Null - - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" - $PSBoundParameters.Remove("Message") | Out-Null - $PSBoundParameters.Add("Message", $Message) - - Write-PipelineTaskError @PSBoundParameters -} - -function Write-PipelineTaskError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - if(!$ci) { - if($Type -eq 'error') { - Write-Host $Message -ForegroundColor Red - return - } - elseif ($Type -eq 'warning') { - Write-Host $Message -ForegroundColor Yellow - return - } - } - - if(($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return - } - if(-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') - } - Write-LogIssue @PSBoundParameters - } - - function Write-PipelineSetVariable { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Name, - [string]$Value, - [switch]$Secret, - [switch]$AsOutput) - - if($ci) { - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ - 'variable' = $Name - 'isSecret' = $Secret - 'isOutput' = 'true' - } -AsOutput:$AsOutput - } - } - - function Write-PipelinePrependPath { - [CmdletBinding()] - param( - [Parameter(Mandatory=$true)] - [string]$Path, - [switch]$AsOutput) - if($ci) { - Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput - } - } - -<######################################## -# Private functions. -########################################> -function Format-LoggingCommandData { - [CmdletBinding()] - param([string]$Value, [switch]$Reverse) - - if (!$Value) { - return '' - } - - if (!$Reverse) { - foreach ($mapping in $script:loggingCommandEscapeMappings) { - $Value = $Value.Replace($mapping.Token, $mapping.Replacement) - } - } else { - for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { - $mapping = $script:loggingCommandEscapeMappings[$i] - $Value = $Value.Replace($mapping.Replacement, $mapping.Token) - } - } - - return $Value -} - -function Format-LoggingCommand { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Area, - [Parameter(Mandatory = $true)] - [string]$Event, - [string]$Data, - [hashtable]$Properties) - - # Append the preamble. - [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder - $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) - - # Append the properties. - if ($Properties) { - $first = $true - foreach ($key in $Properties.Keys) { - [string]$value = Format-LoggingCommandData $Properties[$key] - if ($value) { - if ($first) { - $null = $sb.Append(' ') - $first = $false - } else { - $null = $sb.Append(';') - } - - $null = $sb.Append("$key=$value") - } - } - } - - # Append the tail and output the value. - $Data = Format-LoggingCommandData $Data - $sb.Append(']').Append($Data).ToString() -} - -function Write-LoggingCommand { - [CmdletBinding(DefaultParameterSetName = 'Parameters')] - param( - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Area, - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Event, - [Parameter(ParameterSetName = 'Parameters')] - [string]$Data, - [Parameter(ParameterSetName = 'Parameters')] - [hashtable]$Properties, - [Parameter(Mandatory = $true, ParameterSetName = 'Object')] - $Command, - [switch]$AsOutput) - - if ($PSCmdlet.ParameterSetName -eq 'Object') { - Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput - return - } - - $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties - if ($AsOutput) { - $command - } else { - Write-Host $command - } -} - -function Write-LogIssue { - [CmdletBinding()] - param( - [ValidateSet('warning', 'error')] - [Parameter(Mandatory = $true)] - [string]$Type, - [string]$Message, - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ - 'type' = $Type - 'code' = $ErrCode - 'sourcepath' = $SourcePath - 'linenumber' = $LineNumber - 'columnnumber' = $ColumnNumber - } - if ($AsOutput) { - return $command - } - - if ($Type -eq 'error') { - $foregroundColor = $host.PrivateData.ErrorForegroundColor - $backgroundColor = $host.PrivateData.ErrorBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Red - $backgroundColor = [System.ConsoleColor]::Black - } - } else { - $foregroundColor = $host.PrivateData.WarningForegroundColor - $backgroundColor = $host.PrivateData.WarningBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Yellow - $backgroundColor = [System.ConsoleColor]::Black - } - } - - Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor +# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. + +# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 + +$script:loggingCommandPrefix = '##vso[' +$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? + New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } + New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } + New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } + New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } +) +# TODO: BUG: Escape % ??? +# TODO: Add test to verify don't need to escape "=". + +function Write-PipelineTelemetryError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Category, + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $PSBoundParameters.Remove("Category") | Out-Null + + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + $PSBoundParameters.Remove("Message") | Out-Null + $PSBoundParameters.Add("Message", $Message) + + Write-PipelineTaskError @PSBoundParameters +} + +function Write-PipelineTaskError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + if(!$ci) { + if($Type -eq 'error') { + Write-Host $Message -ForegroundColor Red + return + } + elseif ($Type -eq 'warning') { + Write-Host $Message -ForegroundColor Yellow + return + } + } + + if(($Type -ne 'error') -and ($Type -ne 'warning')) { + Write-Host $Message + return + } + if(-not $PSBoundParameters.ContainsKey('Type')) { + $PSBoundParameters.Add('Type', 'error') + } + Write-LogIssue @PSBoundParameters + } + + function Write-PipelineSetVariable { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput) + + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ + 'variable' = $Name + 'isSecret' = $Secret + 'isOutput' = 'true' + } -AsOutput:$AsOutput + } + } + + function Write-PipelinePrependPath { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string]$Path, + [switch]$AsOutput) + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput + } + } + +<######################################## +# Private functions. +########################################> +function Format-LoggingCommandData { + [CmdletBinding()] + param([string]$Value, [switch]$Reverse) + + if (!$Value) { + return '' + } + + if (!$Reverse) { + foreach ($mapping in $script:loggingCommandEscapeMappings) { + $Value = $Value.Replace($mapping.Token, $mapping.Replacement) + } + } else { + for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { + $mapping = $script:loggingCommandEscapeMappings[$i] + $Value = $Value.Replace($mapping.Replacement, $mapping.Token) + } + } + + return $Value +} + +function Format-LoggingCommand { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Area, + [Parameter(Mandatory = $true)] + [string]$Event, + [string]$Data, + [hashtable]$Properties) + + # Append the preamble. + [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder + $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) + + # Append the properties. + if ($Properties) { + $first = $true + foreach ($key in $Properties.Keys) { + [string]$value = Format-LoggingCommandData $Properties[$key] + if ($value) { + if ($first) { + $null = $sb.Append(' ') + $first = $false + } else { + $null = $sb.Append(';') + } + + $null = $sb.Append("$key=$value") + } + } + } + + # Append the tail and output the value. + $Data = Format-LoggingCommandData $Data + $sb.Append(']').Append($Data).ToString() +} + +function Write-LoggingCommand { + [CmdletBinding(DefaultParameterSetName = 'Parameters')] + param( + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Area, + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Event, + [Parameter(ParameterSetName = 'Parameters')] + [string]$Data, + [Parameter(ParameterSetName = 'Parameters')] + [hashtable]$Properties, + [Parameter(Mandatory = $true, ParameterSetName = 'Object')] + $Command, + [switch]$AsOutput) + + if ($PSCmdlet.ParameterSetName -eq 'Object') { + Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput + return + } + + $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties + if ($AsOutput) { + $command + } else { + Write-Host $command + } +} + +function Write-LogIssue { + [CmdletBinding()] + param( + [ValidateSet('warning', 'error')] + [Parameter(Mandatory = $true)] + [string]$Type, + [string]$Message, + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ + 'type' = $Type + 'code' = $ErrCode + 'sourcepath' = $SourcePath + 'linenumber' = $LineNumber + 'columnnumber' = $ColumnNumber + } + if ($AsOutput) { + return $command + } + + if ($Type -eq 'error') { + $foregroundColor = $host.PrivateData.ErrorForegroundColor + $backgroundColor = $host.PrivateData.ErrorBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Red + $backgroundColor = [System.ConsoleColor]::Black + } + } else { + $foregroundColor = $host.PrivateData.WarningForegroundColor + $backgroundColor = $host.PrivateData.WarningBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Yellow + $backgroundColor = [System.ConsoleColor]::Black + } + } + + Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor } \ No newline at end of file diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh index 6098f9a5438..7868684076b 100644 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -1,102 +1,102 @@ -#!/usr/bin/env bash - -function Write-PipelineTelemetryError { - local telemetry_category='' - local function_args=() - local message='' - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" - case "$opt" in - -category|-c) - telemetry_category=$2 - shift - ;; - -*) - function_args+=("$1 $2") - shift - ;; - *) - message=$* - ;; - esac - shift - done - - if [[ "$ci" != true ]]; then - echo "$message" >&2 - return - fi - - message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" - function_args+=("$message") - - Write-PipelineTaskError $function_args -} - -function Write-PipelineTaskError { - if [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - - message_type="error" - sourcepath='' - linenumber='' - columnnumber='' - error_code='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" - case "$opt" in - -type|-t) - message_type=$2 - shift - ;; - -sourcepath|-s) - sourcepath=$2 - shift - ;; - -linenumber|-ln) - linenumber=$2 - shift - ;; - -columnnumber|-cn) - columnnumber=$2 - shift - ;; - -errcode|-e) - error_code=$2 - shift - ;; - *) - break - ;; - esac - - shift - done - - message="##vso[task.logissue" - - message="$message type=$message_type" - - if [ -n "$sourcepath" ]; then - message="$message;sourcepath=$sourcepath" - fi - - if [ -n "$linenumber" ]; then - message="$message;linenumber=$linenumber" - fi - - if [ -n "$columnnumber" ]; then - message="$message;columnnumber=$columnnumber" - fi - - if [ -n "$error_code" ]; then - message="$message;code=$error_code" - fi - - message="$message]$*" - echo "$message" -} - +#!/usr/bin/env bash + +function Write-PipelineTelemetryError { + local telemetry_category='' + local function_args=() + local message='' + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -category|-c) + telemetry_category=$2 + shift + ;; + -*) + function_args+=("$1 $2") + shift + ;; + *) + message=$* + ;; + esac + shift + done + + if [[ "$ci" != true ]]; then + echo "$message" >&2 + return + fi + + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + + Write-PipelineTaskError $function_args +} + +function Write-PipelineTaskError { + if [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + + message_type="error" + sourcepath='' + linenumber='' + columnnumber='' + error_code='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -type|-t) + message_type=$2 + shift + ;; + -sourcepath|-s) + sourcepath=$2 + shift + ;; + -linenumber|-ln) + linenumber=$2 + shift + ;; + -columnnumber|-cn) + columnnumber=$2 + shift + ;; + -errcode|-e) + error_code=$2 + shift + ;; + *) + break + ;; + esac + + shift + done + + message="##vso[task.logissue" + + message="$message type=$message_type" + + if [ -n "$sourcepath" ]; then + message="$message;sourcepath=$sourcepath" + fi + + if [ -n "$linenumber" ]; then + message="$message;linenumber=$linenumber" + fi + + if [ -n "$columnnumber" ]; then + message="$message;columnnumber=$columnnumber" + fi + + if [ -n "$error_code" ]; then + message="$message;code=$error_code" + fi + + message="$message]$*" + echo "$message" +} + diff --git a/eng/common/post-build/dotnetsymbol-init.ps1 b/eng/common/post-build/dotnetsymbol-init.ps1 index e7659b98c8c..a4e511732a4 100644 --- a/eng/common/post-build/dotnetsymbol-init.ps1 +++ b/eng/common/post-build/dotnetsymbol-init.ps1 @@ -1,29 +1,29 @@ -param ( - $dotnetsymbolVersion = $null -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -$verbosity = "minimal" - -function Installdotnetsymbol ($dotnetsymbolVersion) { - $dotnetsymbolPackageName = "dotnet-symbol" - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { - Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." - } - else { - Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." - Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity $verbosity --global - } -} - -Installdotnetsymbol $dotnetsymbolVersion +param ( + $dotnetsymbolVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function Installdotnetsymbol ($dotnetsymbolVersion) { + $dotnetsymbolPackageName = "dotnet-symbol" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + } + else { + Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity $verbosity --global + } +} + +Installdotnetsymbol $dotnetsymbolVersion diff --git a/eng/common/post-build/sourcelink-cli-init.ps1 b/eng/common/post-build/sourcelink-cli-init.ps1 index 9eaa25b3b50..723dd073f1f 100644 --- a/eng/common/post-build/sourcelink-cli-init.ps1 +++ b/eng/common/post-build/sourcelink-cli-init.ps1 @@ -1,29 +1,29 @@ -param ( - $sourcelinkCliVersion = $null -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -$verbosity = "minimal" - -function InstallSourcelinkCli ($sourcelinkCliVersion) { - $sourcelinkCliPackageName = "sourcelink" - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { - Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." - } - else { - Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." - Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity $verbosity --global - } -} - -InstallSourcelinkCli $sourcelinkCliVersion +param ( + $sourcelinkCliVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function InstallSourcelinkCli ($sourcelinkCliVersion) { + $sourcelinkCliPackageName = "sourcelink" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { + Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." + } + else { + Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity $verbosity --global + } +} + +InstallSourcelinkCli $sourcelinkCliVersion diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 8abd684e9e5..5e9a9404b08 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -1,224 +1,224 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages - [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -# Cache/HashMap (File -> Exist flag) used to consult whether a file exist -# in the repository at a specific commit point. This is populated by inserting -# all files present in the repo at a specific commit point. -$global:RepoFiles = @{} - -$ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - . $using:PSScriptRoot\..\tools.ps1 - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-PipelineTaskError "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @(".dll", ".exe", ".pdb") - - Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith(".resources.dll")) { - return - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" - $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" - $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" - - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - $Status = 0 - } - } - catch { - write-host $_ - $Status = 0 - } - } - - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - } - catch { - - } - finally { - $zip.Dispose() - } - - if ($FailedFiles -eq 0) { - Write-Host "Passed." - } - else { - Write-PipelineTaskError "$PackagePath has broken SourceLink links." - } -} - -function ValidateSourceLinkLinks { - if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { - if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { - Write-PipelineTaskError "GHRepoName should be in the format / or -" - ExitWithExitCode 1 - } - else { - $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; - } - } - - if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { - Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string" - ExitWithExitCode 1 - } - - $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") - $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" - Write-Host $_ - ExitWithExitCode 1 - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - # Process each NuGet package in parallel - $Jobs = @() - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -function CheckExitCode ([string]$stage) { - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } -} - -try { - Write-Host "Installing SourceLink CLI..." - Get-Location - . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion - CheckExitCode "Running sourcelink-cli-init" - - Measure-Command { ValidateSourceLinkLinks } -} -catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages + [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + . $using:PSScriptRoot\..\tools.ps1 + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @(".dll", ".exe", ".pdb") + + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith(".resources.dll")) { + return + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" + $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" + $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" + + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + $Status = 0 + } + } + catch { + write-host $_ + $Status = 0 + } + } + + if ($Status -ne 200) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + } + catch { + + } + finally { + $zip.Dispose() + } + + if ($FailedFiles -eq 0) { + Write-Host "Passed." + } + else { + Write-PipelineTaskError "$PackagePath has broken SourceLink links." + } +} + +function ValidateSourceLinkLinks { + if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { + Write-PipelineTaskError "GHRepoName should be in the format / or -" + ExitWithExitCode 1 + } + else { + $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; + } + } + + if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string" + ExitWithExitCode 1 + } + + $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" + Write-Host $_ + ExitWithExitCode 1 + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + # Process each NuGet package in parallel + $Jobs = @() + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing SourceLink CLI..." + Get-Location + . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion + CheckExitCode "Running sourcelink-cli-init" + + Measure-Command { ValidateSourceLinkLinks } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 69456854e04..663ca636363 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -1,186 +1,186 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -Add-Type -AssemblyName System.IO.Compression.FileSystem - -function FirstMatchingSymbolDescriptionOrDefault { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + "\" + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, ".pdb") - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") - - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") - - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") - - $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" - - & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null - - if (Test-Path $PdbPath) { - return "PDB" - } - elseif (Test-Path $NGenPdb) { - return "NGen PDB" - } - elseif (Test-Path $SODbg) { - return "DBG for SO" - } - elseif (Test-Path $DylibDwarf) { - return "Dwarf for Dylib" - } - elseif (Test-Path $SymbolPath) { - return "Module" - } - else { - return $null - } -} - -function CountMissingSymbols { - param( - [string] $PackagePath # Path to a NuGet package - ) - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-PipelineTaskError "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - # Extensions for which we'll look for symbols - $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") - - # How many files are missing symbol information - $MissingSymbols = 0 - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid - $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" - - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) - - Get-ChildItem -Recurse $ExtractPath | - Where-Object {$RelevantExtensions -contains $_.Extension} | - ForEach-Object { - if ($_.FullName -Match "\\ref\\") { - Write-Host "`t Ignoring reference assembly file" $_.FullName - return - } - - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath - - Write-Host -NoNewLine "`t Checking file" $_.FullName "... " - - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" - } - else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host "No symbols found on MSDL or SymWeb!" - } - else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host "No symbols found on MSDL!" - } - else { - Write-Host "No symbols found on SymWeb!" - } - } - } - } - - Pop-Location - - return $MissingSymbols -} - -function CheckSymbolsAvailable { - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $FileName = $_.Name - - # These packages from Arcade-Services include some native libraries that - # our current symbol uploader can't handle. Below is a workaround until - # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. - if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - - Write-Host "Validating $FileName " - $Status = CountMissingSymbols "$InputPath\$FileName" - - if ($Status -ne 0) { - Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" - ExitWithExitCode $exitCode - } - - Write-Host - } -} - -function CheckExitCode ([string]$stage) { - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } -} - -try { - Write-Host "Installing dotnet symbol ..." - Get-Location - . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion - CheckExitCode "Running dotnetsymbol-init" - - CheckSymbolsAvailable -} -catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function FirstMatchingSymbolDescriptionOrDefault { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + "\" + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") + + $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + + & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return "PDB" + } + elseif (Test-Path $NGenPdb) { + return "NGen PDB" + } + elseif (Test-Path $SODbg) { + return "DBG for SO" + } + elseif (Test-Path $DylibDwarf) { + return "Dwarf for Dylib" + } + elseif (Test-Path $SymbolPath) { + return "Module" + } + else { + return $null + } +} + +function CountMissingSymbols { + param( + [string] $PackagePath # Path to a NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for symbols + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") + + # How many files are missing symbol information + $MissingSymbols = 0 + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $PackageGuid = New-Guid + $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" + + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + + Get-ChildItem -Recurse $ExtractPath | + Where-Object {$RelevantExtensions -contains $_.Extension} | + ForEach-Object { + if ($_.FullName -Match "\\ref\\") { + Write-Host "`t Ignoring reference assembly file" $_.FullName + return + } + + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath + + Write-Host -NoNewLine "`t Checking file" $_.FullName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host "No symbols found on MSDL or SymWeb!" + } + else { + if ($SymbolsOnMSDL -eq $null) { + Write-Host "No symbols found on MSDL!" + } + else { + Write-Host "No symbols found on SymWeb!" + } + } + } + } + + Pop-Location + + return $MissingSymbols +} + +function CheckSymbolsAvailable { + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $FileName = $_.Name + + # These packages from Arcade-Services include some native libraries that + # our current symbol uploader can't handle. Below is a workaround until + # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + + Write-Host "Validating $FileName " + $Status = CountMissingSymbols "$InputPath\$FileName" + + if ($Status -ne 0) { + Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" + ExitWithExitCode $exitCode + } + + Write-Host + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing dotnet symbol ..." + Get-Location + . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion + CheckExitCode "Running dotnetsymbol-init" + + CheckSymbolsAvailable +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index d0eec5163ef..289b5114bdb 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -1,79 +1,79 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $configuration = "Debug", - [string] $task, - [string] $verbosity = "minimal", - [string] $msbuildEngine = $null, - [switch] $restore, - [switch] $prepareMachine, - [switch] $help, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties -) - -$ci = $true -$binaryLog = $true -$warnAsError = $true - -. $PSScriptRoot\tools.ps1 - -function Print-Usage() { - Write-Host "Common settings:" - Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" - Write-Host " -restore Restore dependencies" - Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" - Write-Host " -help Print help and exit" - Write-Host "" - - Write-Host "Advanced settings:" - Write-Host " -prepareMachine Prepare machine for CI run" - Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host "" - Write-Host "Command line arguments not listed above are passed thru to msbuild." -} - -function Build([string]$target) { - $logSuffix = if ($target -eq "Execute") { "" } else { ".$target" } - $log = Join-Path $LogDir "$task$logSuffix.binlog" - $outputPath = Join-Path $ToolsetDir "$task\\" - - MSBuild $taskProject ` - /bl:$log ` - /t:$target ` - /p:Configuration=$configuration ` - /p:RepoRoot=$RepoRoot ` - /p:BaseIntermediateOutputPath=$outputPath ` - @properties -} - -try { - if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { - Print-Usage - exit 0 - } - - if ($task -eq "") { - Write-Host "Missing required parameter '-task '" -ForegroundColor Red - Print-Usage - ExitWithExitCode 1 - } - - $taskProject = GetSdkTaskProject $task - if (!(Test-Path $taskProject)) { - Write-Host "Unknown task: $task" -ForegroundColor Red - ExitWithExitCode 1 - } - - if ($restore) { - Build "Restore" - } - - Build "Execute" -} -catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 -} - -ExitWithExitCode 0 +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $configuration = "Debug", + [string] $task, + [string] $verbosity = "minimal", + [string] $msbuildEngine = $null, + [switch] $restore, + [switch] $prepareMachine, + [switch] $help, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties +) + +$ci = $true +$binaryLog = $true +$warnAsError = $true + +. $PSScriptRoot\tools.ps1 + +function Print-Usage() { + Write-Host "Common settings:" + Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" + Write-Host " -restore Restore dependencies" + Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" + Write-Host " -help Print help and exit" + Write-Host "" + + Write-Host "Advanced settings:" + Write-Host " -prepareMachine Prepare machine for CI run" + Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host "" + Write-Host "Command line arguments not listed above are passed thru to msbuild." +} + +function Build([string]$target) { + $logSuffix = if ($target -eq "Execute") { "" } else { ".$target" } + $log = Join-Path $LogDir "$task$logSuffix.binlog" + $outputPath = Join-Path $ToolsetDir "$task\\" + + MSBuild $taskProject ` + /bl:$log ` + /t:$target ` + /p:Configuration=$configuration ` + /p:RepoRoot=$RepoRoot ` + /p:BaseIntermediateOutputPath=$outputPath ` + @properties +} + +try { + if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { + Print-Usage + exit 0 + } + + if ($task -eq "") { + Write-Host "Missing required parameter '-task '" -ForegroundColor Red + Print-Usage + ExitWithExitCode 1 + } + + $taskProject = GetSdkTaskProject $task + if (!(Test-Path $taskProject)) { + Write-Host "Unknown task: $task" -ForegroundColor Red + ExitWithExitCode 1 + } + + if ($restore) { + Build "Restore" + } + + Build "Execute" +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 0c5451c1141..cfd82eb22f3 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -1,13 +1,13 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 0635f26fb63..d14b2abe934 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -1,97 +1,97 @@ -Param( - [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) - [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) - [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified - [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) - [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located - [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault - [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code - [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts - [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. - [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) - [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed - [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. - [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. - [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 - -#Replace repo names to the format of org/repo -if (!($Repository.contains('/'))) { - $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; -} -else{ - $RepoName = $Repository; -} - -if ($GuardianPackageName) { - $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) -} else { - $guardianCliLocation = $GuardianCliLocation -} - -$ValidPath = Test-Path $guardianCliLocation - -if ($ValidPath -eq $False) -{ - Write-Host "Invalid Guardian CLI Location." - exit 1 -} - -& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel -$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" - -if ($TsaOnboard) { - if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - } else { - Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." - exit 1 - } -} - -if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -} -if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -} - -if ($UpdateBaseline) { - & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Update baseline" -} - -if ($TsaPublish) { - if ($TsaBranchName -and $BuildNumber) { - if (-not $TsaRepositoryName) { - $TsaRepositoryName = "$($Repository)-$($BranchName)" - } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - } else { - Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." - exit 1 - } -} +Param( + [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located + [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located + [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault + [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code + [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +#Replace repo names to the format of org/repo +if (!($Repository.contains('/'))) { + $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; +} +else{ + $RepoName = $Repository; +} + +if ($GuardianPackageName) { + $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) +} else { + $guardianCliLocation = $GuardianCliLocation +} + +$ValidPath = Test-Path $guardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" + +if ($TsaOnboard) { + if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." + exit 1 + } +} + +if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} +if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} + +if ($UpdateBaseline) { + & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Update baseline" +} + +if ($TsaPublish) { + if ($TsaBranchName -and $BuildNumber) { + if (-not $TsaRepositoryName) { + $TsaRepositoryName = "$($Repository)-$($BranchName)" + } + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." + exit 1 + } +} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 26e01c0673c..7e97217e803 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -1,48 +1,48 @@ -Param( - [string] $GuardianCliLocation, - [string] $Repository, - [string] $BranchName="master", - [string] $WorkingDirectory, - [string] $AzureDevOpsAccessToken, - [string] $GuardianLoggerLevel="Standard" -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 - -# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) -$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" -$zipFile = "$WorkingDirectory/gdn.zip" - -Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory ".gdn") -Try -{ - # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead - Write-Host "Downloading gdn folder from internal config repostiory..." - Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile - if (Test-Path $gdnFolder) { - # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) - Remove-Item -Force -Recurse $gdnFolder - } - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) - Write-Host $gdnFolder -} Catch [System.Net.WebException] { - # if the folder does not exist, we'll do a guardian init and push it to the remote repository - Write-Host "Initializing Guardian..." - Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" - & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-Error "Guardian init failed with exit code $LASTEXITCODE." - } - # We create the mainbaseline so it can be edited later - Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" - & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline - if ($LASTEXITCODE -ne 0) { - Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." - } - & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder" +Param( + [string] $GuardianCliLocation, + [string] $Repository, + [string] $BranchName="master", + [string] $WorkingDirectory, + [string] $AzureDevOpsAccessToken, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file +$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) +$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" +$zipFile = "$WorkingDirectory/gdn.zip" + +Add-Type -AssemblyName System.IO.Compression.FileSystem +$gdnFolder = (Join-Path $WorkingDirectory ".gdn") +Try +{ + # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead + Write-Host "Downloading gdn folder from internal config repostiory..." + Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile + if (Test-Path $gdnFolder) { + # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) + Remove-Item -Force -Recurse $gdnFolder + } + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) + Write-Host $gdnFolder +} Catch [System.Net.WebException] { + # if the folder does not exist, we'll do a guardian init and push it to the remote repository + Write-Host "Initializing Guardian..." + Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" + & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian init failed with exit code $LASTEXITCODE." + } + # We create the mainbaseline so it can be edited later + Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" + & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." + } + & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder" } \ No newline at end of file diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index b054737df13..61827640830 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - - - - + + + + diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 index 79c707d6d8a..5b18ed8e652 100644 --- a/eng/common/sdl/push-gdn.ps1 +++ b/eng/common/sdl/push-gdn.ps1 @@ -1,51 +1,51 @@ -Param( - [string] $Repository, - [string] $BranchName="master", - [string] $GdnFolder, - [string] $AzureDevOpsAccessToken, - [string] $PushReason -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 - -# We create the temp directory where we'll store the sdl-config repository -$sdlDir = Join-Path $env:TEMP "sdl" -if (Test-Path $sdlDir) { - Remove-Item -Force -Recurse $sdlDir -} - -Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" -git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir -if ($LASTEXITCODE -ne 0) { - Write-Error "Git clone failed with exit code $LASTEXITCODE." -} -# We copy the .gdn folder from our local run into the git repository so it can be committed -$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" -if (Get-Command Robocopy) { - Robocopy /S $GdnFolder $sdlRepositoryFolder -} else { - rsync -r $GdnFolder $sdlRepositoryFolder -} -# cd to the sdl-config directory so we can run git there -Push-Location $sdlDir -# git add . --> git commit --> git push -Write-Host "git add ." -git add . -if ($LASTEXITCODE -ne 0) { - Write-Error "Git add failed with exit code $LASTEXITCODE." -} -Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" -git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" -if ($LASTEXITCODE -ne 0) { - Write-Error "Git commit failed with exit code $LASTEXITCODE." -} -Write-Host "git push" -git push -if ($LASTEXITCODE -ne 0) { - Write-Error "Git push failed with exit code $LASTEXITCODE." -} - -# Return to the original directory +Param( + [string] $Repository, + [string] $BranchName="master", + [string] $GdnFolder, + [string] $AzureDevOpsAccessToken, + [string] $PushReason +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We create the temp directory where we'll store the sdl-config repository +$sdlDir = Join-Path $env:TEMP "sdl" +if (Test-Path $sdlDir) { + Remove-Item -Force -Recurse $sdlDir +} + +Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" +git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir +if ($LASTEXITCODE -ne 0) { + Write-Error "Git clone failed with exit code $LASTEXITCODE." +} +# We copy the .gdn folder from our local run into the git repository so it can be committed +$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" +if (Get-Command Robocopy) { + Robocopy /S $GdnFolder $sdlRepositoryFolder +} else { + rsync -r $GdnFolder $sdlRepositoryFolder +} +# cd to the sdl-config directory so we can run git there +Push-Location $sdlDir +# git add . --> git commit --> git push +Write-Host "git add ." +git add . +if ($LASTEXITCODE -ne 0) { + Write-Error "Git add failed with exit code $LASTEXITCODE." +} +Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" +git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" +if ($LASTEXITCODE -ne 0) { + Write-Error "Git commit failed with exit code $LASTEXITCODE." +} +Write-Host "git push" +git push +if ($LASTEXITCODE -ne 0) { + Write-Error "Git push failed with exit code $LASTEXITCODE." +} + +# Return to the original directory Pop-Location \ No newline at end of file diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index e6a86d03a21..6f2e9a9dbb9 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -1,65 +1,65 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $TargetDirectory, - [string] $GdnFolder, - [string[]] $ToolsList, - [string] $UpdateBaseline, - [string] $GuardianLoggerLevel="Standard" -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -$LASTEXITCODE = 0 - -# We store config files in the r directory of .gdn -Write-Host $ToolsList -$gdnConfigPath = Join-Path $GdnFolder "r" -$ValidPath = Test-Path $GuardianCliLocation - -if ($ValidPath -eq $False) -{ - Write-Host "Invalid Guardian CLI Location." - exit 1 -} - -foreach ($tool in $ToolsList) { - $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" - $config = $False - Write-Host $tool - # We have to manually configure tools that run on source to look at the source directory only - if ($tool -eq "credscan") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `"" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - $config = $True - } - if ($tool -eq "policheck") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `"" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - $config = $True - } - - Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config" - if ($config) { - & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - } else { - & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE." - exit $LASTEXITCODE - } - } -} - +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + [string[]] $ToolsList, + [string] $UpdateBaseline, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We store config files in the r directory of .gdn +Write-Host $ToolsList +$gdnConfigPath = Join-Path $GdnFolder "r" +$ValidPath = Test-Path $GuardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +foreach ($tool in $ToolsList) { + $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" + $config = $False + Write-Host $tool + # We have to manually configure tools that run on source to look at the source directory only + if ($tool -eq "credscan") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + if ($tool -eq "policheck") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + + Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config" + if ($config) { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } +} + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index acb4c55d735..3a6273c1ba4 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,44 +1,44 @@ -parameters: - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - continueOnError: false # optional: determines whether to continue the build if the step errors; - dependsOn: '' # Optional: dependencies of the job - -jobs: -- job: Run_SDL - dependsOn: ${{ parameters.dependsOn }} - displayName: Run SDL tool - variables: - - group: DotNet-VSTS-Bot - steps: - - checkout: self - clean: true - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: current - downloadType: specific files - matchingPattern: "**" - downloadPath: $(Build.SourcesDirectory)\artifacts - - task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - - task: NuGetCommand@2 - displayName: 'Install Guardian' - inputs: - restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - feedsToUse: config - nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config - externalFeedCredentials: GuardianConnect - restoreDirectory: $(Build.SourcesDirectory)\.packages - - ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} - - ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.3.2 - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) - ${{ parameters.additionalParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} +parameters: + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + continueOnError: false # optional: determines whether to continue the build if the step errors; + dependsOn: '' # Optional: dependencies of the job + +jobs: +- job: Run_SDL + dependsOn: ${{ parameters.dependsOn }} + displayName: Run SDL tool + variables: + - group: DotNet-VSTS-Bot + steps: + - checkout: self + clean: true + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: current + downloadType: specific files + matchingPattern: "**" + downloadPath: $(Build.SourcesDirectory)\artifacts + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + - task: NuGetCommand@2 + displayName: 'Install Guardian' + inputs: + restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + feedsToUse: config + nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config + externalFeedCredentials: GuardianConnect + restoreDirectory: $(Build.SourcesDirectory)\.packages + - ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 + -GuardianPackageName Microsoft.Guardian.Cli.0.6.0 + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/job/generate-graph-files.yml b/eng/common/templates/job/generate-graph-files.yml index e54ce956f90..37aa55fd407 100644 --- a/eng/common/templates/job/generate-graph-files.yml +++ b/eng/common/templates/job/generate-graph-files.yml @@ -1,48 +1,48 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - -jobs: -- job: Generate_Graph_Files - - dependsOn: ${{ parameters.dependsOn }} - - displayName: Generate Graph Files - - pool: ${{ parameters.pool }} - - variables: - # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT - # DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat - - group: Publish-Build-Assets - - group: DotNet-AllOrgs-Darc-Pats - - name: _GraphArguments - value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT) - -azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat) - -barToken $(MaestroAccessToken) - -outputFolder '$(Build.StagingDirectory)/GraphFiles/' - - ${{ if ne(parameters.includeToolset, 'false') }}: - - name: _GraphArguments - value: ${{ variables._GraphArguments }} -includeToolset - - steps: - - task: PowerShell@2 - displayName: Generate Graph Files - inputs: - filePath: eng\common\generate-graph-files.ps1 - arguments: $(_GraphArguments) - continueOnError: true - - task: PublishBuildArtifacts@1 - displayName: Publish Graph to Artifacts - inputs: - PathtoPublish: '$(Build.StagingDirectory)/GraphFiles' - PublishLocation: Container - ArtifactName: GraphFiles - continueOnError: true - condition: always() +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + +jobs: +- job: Generate_Graph_Files + + dependsOn: ${{ parameters.dependsOn }} + + displayName: Generate Graph Files + + pool: ${{ parameters.pool }} + + variables: + # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT + # DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat + - group: Publish-Build-Assets + - group: DotNet-AllOrgs-Darc-Pats + - name: _GraphArguments + value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT) + -azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat) + -barToken $(MaestroAccessToken) + -outputFolder '$(Build.StagingDirectory)/GraphFiles/' + - ${{ if ne(parameters.includeToolset, 'false') }}: + - name: _GraphArguments + value: ${{ variables._GraphArguments }} -includeToolset + + steps: + - task: PowerShell@2 + displayName: Generate Graph Files + inputs: + filePath: eng\common\generate-graph-files.ps1 + arguments: $(_GraphArguments) + continueOnError: true + - task: PublishBuildArtifacts@1 + displayName: Publish Graph to Artifacts + inputs: + PathtoPublish: '$(Build.StagingDirectory)/GraphFiles' + PublishLocation: Container + ArtifactName: GraphFiles + continueOnError: true + condition: always() diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 1814e0ab612..a83e0754283 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,205 +1,205 @@ -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - - condition: '' - - continueOnError: false - - container: '' - - dependsOn: '' - - displayName: '' - - steps: [] - - pool: '' - - strategy: '' - - timeoutInMinutes: '' - - variables: [] - - workspace: '' - -# Job base template specific parameters - # Optional: Enable installing Microbuild plugin - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _TeamName - the name of your team - # _SignType - 'test' or 'real' - enableMicrobuild: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing to the build asset registry - enablePublishBuildAssets: false - - # Optional: Include PublishTestResults task - enablePublishTestResults: false - - # Optional: enable sending telemetry - enableTelemetry: false - - # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') - helixRepo: '' - - # Optional: define the helix type for telemetry (example: 'build/product/') - helixType: '' - - # Required: name of the job - name: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - variables: - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions - - task: sendStartTelemetry@0 - displayName: 'Send Helix Start Telemetry' - inputs: - helixRepo: ${{ parameters.helixRepo }} - ${{ if ne(parameters.helixType, '') }}: - helixType: ${{ parameters.helixType }} - buildConfig: $(_BuildConfig) - runAsPublic: ${{ parameters.runAsPublic }} - continueOnError: ${{ parameters.continueOnError }} - condition: always() - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildSigningPlugin@2 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions - - task: sendEndTelemetry@0 - displayName: 'Send Helix End Telemetry' - continueOnError: ${{ parameters.continueOnError }} - condition: always() - - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: $(Agent.Os)_$(Agent.JobName) - continueOnError: true - condition: always() - - - ${{ if eq(parameters.enablePublishTestResults, 'true') }}: - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(variables['_PublishUsingPipelines'], 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: CopyFiles@2 - displayName: Gather Asset Manifests - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' - TargetFolder: '$(Build.StagingDirectory)/AssetManifests' - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - - task: PublishBuildArtifacts@1 - displayName: Push Asset Manifests - inputs: - PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' - PublishLocation: Container - ArtifactName: AssetManifests - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + + condition: '' + + continueOnError: false + + container: '' + + dependsOn: '' + + displayName: '' + + steps: [] + + pool: '' + + strategy: '' + + timeoutInMinutes: '' + + variables: [] + + workspace: '' + +# Job base template specific parameters + # Optional: Enable installing Microbuild plugin + # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix + # _TeamName - the name of your team + # _SignType - 'test' or 'real' + enableMicrobuild: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing to the build asset registry + enablePublishBuildAssets: false + + # Optional: Include PublishTestResults task + enablePublishTestResults: false + + # Optional: enable sending telemetry + enableTelemetry: false + + # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') + helixRepo: '' + + # Optional: define the helix type for telemetry (example: 'build/product/') + helixType: '' + + # Required: name of the job + name: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions + - task: sendStartTelemetry@0 + displayName: 'Send Helix Start Telemetry' + inputs: + helixRepo: ${{ parameters.helixRepo }} + ${{ if ne(parameters.helixType, '') }}: + helixType: ${{ parameters.helixType }} + buildConfig: $(_BuildConfig) + runAsPublic: ${{ parameters.runAsPublic }} + continueOnError: ${{ parameters.continueOnError }} + condition: always() + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildSigningPlugin@2 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions + - task: sendEndTelemetry@0 + displayName: 'Send Helix End Telemetry' + continueOnError: ${{ parameters.continueOnError }} + condition: always() + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: $(Agent.Os)_$(Agent.JobName) + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enablePublishTestResults, 'true') }}: + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(variables['_PublishUsingPipelines'], 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index ff7346163f4..5acaeb564fa 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -1,85 +1,84 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - -jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - - displayName: Publish to Build Asset Registry - - pool: ${{ parameters.pool }} - - variables: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _BuildConfig - value: ${{ parameters.configuration }} - - group: Publish-Build-Assets - - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:Configuration=$(_BuildConfig) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsInternalBuild) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - - task: PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: $(Agent.Os)_PublishBuildAssets - continueOnError: true - condition: always() +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + + displayName: Publish to Build Asset Registry + + pool: ${{ parameters.pool }} + + variables: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - name: _BuildConfig + value: ${{ parameters.configuration }} + - group: Publish-Build-Assets + + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:Configuration=$(_BuildConfig) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + - task: PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Logs to VSTS + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: $(Agent.Os)_PublishBuildAssets + continueOnError: true + condition: always() diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 6a2f98c036f..f68dbe0df63 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -1,90 +1,90 @@ -parameters: - # Optional: 'true' if failures in job.yml job should not fail the job - continueOnError: false - - # Optional: Enable installing Microbuild plugin - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _TeamName - the name of your team - # _SignType - 'test' or 'real' - enableMicrobuild: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing to the build asset registry - enablePublishBuildAssets: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Optional: Include PublishTestResults task - enablePublishTestResults: false - - # Optional: enable sending telemetry - # if enabled then the 'helixRepo' parameter should also be specified - enableTelemetry: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') - helixRepo: '' - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - pool: - vmImage: vs2017-win2016 - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - -- ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: ../job/generate-graph-files.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - includeToolset: ${{ parameters.graphFileGeneration.includeToolset }} - dependsOn: - - Asset_Registry_Publish - pool: - vmImage: vs2017-win2016 +parameters: + # Optional: 'true' if failures in job.yml job should not fail the job + continueOnError: false + + # Optional: Enable installing Microbuild plugin + # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix + # _TeamName - the name of your team + # _SignType - 'test' or 'real' + enableMicrobuild: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing to the build asset registry + enablePublishBuildAssets: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Optional: Include PublishTestResults task + enablePublishTestResults: false + + # Optional: enable sending telemetry + # if enabled then the 'helixRepo' parameter should also be specified + enableTelemetry: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') + helixRepo: '' + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + pool: + vmImage: vs2017-win2016 + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + +- ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: ../job/generate-graph-files.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + includeToolset: ${{ parameters.graphFileGeneration.includeToolset }} + dependsOn: + - Asset_Registry_Publish + pool: + vmImage: vs2017-win2016 diff --git a/eng/common/templates/phases/base.yml b/eng/common/templates/phases/base.yml index 0123cf43b16..255543cc468 100644 --- a/eng/common/templates/phases/base.yml +++ b/eng/common/templates/phases/base.yml @@ -1,130 +1,130 @@ -parameters: - # Optional: Clean sources before building - clean: true - - # Optional: Git fetch depth - fetchDepth: '' - - # Optional: name of the phase (not specifying phase name may cause name collisions) - name: '' - # Optional: display name of the phase - displayName: '' - - # Optional: condition for the job to run - condition: '' - - # Optional: dependencies of the phase - dependsOn: '' - - # Required: A defined YAML queue - queue: {} - - # Required: build steps - steps: [] - - # Optional: variables - variables: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - ## Telemetry variables - - # Optional: enable sending telemetry - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _HelixBuildConfig - differentiate between Debug, Release, other - # _HelixSource - Example: build/product - # _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch) - enableTelemetry: false - - # Optional: Enable installing Microbuild plugin - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _TeamName - the name of your team - # _SignType - 'test' or 'real' - enableMicrobuild: false - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -phases: -- phase: ${{ parameters.name }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - queue: ${{ parameters.queue }} - - ${{ if ne(parameters.variables, '') }}: - variables: - ${{ insert }}: ${{ parameters.variables }} - - steps: - - checkout: self - clean: ${{ parameters.clean }} - ${{ if ne(parameters.fetchDepth, '') }}: - fetchDepth: ${{ parameters.fetchDepth }} - - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - template: /eng/common/templates/steps/telemetry-start.yml - parameters: - buildConfig: $(_HelixBuildConfig) - helixSource: $(_HelixSource) - helixType: $(_HelixType) - runAsPublic: ${{ parameters.runAsPublic }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - # Internal only resource, and Microbuild signing shouldn't be applied to PRs. - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildSigningPlugin@2 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - # Run provided build steps - - ${{ parameters.steps }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - # Internal only resources - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - env: - TeamName: $(_TeamName) - - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - template: /eng/common/templates/steps/telemetry-end.yml - parameters: - helixSource: $(_HelixSource) - helixType: $(_HelixType) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: CopyFiles@2 - displayName: Gather Asset Manifests - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' - TargetFolder: '$(Build.StagingDirectory)/AssetManifests' - continueOnError: false - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - - task: PublishBuildArtifacts@1 - displayName: Push Asset Manifests - inputs: - PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' - PublishLocation: Container - ArtifactName: AssetManifests - continueOnError: false - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) +parameters: + # Optional: Clean sources before building + clean: true + + # Optional: Git fetch depth + fetchDepth: '' + + # Optional: name of the phase (not specifying phase name may cause name collisions) + name: '' + # Optional: display name of the phase + displayName: '' + + # Optional: condition for the job to run + condition: '' + + # Optional: dependencies of the phase + dependsOn: '' + + # Required: A defined YAML queue + queue: {} + + # Required: build steps + steps: [] + + # Optional: variables + variables: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + ## Telemetry variables + + # Optional: enable sending telemetry + # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix + # _HelixBuildConfig - differentiate between Debug, Release, other + # _HelixSource - Example: build/product + # _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch) + enableTelemetry: false + + # Optional: Enable installing Microbuild plugin + # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix + # _TeamName - the name of your team + # _SignType - 'test' or 'real' + enableMicrobuild: false + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +phases: +- phase: ${{ parameters.name }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + queue: ${{ parameters.queue }} + + ${{ if ne(parameters.variables, '') }}: + variables: + ${{ insert }}: ${{ parameters.variables }} + + steps: + - checkout: self + clean: ${{ parameters.clean }} + ${{ if ne(parameters.fetchDepth, '') }}: + fetchDepth: ${{ parameters.fetchDepth }} + + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + - template: /eng/common/templates/steps/telemetry-start.yml + parameters: + buildConfig: $(_HelixBuildConfig) + helixSource: $(_HelixSource) + helixType: $(_HelixType) + runAsPublic: ${{ parameters.runAsPublic }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + # Internal only resource, and Microbuild signing shouldn't be applied to PRs. + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildSigningPlugin@2 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + + env: + TeamName: $(_TeamName) + continueOnError: false + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + # Run provided build steps + - ${{ parameters.steps }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + # Internal only resources + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + env: + TeamName: $(_TeamName) + + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + - template: /eng/common/templates/steps/telemetry-end.yml + parameters: + helixSource: $(_HelixSource) + helixType: $(_HelixType) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + continueOnError: false + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: false + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml index a0a8074282a..aef29ecf3ea 100644 --- a/eng/common/templates/phases/publish-build-assets.yml +++ b/eng/common/templates/phases/publish-build-assets.yml @@ -1,51 +1,51 @@ -parameters: - dependsOn: '' - queue: {} - configuration: 'Debug' - condition: succeeded() - continueOnError: false - runAsPublic: false - publishUsingPipelines: false -phases: - - phase: Asset_Registry_Publish - displayName: Publish to Build Asset Registry - dependsOn: ${{ parameters.dependsOn }} - queue: ${{ parameters.queue }} - variables: - _BuildConfig: ${{ parameters.configuration }} - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: AzureKeyVault@1 - inputs: - azureSubscription: 'DotNet-Engineering-Services_KeyVault' - KeyVaultName: EngKeyVault - SecretsFilter: 'MaestroAccessToken' - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:Configuration=$(_BuildConfig) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: $(Agent.Os)_Asset_Registry_Publish - continueOnError: true - condition: always() +parameters: + dependsOn: '' + queue: {} + configuration: 'Debug' + condition: succeeded() + continueOnError: false + runAsPublic: false + publishUsingPipelines: false +phases: + - phase: Asset_Registry_Publish + displayName: Publish to Build Asset Registry + dependsOn: ${{ parameters.dependsOn }} + queue: ${{ parameters.queue }} + variables: + _BuildConfig: ${{ parameters.configuration }} + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - task: AzureKeyVault@1 + inputs: + azureSubscription: 'DotNet-Engineering-Services_KeyVault' + KeyVaultName: EngKeyVault + SecretsFilter: 'MaestroAccessToken' + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:Configuration=$(_BuildConfig) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + - task: PublishBuildArtifacts@1 + displayName: Publish Logs to VSTS + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: $(Agent.Os)_Asset_Registry_Publish + continueOnError: true + condition: always() diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml index c61eaa927d9..f924bc979b6 100644 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -1,145 +1,162 @@ -parameters: - enableSymbolValidation: true - -stages: -- stage: Publish - dependsOn: validate - variables: - - template: ../common-variables.yml - displayName: Developer Channel - jobs: - - template: ../setup-maestro-vars.yml - - - job: - displayName: Symbol Publishing - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - variables: - - group: DotNet-Symbol-Server-Pats - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download PDB Artifacts - inputs: - buildType: current - artifactName: PDBArtifacts - continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Publish - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:Configuration=Release - - - job: - displayName: Publish to Static Feed - dependsOn: setupMaestroVars - variables: - - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: DownloadBuildArtifacts@0 - displayName: Download Asset Manifests - inputs: - buildType: current - artifactName: AssetManifests - - - task: PowerShell@2 - displayName: Publish - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet - /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' - /p:OverrideAssetsWithSameName=true - /p:PassIfExistingItemIdentical=true - /p:Configuration=Release - - -- stage: PublishValidation - displayName: Publish Validation - variables: - - template: ../common-variables.yml - jobs: - - template: ../setup-maestro-vars.yml - - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - - job: - displayName: Gather Drop - dependsOn: setupMaestroVars - variables: - BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Setup Darc CLI - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' - - - task: PowerShell@2 - displayName: Run Darc gather-drop - inputs: - targetType: inline - script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location - - - template: ../promote-build.yml - parameters: - ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} +parameters: + enableSymbolValidation: true + +stages: +- stage: Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Developer Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + downloadType: specific files + matchingPattern: "*Artifacts*" + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicDevRelease_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + +- stage: PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 23725c6d620..083c568570c 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -1,91 +1,115 @@ -stages: -- stage: PVR_Publish - dependsOn: validate - variables: - - template: ../common-variables.yml - displayName: Validation Channel - jobs: - - template: ../setup-maestro-vars.yml - - - job: - displayName: Publish to Static Feed - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) - variables: - - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: DownloadBuildArtifacts@0 - displayName: Download Asset Manifests - inputs: - buildType: current - artifactName: AssetManifests - - - task: PowerShell@2 - displayName: Publish - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet - /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' - /p:OverrideAssetsWithSameName=true - /p:PassIfExistingItemIdentical=true - /p:Configuration=Release - - -- stage: PVR_PublishValidation - displayName: Publish Validation - variables: - - template: ../common-variables.yml - jobs: - - template: ../setup-maestro-vars.yml - - - job: - displayName: Gather Drop - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - - group: Publish-Build-Assets - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Setup Darc CLI - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' - - - task: PowerShell@2 - displayName: Run Darc gather-drop - inputs: - targetType: inline - script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) --latest-location - - - template: ../promote-build.yml - parameters: - ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} +stages: +- stage: PVR_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Validation Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + +- stage: PVR_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) --latest-location diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 97b48d97fec..23876e7eb60 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,9 +1,18 @@ -variables: - # .NET Core 3 Dev - PublicDevRelease_30_Channel_Id: 3 - - # .NET Tools - Validation - PublicValidationRelease_30_Channel_Id: 9 - - SourceLinkCLIVersion: 3.0.0 - SymbolToolVersion: 1.0.1 +variables: + # .NET Core 3 Dev + PublicDevRelease_30_Channel_Id: 3 + + # .NET Tools - Validation + PublicValidationRelease_30_Channel_Id: 9 + + # .NET Core 3.0 Internal Servicing + InternalServicing_30_Channel_Id: 184 + + # .NET Core 3.0 Release + PublicRelease_30_Channel_Id: 19 + + # Whether the build is internal or not + IsInternalBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + SourceLinkCLIVersion: 3.0.0 + SymbolToolVersion: 1.0.1 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 2c411dd0098..e79936dccda 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,67 +1,71 @@ -parameters: - enableSourceLinkValidation: true - enableSigningValidation: true - enableSymbolValidation: true - SDLValidationParameters: - enable: false - params: '' - -stages: -- stage: validate - dependsOn: build - displayName: Validate - jobs: - - ${{ if eq(parameters.enableSigningValidation, 'true') }}: - - job: - displayName: Signing Validation - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:Configuration=Release - - - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: - - job: - displayName: SourceLink Validation - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - - - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - -- template: \eng\common\templates\post-build\channels\public-dev-release.yml - parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} - -- template: \eng\common\templates\post-build\channels\public-validation-release.yml +parameters: + enableSourceLinkValidation: true + enableSigningValidation: true + enableSymbolValidation: true + SDLValidationParameters: + enable: false + params: '' + +stages: +- stage: validate + dependsOn: build + displayName: Validate + jobs: + - ${{ if eq(parameters.enableSigningValidation, 'true') }}: + - job: + displayName: Signing Validation + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:Configuration=Release + + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: SourceLink Validation + variables: + - template: common-variables.yml + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + + - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + +- template: \eng\common\templates\post-build\channels\public-dev-release.yml + parameters: + enableSymbolValidation: ${{ parameters.enableSymbolValidation }} + +- template: \eng\common\templates\post-build\channels\public-validation-release.yml + +- template: \eng\common\templates\post-build\channels\public-release.yml + +- template: \eng\common\templates\post-build\channels\internal-servicing.yml diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml index d00317003b7..307b3aad11d 100644 --- a/eng/common/templates/post-build/promote-build.yml +++ b/eng/common/templates/post-build/promote-build.yml @@ -1,28 +1,27 @@ -parameters: - ChannelId: 0 - -jobs: -- job: - displayName: Promote Build - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - - name: ChannelId - value: ${{ parameters.ChannelId }} - - group: Publish-Build-Assets - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Add Build to Channel - inputs: - targetType: inline - script: | - $headers = @{ - "Accept" = "application/json" - "Authorization" = "Bearer $(MaestroAccessToken)" - } - Invoke-RestMethod -Method Post -Headers $headers -Uri https://maestro-prod.westus2.cloudapp.azure.com/api/channels/$(ChannelId)/builds/$(BARBuildId)?api-version=2019-01-16 - enabled: false +parameters: + ChannelId: 0 + +jobs: +- job: + displayName: Promote Build + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: ChannelId + value: ${{ parameters.ChannelId }} + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Add Build to Channel + inputs: + targetType: inline + script: | + $headers = @{ + "Accept" = "application/json" + "Authorization" = "Bearer $(MaestroAccessToken)" + } + Invoke-RestMethod -Method Post -Headers $headers -Uri https://maestro-prod.westus2.cloudapp.azure.com/api/channels/$(ChannelId)/builds/$(BARBuildId)?api-version=2019-01-16 diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0eddd6cd3b7..40cabe5a39f 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,37 +1,35 @@ -jobs: -- job: setupMaestroVars - displayName: Setup Maestro Vars - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - script: | - # This is needed to make Write-PipelineSetVariable works in this context - $ci = $true - - . "$(Build.SourcesDirectory)/eng/common/tools.ps1" - - $Content = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" - - $BarId = $Content | Select -Index 0 - - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } - - $IsInternalBuild = $Content | Select -Index 2 - $IsStableBuild = $Content | Select -Index 3 - - Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId - Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" - Write-PipelineSetVariable -Name 'IsInternalBuild' -Value $IsInternalBuild - Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild +jobs: +- job: setupMaestroVars + displayName: Setup Maestro Vars + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + script: | + # This is needed to make Write-PipelineSetVariable works in this context + $ci = $true + + . "$(Build.SourcesDirectory)/eng/common/tools.ps1" + + $Content = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + + $BarId = $Content | Select -Index 0 + + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsStableBuild = $Content | Select -Index 2 + + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" + Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild diff --git a/eng/common/templates/steps/build-reason.yml b/eng/common/templates/steps/build-reason.yml index eba58109b52..750b5ca1a7c 100644 --- a/eng/common/templates/steps/build-reason.yml +++ b/eng/common/templates/steps/build-reason.yml @@ -1,12 +1,12 @@ -# build-reason.yml -# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons -# to include steps (',' separated). -parameters: - conditions: '' - steps: [] - -steps: - - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: - - ${{ parameters.steps }} - - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: - - ${{ parameters.steps }} +# build-reason.yml +# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons +# to include steps (',' separated). +parameters: + conditions: '' + steps: [] + +steps: + - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: + - ${{ parameters.steps }} + - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-on-unix.yml b/eng/common/templates/steps/run-on-unix.yml index e1733814f65..c322f526265 100644 --- a/eng/common/templates/steps/run-on-unix.yml +++ b/eng/common/templates/steps/run-on-unix.yml @@ -1,7 +1,7 @@ -parameters: - agentOs: '' - steps: [] - -steps: -- ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - - ${{ parameters.steps }} +parameters: + agentOs: '' + steps: [] + +steps: +- ${{ if ne(parameters.agentOs, 'Windows_NT') }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-on-windows.yml b/eng/common/templates/steps/run-on-windows.yml index 73e7e9c275a..2080c5d7ab7 100644 --- a/eng/common/templates/steps/run-on-windows.yml +++ b/eng/common/templates/steps/run-on-windows.yml @@ -1,7 +1,7 @@ -parameters: - agentOs: '' - steps: [] - -steps: -- ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - ${{ parameters.steps }} +parameters: + agentOs: '' + steps: [] + +steps: +- ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-script-ifequalelse.yml b/eng/common/templates/steps/run-script-ifequalelse.yml index 3d1242f5587..2342040ec91 100644 --- a/eng/common/templates/steps/run-script-ifequalelse.yml +++ b/eng/common/templates/steps/run-script-ifequalelse.yml @@ -1,33 +1,33 @@ -parameters: - # if parameter1 equals parameter 2, run 'ifScript' command, else run 'elsescript' command - parameter1: '' - parameter2: '' - ifScript: '' - elseScript: '' - - # name of script step - name: Script - - # display name of script step - displayName: If-Equal-Else Script - - # environment - env: {} - - # conditional expression for step execution - condition: '' - -steps: -- ${{ if and(ne(parameters.ifScript, ''), eq(parameters.parameter1, parameters.parameter2)) }}: - - script: ${{ parameters.ifScript }} - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - env: ${{ parameters.env }} - condition: ${{ parameters.condition }} - -- ${{ if and(ne(parameters.elseScript, ''), ne(parameters.parameter1, parameters.parameter2)) }}: - - script: ${{ parameters.elseScript }} - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - env: ${{ parameters.env }} +parameters: + # if parameter1 equals parameter 2, run 'ifScript' command, else run 'elsescript' command + parameter1: '' + parameter2: '' + ifScript: '' + elseScript: '' + + # name of script step + name: Script + + # display name of script step + displayName: If-Equal-Else Script + + # environment + env: {} + + # conditional expression for step execution + condition: '' + +steps: +- ${{ if and(ne(parameters.ifScript, ''), eq(parameters.parameter1, parameters.parameter2)) }}: + - script: ${{ parameters.ifScript }} + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + env: ${{ parameters.env }} + condition: ${{ parameters.condition }} + +- ${{ if and(ne(parameters.elseScript, ''), ne(parameters.parameter1, parameters.parameter2)) }}: + - script: ${{ parameters.elseScript }} + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + env: ${{ parameters.env }} condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 05df886f55f..9ae41fde17c 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -1,91 +1,91 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - -steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/telemetry-end.yml b/eng/common/templates/steps/telemetry-end.yml index fadc04ca1b9..2cc9e48fcb6 100644 --- a/eng/common/templates/steps/telemetry-end.yml +++ b/eng/common/templates/steps/telemetry-end.yml @@ -1,102 +1,102 @@ -parameters: - maxRetries: 5 - retryDelay: 10 # in seconds - -steps: -- bash: | - if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then - errorCount=0 - else - errorCount=1 - fi - warningCount=0 - - curlStatus=1 - retryCount=0 - # retry loop to harden against spotty telemetry connections - # we don't retry successes and 4xx client errors - until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]] - do - if [ $retryCount -gt 0 ]; then - echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..." - sleep $RetryDelay - fi - - # create a temporary file for curl output - res=`mktemp` - - curlResult=` - curl --verbose --output $res --write-out "%{http_code}"\ - -H 'Content-Type: application/json' \ - -H "X-Helix-Job-Token: $Helix_JobToken" \ - -H 'Content-Length: 0' \ - -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \ - --data-urlencode "errorCount=$errorCount" \ - --data-urlencode "warningCount=$warningCount"` - curlStatus=$? - - if [ $curlStatus -eq 0 ]; then - if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then - curlStatus=$curlResult - fi - fi - - let retryCount++ - done - - if [ $curlStatus -ne 0 ]; then - echo "Failed to Send Build Finish information after $retryCount retries" - vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus" - echo "##$vstsLogOutput" - exit 1 - fi - displayName: Send Unix Build End Telemetry - env: - # defined via VSTS variables in start-job.sh - Helix_JobToken: $(Helix_JobToken) - Helix_WorkItemId: $(Helix_WorkItemId) - MaxRetries: ${{ parameters.maxRetries }} - RetryDelay: ${{ parameters.retryDelay }} - condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT')) -- powershell: | - if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) { - $ErrorCount = 0 - } else { - $ErrorCount = 1 - } - $WarningCount = 0 - - # Basic retry loop to harden against server flakiness - $retryCount = 0 - while ($retryCount -lt $env:MaxRetries) { - try { - Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" ` - -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } - break - } - catch { - $statusCode = $_.Exception.Response.StatusCode.value__ - if ($statusCode -ge 400 -and $statusCode -le 499) { - Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)" - Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message - exit 1 - } - Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..." - $retryCount++ - sleep $env:RetryDelay - continue - } - } - - if ($retryCount -ge $env:MaxRetries) { - Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries." - exit 1 - } - displayName: Send Windows Build End Telemetry - env: - # defined via VSTS variables in start-job.ps1 - Helix_JobToken: $(Helix_JobToken) - Helix_WorkItemId: $(Helix_WorkItemId) - MaxRetries: ${{ parameters.maxRetries }} - RetryDelay: ${{ parameters.retryDelay }} - condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT')) +parameters: + maxRetries: 5 + retryDelay: 10 # in seconds + +steps: +- bash: | + if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then + errorCount=0 + else + errorCount=1 + fi + warningCount=0 + + curlStatus=1 + retryCount=0 + # retry loop to harden against spotty telemetry connections + # we don't retry successes and 4xx client errors + until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]] + do + if [ $retryCount -gt 0 ]; then + echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..." + sleep $RetryDelay + fi + + # create a temporary file for curl output + res=`mktemp` + + curlResult=` + curl --verbose --output $res --write-out "%{http_code}"\ + -H 'Content-Type: application/json' \ + -H "X-Helix-Job-Token: $Helix_JobToken" \ + -H 'Content-Length: 0' \ + -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \ + --data-urlencode "errorCount=$errorCount" \ + --data-urlencode "warningCount=$warningCount"` + curlStatus=$? + + if [ $curlStatus -eq 0 ]; then + if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then + curlStatus=$curlResult + fi + fi + + let retryCount++ + done + + if [ $curlStatus -ne 0 ]; then + echo "Failed to Send Build Finish information after $retryCount retries" + vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus" + echo "##$vstsLogOutput" + exit 1 + fi + displayName: Send Unix Build End Telemetry + env: + # defined via VSTS variables in start-job.sh + Helix_JobToken: $(Helix_JobToken) + Helix_WorkItemId: $(Helix_WorkItemId) + MaxRetries: ${{ parameters.maxRetries }} + RetryDelay: ${{ parameters.retryDelay }} + condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT')) +- powershell: | + if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) { + $ErrorCount = 0 + } else { + $ErrorCount = 1 + } + $WarningCount = 0 + + # Basic retry loop to harden against server flakiness + $retryCount = 0 + while ($retryCount -lt $env:MaxRetries) { + try { + Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" ` + -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } + break + } + catch { + $statusCode = $_.Exception.Response.StatusCode.value__ + if ($statusCode -ge 400 -and $statusCode -le 499) { + Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)" + Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message + exit 1 + } + Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..." + $retryCount++ + sleep $env:RetryDelay + continue + } + } + + if ($retryCount -ge $env:MaxRetries) { + Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries." + exit 1 + } + displayName: Send Windows Build End Telemetry + env: + # defined via VSTS variables in start-job.ps1 + Helix_JobToken: $(Helix_JobToken) + Helix_WorkItemId: $(Helix_WorkItemId) + MaxRetries: ${{ parameters.maxRetries }} + RetryDelay: ${{ parameters.retryDelay }} + condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml index 32c01ef0b55..9bae2af5a22 100644 --- a/eng/common/templates/steps/telemetry-start.yml +++ b/eng/common/templates/steps/telemetry-start.yml @@ -1,241 +1,241 @@ -parameters: - helixSource: 'undefined_defaulted_in_telemetry.yml' - helixType: 'undefined_defaulted_in_telemetry.yml' - buildConfig: '' - runAsPublic: false - maxRetries: 5 - retryDelay: 10 # in seconds - -steps: -- ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@1 - inputs: - azureSubscription: 'HelixProd_KeyVault' - KeyVaultName: HelixProdKV - SecretsFilter: 'HelixApiAccessToken' - condition: always() -- bash: | - # create a temporary file - jobInfo=`mktemp` - - # write job info content to temporary file - cat > $jobInfo < $jobInfo <' | Set-Content $proj - - MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile - - $path = Get-Content $toolsetLocationFile -TotalCount 1 - if (!(Test-Path $path)) { - throw "Invalid toolset path: $path" - } - - return $global:_ToolsetBuildProj = $path -} - -function ExitWithExitCode([int] $exitCode) { - if ($ci -and $prepareMachine) { - Stop-Processes - } - exit $exitCode -} - -function Stop-Processes() { - Write-Host "Killing running build processes..." - foreach ($processName in $processesToStopOnExit) { - Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process - } -} - -# -# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. -# The arguments are automatically quoted. -# Terminates the script if the build fails. -# -function MSBuild() { - if ($pipelinesLog) { - $buildTool = InitializeBuildTool - $toolsetBuildProject = InitializeToolset - $path = Split-Path -parent $toolsetBuildProject - $path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll") - $args += "/logger:$path" - } - - MSBuild-Core @args -} - -# -# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. -# The arguments are automatically quoted. -# Terminates the script if the build fails. -# -function MSBuild-Core() { - if ($ci) { - if (!$binaryLog) { - Write-PipelineTaskError -Message "Binary log must be enabled in CI build." - ExitWithExitCode 1 - } - - if ($nodeReuse) { - Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." - ExitWithExitCode 1 - } - } - - $buildTool = InitializeBuildTool - - $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" - - if ($warnAsError) { - $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" - } - - foreach ($arg in $args) { - if ($arg -ne $null -and $arg.Trim() -ne "") { - $cmdArgs += " `"$arg`"" - } - } - - $exitCode = Exec-Process $buildTool.Path $cmdArgs - - if ($exitCode -ne 0) { - Write-PipelineTaskError -Message "Build failed." - - $buildLog = GetMSBuildBinaryLogCommandLineArgument $args - if ($buildLog -ne $null) { - Write-Host "See log: $buildLog" -ForegroundColor DarkGray - } - - ExitWithExitCode $exitCode - } -} - -function GetMSBuildBinaryLogCommandLineArgument($arguments) { - foreach ($argument in $arguments) { - if ($argument -ne $null) { - $arg = $argument.Trim() - if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { - return $arg.Substring("/bl:".Length) - } - - if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { - return $arg.Substring("/binaryLogger:".Length) - } - } - } - - return $null -} - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") -$EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") -$ArtifactsDir = Join-Path $RepoRoot "artifacts" -$ToolsetDir = Join-Path $ArtifactsDir "toolset" -$ToolsDir = Join-Path $RepoRoot ".tools" -$LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration -$TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration -$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json -# true if global.json contains a "runtimes" section -$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } - -Create-Directory $ToolsetDir -Create-Directory $TempDir -Create-Directory $LogDir - -Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir -Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir -Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir -Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir -Write-PipelineSetVariable -Name 'TMP' -Value $TempDir +# Initialize variables if they aren't already defined. +# These may be defined as parameters of the importing script, or set after importing this script. + +# CI mode - set to true on CI server for PR validation build or official build. +[bool]$ci = if (Test-Path variable:ci) { $ci } else { $false } + +# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. +[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } + +# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. +# Binary log must be enabled on CI. +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } + +# Set to true to use the pipelines logger which will enable Azure logging output. +# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md +# This flag is meant as a temporary opt-opt for the feature while validate it across +# our consumers. It will be deleted in the future. +[bool]$pipelinesLog = if (Test-Path variable:pipelinesLog) { $pipelinesLog } else { $ci } + +# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). +[bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } + +# True to restore toolsets and dependencies. +[bool]$restore = if (Test-Path variable:restore) { $restore } else { $true } + +# Adjusts msbuild verbosity level. +[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" } + +# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. +[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci } + +# Configures warning treatment in msbuild. +[bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true } + +# Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). +[string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } + +# True to attempt using .NET Core already that meets requirements specified in global.json +# installed on the machine instead of downloading one. +[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } + +# True to use global NuGet cache instead of restoring packages to repository-local directory. +[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } + +# An array of names of processes to stop on script exit if prepareMachine is true. +$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") } + +set-strictmode -version 2.0 +$ErrorActionPreference = "Stop" +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +function Create-Directory([string[]] $path) { + if (!(Test-Path $path)) { + New-Item -path $path -force -itemType "Directory" | Out-Null + } +} + +function Unzip([string]$zipfile, [string]$outpath) { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath) +} + +# This will exec a process using the console and return it's exit code. +# This will not throw when the process fails. +# Returns process exit code. +function Exec-Process([string]$command, [string]$commandArgs) { + $startInfo = New-Object System.Diagnostics.ProcessStartInfo + $startInfo.FileName = $command + $startInfo.Arguments = $commandArgs + $startInfo.UseShellExecute = $false + $startInfo.WorkingDirectory = Get-Location + + $process = New-Object System.Diagnostics.Process + $process.StartInfo = $startInfo + $process.Start() | Out-Null + + $finished = $false + try { + while (-not $process.WaitForExit(100)) { + # Non-blocking loop done to allow ctr-c interrupts + } + + $finished = $true + return $global:LASTEXITCODE = $process.ExitCode + } + finally { + # If we didn't finish then an error occured or the user hit ctrl-c. Either + # way kill the process + if (-not $finished) { + $process.Kill() + } + } +} + +function InitializeDotNetCli([bool]$install) { + if (Test-Path variable:global:_DotNetInstallDir) { + return $global:_DotNetInstallDir + } + + # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism + $env:DOTNET_MULTILEVEL_LOOKUP=0 + + # Disable first run since we do not need all ASP.NET packages restored. + $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + + # Disable telemetry on CI. + if ($ci) { + $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 + } + + # Source Build uses DotNetCoreSdkDir variable + if ($env:DotNetCoreSdkDir -ne $null) { + $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir + } + + # Find the first path on %PATH% that contains the dotnet.exe + if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { + $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue + if ($dotnetCmd -ne $null) { + $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent + } + } + + $dotnetSdkVersion = $GlobalJson.tools.dotnet + + # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, + # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. + if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { + $dotnetRoot = $env:DOTNET_INSTALL_DIR + } else { + $dotnetRoot = Join-Path $RepoRoot ".dotnet" + + if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { + if ($install) { + InstallDotNetSdk $dotnetRoot $dotnetSdkVersion + } else { + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" + ExitWithExitCode 1 + } + } + + $env:DOTNET_INSTALL_DIR = $dotnetRoot + } + + # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom + # build steps from using anything other than what we've downloaded. + # It also ensures that VS msbuild will use the downloaded sdk targets. + $env:PATH = "$dotnetRoot;$env:PATH" + + # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + Write-PipelinePrependPath -Path $dotnetRoot + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' + Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' + + return $global:_DotNetInstallDir = $dotnetRoot +} + +function GetDotNetInstallScript([string] $dotnetRoot) { + $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" + if (!(Test-Path $installScript)) { + Create-Directory $dotnetRoot + Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript + } + + return $installScript +} + +function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { + InstallDotNet $dotnetRoot $version $architecture +} + +function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { $installScript = GetDotNetInstallScript $dotnetRoot + $installScript = GetDotNetInstallScript $dotnetRoot + $installParameters = @{ + Version = $version + InstallDir = $dotnetRoot + } + + if ($architecture) { $installParameters.Architecture = $architecture } + if ($runtime) { $installParameters.Runtime = $runtime } + if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } + + & $installScript @installParameters + if ($lastExitCode -ne 0) { + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')." + ExitWithExitCode $lastExitCode + } +} + +# +# Locates Visual Studio MSBuild installation. +# The preference order for MSBuild to use is as follows: +# +# 1. MSBuild from an active VS command prompt +# 2. MSBuild from a compatible VS installation +# 3. MSBuild from the xcopy tool package +# +# Returns full path to msbuild.exe. +# Throws on failure. +# +function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { + if (Test-Path variable:global:_MSBuildExe) { + return $global:_MSBuildExe + } + + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } + $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } + $vsMinVersion = [Version]::new($vsMinVersionStr) + + # Try msbuild command available in the environment. + if ($env:VSINSTALLDIR -ne $null) { + $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue + if ($msbuildCmd -ne $null) { + # Workaround for https://github.com/dotnet/roslyn/issues/35793 + # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ + $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) + + if ($msbuildVersion -ge $vsMinVersion) { + return $global:_MSBuildExe = $msbuildCmd.Path + } + + # Report error - the developer environment is initialized with incompatible VS version. + throw "Developer Command Prompt for VS $($env:VisualStudioVersion) is not recent enough. Please upgrade to $vsMinVersionStr or build from a plain CMD window" + } + } + + # Locate Visual Studio installation or download x-copy msbuild. + $vsInfo = LocateVisualStudio $vsRequirements + if ($vsInfo -ne $null) { + $vsInstallDir = $vsInfo.installationPath + $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] + + InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion + } else { + + if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") { + $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' + $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] + } else { + $vsMajorVersion = $vsMinVersion.Major + $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" + } + + $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install + if ($vsInstallDir -eq $null) { + throw "Unable to find Visual Studio that has required version and components installed" + } + } + + $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } + return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" +} + +function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { + $env:VSINSTALLDIR = $vsInstallDir + Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\") + + $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\" + if (Test-Path $vsSdkInstallDir) { + Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir + $env:VSSDKInstall = $vsSdkInstallDir + } +} + +function InstallXCopyMSBuild([string]$packageVersion) { + return InitializeXCopyMSBuild $packageVersion -install $true +} + +function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { + $packageName = "RoslynTools.MSBuild" + $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" + $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" + + if (!(Test-Path $packageDir)) { + if (!$install) { + return $null + } + + Create-Directory $packageDir + Write-Host "Downloading $packageName $packageVersion" + Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath + Unzip $packagePath $packageDir + } + + return Join-Path $packageDir "tools" +} + +# +# Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. +# +# The following properties of tools.vs are recognized: +# "version": "{major}.{minor}" +# Two part minimal VS version, e.g. "15.9", "16.0", etc. +# "components": ["componentId1", "componentId2", ...] +# Array of ids of workload components that must be available in the VS instance. +# See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017 +# +# Returns JSON describing the located VS instance (same format as returned by vswhere), +# or $null if no instance meeting the requirements is found on the machine. +# +function LocateVisualStudio([object]$vsRequirements = $null){ + if (Get-Member -InputObject $GlobalJson.tools -Name "vswhere") { + $vswhereVersion = $GlobalJson.tools.vswhere + } else { + $vswhereVersion = "2.5.2" + } + + $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion" + $vsWhereExe = Join-Path $vsWhereDir "vswhere.exe" + + if (!(Test-Path $vsWhereExe)) { + Create-Directory $vsWhereDir + Write-Host "Downloading vswhere" + Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + } + + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } + $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") + + if (Get-Member -InputObject $vsRequirements -Name "version") { + $args += "-version" + $args += $vsRequirements.version + } + + if (Get-Member -InputObject $vsRequirements -Name "components") { + foreach ($component in $vsRequirements.components) { + $args += "-requires" + $args += $component + } + } + + $vsInfo =& $vsWhereExe $args | ConvertFrom-Json + + if ($lastExitCode -ne 0) { + return $null + } + + # use first matching instance + return $vsInfo[0] +} + +function InitializeBuildTool() { + if (Test-Path variable:global:_BuildTool) { + return $global:_BuildTool + } + + if (-not $msbuildEngine) { + $msbuildEngine = GetDefaultMSBuildEngine + } + + # Initialize dotnet cli if listed in 'tools' + $dotnetRoot = $null + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { + $dotnetRoot = InitializeDotNetCli -install:$restore + } + + if ($msbuildEngine -eq "dotnet") { + if (!$dotnetRoot) { + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." + ExitWithExitCode 1 + } + + $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild"; Tool = "dotnet"; Framework = "netcoreapp2.1" } + } elseif ($msbuildEngine -eq "vs") { + try { + $msbuildPath = InitializeVisualStudioMSBuild -install:$restore + } catch { + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ + ExitWithExitCode 1 + } + + $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } + } else { + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." + ExitWithExitCode 1 + } + + return $global:_BuildTool = $buildTool +} + +function GetDefaultMSBuildEngine() { + # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. + if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { + return "vs" + } + + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { + return "dotnet" + } + + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." + ExitWithExitCode 1 +} + +function GetNuGetPackageCachePath() { + if ($env:NUGET_PACKAGES -eq $null) { + # Use local cache on CI to ensure deterministic build, + # use global cache in dev builds to avoid cost of downloading packages. + if ($useGlobalNuGetCache) { + $env:NUGET_PACKAGES = Join-Path $env:UserProfile ".nuget\packages" + } else { + $env:NUGET_PACKAGES = Join-Path $RepoRoot ".packages" + } + } + + return $env:NUGET_PACKAGES +} + +# Returns a full path to an Arcade SDK task project file. +function GetSdkTaskProject([string]$taskName) { + return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" +} + +function InitializeNativeTools() { + if (Get-Member -InputObject $GlobalJson -Name "native-tools") { + $nativeArgs= @{} + if ($ci) { + $nativeArgs = @{ + InstallDirectory = "$ToolsDir" + } + } + & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs + } +} + +function InitializeToolset() { + if (Test-Path variable:global:_ToolsetBuildProj) { + return $global:_ToolsetBuildProj + } + + $nugetCache = GetNuGetPackageCachePath + + $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk' + $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" + + if (Test-Path $toolsetLocationFile) { + $path = Get-Content $toolsetLocationFile -TotalCount 1 + if (Test-Path $path) { + return $global:_ToolsetBuildProj = $path + } + } + + if (-not $restore) { + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Toolset version $toolsetVersion has not been restored." + ExitWithExitCode 1 + } + + $buildTool = InitializeBuildTool + + $proj = Join-Path $ToolsetDir "restore.proj" + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } + + '' | Set-Content $proj + + MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile + + $path = Get-Content $toolsetLocationFile -TotalCount 1 + if (!(Test-Path $path)) { + throw "Invalid toolset path: $path" + } + + return $global:_ToolsetBuildProj = $path +} + +function ExitWithExitCode([int] $exitCode) { + if ($ci -and $prepareMachine) { + Stop-Processes + } + exit $exitCode +} + +function Stop-Processes() { + Write-Host "Killing running build processes..." + foreach ($processName in $processesToStopOnExit) { + Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process + } +} + +# +# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. +# The arguments are automatically quoted. +# Terminates the script if the build fails. +# +function MSBuild() { + if ($pipelinesLog) { + $buildTool = InitializeBuildTool + $toolsetBuildProject = InitializeToolset + $path = Split-Path -parent $toolsetBuildProject + $path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll") + $args += "/logger:$path" + } + + MSBuild-Core @args +} + +# +# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. +# The arguments are automatically quoted. +# Terminates the script if the build fails. +# +function MSBuild-Core() { + if ($ci) { + if (!$binaryLog) { + Write-PipelineTaskError -Message "Binary log must be enabled in CI build." + ExitWithExitCode 1 + } + + if ($nodeReuse) { + Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." + ExitWithExitCode 1 + } + } + + $buildTool = InitializeBuildTool + + $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" + + if ($warnAsError) { + $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" + } + + foreach ($arg in $args) { + if ($arg -ne $null -and $arg.Trim() -ne "") { + $cmdArgs += " `"$arg`"" + } + } + + $exitCode = Exec-Process $buildTool.Path $cmdArgs + + if ($exitCode -ne 0) { + Write-PipelineTaskError -Message "Build failed." + + $buildLog = GetMSBuildBinaryLogCommandLineArgument $args + if ($buildLog -ne $null) { + Write-Host "See log: $buildLog" -ForegroundColor DarkGray + } + + ExitWithExitCode $exitCode + } +} + +function GetMSBuildBinaryLogCommandLineArgument($arguments) { + foreach ($argument in $arguments) { + if ($argument -ne $null) { + $arg = $argument.Trim() + if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { + return $arg.Substring("/bl:".Length) + } + + if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { + return $arg.Substring("/binaryLogger:".Length) + } + } + } + + return $null +} + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") +$EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +$ArtifactsDir = Join-Path $RepoRoot "artifacts" +$ToolsetDir = Join-Path $ArtifactsDir "toolset" +$ToolsDir = Join-Path $RepoRoot ".tools" +$LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration +$TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration +$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json +# true if global.json contains a "runtimes" section +$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } + +Create-Directory $ToolsetDir +Create-Directory $TempDir +Create-Directory $LogDir + +Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir +Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir +Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir +Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir +Write-PipelineSetVariable -Name 'TMP' -Value $TempDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 70d92cf85aa..3e06ea3d04b 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -1,393 +1,393 @@ -#!/usr/bin/env bash - -# Initialize variables if they aren't already defined. - -# CI mode - set to true on CI server for PR validation build or official build. -ci=${ci:-false} - -# Set to true to use the pipelines logger which will enable Azure logging output. -# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md -# This flag is meant as a temporary opt-opt for the feature while validate it across -# our consumers. It will be deleted in the future. -if [[ "$ci" == true ]]; then - pipelines_log=${pipelines_log:-true} -else - pipelines_log=${pipelines_log:-false} -fi - -# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. -configuration=${configuration:-'Debug'} - -# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -binary_log=${binary_log:-$ci} - -# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). -prepare_machine=${prepare_machine:-false} - -# True to restore toolsets and dependencies. -restore=${restore:-true} - -# Adjusts msbuild verbosity level. -verbosity=${verbosity:-'minimal'} - -# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. -if [[ "$ci" == true ]]; then - node_reuse=${node_reuse:-false} -else - node_reuse=${node_reuse:-true} -fi - -# Configures warning treatment in msbuild. -warn_as_error=${warn_as_error:-true} - -# True to attempt using .NET Core already that meets requirements specified in global.json -# installed on the machine instead of downloading one. -use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} - -# True to use global NuGet cache instead of restoring packages to repository-local directory. -if [[ "$ci" == true ]]; then - use_global_nuget_cache=${use_global_nuget_cache:-false} -else - use_global_nuget_cache=${use_global_nuget_cache:-true} -fi - -# Resolve any symlinks in the given path. -function ResolvePath { - local path=$1 - - while [[ -h $path ]]; do - local dir="$( cd -P "$( dirname "$path" )" && pwd )" - path="$(readlink "$path")" - - # if $path was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $path != /* ]] && path="$dir/$path" - done - - # return value - _ResolvePath="$path" -} - -# ReadVersionFromJson [json key] -function ReadGlobalVersion { - local key=$1 - - local line=`grep -m 1 "$key" "$global_json_file"` - local pattern="\"$key\" *: *\"(.*)\"" - - if [[ ! $line =~ $pattern ]]; then - Write-PipelineTelemetryError -category 'InitializeTools' "Error: Cannot find \"$key\" in $global_json_file" - ExitWithExitCode 1 - fi - - # return value - _ReadGlobalVersion=${BASH_REMATCH[1]} -} - -function InitializeDotNetCli { - if [[ -n "${_InitializeDotNetCli:-}" ]]; then - return - fi - - local install=$1 - - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - export DOTNET_MULTILEVEL_LOOKUP=0 - - # Disable first run since we want to control all package sources - export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - - # Disable telemetry on CI - if [[ $ci == true ]]; then - export DOTNET_CLI_TELEMETRY_OPTOUT=1 - fi - - # LTTNG is the logging infrastructure used by Core CLR. Need this variable set - # so it doesn't output warnings to the console. - export LTTNG_HOME="$HOME" - - # Source Build uses DotNetCoreSdkDir variable - if [[ -n "${DotNetCoreSdkDir:-}" ]]; then - export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir" - fi - - # Find the first path on $PATH that contains the dotnet.exe - if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then - local dotnet_path=`command -v dotnet` - if [[ -n "$dotnet_path" ]]; then - ResolvePath "$dotnet_path" - export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"` - fi - fi - - ReadGlobalVersion "dotnet" - local dotnet_sdk_version=$_ReadGlobalVersion - local dotnet_root="" - - # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, - # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then - dotnet_root="$DOTNET_INSTALL_DIR" - else - dotnet_root="$repo_root/.dotnet" - - export DOTNET_INSTALL_DIR="$dotnet_root" - - if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then - if [[ "$install" == true ]]; then - InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" - else - Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'" - ExitWithExitCode 1 - fi - fi - fi - - # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom - # build steps from using anything other than what we've downloaded. - export PATH="$dotnet_root:$PATH" - - if [[ $ci == true ]]; then - # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build - echo "##vso[task.prependpath]$dotnet_root" - echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" - echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" - fi - - # return value - _InitializeDotNetCli="$dotnet_root" -} - -function InstallDotNetSdk { - local root=$1 - local version=$2 - local architecture="" - if [[ $# == 3 ]]; then - architecture=$3 - fi - InstallDotNet "$root" "$version" $architecture -} - -function InstallDotNet { - local root=$1 - local version=$2 - - GetDotNetInstallScript "$root" - local install_script=$_GetDotNetInstallScript - - local archArg='' - if [[ -n "${3:-}" ]]; then - archArg="--architecture $3" - fi - local runtimeArg='' - if [[ -n "${4:-}" ]]; then - runtimeArg="--runtime $4" - fi - - local skipNonVersionedFilesArg="" - if [[ "$#" -ge "5" ]]; then - skipNonVersionedFilesArg="--skip-non-versioned-files" - fi - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')." - ExitWithExitCode $exit_code - } -} - -function GetDotNetInstallScript { - local root=$1 - local install_script="$root/dotnet-install.sh" - local install_script_url="https://dot.net/v1/dotnet-install.sh" - - if [[ ! -a "$install_script" ]]; then - mkdir -p "$root" - - echo "Downloading '$install_script_url'" - - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" - else - wget -q -O "$install_script" "$install_script_url" - fi - fi - - # return value - _GetDotNetInstallScript="$install_script" -} - -function InitializeBuildTool { - if [[ -n "${_InitializeBuildTool:-}" ]]; then - return - fi - - InitializeDotNetCli $restore - - # return values - _InitializeBuildTool="$_InitializeDotNetCli/dotnet" - _InitializeBuildToolCommand="msbuild" - _InitializeBuildToolFramework="netcoreapp2.1" -} - -function GetNuGetPackageCachePath { - if [[ -z ${NUGET_PACKAGES:-} ]]; then - if [[ "$use_global_nuget_cache" == true ]]; then - export NUGET_PACKAGES="$HOME/.nuget/packages" - else - export NUGET_PACKAGES="$repo_root/.packages" - fi - fi - - # return value - _GetNuGetPackageCachePath=$NUGET_PACKAGES -} - -function InitializeNativeTools() { - if grep -Fq "native-tools" $global_json_file - then - local nativeArgs="" - if [[ "$ci" == true ]]; then - nativeArgs="-InstallDirectory $tools_dir" - fi - "$_script_dir/init-tools-native.sh" $nativeArgs - fi -} - -function InitializeToolset { - if [[ -n "${_InitializeToolset:-}" ]]; then - return - fi - - GetNuGetPackageCachePath - - ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" - - local toolset_version=$_ReadGlobalVersion - local toolset_location_file="$toolset_dir/$toolset_version.txt" - - if [[ -a "$toolset_location_file" ]]; then - local path=`cat "$toolset_location_file"` - if [[ -a "$path" ]]; then - # return value - _InitializeToolset="$path" - return - fi - fi - - if [[ "$restore" != true ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored." - ExitWithExitCode 2 - fi - - local proj="$toolset_dir/restore.proj" - - local bl="" - if [[ "$binary_log" == true ]]; then - bl="/bl:$log_dir/ToolsetRestore.binlog" - fi - - echo '' > "$proj" - MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" - - local toolset_build_proj=`cat "$toolset_location_file"` - - if [[ ! -a "$toolset_build_proj" ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" - ExitWithExitCode 3 - fi - - # return value - _InitializeToolset="$toolset_build_proj" -} - -function ExitWithExitCode { - if [[ "$ci" == true && "$prepare_machine" == true ]]; then - StopProcesses - fi - exit $1 -} - -function StopProcesses { - echo "Killing running build processes..." - pkill -9 "dotnet" || true - pkill -9 "vbcscompiler" || true - return 0 -} - -function MSBuild { - local args=$@ - if [[ "$pipelines_log" == true ]]; then - InitializeBuildTool - InitializeToolset - local toolset_dir="${_InitializeToolset%/*}" - local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" - args=( "${args[@]}" "-logger:$logger_path" ) - fi - - MSBuild-Core ${args[@]} -} - -function MSBuild-Core { - if [[ "$ci" == true ]]; then - if [[ "$binary_log" != true ]]; then - Write-PipelineTaskError "Binary log must be enabled in CI build." - ExitWithExitCode 1 - fi - - if [[ "$node_reuse" == true ]]; then - Write-PipelineTaskError "Node reuse must be disabled in CI build." - ExitWithExitCode 1 - fi - fi - - InitializeBuildTool - - local warnaserror_switch="" - if [[ $warn_as_error == true ]]; then - warnaserror_switch="/warnaserror" - fi - - "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { - local exit_code=$? - Write-PipelineTaskError "Build failed (exit code '$exit_code')." - ExitWithExitCode $exit_code - } -} - -ResolvePath "${BASH_SOURCE[0]}" -_script_dir=`dirname "$_ResolvePath"` - -. "$_script_dir/pipeline-logging-functions.sh" - -eng_root=`cd -P "$_script_dir/.." && pwd` -repo_root=`cd -P "$_script_dir/../.." && pwd` -artifacts_dir="$repo_root/artifacts" -toolset_dir="$artifacts_dir/toolset" -tools_dir="$repo_root/.tools" -log_dir="$artifacts_dir/log/$configuration" -temp_dir="$artifacts_dir/tmp/$configuration" - -global_json_file="$repo_root/global.json" -# determine if global.json contains a "runtimes" entry -global_json_has_runtimes=false -dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true -if [[ -n "$dotnetlocal_key" ]]; then - global_json_has_runtimes=true -fi - -# HOME may not be defined in some scenarios, but it is required by NuGet -if [[ -z $HOME ]]; then - export HOME="$repo_root/artifacts/.home/" - mkdir -p "$HOME" -fi - -mkdir -p "$toolset_dir" -mkdir -p "$temp_dir" -mkdir -p "$log_dir" - -if [[ $ci == true ]]; then - export TEMP="$temp_dir" - export TMP="$temp_dir" -fi +#!/usr/bin/env bash + +# Initialize variables if they aren't already defined. + +# CI mode - set to true on CI server for PR validation build or official build. +ci=${ci:-false} + +# Set to true to use the pipelines logger which will enable Azure logging output. +# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md +# This flag is meant as a temporary opt-opt for the feature while validate it across +# our consumers. It will be deleted in the future. +if [[ "$ci" == true ]]; then + pipelines_log=${pipelines_log:-true} +else + pipelines_log=${pipelines_log:-false} +fi + +# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. +configuration=${configuration:-'Debug'} + +# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. +# Binary log must be enabled on CI. +binary_log=${binary_log:-$ci} + +# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). +prepare_machine=${prepare_machine:-false} + +# True to restore toolsets and dependencies. +restore=${restore:-true} + +# Adjusts msbuild verbosity level. +verbosity=${verbosity:-'minimal'} + +# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. +if [[ "$ci" == true ]]; then + node_reuse=${node_reuse:-false} +else + node_reuse=${node_reuse:-true} +fi + +# Configures warning treatment in msbuild. +warn_as_error=${warn_as_error:-true} + +# True to attempt using .NET Core already that meets requirements specified in global.json +# installed on the machine instead of downloading one. +use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} + +# True to use global NuGet cache instead of restoring packages to repository-local directory. +if [[ "$ci" == true ]]; then + use_global_nuget_cache=${use_global_nuget_cache:-false} +else + use_global_nuget_cache=${use_global_nuget_cache:-true} +fi + +# Resolve any symlinks in the given path. +function ResolvePath { + local path=$1 + + while [[ -h $path ]]; do + local dir="$( cd -P "$( dirname "$path" )" && pwd )" + path="$(readlink "$path")" + + # if $path was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $path != /* ]] && path="$dir/$path" + done + + # return value + _ResolvePath="$path" +} + +# ReadVersionFromJson [json key] +function ReadGlobalVersion { + local key=$1 + + local line=`grep -m 1 "$key" "$global_json_file"` + local pattern="\"$key\" *: *\"(.*)\"" + + if [[ ! $line =~ $pattern ]]; then + Write-PipelineTelemetryError -category 'InitializeTools' "Error: Cannot find \"$key\" in $global_json_file" + ExitWithExitCode 1 + fi + + # return value + _ReadGlobalVersion=${BASH_REMATCH[1]} +} + +function InitializeDotNetCli { + if [[ -n "${_InitializeDotNetCli:-}" ]]; then + return + fi + + local install=$1 + + # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism + export DOTNET_MULTILEVEL_LOOKUP=0 + + # Disable first run since we want to control all package sources + export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + + # Disable telemetry on CI + if [[ $ci == true ]]; then + export DOTNET_CLI_TELEMETRY_OPTOUT=1 + fi + + # LTTNG is the logging infrastructure used by Core CLR. Need this variable set + # so it doesn't output warnings to the console. + export LTTNG_HOME="$HOME" + + # Source Build uses DotNetCoreSdkDir variable + if [[ -n "${DotNetCoreSdkDir:-}" ]]; then + export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir" + fi + + # Find the first path on $PATH that contains the dotnet.exe + if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then + local dotnet_path=`command -v dotnet` + if [[ -n "$dotnet_path" ]]; then + ResolvePath "$dotnet_path" + export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"` + fi + fi + + ReadGlobalVersion "dotnet" + local dotnet_sdk_version=$_ReadGlobalVersion + local dotnet_root="" + + # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, + # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. + if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then + dotnet_root="$DOTNET_INSTALL_DIR" + else + dotnet_root="$repo_root/.dotnet" + + export DOTNET_INSTALL_DIR="$dotnet_root" + + if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then + if [[ "$install" == true ]]; then + InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" + else + Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'" + ExitWithExitCode 1 + fi + fi + fi + + # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom + # build steps from using anything other than what we've downloaded. + export PATH="$dotnet_root:$PATH" + + if [[ $ci == true ]]; then + # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build + echo "##vso[task.prependpath]$dotnet_root" + echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" + echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" + fi + + # return value + _InitializeDotNetCli="$dotnet_root" +} + +function InstallDotNetSdk { + local root=$1 + local version=$2 + local architecture="" + if [[ $# == 3 ]]; then + architecture=$3 + fi + InstallDotNet "$root" "$version" $architecture +} + +function InstallDotNet { + local root=$1 + local version=$2 + + GetDotNetInstallScript "$root" + local install_script=$_GetDotNetInstallScript + + local archArg='' + if [[ -n "${3:-}" ]]; then + archArg="--architecture $3" + fi + local runtimeArg='' + if [[ -n "${4:-}" ]]; then + runtimeArg="--runtime $4" + fi + + local skipNonVersionedFilesArg="" + if [[ "$#" -ge "5" ]]; then + skipNonVersionedFilesArg="--skip-non-versioned-files" + fi + bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')." + ExitWithExitCode $exit_code + } +} + +function GetDotNetInstallScript { + local root=$1 + local install_script="$root/dotnet-install.sh" + local install_script_url="https://dot.net/v1/dotnet-install.sh" + + if [[ ! -a "$install_script" ]]; then + mkdir -p "$root" + + echo "Downloading '$install_script_url'" + + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" + else + wget -q -O "$install_script" "$install_script_url" + fi + fi + + # return value + _GetDotNetInstallScript="$install_script" +} + +function InitializeBuildTool { + if [[ -n "${_InitializeBuildTool:-}" ]]; then + return + fi + + InitializeDotNetCli $restore + + # return values + _InitializeBuildTool="$_InitializeDotNetCli/dotnet" + _InitializeBuildToolCommand="msbuild" + _InitializeBuildToolFramework="netcoreapp2.1" +} + +function GetNuGetPackageCachePath { + if [[ -z ${NUGET_PACKAGES:-} ]]; then + if [[ "$use_global_nuget_cache" == true ]]; then + export NUGET_PACKAGES="$HOME/.nuget/packages" + else + export NUGET_PACKAGES="$repo_root/.packages" + fi + fi + + # return value + _GetNuGetPackageCachePath=$NUGET_PACKAGES +} + +function InitializeNativeTools() { + if grep -Fq "native-tools" $global_json_file + then + local nativeArgs="" + if [[ "$ci" == true ]]; then + nativeArgs="-InstallDirectory $tools_dir" + fi + "$_script_dir/init-tools-native.sh" $nativeArgs + fi +} + +function InitializeToolset { + if [[ -n "${_InitializeToolset:-}" ]]; then + return + fi + + GetNuGetPackageCachePath + + ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" + + local toolset_version=$_ReadGlobalVersion + local toolset_location_file="$toolset_dir/$toolset_version.txt" + + if [[ -a "$toolset_location_file" ]]; then + local path=`cat "$toolset_location_file"` + if [[ -a "$path" ]]; then + # return value + _InitializeToolset="$path" + return + fi + fi + + if [[ "$restore" != true ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored." + ExitWithExitCode 2 + fi + + local proj="$toolset_dir/restore.proj" + + local bl="" + if [[ "$binary_log" == true ]]; then + bl="/bl:$log_dir/ToolsetRestore.binlog" + fi + + echo '' > "$proj" + MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + + local toolset_build_proj=`cat "$toolset_location_file"` + + if [[ ! -a "$toolset_build_proj" ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" + ExitWithExitCode 3 + fi + + # return value + _InitializeToolset="$toolset_build_proj" +} + +function ExitWithExitCode { + if [[ "$ci" == true && "$prepare_machine" == true ]]; then + StopProcesses + fi + exit $1 +} + +function StopProcesses { + echo "Killing running build processes..." + pkill -9 "dotnet" || true + pkill -9 "vbcscompiler" || true + return 0 +} + +function MSBuild { + local args=$@ + if [[ "$pipelines_log" == true ]]; then + InitializeBuildTool + InitializeToolset + local toolset_dir="${_InitializeToolset%/*}" + local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + args=( "${args[@]}" "-logger:$logger_path" ) + fi + + MSBuild-Core ${args[@]} +} + +function MSBuild-Core { + if [[ "$ci" == true ]]; then + if [[ "$binary_log" != true ]]; then + Write-PipelineTaskError "Binary log must be enabled in CI build." + ExitWithExitCode 1 + fi + + if [[ "$node_reuse" == true ]]; then + Write-PipelineTaskError "Node reuse must be disabled in CI build." + ExitWithExitCode 1 + fi + fi + + InitializeBuildTool + + local warnaserror_switch="" + if [[ $warn_as_error == true ]]; then + warnaserror_switch="/warnaserror" + fi + + "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { + local exit_code=$? + Write-PipelineTaskError "Build failed (exit code '$exit_code')." + ExitWithExitCode $exit_code + } +} + +ResolvePath "${BASH_SOURCE[0]}" +_script_dir=`dirname "$_ResolvePath"` + +. "$_script_dir/pipeline-logging-functions.sh" + +eng_root=`cd -P "$_script_dir/.." && pwd` +repo_root=`cd -P "$_script_dir/../.." && pwd` +artifacts_dir="$repo_root/artifacts" +toolset_dir="$artifacts_dir/toolset" +tools_dir="$repo_root/.tools" +log_dir="$artifacts_dir/log/$configuration" +temp_dir="$artifacts_dir/tmp/$configuration" + +global_json_file="$repo_root/global.json" +# determine if global.json contains a "runtimes" entry +global_json_has_runtimes=false +dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true +if [[ -n "$dotnetlocal_key" ]]; then + global_json_has_runtimes=true +fi + +# HOME may not be defined in some scenarios, but it is required by NuGet +if [[ -z $HOME ]]; then + export HOME="$repo_root/artifacts/.home/" + mkdir -p "$HOME" +fi + +mkdir -p "$toolset_dir" +mkdir -p "$temp_dir" +mkdir -p "$log_dir" + +if [[ $ci == true ]]; then + export TEMP="$temp_dir" + export TMP="$temp_dir" +fi diff --git a/global.json b/global.json index 905c869fc74..12a08128837 100644 --- a/global.json +++ b/global.json @@ -1,24 +1,24 @@ -{ - "tools": { - "dotnet": "3.0.100-preview7-012512", - "runtimes": { - "dotnet": [ - "2.1.7", - "$(MicrosoftNETCoreAppVersion)" - ] - }, - "vs": { - "version": "16.1" - } - }, - "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19324.7", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19324.7" - }, - "native-tools": { - "strawberry-perl": "5.28.1.1-1", - "net-framework-48-ref-assemblies": "0.0.0.1", - "dotnet-api-docs_netcoreapp3.0": "0.0.0.1", - "msvcurt-c1xx": "0.0.0.4" - } -} +{ + "tools": { + "dotnet": "3.0.100-preview7-012512", + "runtimes": { + "dotnet": [ + "2.1.7", + "$(MicrosoftNETCoreAppVersion)" + ] + }, + "vs": { + "version": "16.1" + } + }, + "msbuild-sdks": { + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19326.2", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19326.2" + }, + "native-tools": { + "strawberry-perl": "5.28.1.1-1", + "net-framework-48-ref-assemblies": "0.0.0.1", + "dotnet-api-docs_netcoreapp3.0": "0.0.0.1", + "msvcurt-c1xx": "0.0.0.4" + } +}