From 1fede69022fa63709714d14b604712ffca673fd8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 13:29:32 +0100 Subject: [PATCH 001/215] move customizationcap. --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 83 ++++++- Actions/CheckForUpdates/yamlclass.ps1 | 228 ++++++++++++++++++++ 2 files changed, 306 insertions(+), 5 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 0a1f798fd..8bbbb2d0e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -22,6 +22,18 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper +$anchors = @{ + "_BuildALGoProject.yaml" = @{ + "BuildALGoProject" = @( + @{ "Step" = 'Read settings'; "Before" = $false } + @{ "Step" = 'Read secrets'; "Before" = $false } + @{ "Step" = 'Build'; "Before" = $true } + @{ "Step" = 'Build'; "Before" = $false } + @{ "Step" = 'Cleanup'; "Before" = $true } + ) + } +} + if ($update -eq 'Y') { if (-not $token) { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." @@ -64,19 +76,48 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { $downloadLatest = $true } +$realTemplateFolder = $null $templateFolder = DownloadTemplateRepository -headers $headers -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" $templateBranch = $templateUrl.Split('@')[1] $templateOwner = $templateUrl.Split('/')[3] +$indirectTemplateRepoSettings = @{} +$indirectTemplateProjectSettings = @{} + $isDirectALGo = IsDirectALGo -templateUrl $templateUrl if (-not $isDirectALGo) { $ALGoSettingsFile = Join-Path $templateFolder "*/$repoSettingsFile" if (Test-Path -Path $ALGoSettingsFile -PathType Leaf) { $templateRepoSettings = Get-Content $ALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { - throw "The specified template repository is not a template repository, but instead another AL-Go repository. This is not supported." + # The template repository is a url to another AL-Go repository (an indirect template repository) + # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template reposotiry points to the "real" template repository + # Copy files and folders from the indirect template repository, but grab the unmodified file from the "real" template repository if it exists and apply customizations + Write-Host "Indirect AL-Go template repository detected, downloading the 'real' template repository" + $realTemplateUrl = $templateRepoSettings.templateUrl + if ($templateRepoSettings.Keys -contains "templateSha") { + $realTemplateSha = $templateRepoSettings.templateSha + } + else { + $realTemplateSha = "" + } + # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository + $realTemplateFolder = DownloadTemplateRepository -headers $headers -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') + Write-Host "Real Template Folder: $realTemplateFolder" + + # Set TemplateBranch and TemplateOwner + # Keep TemplateUrl and TemplateSha pointing to the indirect template repository + $templateBranch = $realTemplateUrl.Split('@')[1] + $templateOwner = $realTemplateUrl.Split('/')[3] + + $indirectTemplateRepoSettings = $templateRepoSettings + $projectSettingsFile = Join-Path $templateFolder "*/.AL-Go/settings.json" + if (Test-Path $projectSettingsFile -PathType Leaf) { + Write-Host "Read project settings from indirect template repository" + $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + } } } } @@ -129,9 +170,20 @@ foreach($checkfile in $checkfiles) { $dstPath = $checkfile.dstPath $dstFolder = Join-Path $baseFolder $dstPath $srcFolder = GetSrcFolder -repoSettings $repoSettings -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath + $realSrcFolder = $null + if ($realTemplateFolder) { + $realSrcFolder = GetSrcFolder -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath + } if ($srcFolder) { Push-Location -Path $srcFolder try { + if ($srcPath -eq '.AL-Go' -and $type -eq "script" -and $realSrcFolder) { + Write-Host "Update Project Settings" + # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) + $projectSettingsFile = Join-Path $dstFolder "settings.json" + UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -otherSettings $indirectTemplateProjectSettings + $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } + } # Loop through all files in the template repository matching the pattern Get-ChildItem -Path $srcFolder -Filter $checkfile.pattern | ForEach-Object { # Read the template file and modify it based on the settings @@ -140,24 +192,41 @@ foreach($checkfile in $checkfiles) { Write-Host "- $filename" $dstFile = Join-Path $dstFolder $fileName $srcFile = $_.FullName + $realSrcFile = $srcFile + $isFileDirectALGo = $isDirectALGo Write-Host "SrcFolder: $srcFolder" + if ($realSrcFolder) { + # if SrcFile is an indirect template repository, we need to find the file in the "real" template repository + $fname = Join-Path $realSrcFolder (Resolve-Path $srcFile -Relative) + if (Test-Path -Path $fname -PathType Leaf) { + Write-Host "File is available in the 'real' template repository" + $realSrcFile = $fname + $isFileDirectALGo = IsDirectALGo -templateUrl $realTemplateUrl + } + } if ($type -eq "workflow") { # for workflow files, we might need to modify the file based on the settings - $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $srcFile -repoSettings $repoSettings -depth $depth + $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $realSrcFile -repoSettings $repoSettings -depth $depth } else { # For non-workflow files, just read the file content - $srcContent = Get-ContentLF -Path $srcFile + $srcContent = Get-ContentLF -Path $realSrcFile } # Replace static placeholders $srcContent = $srcContent.Replace('{TEMPLATEURL}', $templateUrl) - if ($isDirectALGo) { + if ($isFileDirectALGo) { # If we are using direct AL-Go repo, we need to change the owner to the remplateOwner, the repo names to AL-Go and AL-Go/Actions and the branch to templateBranch ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } + if ($type -eq 'workflow' -and $realSrcFile -ne $srcFile) { + # Apply customizations from indirect template repository + Write-Host "Apply customizations from indirect template repository: $srcFile" + [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $anchors) + } + $dstFileExists = Test-Path -Path $dstFile -PathType Leaf if ($unusedALGoSystemFiles -contains $fileName) { # file is not used by ALGo, remove it if it exists @@ -167,6 +236,10 @@ foreach($checkfile in $checkfiles) { } } elseif ($dstFileExists) { + if ($type -eq 'workflow') { + Write-Host "Apply customizations from my repository: $dstFile" + [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $anchors) + } # file exists, compare and add to $updateFiles if different $dstContent = Get-ContentLF -Path $dstFile if ($dstContent -cne $srcContent) { @@ -204,7 +277,7 @@ else { invoke-git status - UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } + UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -additionalSettings $indirectTemplateRepoSettings # Update the files # Calculate the release notes, while updating diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 35987b42a..9fbd894b8 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -126,6 +126,45 @@ class Yaml { return $this.Get($line, [ref] $start, [ref] $count) } + # Locate all lines in the next level of a yaml path + # if $line is empty, you get all first level lines + # Example: + # GetNextLevel("jobs:/") returns @("Initialization:","CheckForUpdates:","Build:","Deploy:",...) + [string[]] GetNextLevel([string] $line) { + [int]$start = 0 + [int]$count = 0 + [Yaml] $yaml = $this + if ($line) { + $yaml = $this.Get($line, [ref] $start, [ref] $count) + } + return $yaml.content | Where-Object { $_ -and -not $_.StartsWith(' ') } + } + + # Get the value of a property as a string + # Example: + # GetProperty("jobs:/Build:/needs:") returns "[ Initialization, Build1 ]" + [string] GetProperty([string] $line) { + [int]$start = 0 + [int]$count = 0 + [Yaml] $yaml = $this.Get($line, [ref] $start, [ref] $count) + if ($yaml -and $yaml.content.Count -eq 1) { + return $yaml.content[0].SubString($yaml.content[0].IndexOf(':')+1).Trim() + } + return $null + } + + # Get the value of a property as a string array + # Example: + # GetPropertyArray("jobs:/Build:/needs:") returns @("Initialization", "Build") + [string[]] GetPropertyArray([string] $line) { + $prop = $this.GetProperty($line) + if ($prop) { + # "needs: [ Initialization, Build ]" becomes @("Initialization", "Build") + return $prop.TrimStart('[').TrimEnd(']').Split(',').Trim() + } + return $null + } + # Replace the lines for the specified Yaml path, given by $line with the lines in $content # If $line ends with '/', then the lines for the section are replaced only # If $line doesn't end with '/', then the line + the lines for the section are replaced @@ -182,4 +221,193 @@ class Yaml { $this.content = $this.content[0..($index-1)] + $yamlContent + $this.content[$index..($this.content.Count-1)] } } + + # Locate jobs in YAML based on a name pattern + # Example: + # GetCustomJobsFromYaml() returns @("CustomJob1", "CustomJob2") + # GetCustomJobsFromYaml("Build*") returns @("Build1","Build2","Build") + [hashtable[]] GetCustomJobsFromYaml([string] $name) { + $result = @() + $allJobs = $this.GetNextLevel('jobs:/').Trim(':') + $customJobs = @($allJobs | Where-Object { $_ -like $name }) + if ($customJobs) { + $nativeJobs = @($allJobs | Where-Object { $customJobs -notcontains $_ }) + Write-Host "Native Jobs:" + foreach($nativeJob in $nativeJobs) { + Write-Host "- $nativeJob" + } + Write-Host "Custom Jobs:" + foreach($customJob in $customJobs) { + Write-Host "- $customJob" + $jobsWithDependency = $nativeJobs | Where-Object { $this.GetPropertyArray("jobs:/$($_):/needs:") | Where-Object { $_ -eq $customJob } } + # If any Build Job has a dependency on this CustomJob, add will be added to all build jobs later + if ($jobsWithDependency | Where-Object { $_ -like 'Build*' }) { + $jobsWithDependency = @($jobsWithDependency | Where-Object { $_ -notlike 'Build*' }) + @('Build') + } + if ($jobsWithDependency) { + Write-Host " - Jobs with dependency: $($jobsWithDependency -join ', ')" + $result += @(@{ "Name" = $customJob; "Content" = @($this.Get("jobs:/$($customJob):").content); "NeedsThis" = @($jobsWithDependency) }) + } + } + } + return $result + } + + # Add jobs to Yaml and update Needs section from native jobs which needs this custom Job + # $customJobs is an array of hashtables with Name, Content and NeedsThis + # Example: + # $customJobs = @(@{ "Name" = "CustomJob1"; "Content" = @(" - pwsh"," -File Build1"); "NeedsThis" = @("Initialization", "Build") }) + # AddCustomJobsToYaml($customJobs) + # The function will add the job CustomJob1 to the Yaml file and update the Needs section of Initialization and Build + # The function will not add the job CustomJob1 if it already exists + [void] AddCustomJobsToYaml([hashtable[]] $customJobs) { + $existingJobs = $this.GetNextLevel('jobs:/').Trim(':') + Write-Host "Adding New Jobs" + foreach($customJob in $customJobs) { + if ($existingJobs -contains $customJob.Name) { + Write-Host "Job $($customJob.Name) already exists" + continue + } + Write-Host "$($customJob.Name) has dependencies from $($customJob.NeedsThis -join ',')" + foreach($needsthis in $customJob.NeedsThis) { + if ($needsthis -eq 'Build') { + $existingJobs | Where-Object { $_ -like 'Build*'} | ForEach-Object { + # Add dependency to all build jobs + $this.Replace("jobs:/$($_):/needs:","needs: [ $(@($this.GetPropertyArray("jobs:/$($_):/needs:"))+@($customJob.Name) -join ', ') ]") + } + } + elseif ($existingJobs -contains $needsthis) { + # Add dependency to job + $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) -join ', ') ]") + } + } + $this.content += @('') + @($customJob.content | ForEach-Object { " $_" }) + } + } + + [string[]] GetStepsFromJob([string] $job) { + $steps = $this.GetNextLevel("Jobs:/$($job):/steps:/") | Where-Object { $_ -like '- name: *' } | ForEach-Object { $_.Substring(8).Trim() } + if ($steps | Group-Object | Where-Object { $_.Count -gt 1 }) { + Write-Host "Duplicate step names in job '$job'" + return @() + } + return $steps + } + + [hashtable[]] GetCustomStepsFromAnchor([string] $job, [string] $anchorStep, [bool] $before) { + $steps = $this.GetStepsFromJob($job) + $anchorIdx = $steps.IndexOf($anchorStep) + if ($anchorIdx -lt 0) { + Write-Host "Cannot find anchor step '$anchorStep' in job '$job'" + return @() + } + $idx = $anchorIdx + $customSteps = @() + if ($before) { + while ($idx -gt 0 -and $steps[$idx-1] -like 'CustomStep*') { + $idx-- + } + if ($idx -ne $anchorIdx) { + $customSteps = @($steps[$idx..($anchorIdx-1)]) + # Reverse the order of the custom steps in order to apply in correct order from the anchor step + [array]::Reverse($customSteps) + } + } + else { + while ($idx -lt $steps.Count-1 -and $steps[$idx+1] -like 'CustomStep*') { + $idx++ + } + if ($idx -ne $anchorIdx) { + $customSteps = @($steps[($anchorIdx+1)..$idx]) + } + } + $result = @() + foreach($customStep in $customSteps) { + $stepContent = $this.Get("Jobs:/$($job):/steps:/- name: $($customStep)").content + $result += @(@{"Name" = $customStep; "Content" = $stepContent; "AnchorStep" = $anchorStep; "Before" = $before }) + } + return $result + } + + [hashtable[]] GetCustomStepsFromYaml([string] $job, [hashtable[]] $anchors) { + $steps = $this.GetStepsFromJob($job) + $result = @() + foreach($anchor in $anchors) { + $result += $this.GetCustomStepsFromAnchor($job, $anchor.Step, $anchor.Before) + } + foreach($step in $steps) { + if ($step -like 'CustomStep*') { + if (-not ($result | Where-Object { $_.Name -eq $step })) { + Write-Host "Custom step '$step' does not belong to a supported anchor" + } + } + } + return $result + } + + [void] AddCustomStepsToAnchor([string] $job, [hashtable[]] $customSteps, [string] $anchorStep, [bool] $before) { + $steps = $this.GetStepsFromJob($job) + if (!$steps) { + Write-Host "::Warning::Cannot find job '$job'" + return + } + $anchorIdx = $steps.IndexOf($anchorStep) + if ($anchorIdx -lt 0) { + Write-Host "::Warning::Cannot find anchor step '$anchorStep' in job '$job'" + return + } + foreach($customStep in $customSteps | Where-Object { $_.AnchorStep -eq $anchorStep -and $_.Before -eq $before }) { + if ($steps -contains $customStep.Name) { + Write-Host "Custom step '$($customStep.Name)' already exists in job '$job'" + } + else { + $anchorStart = 0 + $anchorCount = 0 + if ($this.Find("Jobs:/$($job):/steps:/- name: $($anchorStep)", [ref] $anchorStart, [ref] $anchorCount)) { + if ($before) { + $this.Insert($anchorStart-1, @('') + @($customStep.Content | ForEach-Object { " $_" })) + } + else { + $this.Insert($anchorStart+$anchorCount, @('') + @($customStep.Content | ForEach-Object { " $_" })) + } + } + } + # Use added step as anchor for next step + $anchorStep = $customStep.Name + } + } + + [void] AddCustomStepsToYaml([string] $job, [hashtable[]] $customSteps, [hashtable[]] $anchors) { + foreach($anchor in $anchors) { + $this.AddCustomStepsToAnchor($job, $customSteps, $anchor.Step, $anchor.Before) + } + } + + static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile, [hashtable] $anchors) { + $srcYaml = [Yaml]::new($srcContent.Value.Split("`n")) + try { + $yaml = [Yaml]::Load($yamlFile) + } + catch { + return + } + $filename = [System.IO.Path]::GetFileName($yamlFile) + if ($anchors.ContainsKey($filename)) { + $fileAnchors = $anchors."$filename" + foreach($job in $fileAnchors.Keys) { + # Locate custom steps in destination YAML + $customSteps = $yaml.GetCustomStepsFromYaml($job, $fileAnchors."$job") + if ($customSteps) { + $srcYaml.AddCustomStepsToYaml($job, $customSteps, $fileAnchors."$job") + } + } + } + # Locate custom jobs in destination YAML + $customJobs = @($yaml.GetCustomJobsFromYaml('CustomJob*')) + if ($customJobs) { + # Add custom jobs to template YAML + $srcYaml.AddCustomJobsToYaml($customJobs) + } + $srcContent.Value = $srcYaml.content -join "`n" + } } From ad207daa28b623c7d0851eaf86ba49ccc355966c Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 13:45:08 +0100 Subject: [PATCH 002/215] use type --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 4 ++-- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 492e06735..b7872dbd4 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -279,7 +279,7 @@ function IsDirectALGo { function GetSrcFolder { Param( - [hashtable] $repoSettings, + [string] $repoType, [string] $templateUrl, [string] $templateFolder, [string] $srcPath @@ -291,7 +291,7 @@ function GetSrcFolder { return '' } if (IsDirectALGo -templateUrl $templateUrl) { - switch ($repoSettings.type) { + switch ($repoType) { "PTE" { $typePath = "Per Tenant Extension" } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 8bbbb2d0e..51de4f5f0 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -169,10 +169,10 @@ foreach($checkfile in $checkfiles) { $srcPath = $checkfile.srcPath $dstPath = $checkfile.dstPath $dstFolder = Join-Path $baseFolder $dstPath - $srcFolder = GetSrcFolder -repoSettings $repoSettings -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath + $srcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath $realSrcFolder = $null if ($realTemplateFolder) { - $realSrcFolder = GetSrcFolder -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath + $realSrcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath } if ($srcFolder) { Push-Location -Path $srcFolder From 2c0e1c2f312b897abe9a260e291deb7801834a9c Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 13:56:57 +0100 Subject: [PATCH 003/215] use additional --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 51de4f5f0..ad478d7cb 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,7 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - $projectSettingsFile = Join-Path $templateFolder "*/.AL-Go/settings.json" + $projectSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" if (Test-Path $projectSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse @@ -181,7 +181,7 @@ foreach($checkfile in $checkfiles) { Write-Host "Update Project Settings" # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) $projectSettingsFile = Join-Path $dstFolder "settings.json" - UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -otherSettings $indirectTemplateProjectSettings + UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -additionalSettings $indirectTemplateProjectSettings $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } } # Loop through all files in the template repository matching the pattern From e9fc84793863e95f90110ef2ce51d4ec01f90142 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:05:50 +0100 Subject: [PATCH 004/215] realtemplate --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index ad478d7cb..60d8cfd32 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,7 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - $projectSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" + $projectSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" if (Test-Path $projectSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse From cb81cfa5204738e86115c74d80dd347b3c3fbbb8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:08:49 +0100 Subject: [PATCH 005/215] dump --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 60d8cfd32..09727aef9 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,9 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings + Write-Host $ALGoSettingsFile $projectSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" + Write-Host $projectSettingsFile if (Test-Path $projectSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse From 7c3046ba3700161a9b1efa2c2db6681d23a4f6cc Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:11:34 +0100 Subject: [PATCH 006/215] use correct settings --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 09727aef9..1b99de7f5 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -88,9 +88,9 @@ $indirectTemplateProjectSettings = @{} $isDirectALGo = IsDirectALGo -templateUrl $templateUrl if (-not $isDirectALGo) { - $ALGoSettingsFile = Join-Path $templateFolder "*/$repoSettingsFile" - if (Test-Path -Path $ALGoSettingsFile -PathType Leaf) { - $templateRepoSettings = Get-Content $ALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + $myRepoSettingsFile = Join-Path $templateFolder "*/$RepoSettingsFile" + if (Test-Path -Path $myRepoSettingsFile -PathType Leaf) { + $templateRepoSettings = Get-Content $myRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { # The template repository is a url to another AL-Go repository (an indirect template repository) # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template reposotiry points to the "real" template repository @@ -113,12 +113,10 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - Write-Host $ALGoSettingsFile - $projectSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" - Write-Host $projectSettingsFile - if (Test-Path $projectSettingsFile -PathType Leaf) { + $myALGoSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" + if (Test-Path $myALGoSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" - $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + $indirectTemplateProjectSettings = Get-Content $myALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse } } } From cfe76e859dfa47b4d8d98bab97879f8c72c92acf Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:14:45 +0100 Subject: [PATCH 007/215] use correct template --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 1b99de7f5..b02216bd8 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,7 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - $myALGoSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" + $myALGoSettingsFile = Join-Path $templateFolder $ALGoSettingsFile if (Test-Path $myALGoSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $myALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse From 753c7fc81d6a7f54575074dab0c41d7ce2b9a0ef Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:17:27 +0100 Subject: [PATCH 008/215] add subfolder --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index b02216bd8..d8c558184 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,7 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - $myALGoSettingsFile = Join-Path $templateFolder $ALGoSettingsFile + $myALGoSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" if (Test-Path $myALGoSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $myALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse From f03dbd3e4329e31448431a26f198cd93c35ea2c2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 22:57:23 +0100 Subject: [PATCH 009/215] release notes --- RELEASENOTES.md | 17 ++++++++++++++++- Scenarios/settings.md | 6 ++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 8a8e5141b..cb81e48d8 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -13,6 +13,21 @@ Note that when using the preview version of AL-Go for GitHub, we recommend you U Add another parameter when running Update AL-Go System Files, called downloadLatest, used to indicate whether to download latest version from template repository. Default value is true. If false, the templateSha repository setting is used to download specific AL-Go System Files when calculating new files. +### Add custom jobs to AL-Go workflows +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go system files. + +### Add custom steps to the AL-Go build workflow +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go system files. + +### Indirect AL-Go template repository +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + ### Issues - Issue 782 Exclude '.altestrunner/' from template .gitignore - Issue 823 Dependencies from prior build jobs are not included when using useProjectDependencies @@ -22,7 +37,7 @@ If false, the templateSha repository setting is used to download specific AL-Go ### New Settings -- `alDoc` : JSON object with properties for the ALDoc reference document generation +- `[alDoc](https://aka.ms/algosettings#aldoc)` : JSON object with properties for the ALDoc reference document generation - **continuousDeployment** = Determines if reference documentation will be deployed continuously as part of CI/CD. You can run the **Deploy Reference Documentation** workflow to deploy manually or on a schedule. (Default false) - **deployToGitHubPages** = Determines whether or not the reference documentation site should be deployed to GitHub Pages for the repository. In order to deploy to GitHub Pages, GitHub Pages must be enabled and set to GitHub Actions. (Default true) - **maxReleases** = Maximum number of releases to include in the reference documentation. (Default 3) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index c83b4262b..f462981de 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -294,6 +294,12 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | +## Custom jobs in AL-Go for GitHub workflows + +## Custom steps in the _BuildALGoProject workflow + +## Indirect template repositories + ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. From c5ee954924ae94e9605ef3c867dd9e4bcf5fba3d Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 22:58:35 +0100 Subject: [PATCH 010/215] lf --- Scenarios/settings.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index f462981de..b38163a29 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -294,11 +294,14 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | -## Custom jobs in AL-Go for GitHub workflows + +## Custom jobs in AL-Go for GitHub workflows -## Custom steps in the _BuildALGoProject workflow + +## Custom steps in the _BuildALGoProject workflow -## Indirect template repositories + +## Indirect template repositories ## Your own version of AL-Go for GitHub From 811cc63cb3487778109d27383f69ceca111bf576 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 22:59:49 +0100 Subject: [PATCH 011/215] lines --- Scenarios/settings.md | 127 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index b38163a29..19a0784fe 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -297,12 +297,139 @@ Settings, which might be relevant to set in the settings file includes ## Custom jobs in AL-Go for GitHub workflows +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + ## Custom steps in the _BuildALGoProject workflow +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + + ## Indirect template repositories +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + + + ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. From 5971157d686bb2505fb554dce4675c81958746e3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 23:07:18 +0100 Subject: [PATCH 012/215] exp --- Scenarios/settings.md | 158 ++++++------------------------------------ 1 file changed, 20 insertions(+), 138 deletions(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 19a0784fe..bd5764e15 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -21,6 +21,7 @@ When running a workflow or a local script, the settings are applied by reading s 1. `.AL-Go/.settings.json` is the **user-specific settings file**. This option is rarely used, but if you have special settings, which should only be used for one specific user (potentially in the local scripts), these settings can be added to a settings file with the name of the user followed by `.settings.json`. + ## Basic settings | Name | Description | Default value | @@ -68,6 +69,7 @@ The repository settings are only read from the repository settings file (.github | UpdateGitHubGoSystemFilesSchedule | CRON schedule for when Update AL-Go System Files should run. When Update AL-Go System Files runs on a schedule, it uses direct Commit instead of creating a PR. Default is no scheduled run, only manual trigger. Build your CRON string here: [https://crontab.guru](https://crontab.guru) | | buildModes | A list of build modes to use when building the AL-Go projects. Every AL-Go projects will be built using each built mode. Available build modes are:
**Default**: Apps are compiled as they are in the source code.
**Clean**: _PreprocessorSymbols_ are enabled when compiling the apps. The values for the symbols correspond to the `cleanModePreprocessorSymbols` setting of the AL-Go project.
**Translated**: `TranslationFile` compiler feature is enabled when compiling the apps. | + ## Advanced settings | Name | Description | Default value | @@ -115,6 +117,7 @@ The repository settings are only read from the repository settings file (.github | appSourceContextSecretName | This setting specifies the name (**NOT the secret**) of a secret containing a json string with ClientID, TenantID and ClientSecret or RefreshToken. If this secret exists, AL-Go will can upload builds to AppSource validation. | AppSourceContext | | keyVaultCertificateUrlSecretName
keyVaultCertificatePasswordSecretName
keyVaultClientIdSecretName | If you want to enable KeyVault access for your AppSource App, you need to provide 3 secrets as GitHub Secrets or in the Azure KeyVault. The names of those secrets (**NOT the secrets**) should be specified in the settings file with these 3 settings. Default is to not have KeyVault access from your AppSource App. Read [this](EnableKeyVaultForAppSourceApp.md) for more information. | | + ## Conditional Settings In any of the settings files, you can add conditional settings by using the ConditionalSettings setting. @@ -161,6 +164,11 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza **Note:** that you can have conditional settings on any level and all conditional settings which has all conditions met will be applied in the order of settings file + appearance. + +# Expert level + +The following settings and functionality requires knowledge in GitHub Workflows/Actions and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. + ## Expert settings (rarely used) | Name | Description | Default value | @@ -181,8 +189,7 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | [ ] | -# Expert level - + ## Custom Delivery You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named DeliverTo*.ps1 in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. @@ -222,6 +229,7 @@ Here are the parameters to use in your custom script: | `$parameters.testAppsFolders` | The folders that contain the build artifacts from all builds (from different build modes) of the test apps in the AL-Go project | AllProjects_MyProject-main-TestApps-1.0.0.0, AllProjects_MyProject-main-CleanTestApps-1.0.0.0 | | `$parameters.dependenciesFolders` | The folders that contain the dependencies of the AL-Go project for all builds (from different build modes) | AllProjects_MyProject-main-Dependencies-1.0.0.0, AllProjects_MyProject-main-CleanDependencies-1.0.0.0 | + ## Custom Deployment You can override existing AL-Go Deployment functionality or you can define your own custom deployment mechanism for AL-Go for GitHub. By specifying a PowerShell script named `DeployTo.ps1` in the .github folder. Default Environment Type is SaaS, but you can define your own type by specifying EnvironmentType in the `DeployTo` setting. The following example will spin up a deployment job to SharePoint on CI/CD and Publish To Environment. @@ -255,6 +263,16 @@ Here are the parameters to use in your custom script: | `$parameters.ContinuousDeployment` | Is this environment setup for continuous deployment | false | | `$parameters."runs-on"` | GitHub runner to be used to run the deployment script | windows-latest | + + +## Custom jobs in AL-Go for GitHub workflows + + +## Custom steps in the _BuildALGoProject workflow + + +## Indirect template repositories + ## Run-AlPipeline script override AL-Go for GitHub utilizes the Run-AlPipeline function from BcContainerHelper to perform the actual build (compile, publish, test etc). The Run-AlPipeline function supports overriding functions for creating containers, compiling apps and a lot of other things. @@ -294,142 +312,6 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | - -## Custom jobs in AL-Go for GitHub workflows - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - - -## Custom steps in the _BuildALGoProject workflow - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - - - -## Indirect template repositories - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - - - ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. From 8fe0008be796ddf522fbe97c95c515b778d6530f Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 13 Dec 2023 06:38:34 +0100 Subject: [PATCH 013/215] docs --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- RELEASENOTES.md | 4 +- Scenarios/settings.md | 97 ++++++++++++++++++--- Workshop/Index.md | 7 ++ 4 files changed, 95 insertions(+), 15 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index d8c558184..05fd2b9ab 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -226,7 +226,7 @@ foreach($checkfile in $checkfiles) { Write-Host "Apply customizations from indirect template repository: $srcFile" [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $anchors) } - + $dstFileExists = Test-Path -Path $dstFile -PathType Leaf if ($unusedALGoSystemFiles -contains $fileName) { # file is not used by ALGo, remove it if it exists diff --git a/RELEASENOTES.md b/RELEASENOTES.md index cb81e48d8..1c777dcf3 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -16,12 +16,12 @@ If false, the templateSha repository setting is used to download specific AL-Go ### Add custom jobs to AL-Go workflows It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go system files. +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. ### Add custom steps to the AL-Go build workflow It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go system files. +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. ### Indirect AL-Go template repository Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index bd5764e15..c78b044ab 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -167,7 +167,9 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza # Expert level -The following settings and functionality requires knowledge in GitHub Workflows/Actions and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. +The settings and functionality in the expert section might requires knowledge about GitHub Workflows/Actions, yaml, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. + +Please read the release notes carefully when installing new versions of AL-Go for GitHub. ## Expert settings (rarely used) @@ -263,22 +265,14 @@ Here are the parameters to use in your custom script: | `$parameters.ContinuousDeployment` | Is this environment setup for continuous deployment | false | | `$parameters."runs-on"` | GitHub runner to be used to run the deployment script | windows-latest | - - -## Custom jobs in AL-Go for GitHub workflows - - -## Custom steps in the _BuildALGoProject workflow - - -## Indirect template repositories - ## Run-AlPipeline script override AL-Go for GitHub utilizes the Run-AlPipeline function from BcContainerHelper to perform the actual build (compile, publish, test etc). The Run-AlPipeline function supports overriding functions for creating containers, compiling apps and a lot of other things. This functionality is also available in AL-Go for GitHub, by adding a file to the .AL-Go folder, you automatically override the function. +Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the future might break the usage of these overrides. + | Override | Description | | :-- | :-- | | DockerPull.ps1 | Pull the image specified by the parameter $imageName | @@ -302,6 +296,8 @@ The repo settings file (.github\\AL-Go-Settings.json) can contain BcContainerHel Settings, which might be relevant to set in the settings file includes +Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the future might break the usage of these overrides. + | Setting | Description | Default | | :-- | :-- | :-- | | baseUrl | The Base Url for the online Business Central Web Client. This should be changed when targetting embed apps. | [https://businesscentral.dynamics.com](https://businesscentral.dynamics.com) | @@ -312,11 +308,88 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | + +## Custom jobs in AL-Go for GitHub workflows + +Adding a custom job to any AL-Go for GitHub workflow is done by adding a job with the name `CustomJob` to the end of an AL-Go for GitHub workflow, like this: + +``` + CustomJob-PrepareDeploy: + name: My Job + needs: [ Build ] + runs-on: [ ubuntu-latest ] + defaults: + run: + shell: pwsh + steps: + - name: This is my job + run: | + Write-Host "This is my job" +``` + +In the `needs` property, you specify which jobs should be complete before this job is run. If you require this job to run before other AL-Go for GitHub jobs are complete, you can add the name of this job in the `needs` property of that job, like: + +``` + Deploy: + needs: [ Initialization, Build, **CustomJob-PrepareDeploy** ] + if: always() && needs.Build.result == 'Success' && needs.Initialization.outputs.environmentCount > 0 + strategy: ${{ fromJson(needs.Initialization.outputs.environmentsMatrixJson) }} +``` + +Custom jobs will be preserved when running Update AL-Go System Files. + +**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + + +## Custom steps in the _BuildALGoProject workflow + +Adding a custom step is done by adding a step with the name `CustomStep` to the _BuildALGoProject.yaml workflow at one of these anchor points: +- Before Read Settings +- Before Read Secrets +- Before or After Build +- Before Cleanup + +Example, insert the following step before the Build step: + +``` + - name: CustomStep that will run before the Build step + run: | + Write-Host "before build" + + - name: Build + uses: ... +``` + +Custom steps will be preserved when running Update AL-Go System Files. + +**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + + +## Indirect template repositories + +If you are utilizing script overrides, custom jobs, custom steps, custom delivery or like in many repositories, you might want to take advantage of the indirect template repository feature. + +An indirect template repository is an AL-Go for GitHub repository (without any apps), which is used as a template for the remaining AL-Go for GitHub repositories. As an example, if you are using a custom delivery script, which you want to have in all your repositories, you can create an empty AL-Go for GitHub repository, place the delivery script in the .github folder and use that repository as a template when running Update AL-Go system files in your other repositories. + +This would make sure that all repositories would have this script (and updated versions of the script) in the future. + +The items, which are currently supported from indirect template repositories are: +- Repository script overrides in the .github folder +- Project script overrides in the .AL-Go folder +- Custom workflows in the .github/workflows folder +- Custom jobs in any AL-Go for GitHub workflow +- Custom steps in the _BuildALGoProject workflow +- New repository settings +- New project settings + +**Note** that an AL-Go for GitHub indirect template repository can be private or public. + ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. -**Note:** Our goal is to never break repositories, which are using AL-Go for GitHub as their template. We almost certainly will break you if you create local modifications to scripts and pipelines. +> [!NOTE] +> Our goal is to never break repositories, which are using standard AL-Go for GitHub as their template. We almost certainly will break you if you create local modifications to scripts and pipelines. --- [back](../README.md) diff --git a/Workshop/Index.md b/Workshop/Index.md index 717334675..fbfc8fb44 100644 --- a/Workshop/Index.md +++ b/Workshop/Index.md @@ -21,6 +21,13 @@ This workshop shows you how to take advantage of the functionality, which is pro 1. [The Development Process](TheDevelopmentProcess.md) - *FUTURE TOPIC: The recommended way to work with feature branches, pull requests, code reviews and branch protection rules.* 1. [Keeping your Repository Up-to-date](KeepUpToDate.md) - *FUTURE TOPIC: Updating AL-Go for GitHub to the latest version by running a workflow.* +## Expert level + +1. [Custom Delivery](CustomDelivery.md) - *FUTURE TOPIC: Setting up custom delivery to f.ex. a Teams channel.* +1. [Custom Delivery](CustomDeployment.md) - *FUTURE TOPIC: Setting up custom deployment to f.ex. an on-premises environment.* +1. [Custom Jobs](CustomJobs.md) - *FUTURE TOPIC: Adding a custom job to an AL-Go for GitHub workflows.* +1. [Custom Steps](CustomSteps.md) - *FUTURE TOPIC: Adding a custom step to the BuildALGoProject workflow.* +1. [Using Indirect Templates](IndirectTemplates.md) - *FUTURE TOPIC: Using indirect templates to ensure that all repositories are using the same customizations* ## Additional Future topics 1. Dependencies to other apps From 32b5586c292378a5f60023d937f5fc81557c6535 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 13 Dec 2023 06:39:39 +0100 Subject: [PATCH 014/215] aspit --- Scenarios/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index c78b044ab..f22ebb23c 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -389,7 +389,7 @@ The items, which are currently supported from indirect template repositories are For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. > [!NOTE] -> Our goal is to never break repositories, which are using standard AL-Go for GitHub as their template. We almost certainly will break you if you create local modifications to scripts and pipelines. +> Our goal is to never break repositories, which are using standard AL-Go for GitHub as their template. We almost certainly will break you at some point in time if you create local modifications to scripts and pipelines. --- [back](../README.md) From 2bcc4ae09f921c3df085f60381b676145b4245f5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 13 Dec 2023 06:40:13 +0100 Subject: [PATCH 015/215] remove bold --- Scenarios/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index f22ebb23c..76bdcf284 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -331,7 +331,7 @@ In the `needs` property, you specify which jobs should be complete before this j ``` Deploy: - needs: [ Initialization, Build, **CustomJob-PrepareDeploy** ] + needs: [ Initialization, Build, CustomJob-PrepareDeploy ] if: always() && needs.Build.result == 'Success' && needs.Initialization.outputs.environmentCount > 0 strategy: ${{ fromJson(needs.Initialization.outputs.environmentsMatrixJson) }} ``` From 3c6f7cd1c88124f0ed292768bf725566faf64ba9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 15 Dec 2023 22:16:06 +0100 Subject: [PATCH 016/215] do not use AL-Go name in package name --- Actions/Deliver/Deliver.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/Actions/Deliver/Deliver.ps1 b/Actions/Deliver/Deliver.ps1 index b10aac177..a4ebefddd 100644 --- a/Actions/Deliver/Deliver.ps1 +++ b/Actions/Deliver/Deliver.ps1 @@ -218,9 +218,6 @@ try { Get-Item -Path (Join-Path $folder[0] "*.app") | ForEach-Object { $parameters = @{ "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY" - "includeNuGetDependencies" = $true - "dependencyIdTemplate" = "AL-Go-{id}" - "packageId" = "AL-Go-{id}" } $parameters.appFiles = $_.FullName $package = New-BcNuGetPackage @parameters From 418d40f6786b8c91bffa32d0c4693d704ac31ee4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Jun 2024 19:38:25 +0200 Subject: [PATCH 017/215] pre-commit --- RELEASENOTES.md | 33 ++++++++++++++++++--------------- Scenarios/settings.md | 17 ++++++++++++++--- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 93071e0a6..fa2419b82 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -17,6 +17,24 @@ In the summary after a Test Run, you now also have the result of performance tes > \[!NOTE\] > Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Add custom steps to the AL-Go build workflow + +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Indirect AL-Go template repository + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + ## v5.2 ### Issues @@ -137,21 +155,6 @@ AL-Go ships with Default, Translated and Clean mode out of the box. Now you can Add another parameter when running Update AL-Go System Files, called downloadLatest, used to indicate whether to download latest version from template repository. Default value is true. If false, the templateSha repository setting is used to download specific AL-Go System Files when calculating new files. -### Add custom jobs to AL-Go workflows -It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. - -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Add custom steps to the AL-Go build workflow -It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. - -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Indirect AL-Go template repository -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. - -Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. - ### Issues - Issue 782 Exclude '.altestrunner/' from template .gitignore diff --git a/Scenarios/settings.md b/Scenarios/settings.md index f6fe68d7f..7aac0c392 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -23,6 +23,7 @@ When running a workflow or a local script, the settings are applied by reading s 1. `.AL-Go/.settings.json` is the **user-specific settings file**. This option is rarely used, but if you have special settings, which should only be used for one specific user (potentially in the local scripts), these settings can be added to a settings file with the name of the user followed by `.settings.json`. + ## Basic Project settings | Name | Description | Default value | @@ -78,6 +79,7 @@ The repository settings are only read from the repository settings file (.github | buildModes | A list of build modes to use when building the AL-Go projects. Every AL-Go project will be built using each build mode. AL-Go ships with the following build modes out of the box:
**Default**: Apps are compiled as they are in the source code.
**Clean**: _PreprocessorSymbols_ are enabled when compiling the apps. The values for the symbols correspond to the `cleanModePreprocessorSymbols` setting of the AL-Go project.
**Translated**: `TranslationFile` compiler feature is enabled when compiling the apps.

It is also possible to specify custom build modes by adding a build mode that is different than 'Default', 'Clean' or 'Translated'. | + ## Advanced settings | Name | Description | Default value | @@ -127,6 +129,7 @@ The repository settings are only read from the repository settings file (.github | keyVaultCertificateUrlSecretName
keyVaultCertificatePasswordSecretName
keyVaultClientIdSecretName | If you want to enable KeyVault access for your AppSource App, you need to provide 3 secrets as GitHub Secrets or in the Azure KeyVault. The names of those secrets (**NOT the secrets**) should be specified in the settings file with these 3 settings. Default is to not have KeyVault access from your AppSource App. Read [this](EnableKeyVaultForAppSourceApp.md) for more information. | | + ## Conditional Settings In any of the settings files, you can add conditional settings by using the ConditionalSettings setting. @@ -179,6 +182,7 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza > You can have conditional settings on any level and all conditional settings which has all conditions met will be applied in the order of settings file + appearance. + # Expert level The settings and functionality in the expert section might requires knowledge about GitHub Workflows/Actions, yaml, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. @@ -206,6 +210,7 @@ Please read the release notes carefully when installing new versions of AL-Go fo | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | \[ \] | + ## Custom Delivery You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named DeliverTo\*.ps1 in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. @@ -248,6 +253,7 @@ Here are the parameters to use in your custom script: | `$parameters.dependenciesFolders` | The folders that contain the dependencies of the AL-Go project for all builds (from different build modes) | AllProjects_MyProject-main-Dependencies-1.0.0.0, AllProjects_MyProject-main-CleanDependencies-1.0.0.0 | + ## Custom Deployment You can override existing AL-Go Deployment functionality or you can define your own custom deployment mechanism for AL-Go for GitHub. By specifying a PowerShell script named `DeployTo.ps1` in the .github folder. Default Environment Type is SaaS, but you can define your own type by specifying EnvironmentType in the `DeployTo` setting. The following example will spin up a deployment job to SharePoint on CI/CD and Publish To Environment. @@ -328,6 +334,7 @@ Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the fut | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | + ## Custom jobs in AL-Go for GitHub workflows Adding a custom job to any AL-Go for GitHub workflow is done by adding a job with the name `CustomJob` to the end of an AL-Go for GitHub workflow, like this: @@ -360,9 +367,11 @@ Custom jobs will be preserved when running Update AL-Go System Files. **Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). -## Custom steps in the _BuildALGoProject workflow -Adding a custom step is done by adding a step with the name `CustomStep` to the _BuildALGoProject.yaml workflow at one of these anchor points: +## Custom steps in the \_BuildALGoProject workflow + +Adding a custom step is done by adding a step with the name `CustomStep` to the \_BuildALGoProject.yaml workflow at one of these anchor points: + - Before Read Settings - Before Read Secrets - Before or After Build @@ -384,6 +393,7 @@ Custom steps will be preserved when running Update AL-Go System Files. **Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + ## Indirect template repositories If you are utilizing script overrides, custom jobs, custom steps, custom delivery or like in many repositories, you might want to take advantage of the indirect template repository feature. @@ -393,11 +403,12 @@ An indirect template repository is an AL-Go for GitHub repository (without any a This would make sure that all repositories would have this script (and updated versions of the script) in the future. The items, which are currently supported from indirect template repositories are: + - Repository script overrides in the .github folder - Project script overrides in the .AL-Go folder - Custom workflows in the .github/workflows folder - Custom jobs in any AL-Go for GitHub workflow -- Custom steps in the _BuildALGoProject workflow +- Custom steps in the \_BuildALGoProject workflow - New repository settings - New project settings From 2efb6a5929f8e32c29c989c00cc5956446777ce0 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 24 Jul 2024 12:29:34 +0200 Subject: [PATCH 018/215] back --- Actions/Deliver/Deliver.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Actions/Deliver/Deliver.ps1 b/Actions/Deliver/Deliver.ps1 index 76c3d52fe..0d22b793d 100644 --- a/Actions/Deliver/Deliver.ps1 +++ b/Actions/Deliver/Deliver.ps1 @@ -246,6 +246,9 @@ try { Get-Item -Path (Join-Path $folder[0] "*.app") | ForEach-Object { $parameters = @{ "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY" + "includeNuGetDependencies" = $true + "dependencyIdTemplate" = "AL-Go-{id}" + "packageId" = "AL-Go-{id}" } $parameters.appFiles = $_.FullName $package = New-BcNuGetPackage @parameters From bcc2f3e2a9eae55dba8bf63803dd65d43e34f461 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 11:13:41 +0200 Subject: [PATCH 019/215] add permissions --- Actions/CheckForUpdates/yamlclass.ps1 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 69f1ebb76..331837edb 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -394,6 +394,15 @@ class Yaml { catch { return } + $srcPermissions = $srcYaml.GetPropertyArray('permissions:') + $yamlPermissions = $yaml.GetPropertyArray('permissions:') + if ($srcPermissions) { + $yamlPermissions | ForEach-Object { + if ($srcPermissions -notcontains $_) { + $srcPermissions.Add($_) + } + } + } $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { $fileAnchors = $anchors."$filename" From 2c62b84f167c754aee0abcc6a72a99b221c44742 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 11:24:48 +0200 Subject: [PATCH 020/215] add --- Actions/CheckForUpdates/yamlclass.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 331837edb..537d22401 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -394,8 +394,8 @@ class Yaml { catch { return } - $srcPermissions = $srcYaml.GetPropertyArray('permissions:') - $yamlPermissions = $yaml.GetPropertyArray('permissions:') + $srcPermissions = $srcYaml.Get('permissions:/') + $yamlPermissions = $yaml.Get('permissions:/') if ($srcPermissions) { $yamlPermissions | ForEach-Object { if ($srcPermissions -notcontains $_) { From 496db9104f15ebd713faea7555d706bfa684948b Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 11:29:16 +0200 Subject: [PATCH 021/215] add --- Actions/CheckForUpdates/yamlclass.ps1 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 537d22401..b190113d3 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -225,6 +225,14 @@ class Yaml { } } + # Add lines to Yaml content + [void] Add([string[]] $yamlContent) { + if (!$yamlContent) { + return + } + $this.Insert($this.content.Count, $yamlContent) + } + # Locate jobs in YAML based on a name pattern # Example: # GetCustomJobsFromYaml() returns @("CustomJob1", "CustomJob2") @@ -394,6 +402,7 @@ class Yaml { catch { return } + # Merge permissions $srcPermissions = $srcYaml.Get('permissions:/') $yamlPermissions = $yaml.Get('permissions:/') if ($srcPermissions) { From 4445277b86f72a4cd05442743b02b2856cdfb83d Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 11:48:26 +0200 Subject: [PATCH 022/215] dumps --- Actions/CheckForUpdates/yamlclass.ps1 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index b190113d3..a85f62671 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -403,15 +403,25 @@ class Yaml { return } # Merge permissions + Write-host "Merge permissions" $srcPermissions = $srcYaml.Get('permissions:/') $yamlPermissions = $yaml.Get('permissions:/') + Write-Host "yamlPermissions:" + $yamlPermissions.content | Out-Host + Write-Host "srcPermissions:" + $srcPermissions.content | Out-Host if ($srcPermissions) { - $yamlPermissions | ForEach-Object { - if ($srcPermissions -notcontains $_) { + $yamlPermissions.content | ForEach-Object { + Write-Host $_ + if ($srcPermissions.content -notcontains $_) { + Write-Host "Add permission $_" $srcPermissions.Add($_) } } } + Write-Host "srcPermissions (after):" + $srcPermissions.content | Out-Host + $srcYaml.Replace('permissions:/', $srcPermissions.content) $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { $fileAnchors = $anchors."$filename" From 7ef5d3d186c7f5d5776bad1f3a0944f32d95a588 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:05:02 +0200 Subject: [PATCH 023/215] add merge --- Actions/CheckForUpdates/yamlclass.ps1 | 59 +++++++++++++++++++-------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index a85f62671..52839ad50 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -394,6 +394,44 @@ class Yaml { } } + static [hashtable] GetPermissionsFromArray([string[]] $permissionsArray) { + $permissions = @{} + $permissionsArray | ForEach-Object { + $permissions += @{ $_.Split(':')[0].Trim() = $_.Split(':')[1].Trim() } + } + return $permissions + } + + static [string[]] GetPermissionsArray([hashtable] $permissions) { + $permissionsArray = @() + $permissions.Keys | ForEach-Object { + $permissionsArray += "$($_): $($permissions[$_])" + } + return $permissionsArray + } + + static [hashtable] MergePermissions([hashtable] $permissions, [hashtable] $permissions2) { + $permissions2.Keys | ForEach-Object { + if ($permissions.ContainsKey($_)) { + $permission = $permissions[$_] + $permission2 = $permissions2[$_] + if ($permission -eq 'Write' -or $permission2 -eq 'Write') { + $permissions[$_] = 'Write' + } + elseif ($permission -eq 'Read' -or $permission2 -eq 'Read') { + $permissions[$_] = 'Read' + } + else { + $permissions[$_] = 'None' + } + } + else { + $permissions += @{ $_ = $permissions2[$_] } + } + } + return $permissions + } + static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile, [hashtable] $anchors) { $srcYaml = [Yaml]::new($srcContent.Value.Split("`n")) try { @@ -404,24 +442,9 @@ class Yaml { } # Merge permissions Write-host "Merge permissions" - $srcPermissions = $srcYaml.Get('permissions:/') - $yamlPermissions = $yaml.Get('permissions:/') - Write-Host "yamlPermissions:" - $yamlPermissions.content | Out-Host - Write-Host "srcPermissions:" - $srcPermissions.content | Out-Host - if ($srcPermissions) { - $yamlPermissions.content | ForEach-Object { - Write-Host $_ - if ($srcPermissions.content -notcontains $_) { - Write-Host "Add permission $_" - $srcPermissions.Add($_) - } - } - } - Write-Host "srcPermissions (after):" - $srcPermissions.content | Out-Host - $srcYaml.Replace('permissions:/', $srcPermissions.content) + $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) + $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) + $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { $fileAnchors = $anchors."$filename" From 6e8b95cbd33b109755b1e56537418ce7fd6804d0 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:05:54 +0200 Subject: [PATCH 024/215] merge --- Actions/CheckForUpdates/yamlclass.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 52839ad50..b56bd2ed7 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,6 +445,7 @@ class Yaml { $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + Write-Host "Apply custom steps" $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { $fileAnchors = $anchors."$filename" @@ -457,6 +458,7 @@ class Yaml { } } # Locate custom jobs in destination YAML + Write-Host "Apply custom jobs" $customJobs = @($yaml.GetCustomJobsFromYaml('CustomJob*')) if ($customJobs) { # Add custom jobs to template YAML From 422386d50aab33206368a860084bebf9312d34b8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:14:42 +0200 Subject: [PATCH 025/215] use pso --- Actions/CheckForUpdates/yamlclass.ps1 | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index b56bd2ed7..59cd119b9 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -394,39 +394,39 @@ class Yaml { } } - static [hashtable] GetPermissionsFromArray([string[]] $permissionsArray) { - $permissions = @{} + static [PSCustomObject] GetPermissionsFromArray([string[]] $permissionsArray) { + $permissions = [PSCustomObject]@{} $permissionsArray | ForEach-Object { - $permissions += @{ $_.Split(':')[0].Trim() = $_.Split(':')[1].Trim() } + $permissions | Add-Member -MemberType NoteProperty -Name $_.Split(':')[0].Trim() -Value $_.Split(':')[1].Trim() } return $permissions } - static [string[]] GetPermissionsArray([hashtable] $permissions) { + static [string[]] GetPermissionsArray([PSCustomObject] $permissions) { $permissionsArray = @() - $permissions.Keys | ForEach-Object { + $permissions.PSObject.Properties.Name | ForEach-Object { $permissionsArray += "$($_): $($permissions[$_])" } return $permissionsArray } - static [hashtable] MergePermissions([hashtable] $permissions, [hashtable] $permissions2) { - $permissions2.Keys | ForEach-Object { - if ($permissions.ContainsKey($_)) { + static [PSCustomObject] MergePermissions([PSCustomObject] $permissions, [PSCustomObject] $permissions2) { + $permissions2.PSObject.Properties.Name | ForEach-Object { + if ($permissions.PSObject.Properties.Name -eq $_) { $permission = $permissions[$_] $permission2 = $permissions2[$_] if ($permission -eq 'Write' -or $permission2 -eq 'Write') { - $permissions[$_] = 'Write' + $permissions."$_" = 'Write' } elseif ($permission -eq 'Read' -or $permission2 -eq 'Read') { - $permissions[$_] = 'Read' + $permissions."$_" = 'Read' } else { - $permissions[$_] = 'None' + $permissions."$_" = 'None' } } else { - $permissions += @{ $_ = $permissions2[$_] } + $permissions | Add-Member -MemberType NoteProperty -Name $_ -Value $permissions2[$_] } } return $permissions From 40e20fd6864f8215cf0f7f58dd3321b90d8ff672 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:16:41 +0200 Subject: [PATCH 026/215] use prop --- Actions/CheckForUpdates/yamlclass.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 59cd119b9..7e6fb9dba 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -413,8 +413,8 @@ class Yaml { static [PSCustomObject] MergePermissions([PSCustomObject] $permissions, [PSCustomObject] $permissions2) { $permissions2.PSObject.Properties.Name | ForEach-Object { if ($permissions.PSObject.Properties.Name -eq $_) { - $permission = $permissions[$_] - $permission2 = $permissions2[$_] + $permission = $permissions."$_" + $permission2 = $permissions2."$_" if ($permission -eq 'Write' -or $permission2 -eq 'Write') { $permissions."$_" = 'Write' } From 846626e3e92b4c9d62ccdc40d429f71052042e3c Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:17:05 +0200 Subject: [PATCH 027/215] test --- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index 214e500ad..9b5a2dc6f 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -73,7 +73,7 @@ on: type: string permissions: - contents: read + contents: write actions: read id-token: write From 82361a814f13f775fdab45ebfaf814bc5b00470c Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:18:26 +0200 Subject: [PATCH 028/215] no idx --- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 7e6fb9dba..049c48bc1 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -405,7 +405,7 @@ class Yaml { static [string[]] GetPermissionsArray([PSCustomObject] $permissions) { $permissionsArray = @() $permissions.PSObject.Properties.Name | ForEach-Object { - $permissionsArray += "$($_): $($permissions[$_])" + $permissionsArray += "$($_): $($permissions."$_")" } return $permissionsArray } From 09362823e445f22fd937ac367b7bb309d1e517e9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:20:07 +0200 Subject: [PATCH 029/215] no idx --- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 049c48bc1..f05ccbe38 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -426,7 +426,7 @@ class Yaml { } } else { - $permissions | Add-Member -MemberType NoteProperty -Name $_ -Value $permissions2[$_] + $permissions | Add-Member -MemberType NoteProperty -Name $_ -Value $permissions2."$_" } } return $permissions From b1566937b758c3c36f045bb868e7aab5c3e18601 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:22:42 +0200 Subject: [PATCH 030/215] lower case --- Actions/CheckForUpdates/yamlclass.ps1 | 10 +++++----- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index f05ccbe38..dbce79590 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -415,14 +415,14 @@ class Yaml { if ($permissions.PSObject.Properties.Name -eq $_) { $permission = $permissions."$_" $permission2 = $permissions2."$_" - if ($permission -eq 'Write' -or $permission2 -eq 'Write') { - $permissions."$_" = 'Write' + if ($permission -eq 'write' -or $permission2 -eq 'write') { + $permissions."$_" = 'write' } - elseif ($permission -eq 'Read' -or $permission2 -eq 'Read') { - $permissions."$_" = 'Read' + elseif ($permission -eq 'read' -or $permission2 -eq 'read') { + $permissions."$_" = 'read' } else { - $permissions."$_" = 'None' + $permissions."$_" = 'none' } } else { diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index 9b5a2dc6f..214e500ad 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -73,7 +73,7 @@ on: type: string permissions: - contents: write + contents: read actions: read id-token: write From af146b731832634e88134ee990aca0f3e687b27d Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 18:39:46 +0200 Subject: [PATCH 031/215] add permissions --- Scenarios/Contribute.md | 1 - .../.github/workflows/_BuildPowerPlatformSolution.yaml | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Scenarios/Contribute.md b/Scenarios/Contribute.md index cd385fa8b..8ff23d670 100644 --- a/Scenarios/Contribute.md +++ b/Scenarios/Contribute.md @@ -80,7 +80,6 @@ You can also run the end to end tests directly from VS Code, by providing the fo |$global:E2EgitHubOwner| String | The GitHub owner of the test repositories (like `freddydk` or `microsoft`) | |$global:SecureE2EPAT| SecureString | A personal access token with workflow permissions | |$global:SecureAdminCenterApiToken| SecureString | Admin Center API Credentials | -|$global:SecureLicenseFileUrl| SecureString | Direct download URL to a license file | |$global:pteTemplate| String | URL for your PTE template (like `freddyk/AL-Go-PTE@main` or `freddydk/AL-Go@main\|Templates/Per Tenant Extension` for using your AL-Go fork directly) | |$global:appSourceTemplate| String | URL for your PTE template (like `freddyk/AL-Go-AppSource@main` or `freddydk/AL-Go@main\|Templates/AppSource App` for using your AL-Go fork directly) | diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml index 314365f96..28dfbd066 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml @@ -41,6 +41,10 @@ on: required: false type: string +permissions: + contents: read + actions: read + env: ALGoOrgSettings: ${{ vars.ALGoOrgSettings }} ALGoRepoSettings: ${{ vars.ALGoRepoSettings }} From b9219b78ec33d912ec41c9d325ea791a153e24c3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 27 Jul 2024 09:24:17 +0200 Subject: [PATCH 032/215] updates --- .../CheckForUpdates.HelperFunctions.ps1 | 14 ++++++++++++++ Actions/CheckForUpdates/CheckForUpdates.ps1 | 17 +++-------------- Actions/CheckForUpdates/yamlclass.ps1 | 2 ++ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 560a7be49..466993bea 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -1,3 +1,17 @@ +function GetCustomizationAnchors { + return @{ + "_BuildALGoProject.yaml" = @{ + "BuildALGoProject" = @( + @{ "Step" = 'Read settings'; "Before" = $false } + @{ "Step" = 'Read secrets'; "Before" = $false } + @{ "Step" = 'Build'; "Before" = $true } + @{ "Step" = 'Build'; "Before" = $false } + @{ "Step" = 'Cleanup'; "Before" = $true } + ) + } + } +} + <# .SYNOPSIS Downloads a template repository and returns the path to the downloaded folder diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index a9683250b..1d84a6c40 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -22,18 +22,6 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -$anchors = @{ - "_BuildALGoProject.yaml" = @{ - "BuildALGoProject" = @( - @{ "Step" = 'Read settings'; "Before" = $false } - @{ "Step" = 'Read secrets'; "Before" = $false } - @{ "Step" = 'Build'; "Before" = $true } - @{ "Step" = 'Build'; "Before" = $false } - @{ "Step" = 'Cleanup'; "Before" = $true } - ) - } -} - if ($update -eq 'Y') { if (-not $token) { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." @@ -227,10 +215,11 @@ foreach($checkfile in $checkfiles) { ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } + $customizationAnchors = GetCustomizationAnchors if ($type -eq 'workflow' -and $realSrcFile -ne $srcFile) { # Apply customizations from indirect template repository Write-Host "Apply customizations from indirect template repository: $srcFile" - [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $anchors) + [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $customizationAnchors) } $dstFileExists = Test-Path -Path $dstFile -PathType Leaf @@ -244,7 +233,7 @@ foreach($checkfile in $checkfiles) { elseif ($dstFileExists) { if ($type -eq 'workflow') { Write-Host "Apply customizations from my repository: $dstFile" - [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $anchors) + [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $customizationAnchors) } # file exists, compare and add to $updateFiles if different $dstContent = Get-ContentLF -Path $dstFile diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index dbce79590..f3b405b1a 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,6 +445,8 @@ class Yaml { $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + + # Apply cystom steps Write-Host "Apply custom steps" $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { From dd36934a6f1a589ae41e72e34249957f3b7eb3a4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 27 Jul 2024 09:42:58 +0200 Subject: [PATCH 033/215] content parameter --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- Actions/CreateApp/AppHelper.psm1 | 2 +- Actions/Github-Helper.psm1 | 9 ++++++--- e2eTests/e2eTestHelper.psm1 | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 1d84a6c40..b9b3dd6d8 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -308,7 +308,7 @@ else { } } Write-Host "Update $($_.DstFile)" - $_.Content | Set-ContentLF -Path $_.DstFile + Set-ContentLF -Path $_.DstFile -Content $_.Content } if ($releaseNotes -eq "") { $releaseNotes = "No release notes available!" diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index f3b405b1a..908b20926 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -20,7 +20,7 @@ class Yaml { # Save the Yaml file with LF line endings using UTF8 encoding Save([string] $filename) { - $this.content | Set-ContentLF -Path $filename + Set-ContentLF -Path $filename -Content $this.content } # Find the lines for the specified Yaml path, given by $line diff --git a/Actions/CreateApp/AppHelper.psm1 b/Actions/CreateApp/AppHelper.psm1 index 19221df37..bcb3ad8e6 100644 --- a/Actions/CreateApp/AppHelper.psm1 +++ b/Actions/CreateApp/AppHelper.psm1 @@ -97,7 +97,7 @@ function UpdateALFile $al = $al.Replace("$_", $startId) $startId++ } - $al | Set-ContentLF -Path (Join-Path $destinationFolder $alFileName) + Set-ContentLF -Path (Join-Path $destinationFolder $alFileName) -Content $al } <# diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index c4439d7f4..ecd5f5926 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -715,12 +715,15 @@ function Set-ContentLF { Process { $path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($path) if ($content -is [array]) { - $content = $content -join "`n" + Write-Host "array" + $contentStr = $content -join "`n" + Write-Host $contentStr } else { - $content = "$content".Replace("`r", "") + Write-Host "String" + $contentStr = "$content".Replace("`r", "") } - [System.IO.File]::WriteAllText($path, "$content`n") + [System.IO.File]::WriteAllText($path, "$contentStr`n") } } diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 9900c4958..a79048c9a 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -337,7 +337,7 @@ function CreateNewAppInFolder { $folder = Join-Path $folder $name New-Item -Path $folder -ItemType Directory | Out-Null $appJson | Set-JsonContentLF -Path (Join-Path $folder "app.json") - $al -join "`n" | Set-ContentLF -Path (Join-Path $folder "$name.al") + Set-ContentLF -Path (Join-Path $folder "$name.al") -Content $al $id } From f6c2dfb66673449be31f37ed23081a3cc832f37f Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 27 Jul 2024 09:50:51 +0200 Subject: [PATCH 034/215] use str --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- Actions/CreateApp/AppHelper.psm1 | 2 +- Actions/Github-Helper.psm1 | 14 +++----------- e2eTests/e2eTestHelper.psm1 | 2 +- 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index b9b3dd6d8..1d84a6c40 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -308,7 +308,7 @@ else { } } Write-Host "Update $($_.DstFile)" - Set-ContentLF -Path $_.DstFile -Content $_.Content + $_.Content | Set-ContentLF -Path $_.DstFile } if ($releaseNotes -eq "") { $releaseNotes = "No release notes available!" diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 908b20926..676ac8a4c 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -20,7 +20,7 @@ class Yaml { # Save the Yaml file with LF line endings using UTF8 encoding Save([string] $filename) { - Set-ContentLF -Path $filename -Content $this.content + $this.content -join "`n" | Set-ContentLF -Path $filename } # Find the lines for the specified Yaml path, given by $line diff --git a/Actions/CreateApp/AppHelper.psm1 b/Actions/CreateApp/AppHelper.psm1 index bcb3ad8e6..19221df37 100644 --- a/Actions/CreateApp/AppHelper.psm1 +++ b/Actions/CreateApp/AppHelper.psm1 @@ -97,7 +97,7 @@ function UpdateALFile $al = $al.Replace("$_", $startId) $startId++ } - Set-ContentLF -Path (Join-Path $destinationFolder $alFileName) -Content $al + $al | Set-ContentLF -Path (Join-Path $destinationFolder $alFileName) } <# diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index ecd5f5926..399b8073d 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -709,21 +709,13 @@ function Set-ContentLF { [parameter(mandatory = $true, ValueFromPipeline = $false)] [string] $path, [parameter(mandatory = $true, ValueFromPipeline = $true)] - $content + [string] $content ) Process { $path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($path) - if ($content -is [array]) { - Write-Host "array" - $contentStr = $content -join "`n" - Write-Host $contentStr - } - else { - Write-Host "String" - $contentStr = "$content".Replace("`r", "") - } - [System.IO.File]::WriteAllText($path, "$contentStr`n") + $content = "$content".Replace("`r", "").TrimEnd("`n") + [System.IO.File]::WriteAllText($path, "$content`n") } } diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index a79048c9a..9900c4958 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -337,7 +337,7 @@ function CreateNewAppInFolder { $folder = Join-Path $folder $name New-Item -Path $folder -ItemType Directory | Out-Null $appJson | Set-JsonContentLF -Path (Join-Path $folder "app.json") - Set-ContentLF -Path (Join-Path $folder "$name.al") -Content $al + $al -join "`n" | Set-ContentLF -Path (Join-Path $folder "$name.al") $id } From 2992605e7551c145040857740bf2e63b85478841 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 27 Jul 2024 10:32:50 +0200 Subject: [PATCH 035/215] fix needs --- Actions/CheckForUpdates/yamlclass.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 676ac8a4c..7f463ed47 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -289,7 +289,8 @@ class Yaml { } elseif ($existingJobs -contains $needsthis) { # Add dependency to job - $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) -join ', ') ]") + $needs = @(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) | Where-Object { $_ }) -join ', ' + $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $needs ]") } } $this.content += @('') + @($customJob.content | ForEach-Object { " $_" }) From b3534d50ef7b86560aa6dd64626ca849747dff45 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 30 Jul 2024 09:31:59 +0200 Subject: [PATCH 036/215] add end 2 end test scenario and unit tests --- Tests/CheckForUpdates.Action.Test.ps1 | 29 ++ Tests/CustomizedYamlSnippet.txt | 97 ++++++ .../scenarios/IndirectTemplate/runtest.ps1 | 296 ++++++++++++++++++ 3 files changed, 422 insertions(+) create mode 100644 Tests/CustomizedYamlSnippet.txt create mode 100644 e2eTests/scenarios/IndirectTemplate/runtest.ps1 diff --git a/Tests/CheckForUpdates.Action.Test.ps1 b/Tests/CheckForUpdates.Action.Test.ps1 index ea3cb62b7..6fff8f3a2 100644 --- a/Tests/CheckForUpdates.Action.Test.ps1 +++ b/Tests/CheckForUpdates.Action.Test.ps1 @@ -121,5 +121,34 @@ Describe "CheckForUpdates Action Tests" { $permissionsContent.content[1].Trim() | Should -be 'actions: read' } + It 'Test YamlClass Customizations' { + . (Join-Path $scriptRoot "yamlclass.ps1") + + $customizedYaml = [Yaml]::load((Join-Path $PSScriptRoot 'CustomizedYamlSnippet.txt')) + $yaml = [Yaml]::load((Join-Path $PSScriptRoot 'YamlSnippet.txt')) + + # Get Custom jobs from yaml + $customJobs = $customizedYaml.GetCustomJobsFromYaml('CustomJob*') + $customJobs | Should -Not -BeNullOrEmpty + $customJobs.Count | Should -be 1 + + # Get Custom steps from yaml + $customStep1 = $customizedYaml.GetCustomStepsFromAnchor('Initialization', 'Read settings', $true) + $customStep1 | Should -Not -BeNullOrEmpty + $customStep1.Count | Should -be 2 + + $customStep2 = $customizedYaml.GetCustomStepsFromAnchor('Initialization', 'Read settings', $false) + $customStep2 | Should -Not -BeNullOrEmpty + $customStep2.Count | Should -be 1 + + # Apply Custom jobs and steps to yaml + $yaml.AddCustomJobsToYaml($customJobs) + $yaml.AddCustomStepsToAnchor('Initialization', $customStep1, 'Read settings', $true) + $yaml.AddCustomStepsToAnchor('Initialization', $customStep2, 'Read settings', $false) + + # Check if new yaml content is equal to customized yaml content + ($yaml.content -join "`r`n") | Should -be ($customizedYaml.content -join "`r`n") + } + # Call action } diff --git a/Tests/CustomizedYamlSnippet.txt b/Tests/CustomizedYamlSnippet.txt new file mode 100644 index 000000000..ae1b5ba46 --- /dev/null +++ b/Tests/CustomizedYamlSnippet.txt @@ -0,0 +1,97 @@ +name: 'CI/CD' + +on: + workflow_dispatch: + workflow_run: + workflows: ["Pull Request Handler"] + types: + - completed + push: + paths-ignore: + - '**.md' + - '.github/workflows/*.yaml' + - '!.github/workflows/CICD.yaml' + branches: [ 'main', 'release/*', 'feature/*' ] + +run-name: ${{ fromJson(format('["","Check pull request from {1}/{2}{0} {3}"]',':',github.event.workflow_run.head_repository.owner.login,github.event.workflow_run.head_branch,github.event.workflow_run.display_title))[github.event_name == 'workflow_run'] }} + +permissions: + contents: read + actions: read + pull-requests: write + checks: write + +defaults: + run: + shell: powershell + +env: + workflowDepth: 1 + +jobs: + Initialization: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: [ windows-latest ] + outputs: + telemetryScopeJson: ${{ steps.init.outputs.telemetryScopeJson }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: true + + - name: Initialize the workflow + id: init + uses: microsoft/AL-Go-Actions/WorkflowInitialize@main + with: + shell: powershell + eventId: "DO0091" + + - name: CustomStep-MyStep1 + run: | + Write-Host 'My own step1!' + + - name: CustomStep-MyStepX + run: | + Write-Host 'Some stepX!' + + - name: Read settings + id: ReadSettings + uses: microsoft/AL-Go-Actions/ReadSettings@main + with: + shell: powershell + parentTelemetryScopeJson: ${{ steps.init.outputs.telemetryScopeJson }} + + - name: CustomStep-MyStep2 + run: | + Write-Host 'My own step2!' + + CheckForUpdates: + runs-on: [ windows-latest ] + needs: [ Initialization, CustomJob-MyJob ] + if: github.event_name != 'workflow_run' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read settings + uses: microsoft/AL-Go-Actions/ReadSettings@main + with: + shell: powershell + parentTelemetryScopeJson: ${{ needs.Initialization.outputs.telemetryScopeJson }} + get: templateUrl + + - name: Check for updates to AL-Go system files + uses: microsoft/AL-Go-Actions/CheckForUpdates@main + with: + shell: powershell + parentTelemetryScopeJson: ${{ needs.Initialization.outputs.telemetryScopeJson }} + templateUrl: ${{ env.templateUrl }} + + CustomJob-MyJob: + needs: [ Initialization ] + runs-on: [ windows-latest ] + steps: + - name: MyStep + run: | + Write-Host 'My own job!' diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 new file mode 100644 index 000000000..101d1862a --- /dev/null +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -0,0 +1,296 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidGlobalVars', '', Justification = 'Global vars used for local test execution only.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'All scenario tests have equal parameter set.')] +Param( + [switch] $github, + [switch] $linux, + [string] $githubOwner = $global:E2EgithubOwner, + [string] $repoName = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName()), + [string] $token = ($Global:SecureE2EPAT | Get-PlainText), + [string] $pteTemplate = $global:pteTemplate, + [string] $appSourceTemplate = $global:appSourceTemplate, + [string] $adminCenterApiToken = ($global:SecureAdminCenterApiToken | Get-PlainText) +) + +Write-Host -ForegroundColor Yellow @' +# _____ _ _ _ _______ _ _ +# |_ _| | (_) | | |__ __| | | | | +# | | _ __ __| |_ _ __ ___ ___| |_ | | ___ _ __ ___ _ __ | | __ _| |_ ___ +# | | | '_ \ / _` | | '__/ _ \/ __| __| | |/ _ \ '_ ` _ \| '_ \| |/ _` | __/ _ \ +# _| |_| | | | (_| | | | | __/ (__| |_ | | __/ | | | | | |_) | | (_| | || __/ +# |_____|_| |_|\__,_|_|_| \___|\___|\__| |_|\___|_| |_| |_| .__/|_|\__,_|\__\___| +# | | +# |_| +# This test tests the following scenario: +# +# - Create a new repository based on the PTE template with no apps (this will be the "indirect" template repository) +# - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) +# - Run Update AL-Go System Files in final repo (using indirect repo as template) +# - Add CustomStep to indirect repo +# - Run Update AL-Go System files in indirect repo +# - Validate that custom step is present in indirect repo +# - Run Update AL-Go System files in final repo +# - Validate that custom step is present in final repo +# - Add CustomStep in final repo +# - Run Update AL-Go System files in final repo +# - Validate that both custom steps is present in final repo +# +'@ + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 +$prevLocation = Get-Location + +Remove-Module e2eTestHelper -ErrorAction SilentlyContinue +Import-Module (Join-Path $PSScriptRoot "..\..\e2eTestHelper.psm1") -DisableNameChecking +. (Join-Path -Path $PSScriptRoot -ChildPath '..\..\..\Actions\CheckForUpdates\yamlclass.ps1') +. (Join-Path -Path $PSScriptRoot -ChildPath "..\..\..\Actions\CheckForUpdates\CheckForUpdates.HelperFunctions.ps1") + +$templateRepository = "$githubOwner/$repoName-template" +$repository = "$githubOwner/$repoName" +$branch = "main" + +$template = "https://github.com/$pteTemplate" + +# Login +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository + +# Create tempolate repository +CreateAlGoRepository ` + -github:$github ` + -linux:$linux ` + -template $template ` + -repository $templateRepository ` + -branch $branch +$templateRepoPath = (Get-Location).Path + +Set-Location $prevLocation + +$appName = 'MyApp' +$publisherName = 'Contoso' + +# Create repository +CreateAlGoRepository ` + -github:$github ` + -linux:$linux ` + -template $template ` + -repository $repository ` + -branch $branch ` + -contentScript { + Param([string] $path) + $null = CreateNewAppInFolder -folder $path -name $appName -publisher $publisherName + } +$repoPath = (Get-Location).Path + +# Update AL-Go System Files to uptake UseProjectDependencies setting +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null + +Set-Location $templateRepoPath + +Pull + +# Make modifications to the template repository +$buildALGoProjectWorkflow = Join-Path $templateRepoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty + +# Modify the permissions +$buildYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) + +# Add customization steps +$customizationAnchors = GetCustomizationAnchors +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $customStep = @{ + "Name" = "CustomStep-Template$idx" + "Content" = @( + "- name: CustomStep-Template$idx" + " run: |" + " Write-Host 'CustomStep-Template$idx was here!'" + ) + "AnchorStep" = $anchor.Step + "Before" = $anchor.Before + } + $buildYaml.AddCustomStepsToAnchor('BuildALGoProject', $customStep, $anchor.Step, $anchor.Before) +} +$buildYaml.Save($buildALGoProjectWorkflow) + +# Add Custom Jobs to CICD.yaml +$cicdWorkflow = Join-Path $templateRepoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty +# Modify the permissions +$cicdYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) +$customJobs = @( + @{ + "Name" = "CustomJob-TemplateInit" + "Content" = @( + "CustomJob-TemplateInit:" + " runs-on: [ windows-latest ]" + " steps:" + " - name: Init" + " run: |" + " Write-Host 'CustomJob-TemplateInit was here!'" + ) + "NeedsThis" = @( 'Initialization' ) + } + @{ + "Name" = "CustomJob-TemplateDeploy" + "Content" = @( + "CustomJob-TemplateDeploy:" + " needs: [ Initialization, Build ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: Deploy" + " run: |" + " Write-Host 'CustomJob-TemplateDeploy was here!'" + ) + "NeedsThis" = @( 'PostProcess' ) + } +) +# Add custom Jobs +$cicdYaml.AddCustomJobsToYaml($customJobs) +$cicdYaml.Save($cicdWorkflow) + +# Push +CommitAndPush -commitMessage 'Add template customizations' + +# Do not run workflows on template repository +CancelAllWorkflows -repository $templateRepository + +# Add local customizations to the final repository +Set-Location $repoPath +Pull + +# Make modifications to the template repository +$buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty + +# Add customization steps +$customizationAnchors = GetCustomizationAnchors +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $customStep = @{ + "Name" = "CustomStep-Final$idx" + "Content" = @( + "- name: CustomStep-Final$idx" + " run: |" + " Write-Host 'CustomStep-Final$idx was here!'" + ) + "AnchorStep" = $anchor.Step + "Before" = $anchor.Before + } + $buildYaml.AddCustomStepsToAnchor('BuildALGoProject', $customStep, $anchor.Step, $anchor.Before) +} + +# save +$buildYaml.Save($buildALGoProjectWorkflow) + +# Add Custom Jobs to CICD.yaml +$cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty +# Modify the permissions +$cicdYaml.Replace('permissions:/contents: read', @('contents: read', 'issues: write')) + +$customJobs = @( + @{ + "Name" = "CustomJob-PreDeploy" + "Content" = @( + "CustomJob-PreDeploy:" + " needs: [ Initialization, Build ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: PreDeploy" + " run: |" + " Write-Host 'CustomJob-PreDeploy was here!'" + ) + "NeedsThis" = @( 'Deploy' ) + } + @{ + "Name" = "CustomJob-PostDeploy" + "Content" = @( + "CustomJob-PostDeploy:" + " needs: [ Initialization, Build, Deploy ]" + " if: (!cancelled())" + " runs-on: [ windows-latest ]" + " steps:" + " - name: PostDeploy" + " run: |" + " Write-Host 'CustomJob-PostDeploy was here!'" + ) + "NeedsThis" = @( 'PostProcess' ) + } +) +# Add custom Jobs +$cicdYaml.AddCustomJobsToYaml($customJobs) + +# save +$cicdYaml.Save($cicdWorkflow) + + +# Push +CommitAndPush -commitMessage 'Add final repo customizations' + +# Update AL-Go System Files to uptake UseProjectDependencies setting +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null + +# Stop all currently running workflows and run a new CI/CD workflow +CancelAllWorkflows -repository $repository + +# Pull changes +Pull + +# Run CICD +$run = RunCICD -repository $repository -branch $branch -wait + +# Check Custom Steps +1..$idx | ForEach-Object { + Test-LogContainsFromRun -runid $run.id -jobName 'Build . (Default) . (Default)' -stepName "CustomStep-Template$_" -expectedText "CustomStep-Template$_ was here!" +} +1..$idx | ForEach-Object { + Test-LogContainsFromRun -runid $run.id -jobName 'Build . (Default) . (Default)' -stepName "CustomStep-Final$_" -expectedText "CustomStep-Final$_ was here!" +} + +# Check correct order of custom steps +DownloadWorkflowLog -repository $repository -runid $run.id -path 'logs' +$logcontent = Get-Content -Path 'logs/0_Build . (Default) . (Default).txt' -Encoding utf8 -Raw +Remove-Item -Path 'logs' -Recurse -Force +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $templateStepIdx = $logcontent.IndexOf("CustomStep-Template$idx was here!") + $finalStepIdx = $logcontent.IndexOf("CustomStep-Final$idx was here!") + if ($anchor.Before) { + $finalStepIdx | Should -BeGreaterThan $templateStepIdx -Because "CustomStep-Final$idx should be after CustomStep-Template$idx" + } + else { + $finalStepIdx | Should -BeLessThan $templateStepIdx -Because "CustomStep-Final$idx should be before CustomStep-Template$idx" + } +} + +# Check Custom Jobs +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateInit' -stepName 'Init' -expectedText 'CustomJob-TemplateInit was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateDeploy' -stepName 'Deploy' -expectedText 'CustomJob-TemplateDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PreDeploy' -stepName 'PreDeploy' -expectedText 'CustomJob-PreDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PostDeploy' -stepName 'PostDeploy' -expectedText 'CustomJob-PostDeploy was here!' + +# Check Permissions +# TODO: check issues: write in cicd.yaml (from final) and issues: read in _buildALGoProject.yaml (from template) +$buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty +$buildYaml.get('Permissions:/issues:').content | Should -Be 'issues: read' + +$cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty +$cicdYaml.get('Permissions:/issues:').content | Should -Be 'issues: write' + +Set-Location $prevLocation + +Read-Host "Press Enter to continue" + +RemoveRepository -repository $repository -path $repoPath +RemoveRepository -repository $templateRepository -path $templateRepoPath From 5f34aede27741a15e215072c13576edc30896460 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 30 Jul 2024 09:55:55 +0200 Subject: [PATCH 037/215] precommit --- .../scenarios/IndirectTemplate/runtest.ps1 | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 index 101d1862a..432e5d9ab 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -12,14 +12,14 @@ Param( ) Write-Host -ForegroundColor Yellow @' -# _____ _ _ _ _______ _ _ -# |_ _| | (_) | | |__ __| | | | | -# | | _ __ __| |_ _ __ ___ ___| |_ | | ___ _ __ ___ _ __ | | __ _| |_ ___ +# _____ _ _ _ _______ _ _ +# |_ _| | (_) | | |__ __| | | | | +# | | _ __ __| |_ _ __ ___ ___| |_ | | ___ _ __ ___ _ __ | | __ _| |_ ___ # | | | '_ \ / _` | | '__/ _ \/ __| __| | |/ _ \ '_ ` _ \| '_ \| |/ _` | __/ _ \ # _| |_| | | | (_| | | | | __/ (__| |_ | | __/ | | | | | |_) | | (_| | || __/ # |_____|_| |_|\__,_|_|_| \___|\___|\__| |_|\___|_| |_| |_| .__/|_|\__,_|\__\___| -# | | -# |_| +# | | +# |_| # This test tests the following scenario: # # - Create a new repository based on the PTE template with no apps (this will be the "indirect" template repository) @@ -92,7 +92,7 @@ $buildALGoProjectWorkflow = Join-Path $templateRepoPath '.github/workflows/_Buil $buildYaml = [yaml]::Load($buildALGoProjectWorkflow) $buildYaml | Should -Not -BeNullOrEmpty -# Modify the permissions +# Modify the permissions $buildYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) # Add customization steps @@ -118,7 +118,8 @@ $buildYaml.Save($buildALGoProjectWorkflow) $cicdWorkflow = Join-Path $templateRepoPath '.github/workflows/CICD.yaml' $cicdYaml = [yaml]::Load($cicdWorkflow) $cicdYaml | Should -Not -BeNullOrEmpty -# Modify the permissions + +# Modify the permissions $cicdYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) $customJobs = @( @{ @@ -191,7 +192,8 @@ $buildYaml.Save($buildALGoProjectWorkflow) $cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' $cicdYaml = [yaml]::Load($cicdWorkflow) $cicdYaml | Should -Not -BeNullOrEmpty -# Modify the permissions + +# Modify the permissions $cicdYaml.Replace('permissions:/contents: read', @('contents: read', 'issues: write')) $customJobs = @( From f19dda62f97766e18d8f77b4a6adff358c04f39e Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 7 Aug 2024 22:04:58 +0200 Subject: [PATCH 038/215] add pipeline init and finalize --- Actions/AL-Go-Helper.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 3c7fbdc03..d34ae33c5 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -18,7 +18,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent -$defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" +$defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/archive/refs/heads/runpipeline.zip" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" $notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName") $runAlPipelineOverrides = @( @@ -38,6 +38,8 @@ $runAlPipelineOverrides = @( "InstallMissingDependencies" "PreCompileApp" "PostCompileApp" + "PipelineInitialize" + "PipelineFinalize" ) # Well known AppIds From a2e7b2b642e34ab2bf1f2894d07304be071c3cdb Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 8 Aug 2024 13:24:51 +0200 Subject: [PATCH 039/215] support cicdauthcontext --- Actions/AL-Go-Helper.ps1 | 2 +- Actions/RunPipeline/RunPipeline.ps1 | 15 ++++++++++++++- .../.github/workflows/_BuildALGoProject.yaml | 2 +- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index d34ae33c5..1c1bc0d48 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -19,7 +19,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent $defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/archive/refs/heads/runpipeline.zip" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" -$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName") +$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","Environment") $runAlPipelineOverrides = @( "DockerPull" diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index b102a3a91..c76d943f0 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -80,7 +80,7 @@ try { $appBuild = $settings.appBuild $appRevision = $settings.appRevision - 'licenseFileUrl','codeSignCertificateUrl','*codeSignCertificatePassword','keyVaultCertificateUrl','*keyVaultCertificatePassword','keyVaultClientId','gitHubPackagesContext','applicationInsightsConnectionString' | ForEach-Object { + 'licenseFileUrl','codeSignCertificateUrl','*codeSignCertificatePassword','keyVaultCertificateUrl','*keyVaultCertificatePassword','keyVaultClientId','gitHubPackagesContext','applicationInsightsConnectionString','cicdAuthContext' | ForEach-Object { # Secrets might not be read during Pull Request runs if ($secrets.Keys -contains $_) { $value = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$_")) @@ -186,6 +186,19 @@ try { } $authContext = $null $environmentName = "" + if ($cicdAuthContext) { + $authContext = $cicdAuthContext | ConvertFrom-Json | ConvertTo-HashTable + if ($authContext.ContainsKey('environmentName')) { + $environmentName = $authContext.environmentName + $authContext.Remove('environmentName') + if ($environmentName -notlike 'https://*') { + $authContext = New-BcAuthContext @authContext + } + } + else { + Write-Host "::WARNING::CI/CD AuthContext is missing environmentName, ignoring cicdAuthContext secret." + } + } $CreateRuntimePackages = $false if ($settings.versioningStrategy -eq -1) { diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index cafa9d978..a875964ff 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -106,7 +106,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS' + getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS,cicdAuthContext' - name: Determine ArtifactUrl uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index cafa9d978..a875964ff 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -106,7 +106,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS' + getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS,cicdAuthContext' - name: Determine ArtifactUrl uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main From 91041b11b0c27f7dbfae57b341231a667bd3db02 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 8 Aug 2024 13:34:27 +0200 Subject: [PATCH 040/215] env name --- Actions/AL-Go-Helper.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 1c1bc0d48..3a02f9ef7 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -19,7 +19,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent $defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/archive/refs/heads/runpipeline.zip" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" -$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","Environment") +$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","EnvironmentName") $runAlPipelineOverrides = @( "DockerPull" From 492cfb5ddb5897b061d3475a13f65292067fe981 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 8 Aug 2024 22:22:01 +0200 Subject: [PATCH 041/215] dump --- Actions/RunPipeline/RunPipeline.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index c76d943f0..9be100f21 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -193,6 +193,7 @@ try { $authContext.Remove('environmentName') if ($environmentName -notlike 'https://*') { $authContext = New-BcAuthContext @authContext + Write-Host $authContext.accesstoken } } else { From 699ff4d562f1dccd652dca7c43e80fa7452cc517 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 9 Aug 2024 06:23:08 +0200 Subject: [PATCH 042/215] remove dump --- Actions/RunPipeline/RunPipeline.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index 9be100f21..c76d943f0 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -193,7 +193,6 @@ try { $authContext.Remove('environmentName') if ($environmentName -notlike 'https://*') { $authContext = New-BcAuthContext @authContext - Write-Host $authContext.accesstoken } } else { From b2e47b1e7dc7f49249374d9a0255ab0356e224ef Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 9 Aug 2024 14:23:07 +0200 Subject: [PATCH 043/215] fix tests --- Tests/CustomizedYamlSnippet.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Tests/CustomizedYamlSnippet.txt b/Tests/CustomizedYamlSnippet.txt index ae1b5ba46..0ccb32cb5 100644 --- a/Tests/CustomizedYamlSnippet.txt +++ b/Tests/CustomizedYamlSnippet.txt @@ -45,7 +45,6 @@ jobs: uses: microsoft/AL-Go-Actions/WorkflowInitialize@main with: shell: powershell - eventId: "DO0091" - name: CustomStep-MyStep1 run: | @@ -60,7 +59,6 @@ jobs: uses: microsoft/AL-Go-Actions/ReadSettings@main with: shell: powershell - parentTelemetryScopeJson: ${{ steps.init.outputs.telemetryScopeJson }} - name: CustomStep-MyStep2 run: | @@ -78,14 +76,12 @@ jobs: uses: microsoft/AL-Go-Actions/ReadSettings@main with: shell: powershell - parentTelemetryScopeJson: ${{ needs.Initialization.outputs.telemetryScopeJson }} get: templateUrl - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - parentTelemetryScopeJson: ${{ needs.Initialization.outputs.telemetryScopeJson }} templateUrl: ${{ env.templateUrl }} CustomJob-MyJob: From 1cc3a947178e3c4f271500dceb18ab69f3597c7a Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 06:52:02 +0200 Subject: [PATCH 044/215] fix aldoc if --- Templates/AppSource App/.github/workflows/CICD.yaml | 2 +- Templates/Per Tenant Extension/.github/workflows/CICD.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index d57c5c899..c6f19ed66 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -166,7 +166,7 @@ jobs: DeployALDoc: needs: [ Initialization, Build ] - if: (!cancelled()) && needs.Build.result == 'Success' && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' + if: (!cancelled()) && (needs.Build.result == 'success' || needs.Build.result == 'skipped') && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' runs-on: [ windows-latest ] name: Deploy Reference Documentation permissions: diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 3d9933ee4..0dc128768 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -180,7 +180,7 @@ jobs: DeployALDoc: needs: [ Initialization, Build ] - if: (!cancelled()) && needs.Build.result == 'Success' && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' + if: (!cancelled()) && (needs.Build.result == 'success' || needs.Build.result == 'skipped') && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' runs-on: [ windows-latest ] name: Deploy Reference Documentation permissions: From b48f6753ddb1c05536234fdf461409087a831f95 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:19:38 +0200 Subject: [PATCH 045/215] Replace template owner --- Internal/Deploy.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 62eba2435..678dc1033 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -252,6 +252,12 @@ try { } Set-Content -Path (Join-Path "./.github" "RELEASENOTES.copy.md") -Value $releaseNotes -Encoding utf8 } + # Replace template_owner in README.md + $readmeFile = Join-Path $baseRepoPath "README.md" + $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" + $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") + Set-Content -Path $readmeFile -Encoding utf8 -Value $readme + # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit } From 7c986d9f5408e8a417f597477a788e1cab02f646 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:24:05 +0200 Subject: [PATCH 046/215] remove jekyll --- .github/workflows/jekyll-gh-pages.yml | 49 --------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml deleted file mode 100644 index f5916b31d..000000000 --- a/.github/workflows/jekyll-gh-pages.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll with GitHub Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@b178f9334b208360999a0a57b523613563698c66 # v1.0.12 - with: - source: . - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 - with: - preview: true From 39023cfe63e539cc6034b1dc507a6ede89ced238 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:29:40 +0200 Subject: [PATCH 047/215] modify readme --- Internal/Deploy.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 678dc1033..c7b938acf 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -116,7 +116,7 @@ try { "appSourceAppRepo" = "$($config.githubOwner)/$($config.appSourceAppRepo)" } - if ($config.branch -eq 'preview') { + if ($config.branch -eq 'preview' -or $config.githubOwner -ne 'microsoft') { # When deploying to preview, we are NOT going to deploy to a branch in the AL-Go-Actions repository # Instead, we are going to have AL-Go-PTE and AL-Go-AppSource point directly to the SHA in AL-Go $dstOwnerAndRepo += @{ @@ -252,11 +252,13 @@ try { } Set-Content -Path (Join-Path "./.github" "RELEASENOTES.copy.md") -Value $releaseNotes -Encoding utf8 } + # Replace template_owner in README.md - $readmeFile = Join-Path $baseRepoPath "README.md" + $readmeFile = './README.md' $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") Set-Content -Path $readmeFile -Encoding utf8 -Value $readme + # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit From 0c69b29608012ffb393159f584652ba810072d4e Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:33:00 +0200 Subject: [PATCH 048/215] dump readme --- Internal/Deploy.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index c7b938acf..e60f5533e 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -256,8 +256,10 @@ try { # Replace template_owner in README.md $readmeFile = './README.md' $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" + Write-Host $readme $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") Set-Content -Path $readmeFile -Encoding utf8 -Value $readme + Write-Host $readme # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit From 821aa813402d94deaa8d84d71963e1e461e30328 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:37:09 +0200 Subject: [PATCH 049/215] dump --- Internal/Deploy.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index e60f5533e..6c5f371a7 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -256,9 +256,11 @@ try { # Replace template_owner in README.md $readmeFile = './README.md' $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" + Write-Host "------------------------------------------" Write-Host $readme $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") Set-Content -Path $readmeFile -Encoding utf8 -Value $readme + Write-Host "==========================================" Write-Host $readme # Push changes From 525048b6aab1204cdd927f844212acd0aea1b04c Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:39:38 +0200 Subject: [PATCH 050/215] replace --- Internal/Deploy.ps1 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 6c5f371a7..122cab4ee 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -256,12 +256,8 @@ try { # Replace template_owner in README.md $readmeFile = './README.md' $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" - Write-Host "------------------------------------------" - Write-Host $readme - $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") + $readme = $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") Set-Content -Path $readmeFile -Encoding utf8 -Value $readme - Write-Host "==========================================" - Write-Host $readme # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit From 1066788650f7e47cc3b26b25a459df8d0b794550 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 10:39:32 +0200 Subject: [PATCH 051/215] merge if permissions are present --- Actions/CheckForUpdates/yamlclass.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 7f463ed47..d58f41fd0 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,7 +445,9 @@ class Yaml { Write-host "Merge permissions" $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) - $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + if ($srcPermissions -and $yamlPermissions) { + $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + } # Apply cystom steps Write-Host "Apply custom steps" From 82585722f4245092e9b6623a82c04739819e1a8a Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 10:58:27 +0200 Subject: [PATCH 052/215] check --- Actions/CheckForUpdates/yamlclass.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index d58f41fd0..c171b6d83 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,6 +445,8 @@ class Yaml { Write-host "Merge permissions" $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) + Write-Host "'$srcPermissions'" + Write-Host "'$yamlPermissions'" if ($srcPermissions -and $yamlPermissions) { $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) } From 32e7f1bae09765c4bcc53a835a865e1e72ddc0db Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 11:07:20 +0200 Subject: [PATCH 053/215] check not '' --- Actions/CheckForUpdates/yamlclass.ps1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index c171b6d83..d8ae1016d 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,9 +445,7 @@ class Yaml { Write-host "Merge permissions" $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) - Write-Host "'$srcPermissions'" - Write-Host "'$yamlPermissions'" - if ($srcPermissions -and $yamlPermissions) { + if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) } From ddd95b158bca22906256f9534e46d559246d4ddc Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 11:22:23 +0200 Subject: [PATCH 054/215] add dump --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 1d84a6c40..6c84d6814 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -222,7 +222,12 @@ foreach($checkfile in $checkfiles) { [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $customizationAnchors) } + Write-Host "Checking $dstFile" $dstFileExists = Test-Path -Path $dstFile -PathType Leaf + Write-Host "DstFileExists: $dstFileExists" + Write-Host "Filename: $fileName" + Write-Host "DstPath: $dstPath" + $unusedALGoSystemFiles | Out-Host if ($unusedALGoSystemFiles -contains $fileName) { # file is not used by ALGo, remove it if it exists # do not add it to $updateFiles if it does not exist From 96818d74acc0ec40d5a8bafc2b4500d756268037 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 11:35:01 +0200 Subject: [PATCH 055/215] remove unused files not present in template --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 6c84d6814..42dcdc670 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -178,6 +178,14 @@ foreach($checkfile in $checkfiles) { UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -additionalSettings $indirectTemplateProjectSettings $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } } + + $unusedALGoSystemFiles | ForEach-Object { + if (Test-Path -Path (Join-Path $dstFolder $_) -PathType Leaf) { + Write-Host "Remove unused AL-Go system file: $_" + $removeFiles += @(Join-Path $dstPath $_) + } + } + # Loop through all files in the template repository matching the pattern Get-ChildItem -Path $srcFolder -Filter $checkfile.pattern | ForEach-Object { # Read the template file and modify it based on the settings From c97f705fb9e52a022bcdf5b3a222ecc65e9f1927 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 11:42:43 +0200 Subject: [PATCH 056/215] move unused check --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 42 ++++++++------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 42dcdc670..9d734f475 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -179,6 +179,7 @@ foreach($checkfile in $checkfiles) { $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } } + # Remove unused AL-Go system files $unusedALGoSystemFiles | ForEach-Object { if (Test-Path -Path (Join-Path $dstFolder $_) -PathType Leaf) { Write-Host "Remove unused AL-Go system file: $_" @@ -230,36 +231,25 @@ foreach($checkfile in $checkfiles) { [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $customizationAnchors) } - Write-Host "Checking $dstFile" - $dstFileExists = Test-Path -Path $dstFile -PathType Leaf - Write-Host "DstFileExists: $dstFileExists" - Write-Host "Filename: $fileName" - Write-Host "DstPath: $dstPath" - $unusedALGoSystemFiles | Out-Host - if ($unusedALGoSystemFiles -contains $fileName) { - # file is not used by ALGo, remove it if it exists - # do not add it to $updateFiles if it does not exist - if ($dstFileExists) { - $removeFiles += @(Join-Path $dstPath $filename) + if ($unusedALGoSystemFiles -notcontains $fileName) { + if (Test-Path -Path $dstFile -PathType Leaf) { + if ($type -eq 'workflow') { + Write-Host "Apply customizations from my repository: $dstFile" + [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $customizationAnchors) + } + # file exists, compare and add to $updateFiles if different + $dstContent = Get-ContentLF -Path $dstFile + if ($dstContent -cne $srcContent) { + Write-Host "Updated $type ($(Join-Path $dstPath $filename)) available" + $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } + } } - } - elseif ($dstFileExists) { - if ($type -eq 'workflow') { - Write-Host "Apply customizations from my repository: $dstFile" - [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $customizationAnchors) - } - # file exists, compare and add to $updateFiles if different - $dstContent = Get-ContentLF -Path $dstFile - if ($dstContent -cne $srcContent) { - Write-Host "Updated $type ($(Join-Path $dstPath $filename)) available" + else { + # new file, add to $updateFiles + Write-Host "New $type ($(Join-Path $dstPath $filename)) available" $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } } } - else { - # new file, add to $updateFiles - Write-Host "New $type ($(Join-Path $dstPath $filename)) available" - $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } - } } } finally { From 01b6d49b904f0b54eacbfd5d0d9759398bcdb17d Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 18 Aug 2024 21:32:50 +0200 Subject: [PATCH 057/215] add customALGoSystemFiles --- Actions/AL-Go-Helper.ps1 | 1 + .../CheckForUpdates.HelperFunctions.ps1 | 96 ++++++ Actions/CheckForUpdates/CheckForUpdates.ps1 | 3 + README.md | 1 + RELEASENOTES.md | 7 + Scenarios/Contribute.md | 17 +- Scenarios/CustomizingALGoForGitHub.md | 304 ++++++++++++++++++ Scenarios/settings.md | 10 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 4 +- 9 files changed, 429 insertions(+), 14 deletions(-) create mode 100644 Scenarios/CustomizingALGoForGitHub.md diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 3a02f9ef7..73c731ad9 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -554,6 +554,7 @@ function ReadSettings { $settings = [ordered]@{ "type" = "PTE" "unusedALGoSystemFiles" = @() + "customALGoSystemFiles" = @() "projects" = @() "powerPlatformSolutionFolder" = "" "country" = "us" diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 466993bea..ead9cc0fb 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -463,3 +463,99 @@ function UpdateSettingsFile { # Save the file with LF line endings and UTF8 encoding $settings | Set-JsonContentLF -path $settingsFile } + +function ApplyCustomALGoSystemFiles { + Param( + [string] $path, + [hashtable] $settings + ) + + Push-Location -Path $path + try { + if ($settings.customALGoSystemFiles -isnot [Array]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + foreach($customspec in $settings.customALGoSystemFiles) { + if ($customspec -isnot [Hashtable]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." + } + $source = $customspec.Source + $destination = $customSpec.Destination + if ($source -isnot [string] -or $destination -isnot [string]) { + throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be strings. See https://aka.ms/algosettings#customalgosystemfiles." + } + + $finalDestination = Join-Path $path $destination + $ext = [System.IO.Path]::GetExtension($source) + if ($ext -eq '.zip') { + Write-Host $destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) + if ($customSpec.ContainsKey('FileSpec')) { $fileSpec = $customSpec.FileSpec } else { $fileSpec = '*' } + if ($customSpec.ContainsKey('Recurse')) { $recurse = $customSpec.Recurse } else { $recurse = $true } + if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { + throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (Test-Path $finalDestination -PathType Leaf) { + throw "A file with that name already exists - destination must point to a folder when adding custom AL-Go System Files from a .zip file. See https://aka.ms/algosettings#customalgosystemfiles." + } + $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + New-Item -Path $tempFolder -ItemType Directory | Out-Null + $zipName = "$tempFolder$ext" + try { + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName + Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force + $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve + Push-Location -Path $subFolder + try { + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFinalFileName = Join-Path $finalDestination $destRelativeFileName + $destFinalFolder = [System.IO.Path]::GetDirectoryName($destFinalFileName) + if (Test-Path -Path $destFinalFolder -PathType Leaf) { + throw "A file ($destFinalFolder) already exists, where a folder is expected when applying CustomALGoSystemFiles." + } + if (!(Test-Path -Path $destFinalFolder)) { + New-Item -path $destFinalFolder -ItemType Directory | Out-Null + } + Write-Host "- $($destRelativeFileName.Substring(2))" + Copy-Item -Path $_.FullName -Destination $destFinalFileName + } + } + finally { + Pop-Location + } + } + finally { + if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } + Remove-Item -Path $tempFolder -Recurse -Force + } + } + else { + if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { + throw "customALGoSystemFiles setting is wrongly formatted, FileSpec and Recurse are only allowed with .zip files. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (Test-Path -Path $finalDestination -PathType Container) { + $finalDestination = Join-Path $finalDestination ([System.IO.Path]::GetFileName($source)) + } + else { + if ($destination.endsWith('\') -or $destination.endsWith('/')) { + New-Item -Path $finalDestination -ItemType Directory | Out-Null + $finalDestination = Join-Path $finalDestination ([System.IO.Path]::GetFileName($source)) + } + } + $overridingStr = '' + if (Test-Path -Path $finalDestination -PathType Leaf) { + $overridingStr = ' (Overriding existing file)' + } + $relativeFolder = Resolve-Path -Path ([System.IO.Path]::GetDirectoryName($finalDestination)) -Relative + Write-Host "$(Join-Path $relativeFolder ([System.IO.Path]::GetFileName($finalDestination)))$overridingStr" + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $finalDestination + } + } + } + finally { + Pop-Location + } +} \ No newline at end of file diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 9d734f475..ff4c367c6 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -116,6 +116,9 @@ if (-not $isDirectALGo) { } } +# Apply Custom AL-Go System Files from settings +ApplyCustomALGoSystemFiles -path $templateFolder -settings $repoSettings + # CheckFiles is an array of hashtables with the following properties: # dstPath: The path to the file in the current repository # srcPath: The path to the file in the template repository diff --git a/README.md b/README.md index 2ff4b733e..d67e0bf67 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Try out the [AL-Go workshop](https://aka.ms/algoworkshop) for an in-depth worksh 1. [Connect your GitHub repository to Power Platform](Scenarios/SetupPowerPlatform.md) 1. [How to set up Service Principal for Power Platform](Scenarios/SetupServicePrincipalForPowerPlatform.md) 1. [Try one of the Business Central and Power Platform samples](Scenarios/TryPowerPlatformSamples.md) +1. [Customizing AL-Go for GitHub](Scenarios/CustomizingALGoForGitHub.md) ## Migration scenarios diff --git a/RELEASENOTES.md b/RELEASENOTES.md index c88af836a..9abaac27e 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -47,6 +47,13 @@ AL-Go for GitHub now includes a new telemetry module. For detailed information o - **NumberOfSqlStmtsWarning** - a warning is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 5) - **NumberOfSqlStmtsError** - an error is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 10) +- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: + + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) + - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + > \[!NOTE\] > Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. diff --git a/Scenarios/Contribute.md b/Scenarios/Contribute.md index 8ff23d670..0bca431d8 100644 --- a/Scenarios/Contribute.md +++ b/Scenarios/Contribute.md @@ -5,16 +5,16 @@ This section describes how to contribute to AL-Go. How to set up your own enviro You can do this in two ways: - Use a fork of AL-Go for GitHub in your own **personal GitHub account** in development mode -- Use 3 public repositories in your own **personal GitHub account** (AL-Go-PTE, AL-Go-AppSource and AL-Go-Actions, much like in production) +- Use 2 public repositories in your own **personal GitHub account** (AL-Go-PTE, AL-Go-AppSource and AL-Go-Actions, much like in production) ## Use a fork of AL-Go for GitHub in "development mode" 1. Fork the [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go) repository to your **personal GitHub account**. 1. You can optionally also create a branch in the AL-Go fork for the feature you are working on. -**https://github.com//AL-Go@** can now be used as a template in your AL-Go project when running _Update AL-Go System Files_ to use the actions/workflows from this fork. +**/AL-Go@** can now be used as a template in your AL-Go project when running _Update AL-Go System Files_ to use the actions/workflows from this fork. -## Use 3 public repositories in "production mode" +## Use 2 public repositories in "production mode" 1. Fork the [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go) repository to your **personal GitHub account**. 1. Navigate to [https://github.com/settings/tokens/new](https://github.com/settings/tokens/new) and create a new personal access token with **Full control of private repositories** and **workflow** permissions. @@ -22,15 +22,11 @@ You can do this in two ways: 1. In your personal fork of AL-Go, navigate to **Actions**, select the **Deploy** workflow and choose **Run Workflow**. 1. Using the default settings press **Run workflow**. Select the AL-Go branch to run from and the branch to deploy to. -Now you should have 3 new public repositories: +Now you should have 2 new public repositories: -- [https://github.com/yourGitHubUserName/AL-Go-Actions](https://github.com/yourGitHubUserName/AL-Go-Actions) - [https://github.com/yourGitHubUserName/AL-Go-AppSource](https://github.com/yourGitHubUserName/AL-Go-AppSource) - [https://github.com/yourGitHubUserName/AL-Go-PTE](https://github.com/yourGitHubUserName/AL-Go-PTE) -> \[!NOTE\] -> Deploying to a branch called **preview** will only update the two template repositories (and use your AL-Go project with the current SHA as actions repository). - You can optionally also create a branch in the AL-Go fork for the feature you are working on and then select that branch when running Deploy (both as **Use workflow from** and as **Branch to deploy to**). **yourGitHubUserName/AL-Go-PTE@yourBranch** or **yourGitHubUserName/AL-Go-AppSource@yourBranch** can now be used in your AL project when running Update AL-Go System Files to use the actions/workflows from this area for your AL project. @@ -38,7 +34,7 @@ You can optionally also create a branch in the AL-Go fork for the feature you ar Please ensure that all unit tests run and create a Pull Request against [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go). You are very welcome to run the end to end tests as well, but we will also run the end to end tests as part of the code review process. > \[!NOTE\] -> You can also deploy to a different branch in the 3 public repositories by specifying a branch name under **Branch to deploy** to when running the **Deploy** workflow. The branch you specify in **Use workflow from** indicates which branch in **your personal fork of the AL-Go repository** you publish to the 3 repositories. +> You can also deploy to a different branch in the 2 public repositories by specifying a branch name under **Branch to deploy** to when running the **Deploy** workflow. The branch you specify in **Use workflow from** indicates which branch in **your personal fork of the AL-Go repository** you publish to the 2 repositories. ## Pre-Commit @@ -57,9 +53,8 @@ In the e2eTests folder, in the AL-Go repository, there are 3 types of end to end - The scenarios folder contains a set of AL-Go scenarios, which tests specific functionality end to end. Every folder under e2eTests/scenarios, which contains a runtests.ps1 will be run as a scenario test, like: - UseProjectDependencies - create a repo with multiple projects and set **UseProjectDependencies** to modify CI/CD and other build workflows to build projects in the right order - GitHubPackages - create 3 repositories using GitHub Packages as dependency resolver and check that artifacts are built properly - - BuildModes - create a repository, set buildModes and test that generated artifacts are as expected. - - ReleaseBranches - testing that create release works, release branches are create and subsequently found correctly as previous build - SpecialCharacters - testing that various settings (+ publisher name and app name) can contain special national characters + - and more... In your personal fork, you can now run the end to end tests, if the following pre-requisites are available: diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md new file mode 100644 index 000000000..3867f89be --- /dev/null +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -0,0 +1,304 @@ +# Customizing AL-Go for GitHub + +AL-Go for GitHub is a plug-and-play DevOps solution, intended to support 100% of the functionality needed by 90% of the people developing applications for Microsoft Dynamics 365 Business Central out-of-the-box. + +If AL-Go functionality out-of-the-box doesn't match your needs, you really three options: + +1. Customize AL-Go for GitHub to fit your needs +1. Select another managed DevOps solution +1. Create your own DevOps solution from scratch (not recommended) + +Creating your own DevOps solution from scratch requires dedicated resources to develop and maintain workflows, processes etc. **This is not a small task**. There are many moving parts in a DevOps solution, which might require you to make changes to workflows and scripts over time and stay secure and having to maintain many repositories is tedious and time consuming, even when using templates and other advanced features. + +Microsoft will continuously develop and maintain AL-Go for GitHub and ensure that we always use the latest versions of GitHub actions, which are under our control. Microsoft will never add dependencies to any third party GitHub action, which are not under our control. + +Keeping your repositories up-to-date can be done manually or on a schedule (like Windows update really). You will be notified when an update is available and we recommend that you keep your repositories up-to-date at all time. If you make modifications to the AL-Go System Files (scripts and workflows) in your repository, in other ways than described in this document, these changes will be removed with the next AL-Go update. + +> \[!TIP] +> If for some reason the updated version of AL-Go for GitHub doesn't work for you, we recommend that you file an issue [here](https://github.com/microsoft/AL-Go/issues) with a detailed description of the problem and full logs of the failing workflows. You can then revert back to the prior version of AL-Go for GitHub until the issue is resolved. +> +> It is important to get back to the mainstream version of AL-Go for GitHub as soon as the issue is resolved. + +There are three ways you can customize AL-Go for GitHub to fit your needs. You can + +1. customize the repository with custom scripts, workflows, jobs or steps following the guidelines below +1. create a customized repository and use this as your template repository (indirect template) +1. fork the AL-Go for GitHub and create your "own" version + +> \[!CAUTION\] +> The more you customize AL-Go for GitHub, the more likely you are to be broken by future updates to AL-Go for GitHub, meaning that you will have to update your customizations to match the changes in AL-Go for GitHub. + +## Customizing your repository + +There are several ways you can customize your AL-Go repository and ensure that the changes you make, will survive an update of AL-Go for GitHub. + +### Hide/Remove unused workflows + +By adding a setting called [`unusedALGoSystemFiles`](https://aka.ms/algosettings#unusedalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these system files are not used. Example: + +``` + "unusedALGoSystemFiles": [ + "AddExistingAppOrTestApp.yaml", + "CreateApp.yaml", + "CreatePerformanceTestApp.yaml", + "CreateTestApp.yaml", + "cloudDevEnv.ps1" + ] +``` + +This setting will cause AL-Go for GitHub to remove these files during the next update. Note that if you remove files like `_BuildALGoProject.yaml`, AL-Go will obviously stop working as intended - so please use with care. + +### Custom delivery + +You can setup [custom delivery](https://aka.ms/algosettings#customdelivery) in order to deliver your apps to locations not supported by AL-Go for GitHub out-of-the-box, by adding a custom delivery powershell script (named `.github/DeliverTo.ps1`) and a context secret (called `Context`) formatted as compressed json, you can define the delivery functionality as you like. Example: + +```powershell +Param([Hashtable] $parameters) + +Get-ChildItem -Path $parameters.appsFolder | Out-Host +$context = $parameters.context | ConvertFrom-Json +Write-Host "Token Length: $($context.Token.Length)" +``` + +In this example the context secret is assumed to contain a Token property. Read [this](https://aka.ms/algosettings#customdelivery) for more information. + +### Custom deployment + +You can setup [custom deployment](https://aka.ms/algosettings#customdeployment) to environment types not supported by AL-Go for GitHub out-of-the-box. You can also override deployment functionality to environment Type `SaaS` if you like. You can add an environment called `` and a `DeployTo` setting, defining which environment Type should be used. Example: + +```json + "Environments": [ + "" + ], + "DeployTo": { + "EnvironmentType": "" + } +``` + +You also need to create an AuthContext secret (called `_AuthContext`) and a powershell script (named `.github/DeployTo.ps1`), which defines the deployment functionality. Example: + +```powershell +Param([Hashtable] $parameters) + +$parameters | ConvertTo-Json -Depth 99 | Out-Host +$tempPath = Join-Path ([System.IO.Path]::GetTempPath()) ([GUID]::NewGuid().ToString()) +New-Item -ItemType Directory -Path $tempPath | Out-Null +Copy-AppFilesToFolder -appFiles $parameters.apps -folder $tempPath | Out-Null +Get-ChildItem -Path $tempPath -Filter *.app | Out-Host +$authContext = $parameters.authContext | ConvertFrom-Json +Write-Host "Token Length: $($authContext.Token.Length)" +``` + +In this example the AuthContext secret is assumed to contain a Token property. Read [this](https://aka.ms/algosettings#customdeployment) for more information. + +### Adding custom workflows + +If you add new workflows to the `.github/workflows` folder, which is unknown to AL-Go for GitHub, AL-Go will leave them un-touched. These workflows needs to follow standard GitHub Actions schema (yaml) and can be triggered as any other workflows. Example: + +```yaml +name: 'Create Build Tag' + +on: + workflow_run: + workflows: [' CI/CD','CI/CD'] + types: [completed] + branches: [ 'main' ] + +run-name: "[${{ github.ref_name }}] Create build tag" + +permissions: read-all + +jobs: + CreateTag: + if: github.event.workflow_run.conclusion == 'success' + runs-on: windows-latest + steps: + - name: mystep + run: | + Write-Host "Create tag" +``` + +It is recommended to prefix your workflows with `my`, `our`, your name or your organization name in order to avoid that the workflow suddenly gets overridden by a new workflow in AL-Go for GitHub. The above workflow is a real example from [here](https://github.com/microsoft/BCApps/blob/main/.github/workflows/CreateBuildTag.yaml). + +> \[!CAUTION\] +> This workflow gets triggered when the CI/CD workflow has completed. Note that the name of the CI/CD workflow currently is prefixed with a space, this space will very likely be removed in the future, which is why we specify both names in this example. Obviously this workflow would break if we decide to rename the CI/CD workflow to something different. + +### Adding custom scripts + +You can add custom powershell scripts under the .github folder for repository scoped scripts or in the .AL-Go folder for project scoped scripts. Specially named scripts in the .AL-Go folder can override standard functionality in AL-Go for GitHub workflows. A list of these script overrides can be found [here](https://aka.ms/algosettings#customdeployment). Scripts under the .github folder can be used in custom workflows instead of using inline scripts inside the workflow. + +One example of a script override is the NewBcContainer override used in the System Application project in BCApps (can be found [here](https://github.com/microsoft/BCApps/blob/main/build/projects/System%20Application/.AL-Go/NewBcContainer.ps1)). This override looks like: + +```powershell +Param([Hashtable] $parameters) + +$script = Join-Path $PSScriptRoot "../../../scripts/NewBcContainer.ps1" -Resolve +. $script -parameters $parameters +``` + +Which basically launches a script located in the script folder in the repository for creating the build container needed for building and testing the System Application. + +> \[!CAUTION\] +> Script overrides will almost certainly be broken in the future. The current script overrides is very much tied to the current implementation of the `Run-AlPipeline` function in BcContainerHelper. In the future, we will move this functionality to GitHub actions and no longer depend on BcContainerHelper and Run-AlPipeline. At that time, these script overrides will have to be changed to follow the new implementation. + + + +### Adding custom workflows and/or scripts using a URL + +By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings#customalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these files should be included in the update. Example: + +``` + "customALGoSystemFiles": [ + { + "Destionation": ".AL-Go/", + "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1", + }, + { + "Destionation": ".AL-Go/", + "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1", + }, + { + "Destionation": ".AL-Go/NewBcContainer.ps1", + "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1", + }, + { + "Destination": ".github/", + "Source": "https://github.com/freddydk/CustomALGoSystemFiles/archive/refs/heads/main.zip", + "FileSpec": "*/.github/*", + "Recurse": true + } + ] +``` + +`customALGoSystemFiles` is an array of objects, which currently can have 4 properties: + +| Property | Description | Mandatory | Default | +| :-- | :-- | :-: | :-- | +| Destination | Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) | Yes | | +| Source | URL to a either a single file or a .zip file containing | Yes | | +| FileSpec | If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | +| Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | + +This setting will cause AL-Go for GitHub to include these files during the next update. + +> \[!WARNING\] +> You can override existing AL-Go for GitHub system files this way, please prefix files in your repository with `my` or your organization name (except for DeployTo and DeliverTo) in order to avoid overriding future workflows from AL-Go for GitHub. + +> \[!NOTE\] +> If the destination is in the .AL-Go folder, the file(s) will be copied to all .AL-Go folders in multi-project repositories. + +### Adding custom jobs + +You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Custom jobs can depend on other jobs and other jobs can made to depend on custom jobs. Custom jobs needs to be named `CustomJob`, but can specify another name to be shown in the UI. Example: + +```yaml + CustomJob-CreateBuildTag: + name: Create Build Tag + needs: [ Initialization, Build ] + if: (!cancelled()) && (needs.Build.result == 'success') + runs-on: [ ubuntu-latest ] + steps: + - name: Create Tag + run: | + Write-Host "Create Tag" + + PostProcess: + needs: [ Initialization, Build2, Build1, Build, Deploy, Deliver, DeployALDoc, CustomJob-CreateBuildTag ] + if: (!cancelled()) + runs-on: [ windows-latest ] + steps: + ... +``` + +Adding a custom job like this, will cause this job to run simultaneously with the deploy and the deliver jobs. + +> \[!NOTE\] +> All custom jobs will be moved to the tail of the yaml file when running Update AL-Go System Files, but dependencies to/from the custom jobs will be maintained. + +> \[!CAUTION\] +> Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. + +### Adding custom steps + +You can also add custom steps to AL-Go for GitHub Workflows, but only in pre-defined anchor-points. The reason for only allowing custom steps at pre-defined anchor-points is that we want to limit the number of places where steps can be added in order to have some level of freedom to refactor, develop and maintain the AL-Go for GitHub workflows, without breaking customizations constantly. + +At this time, the anchor-points where you can add custom steps are: + +| Workflow | Job | Step | Before or AFter | +| :-- | :-- | :-- | :-: | +| _BuildALGoProject.yaml | BuildALGoProject | Read settings | After | +| | | Read secrets | After | +| | | Build | Before | +| | | Read secrets | After | +| | | Cleanup | Before | + +The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic + +```yaml + - name: CustomStep-ModifySettings + run: | + $settings = $env:Settings | ConvertFrom-Json + $settings.artifact = Invoke-RestMethod -Method GET -UseBasicParsing -Uri "https://bca-url-proxy.azurewebsites.net/bca-url/sandbox/us?select=weekly&doNotRedirect=true" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "Settings=$($$settings | ConvertTo-Json -Depth 99 -Compress)" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "artifact=$($settings.artifact)" +``` + +> \[!TIP\] +> Create a feature request [here](https://github.com/microsoft/AL-Go/issues/new?assignees=&labels=enhancement&projects=&template=enhancement.yaml&title=%5BEnhancement%5D%3A+) with a description on where you would like additional anchor-points and what you want to use it for. + +> \[!CAUTION\] +> Please be aware that changes to AL-Go for GitHub might break with future versions of AL-Go for GitHub. We will of course try to keep these breaking changes to a minimum, but the only way you can be sure to NOT be broken is by NOT customizing AL-Go for GitHub. + +### Modifying workflow permissions + +If any of your custom jobs require permissions, which exceeds the permissions already assigned in the workflow, then these permissions can be specified directly on the custom job. + +If any of your custom steps require permissions, which exceeds the permissions already assigned in the workflow, you can modify the permissions of the workflow and assign additional permissions. AL-Go for GitHub will not allow you to remove permissions, which might be needed in other steps/jobs, but additional permissions will be included when running Update AL-Go System Files. + +## Using indirect templates + +If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. + +> \[!NOTE\] +> Indirect templates can be public or private. + + + +> \[!TIP\] +> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specied. + + + + +> \[!NOTE\] +> If you use the indirect template as a GitHub template, for creating the repository, then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. + + + + + +## Forking AL-Go for GitHub and making your "own" **public** version + +Using a fork of AL-Go for GitHub to have your "own" public version of AL-Go for GitHub gives you the maximum customization capabilities. It does however also come with the most work. + +> \[!NOTE\] +> When customizing AL-Go for GitHub using a fork, your customizations are public and will be visible to everyone. For more information, [read this](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-permissions-and-visibility-of-forks). + +There are two ways of forking AL-Go for GitHub. You can fork the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository or you can fork the template repositories [AL-Go PTE](https://github.com/microsoft/AL-Go-PTE) and/or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource). + +For simple changes to the templates, you can fork the template repositories and make the changes directly in your fork. Note that we do not accept any pull requests to the template repositories as they are deployed from the main AL-Go repository. We do not actually develop anything in the template repositories ourself. In the template repositories you will find a branch for every version of AL-Go we have shipped. The main branch is the latest version and the preview branch is the next version. You can customize the preview branch and/or the main branch and then use your fork as the template repository when running Update AL-Go System Files from your app repositories. + +> \[!NOTE\] +> We do NOT accept pull requests to the template repositories. You need to follow the guidelines [here](Contribute.md) in order to contribute to AL-Go development. + +> \[!TIP\] +> When forking the template repositories, you should include all branches in order to be able to use either the latest version of AL-Go or the preview version of AL-Go. + +When forking the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository, you are basically developing AL-Go in the same way as we are doing in Microsoft. Please follow the guidelines [here](Contribute.md) on how to develop. This gives you maximum customization capabilities, but if your changes are not being contributed to AL-Go, then you will have to merge our changes all the time. + +> \[!CAUTION\] +> We strongly suggest that you keep your changes to a minimum and that you keep your fork up-to-date with the latest changes of AL-Go for GitHub at all time. + +______________________________________________________________________ + +[back](../README.md) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 67927c0be..6deef695e 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -4,6 +4,8 @@ The behavior of AL-Go for GitHub is very much controlled by settings and secrets To learn more about the secrets used by AL-Go for GitHub, please navigate to [Secrets](secrets.md). + + ## Where are the settings located Settings can be defined in GitHub variables or in various settings file. An AL-Go repository can consist of a single project (with multiple apps) or multiple projects (each with multiple apps). Settings can be applied on the project level or on the repository level. Multiple projects in a single repository are comparable to multiple repositories; they are built, deployed, and tested separately. All apps in each project (single or multiple) are built together in the same pipeline, published and tested together. If a repository is multiple projects, each project is stored in a separate folder in the root of the repository. @@ -210,12 +212,13 @@ Please read the release notes carefully when installing new versions of AL-Go fo | memoryLimit | Specifies the memory limit for the build container. By default, this is left to BcContainerHelper to handle and will currently be set to 8G | 8G | | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | \[ \] | +| customALGoSystemFiles | An array of objects, specifying custom AL-Go System Files. Each object must contain **Destination** and **Source** url and if the source url points to a .zip file, you can also add **FileSpec** and **Recurse**. See more [here](./CustomizingALGoForGitHub.md#customALGoSystemFiles) | \[ \] | ## Custom Delivery -You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named DeliverTo\*.ps1 in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. +You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named `DeliverTo.ps1` in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. Beside the script, you also need to create a secret called `Context`, formatted as compressed json, containing delivery information for your delivery target. ### DeliverToSharePoint.ps1 @@ -227,6 +230,7 @@ Param( Write-Host "Current project path: $($parameters.project)" Write-Host "Current project name: $($parameters.projectName)" Write-Host "Delivery Type (CD or Release): $($parameters.type)" +Write-Host "Delivery Context: $($parameters.context)" Write-Host "Folder containing apps: $($parameters.appsFolder)" Write-Host "Folder containing test apps: $($parameters.testAppsFolder)" Write-Host "Folder containing dependencies (requires generateDependencyArtifact set to true): $($parameters.dependenciesFolder)" @@ -306,6 +310,8 @@ Here are the parameters to use in your custom script: | `$parameters."runs-on"` | GitHub runner to be used to run the deployment script | windows-latest | | `$parameters."shell"` | Shell used to run the deployment script, pwsh or powershell | powershell | + + ## Run-AlPipeline script override AL-Go for GitHub utilizes the Run-AlPipeline function from BcContainerHelper to perform the actual build (compile, publish, test etc). The Run-AlPipeline function supports overriding functions for creating containers, compiling apps and a lot of other things. @@ -316,6 +322,7 @@ Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the fut | Override | Description | | :-- | :-- | +| PipelineInitialize.ps1 | Initialize the pipeline | | DockerPull.ps1 | Pull the image specified by the parameter $imageName | | NewBcContainer.ps1 | Create the container using the parameters transferred in the $parameters hashtable | | ImportTestToolkitToBcContainer.ps1 | Import the test toolkit apps specified by the $parameters hashtable | @@ -330,6 +337,7 @@ Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the fut | GetBcContainerAppRuntimePackage.ps1 | Get the runtime package specified by the $parameters hashtable | | RemoveBcContainer.ps1 | Cleanup based on the $parameters hashtable | | InstallMissingDependencies | Install missing dependencies | +| PipelineFinalize.ps1 | Finalize the pipeline | ## BcContainerHelper settings diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 9ded4de35..bfdf73edd 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -65,10 +65,10 @@ jobs: - name: Override templateUrl env: - templateUrl: ${{ github.event.inputs.templateUrl }} + newTemplateUrl: ${{ github.event.inputs.templateUrl }} run: | $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 - $templateUrl = $ENV:templateUrl + $templateUrl = $ENV:newTemplateUrl if ($templateUrl) { Write-Host "Using Template Url: $templateUrl" Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "templateUrl=$templateUrl" From c43ea9ac87752bbd1e47814e4f22da55255c143b Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 06:47:49 +0200 Subject: [PATCH 058/215] Apply custom files --- .../CheckForUpdates.HelperFunctions.ps1 | 141 ++++++++++-------- Actions/CheckForUpdates/CheckForUpdates.ps1 | 6 +- Scenarios/CustomizingALGoForGitHub.md | 12 +- 3 files changed, 84 insertions(+), 75 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ead9cc0fb..ca19d166c 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -467,95 +467,108 @@ function UpdateSettingsFile { function ApplyCustomALGoSystemFiles { Param( [string] $path, - [hashtable] $settings + [hashtable] $settings, + [string[]] $projects ) - Push-Location -Path $path - try { - if ($settings.customALGoSystemFiles -isnot [Array]) { - throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + function CopyItem{ + Param( + [string] $source, + [string] $basePath, + [string] $destination, + [string[]] $projects + ) + + if ($destination -like ".AL-Go$([IO.Path]::DirectorySeparatorChar)*") { + $destinations = $projects | ForEach-Object { Join-Path $_ $destination } + } + else { + $destinations = @($destination) } - foreach($customspec in $settings.customALGoSystemFiles) { - if ($customspec -isnot [Hashtable]) { - throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + $destinations | ForEach-Object { + $finalDestination = Join-Path $basePath $_ + $finalDestinationFolder = [System.IO.Path]::GetDirectoryName($finalDestination) + if (!(Test-Path -Path $finalDestinationFolder -PathType Container)) { + New-Item -path $finalDestinationFolder -ItemType Directory | Out-Null } - if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { - throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." + if (Test-Path $finalDestination) { + Write-Host "- $_ (overriding existing file)" } - $source = $customspec.Source - $destination = $customSpec.Destination - if ($source -isnot [string] -or $destination -isnot [string]) { - throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be strings. See https://aka.ms/algosettings#customalgosystemfiles." + else { + Write-Host "- $_" } - - $finalDestination = Join-Path $path $destination - $ext = [System.IO.Path]::GetExtension($source) + Copy-Item -Path $source -Destination $finalDestination + } + } + + if ($settings.customALGoSystemFiles -isnot [Array]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + foreach($customspec in $settings.customALGoSystemFiles) { + if ($customspec -isnot [Hashtable]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." + } + $source = $customspec.Source + $destination = $customSpec.Destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) + if ($destination -isnot [string] -or $destination -eq '') { + throw "customALGoSystemFiles setting is wrongly formatted, Destination must be a string, which isn't blank. See https://aka.ms/algosettings#customalgosystemfiles." + } + if ($source -isnot [string] -or $source -notlike 'https://*' -or (-not [System.Uri]::IsWellFormedUriString($source,1))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source must secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." + } + + $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + New-Item -Path $tempFolder -ItemType Directory | Out-Null + $ext = [System.IO.Path]::GetExtension($source) + $zipName = "$tempFolder$ext" + Push-Location -Path $path + try { if ($ext -eq '.zip') { - Write-Host $destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) + Write-Host "$($destination):" if ($customSpec.ContainsKey('FileSpec')) { $fileSpec = $customSpec.FileSpec } else { $fileSpec = '*' } if ($customSpec.ContainsKey('Recurse')) { $recurse = $customSpec.Recurse } else { $recurse = $true } if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." } - if (Test-Path $finalDestination -PathType Leaf) { - throw "A file with that name already exists - destination must point to a folder when adding custom AL-Go System Files from a .zip file. See https://aka.ms/algosettings#customalgosystemfiles." + if (!($destination.EndsWith([IO.Path]::DirectorySeparatorChar))) { + throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." } - $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - New-Item -Path $tempFolder -ItemType Directory | Out-Null - $zipName = "$tempFolder$ext" + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName + Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force + $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve + Push-Location -Path $subFolder try { - Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName - Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force - $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve - Push-Location -Path $subFolder - try { - Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { - $destRelativeFileName = Resolve-Path $_.FullName -Relative - $destFinalFileName = Join-Path $finalDestination $destRelativeFileName - $destFinalFolder = [System.IO.Path]::GetDirectoryName($destFinalFileName) - if (Test-Path -Path $destFinalFolder -PathType Leaf) { - throw "A file ($destFinalFolder) already exists, where a folder is expected when applying CustomALGoSystemFiles." - } - if (!(Test-Path -Path $destFinalFolder)) { - New-Item -path $destFinalFolder -ItemType Directory | Out-Null - } - Write-Host "- $($destRelativeFileName.Substring(2))" - Copy-Item -Path $_.FullName -Destination $destFinalFileName - } - } - finally { - Pop-Location + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFileName = Join-Path $destination $destRelativeFileName + $destFileName = $destFileName.TrimStart('\/') + CopyItem -source $_.FullName -basePath $path -destination $destFileName -projects $projects } } finally { - if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } - Remove-Item -Path $tempFolder -Recurse -Force + Pop-Location } } else { if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { throw "customALGoSystemFiles setting is wrongly formatted, FileSpec and Recurse are only allowed with .zip files. See https://aka.ms/algosettings#customalgosystemfiles." } - if (Test-Path -Path $finalDestination -PathType Container) { - $finalDestination = Join-Path $finalDestination ([System.IO.Path]::GetFileName($source)) + if ($destination.endsWith([IO.Path]::DirectorySeparatorChar)) { + $destination = Join-Path $destination ([System.IO.Path]::GetFileName($source)) } - else { - if ($destination.endsWith('\') -or $destination.endsWith('/')) { - New-Item -Path $finalDestination -ItemType Directory | Out-Null - $finalDestination = Join-Path $finalDestination ([System.IO.Path]::GetFileName($source)) - } - } - $overridingStr = '' - if (Test-Path -Path $finalDestination -PathType Leaf) { - $overridingStr = ' (Overriding existing file)' - } - $relativeFolder = Resolve-Path -Path ([System.IO.Path]::GetDirectoryName($finalDestination)) -Relative - Write-Host "$(Join-Path $relativeFolder ([System.IO.Path]::GetFileName($finalDestination)))$overridingStr" - Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $finalDestination + Write-Host "$($destination):" + $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $tempFilename + CopyItem -source $tempFilename -basePath $path -destination $destination -projects $projects } } - } - finally { - Pop-Location + finally { + Pop-Location + if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } + Remove-Item -Path $tempFolder -Recurse -Force + } } } \ No newline at end of file diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index ff4c367c6..cc9928ac4 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -116,9 +116,6 @@ if (-not $isDirectALGo) { } } -# Apply Custom AL-Go System Files from settings -ApplyCustomALGoSystemFiles -path $templateFolder -settings $repoSettings - # CheckFiles is an array of hashtables with the following properties: # dstPath: The path to the file in the current repository # srcPath: The path to the file in the template repository @@ -261,6 +258,9 @@ foreach($checkfile in $checkfiles) { } } +# Apply Custom AL-Go System Files from settings +ApplyCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects + if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 3867f89be..f69ede4e2 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -150,16 +150,12 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings ``` "customALGoSystemFiles": [ { - "Destionation": ".AL-Go/", - "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1", + "Destination": ".AL-Go/", + "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1" }, { - "Destionation": ".AL-Go/", - "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1", - }, - { - "Destionation": ".AL-Go/NewBcContainer.ps1", - "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1", + "Destination": ".AL-Go/NewBcContainer.ps1", + "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1" }, { "Destination": ".github/", From fd03c25165354ae69194385313aeebc68f955aee Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 06:59:41 +0200 Subject: [PATCH 059/215] add to updates files --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ca19d166c..5e2a5cf7f 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -497,7 +497,7 @@ function ApplyCustomALGoSystemFiles { else { Write-Host "- $_" } - Copy-Item -Path $source -Destination $finalDestination + Write-Output @{ "DstFile" = $finalDestination; "content" = (Get-Content -Raw -Encoding UTF8 -Path $source) } } } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index cc9928ac4..8f987720a 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -259,7 +259,7 @@ foreach($checkfile in $checkfiles) { } # Apply Custom AL-Go System Files from settings -ApplyCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects +$updateFiles += @(ApplyCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects) if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available From 9ac3f24140066de8c32ac2c6e52dcca37417692f Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 07:11:31 +0200 Subject: [PATCH 060/215] relative --- .../CheckForUpdates.HelperFunctions.ps1 | 25 +++++-------------- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 5e2a5cf7f..8da651797 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -464,17 +464,16 @@ function UpdateSettingsFile { $settings | Set-JsonContentLF -path $settingsFile } -function ApplyCustomALGoSystemFiles { +function GetCustomALGoSystemFiles { Param( [string] $path, [hashtable] $settings, [string[]] $projects ) - function CopyItem{ + function YieldItem{ Param( [string] $source, - [string] $basePath, [string] $destination, [string[]] $projects ) @@ -486,18 +485,8 @@ function ApplyCustomALGoSystemFiles { $destinations = @($destination) } $destinations | ForEach-Object { - $finalDestination = Join-Path $basePath $_ - $finalDestinationFolder = [System.IO.Path]::GetDirectoryName($finalDestination) - if (!(Test-Path -Path $finalDestinationFolder -PathType Container)) { - New-Item -path $finalDestinationFolder -ItemType Directory | Out-Null - } - if (Test-Path $finalDestination) { - Write-Host "- $_ (overriding existing file)" - } - else { - Write-Host "- $_" - } - Write-Output @{ "DstFile" = $finalDestination; "content" = (Get-Content -Raw -Encoding UTF8 -Path $source) } + Write-Host "- $_" + Write-Output @{ "DstFile" = $_; "content" = (Get-Content -Raw -Encoding UTF8 -Path $source) } } } @@ -524,7 +513,6 @@ function ApplyCustomALGoSystemFiles { New-Item -Path $tempFolder -ItemType Directory | Out-Null $ext = [System.IO.Path]::GetExtension($source) $zipName = "$tempFolder$ext" - Push-Location -Path $path try { if ($ext -eq '.zip') { Write-Host "$($destination):" @@ -545,7 +533,7 @@ function ApplyCustomALGoSystemFiles { $destRelativeFileName = Resolve-Path $_.FullName -Relative $destFileName = Join-Path $destination $destRelativeFileName $destFileName = $destFileName.TrimStart('\/') - CopyItem -source $_.FullName -basePath $path -destination $destFileName -projects $projects + YieldItem -source $_.FullName -destination $destFileName -projects $projects } } finally { @@ -562,11 +550,10 @@ function ApplyCustomALGoSystemFiles { Write-Host "$($destination):" $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $tempFilename - CopyItem -source $tempFilename -basePath $path -destination $destination -projects $projects + YieldItem -source $tempFilename -destination $destination -projects $projects } } finally { - Pop-Location if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } Remove-Item -Path $tempFolder -Recurse -Force } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 8f987720a..99b8f99f6 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -259,7 +259,9 @@ foreach($checkfile in $checkfiles) { } # Apply Custom AL-Go System Files from settings -$updateFiles += @(ApplyCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects) +$updateFiles += @(GetCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects) + +$updateFiles | Out-Host if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available From 362ec0f0ac7d8883f8a1701b9226a4aae8b6445b Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 07:12:14 +0200 Subject: [PATCH 061/215] remove path --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 1 - Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 8da651797..38770b9e9 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -466,7 +466,6 @@ function UpdateSettingsFile { function GetCustomALGoSystemFiles { Param( - [string] $path, [hashtable] $settings, [string[]] $projects ) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 99b8f99f6..560189afd 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -259,7 +259,7 @@ foreach($checkfile in $checkfiles) { } # Apply Custom AL-Go System Files from settings -$updateFiles += @(GetCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects) +$updateFiles += @(GetCustomALGoSystemFiles -settings $repoSettings -projects $projects) $updateFiles | Out-Host From 977f94b9f00dcee1eb1960ad9f831fc08312803e Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 07:18:00 +0200 Subject: [PATCH 062/215] do not dump custom algo system files --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 560189afd..772ad3332 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -261,8 +261,6 @@ foreach($checkfile in $checkfiles) { # Apply Custom AL-Go System Files from settings $updateFiles += @(GetCustomALGoSystemFiles -settings $repoSettings -projects $projects) -$updateFiles | Out-Host - if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { From 4ccfc85250a76fe10d1ec953ae8c204c293e889f Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 07:36:35 +0200 Subject: [PATCH 063/215] compare --- .../CheckForUpdates.HelperFunctions.ps1 | 27 +++++++++++-------- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 38770b9e9..83c90d95b 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -466,6 +466,7 @@ function UpdateSettingsFile { function GetCustomALGoSystemFiles { Param( + [string] $baseFolder, [hashtable] $settings, [string[]] $projects ) @@ -485,10 +486,17 @@ function GetCustomALGoSystemFiles { } $destinations | ForEach-Object { Write-Host "- $_" - Write-Output @{ "DstFile" = $_; "content" = (Get-Content -Raw -Encoding UTF8 -Path $source) } + $content = Get-ContentLF -Path $source + if (Test-Path -Path $_) { + $existingContent = Get-ContentLF -Path $_ + } + if ($content -ne $existingContent) { + Write-Output @{ "DstFile" = $_; "content" = $content } + } } } + Push-Location $baseFolder if ($settings.customALGoSystemFiles -isnot [Array]) { throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." } @@ -527,17 +535,13 @@ function GetCustomALGoSystemFiles { Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve Push-Location -Path $subFolder - try { - Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { - $destRelativeFileName = Resolve-Path $_.FullName -Relative - $destFileName = Join-Path $destination $destRelativeFileName - $destFileName = $destFileName.TrimStart('\/') - YieldItem -source $_.FullName -destination $destFileName -projects $projects - } - } - finally { - Pop-Location + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFileName = Join-Path $destination $destRelativeFileName + $destFileName = $destFileName.TrimStart('\/') + YieldItem -source $_.FullName -destination $destFileName -projects $projects } + Pop-Location } else { if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { @@ -557,4 +561,5 @@ function GetCustomALGoSystemFiles { Remove-Item -Path $tempFolder -Recurse -Force } } + Pop-Location } \ No newline at end of file diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 772ad3332..a5d5eaa8f 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -259,7 +259,7 @@ foreach($checkfile in $checkfiles) { } # Apply Custom AL-Go System Files from settings -$updateFiles += @(GetCustomALGoSystemFiles -settings $repoSettings -projects $projects) +$updateFiles += @(GetCustomALGoSystemFiles -baseFolder $baseFolder -settings $repoSettings -projects $projects) if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available From 48cd019e893f005ab7eccc3f43b87c6ebadb2733 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 09:12:23 +0200 Subject: [PATCH 064/215] define existing --- .../CheckForUpdates.HelperFunctions.ps1 | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 83c90d95b..6fb7f0e48 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -487,6 +487,7 @@ function GetCustomALGoSystemFiles { $destinations | ForEach-Object { Write-Host "- $_" $content = Get-ContentLF -Path $source + $existingContent = '' if (Test-Path -Path $_) { $existingContent = Get-ContentLF -Path $_ } @@ -535,13 +536,17 @@ function GetCustomALGoSystemFiles { Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve Push-Location -Path $subFolder - Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { - $destRelativeFileName = Resolve-Path $_.FullName -Relative - $destFileName = Join-Path $destination $destRelativeFileName - $destFileName = $destFileName.TrimStart('\/') - YieldItem -source $_.FullName -destination $destFileName -projects $projects + try { + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFileName = Join-Path $destination $destRelativeFileName + $destFileName = $destFileName.TrimStart('\/') + YieldItem -source $_.FullName -destination $destFileName -projects $projects + } + } + finally { + Pop-Location } - Pop-Location } else { if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { From 5b70630da85cfe5fef9e7aa429da4ad79d6d0258 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 09:25:27 +0200 Subject: [PATCH 065/215] dump modified files --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index a5d5eaa8f..af83e2cad 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -264,6 +264,14 @@ $updateFiles += @(GetCustomALGoSystemFiles -baseFolder $baseFolder -settings $re if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { + if ($updateFiles) { + Write-Host "Updated files:" + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" + } + if ($removeFiles) { + Write-Host "Removed files:" + $removeFiles | ForEach-Object { Write-Host "- $_" + } OutputWarning -message "There are updates for your AL-Go system, run 'Update AL-Go System Files' workflow to download the latest version of AL-Go." } else { From f83caa2afa39e22f6b48eb3c47c86710a3330160 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 09:30:58 +0200 Subject: [PATCH 066/215] syntax --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index af83e2cad..6b7aa0ffa 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -266,11 +266,11 @@ if ($update -ne 'Y') { if (($updateFiles) -or ($removeFiles)) { if ($updateFiles) { Write-Host "Updated files:" - $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" } } if ($removeFiles) { Write-Host "Removed files:" - $removeFiles | ForEach-Object { Write-Host "- $_" + $removeFiles | ForEach-Object { Write-Host "- $_" } } OutputWarning -message "There are updates for your AL-Go system, run 'Update AL-Go System Files' workflow to download the latest version of AL-Go." } From 7646df4481f469470030a04941160ca3c7bd7ff5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 09:42:40 +0200 Subject: [PATCH 067/215] use basefolder --- .../CheckForUpdates.HelperFunctions.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 6fb7f0e48..40efb10d3 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -473,6 +473,7 @@ function GetCustomALGoSystemFiles { function YieldItem{ Param( + [string] $baseFolder, [string] $source, [string] $destination, [string[]] $projects @@ -487,9 +488,10 @@ function GetCustomALGoSystemFiles { $destinations | ForEach-Object { Write-Host "- $_" $content = Get-ContentLF -Path $source + $existingFile = Join-Path $baseFolder $_ $existingContent = '' - if (Test-Path -Path $_) { - $existingContent = Get-ContentLF -Path $_ + if (Test-Path -Path $existingFile) { + $existingContent = Get-ContentLF -Path $existingFile } if ($content -ne $existingContent) { Write-Output @{ "DstFile" = $_; "content" = $content } @@ -497,7 +499,6 @@ function GetCustomALGoSystemFiles { } } - Push-Location $baseFolder if ($settings.customALGoSystemFiles -isnot [Array]) { throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." } @@ -541,7 +542,7 @@ function GetCustomALGoSystemFiles { $destRelativeFileName = Resolve-Path $_.FullName -Relative $destFileName = Join-Path $destination $destRelativeFileName $destFileName = $destFileName.TrimStart('\/') - YieldItem -source $_.FullName -destination $destFileName -projects $projects + YieldItem -baseFolder $baseFolder -source $_.FullName -destination $destFileName -projects $projects } } finally { @@ -558,7 +559,7 @@ function GetCustomALGoSystemFiles { Write-Host "$($destination):" $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $tempFilename - YieldItem -source $tempFilename -destination $destination -projects $projects + YieldItem -baseFolder $baseFolder -source $tempFilename -destination $destination -projects $projects } } finally { @@ -566,5 +567,4 @@ function GetCustomALGoSystemFiles { Remove-Item -Path $tempFolder -Recurse -Force } } - Pop-Location } \ No newline at end of file From c23d28c4f166ac6ac4a03a07e07a1bb49a2d94a1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 10:02:34 +0200 Subject: [PATCH 068/215] doc --- Scenarios/CustomizingALGoForGitHub.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index f69ede4e2..e8ff18319 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -170,8 +170,8 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings | Property | Description | Mandatory | Default | | :-- | :-- | :-: | :-- | -| Destination | Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) | Yes | | -| Source | URL to a either a single file or a .zip file containing | Yes | | +| Destination | Path in which the file should be placed. Can include the filename if the source doesn't point to a .zip file, must include a terminating / or \\ if a filename is not included. | Yes | | +| Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | | FileSpec | If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | | Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | From c7c49d84e8f53cefb623947208510762a7e14fee Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 14:49:58 +0200 Subject: [PATCH 069/215] upd --- Scenarios/CustomizingALGoForGitHub.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index e8ff18319..aa784793d 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -253,7 +253,7 @@ If any of your custom steps require permissions, which exceeds the permissions a ## Using indirect templates -If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. +If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. > \[!NOTE\] > Indirect templates can be public or private. @@ -261,8 +261,7 @@ If you have have customizations you want to apply to multiple repositories, you > \[!TIP\] -> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specied. - +> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. From f8a1310060c5a6c5e129600159def0a0a64fe5ae Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 07:33:02 +0200 Subject: [PATCH 070/215] update custom and unused algo system files --- .../CheckForUpdates.HelperFunctions.ps1 | 14 ++++++++++---- Actions/CheckForUpdates/CheckForUpdates.ps1 | 8 ++++++-- Scenarios/CustomizingALGoForGitHub.md | 3 +++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 40efb10d3..72d9e191c 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -434,7 +434,7 @@ function UpdateSettingsFile { Param( [string] $settingsFile, [hashtable] $updateSettings, - [hashtable] $additionalSettings = @{} + [hashtable] $indirectTemplateSettings = @{} ) # Update Repo Settings file with the template URL @@ -453,13 +453,19 @@ function UpdateSettingsFile { $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $updateSettings."$key" } } - # Grab settings from additionalSettings if they are not already in settings - foreach($key in $additionalSettings.Keys) { + # Grab settings from indirectTemplateSettings if they are not already in settings + foreach($key in $indirectTemplateSettings.Keys) { + # CustomALGoSystemFiles will not be copied from the indirect template settings - they will be applied to the indirect template + # UnusedALGoSystemFiles will not be copied from the indirect template settings - they will be used during the update process + if (@('customALGoSystemFiles','unusedALGoSystemFiles') -contains $key) { + continue + } if (!($settings.PSObject.Properties.Name -eq $key)) { # Add the property if it doesn't exist - $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $additionalSettings."$key" + $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $indirectTemplateSettings."$key" } } + # Save the file with LF line endings and UTF8 encoding $settings | Set-JsonContentLF -path $settingsFile } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 6b7aa0ffa..af09982e7 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -107,6 +107,10 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings + # If the indirect template contains unusedALGoSystemFiles, we need to remove them from the current repository + if ($indirectTemplateRepoSettings.ContainsKey('unusedALGoSystemFiles')) { + $unusedALGoSystemFiles += $indirectTemplateRepoSettings.unusedALGoSystemFiles + } $myALGoSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" if (Test-Path $myALGoSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" @@ -175,7 +179,7 @@ foreach($checkfile in $checkfiles) { Write-Host "Update Project Settings" # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) $projectSettingsFile = Join-Path $dstFolder "settings.json" - UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -additionalSettings $indirectTemplateProjectSettings + UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -indirectTemplateSettings $indirectTemplateProjectSettings $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } } @@ -295,7 +299,7 @@ else { invoke-git status - UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -additionalSettings $indirectTemplateRepoSettings + UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -indirectTemplateSettings $indirectTemplateRepoSettings # Update the files # Calculate the release notes, while updating diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index aa784793d..acfd9854e 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -258,7 +258,10 @@ If you have have customizations you want to apply to multiple repositories, you > \[!NOTE\] > Indirect templates can be public or private. +Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. +> \[!NOTE\] +> CustomALGoSystemFiles will NOT be copied from the indirect template, they will be applied there and UnusedALGoSystemFiles will be merged. > \[!TIP\] > The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. From 4317c17740e68cdf67150f812d43fd5ebdbfc305 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 07:56:26 +0200 Subject: [PATCH 071/215] Support private template repo --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 5 +++++ Templates/AppSource App/.github/workflows/CICD.yaml | 9 +++++++++ .../Per Tenant Extension/.github/workflows/CICD.yaml | 9 +++++++++ 3 files changed, 23 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index af09982e7..e736eebdc 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -30,6 +30,11 @@ if ($update -eq 'Y') { $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } } +else { + if (-not $token) { + $token = $env:GITHUB_TOKEN + } +} # Use Authenticated API request to avoid the 60 API calls per hour limit $headers = @{ diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index c6f19ed66..851f956ed 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -134,10 +134,19 @@ jobs: shell: powershell get: templateUrl + - name: Read secrets + id: ReadSecrets + uses: microsoft/AL-Go-Actions/ReadSecrets@main + with: + shell: powershell + gitHubSecrets: ${{ toJson(secrets) }} + getSecrets: 'ghTokenWorkflow' + - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 0dc128768..1ee6dc260 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -134,10 +134,19 @@ jobs: shell: powershell get: templateUrl + - name: Read secrets + id: ReadSecrets + uses: microsoft/AL-Go-Actions/ReadSecrets@main + with: + shell: powershell + gitHubSecrets: ${{ toJson(secrets) }} + getSecrets: 'ghTokenWorkflow' + - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true From 57acb7cc481cd1d023186a07a17000dd74349626 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 08:14:05 +0200 Subject: [PATCH 072/215] use ghtokenworkflow --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 10 ++++------ Actions/CheckForUpdates/action.yaml | 2 +- Scenarios/CustomizingALGoForGitHub.md | 4 ++-- .../AppSource App/.github/workflows/CreateRelease.yaml | 3 ++- .../.github/workflows/CreateRelease.yaml | 3 ++- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index e736eebdc..2c87853de 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -22,18 +22,16 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -if ($update -eq 'Y') { - if (-not $token) { +if (-not $token) { + if ($update -eq 'Y') { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } else { - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + $token = $env:GITHUB_TOKEN } } else { - if (-not $token) { - $token = $env:GITHUB_TOKEN - } + $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } # Use Authenticated API request to avoid the 60 API calls per hour limit diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 4b12bc6f7..28ea8e24e 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -16,7 +16,7 @@ inputs: token: description: The GitHub token running the action required: false - default: ${{ github.token }} + default: '' templateUrl: description: URL of the template repository (default is the template repository used to create the repository) required: false diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index acfd9854e..5badcc887 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -256,12 +256,12 @@ If any of your custom steps require permissions, which exceeds the permissions a If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. > \[!NOTE\] -> Indirect templates can be public or private. +> Indirect templates can be public or private. If you are using a private indirect template, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. > \[!NOTE\] -> CustomALGoSystemFiles will NOT be copied from the indirect template, they will be applied there and UnusedALGoSystemFiles will be merged. +> UnusedALGoSystemFiles and CustomALGoSystemFiles will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. > \[!TIP\] > The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index a69bbe356..c3d74ff85 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -99,7 +99,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'TokenForPush' + getSecrets: 'TokenForPush,GhTokenWorkflow' useGhTokenWorkflowForPush: '${{ github.event.inputs.useGhTokenWorkflow }}' - name: Determine Projects @@ -112,6 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index 5cb6d41e4..b84620364 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -99,7 +99,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'TokenForPush' + getSecrets: 'TokenForPush,GhTokenWorkflow' useGhTokenWorkflowForPush: '${{ github.event.inputs.useGhTokenWorkflow }}' - name: Determine Projects @@ -112,6 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true From 557350608d1c08ee7bc98fa80dde1f5a514b312c Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 08:20:00 +0200 Subject: [PATCH 073/215] dump content --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 2c87853de..986deda7a 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -273,7 +273,8 @@ if ($update -ne 'Y') { if (($updateFiles) -or ($removeFiles)) { if ($updateFiles) { Write-Host "Updated files:" - $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" } + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)"; Write-Host "'$($_.Content)'" } + } if ($removeFiles) { Write-Host "Removed files:" From 3ec676b913d580d7e05ce2a06a5857a9f9534908 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 08:25:47 +0200 Subject: [PATCH 074/215] only update changed settings files --- .../CheckForUpdates.HelperFunctions.ps1 | 16 +++++++++++++--- Actions/CheckForUpdates/CheckForUpdates.ps1 | 12 ++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 72d9e191c..11a29212b 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -437,20 +437,26 @@ function UpdateSettingsFile { [hashtable] $indirectTemplateSettings = @{} ) + $modified = $false # Update Repo Settings file with the template URL if (Test-Path $settingsFile) { $settings = Get-Content $settingsFile -Encoding UTF8 | ConvertFrom-Json } else { $settings = [PSCustomObject]@{} + $modified = $true } foreach($key in $updateSettings.Keys) { if ($settings.PSObject.Properties.Name -eq $key) { - $settings."$key" = $updateSettings."$key" + if ($settings."$key" -ne $updateSettings."$key") { + $settings."$key" = $updateSettings."$key" + $modified = $true + } } else { # Add the property if it doesn't exist $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $updateSettings."$key" + $modified = $true } } # Grab settings from indirectTemplateSettings if they are not already in settings @@ -463,11 +469,15 @@ function UpdateSettingsFile { if (!($settings.PSObject.Properties.Name -eq $key)) { # Add the property if it doesn't exist $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $indirectTemplateSettings."$key" + $modified = $true } } - # Save the file with LF line endings and UTF8 encoding - $settings | Set-JsonContentLF -path $settingsFile + if ($modified) { + # Save the file with LF line endings and UTF8 encoding + $settings | Set-JsonContentLF -path $settingsFile + } + return $modified } function GetCustomALGoSystemFiles { diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 986deda7a..2114c23be 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -182,8 +182,9 @@ foreach($checkfile in $checkfiles) { Write-Host "Update Project Settings" # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) $projectSettingsFile = Join-Path $dstFolder "settings.json" - UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -indirectTemplateSettings $indirectTemplateProjectSettings - $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } + if (UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -indirectTemplateSettings $indirectTemplateProjectSettings) { + $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } + } } # Remove unused AL-Go system files @@ -273,7 +274,7 @@ if ($update -ne 'Y') { if (($updateFiles) -or ($removeFiles)) { if ($updateFiles) { Write-Host "Updated files:" - $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)"; Write-Host "'$($_.Content)'" } + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" } } if ($removeFiles) { @@ -303,7 +304,10 @@ else { invoke-git status - UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -indirectTemplateSettings $indirectTemplateRepoSettings + $repoSettingsFile = Join-Path ".github" "AL-Go-Settings.json" + if (UpdateSettingsFile -settingsFile $repoSettingsFile -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -indirectTemplateSettings $indirectTemplateRepoSettings) { + $updateFiles += @{ "DstFile" = $repoSettingsFile; "content" = (Get-Content -Path $repoSettingsFile -Encoding UTF8 -Raw) } + } # Update the files # Calculate the release notes, while updating From 0d39e968aebee2c236c1c5efc126fb240e54b57b Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 08:56:58 +0200 Subject: [PATCH 075/215] decode --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 2114c23be..b1bf85328 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -31,6 +31,7 @@ if (-not $token) { } } else { + # Specified token is GhTokenWorkflow secret - decode from base 64 $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } From 3451a4a24cd827db6cc181e967fe6416043fe3aa Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:11:03 +0200 Subject: [PATCH 076/215] indirect --- Scenarios/CustomizingALGoForGitHub.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 5badcc887..3aa08539f 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -258,22 +258,21 @@ If you have have customizations you want to apply to multiple repositories, you > \[!NOTE\] > Indirect templates can be public or private. If you are using a private indirect template, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. -Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. - -> \[!NOTE\] -> UnusedALGoSystemFiles and CustomALGoSystemFiles will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. +Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. **UnusedALGoSystemFiles** and **CustomALGoSystemFiles** will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. > \[!TIP\] -> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. - - +> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a **GhTokenWorkflow** secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. > \[!NOTE\] -> If you use the indirect template as a GitHub template, for creating the repository, then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. - +> If you use the indirect template as a GitHub template for creating the repository, by clicking use this template in your indirect template - then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. +Repositories based on your indirect template will notify you that changes are available for your AL-Go System Files when you update the indirect template only. You will not be notified when new versions of AL-Go for GitHub is released in every repository - only in the indirect template repository. +> \[!WARNING\] +> You should ensure that your indirect template repository is kept up-to-date with the latest changes in AL-Go for GitHub. +> \[!TIP\] +> You can setup the Update AL-Go System Files workflow to run on a schedule to uptake new releases of AL-Go for GitHub regularly. ## Forking AL-Go for GitHub and making your "own" **public** version From 37b7ecbfa5e6f72e346b205bf879a42f4ad57971 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:12:46 +0200 Subject: [PATCH 077/215] insert --- .github/workflows/jekyll-gh-pages.yml | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml new file mode 100644 index 000000000..ded8688cc --- /dev/null +++ b/.github/workflows/jekyll-gh-pages.yml @@ -0,0 +1,49 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy Jekyll with GitHub Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup Pages + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 + with: + source: . + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 + with: + preview: true \ No newline at end of file From dccb8e4f78f2609ec0070f26b23b653adaf47ba1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:14:26 +0200 Subject: [PATCH 078/215] remove jekyll --- .github/workflows/jekyll-gh-pages.yml | 49 --------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml deleted file mode 100644 index ded8688cc..000000000 --- a/.github/workflows/jekyll-gh-pages.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll with GitHub Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 - with: - source: . - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 - with: - preview: true \ No newline at end of file From 570628a7a293dc41eb64da1ac64470be4eef7cb8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:20:00 +0200 Subject: [PATCH 079/215] revert --- .github/workflows/powershell.yaml | 2 +- .../.github/workflows/_BuildALGoProject.yaml | 20 +++++++++---------- .../.github/workflows/_BuildALGoProject.yaml | 20 +++++++++---------- .../_BuildPowerPlatformSolution.yaml | 2 +- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/powershell.yaml b/.github/workflows/powershell.yaml index 6fe36a18e..5a7883ad4 100644 --- a/.github/workflows/powershell.yaml +++ b/.github/workflows/powershell.yaml @@ -31,6 +31,6 @@ jobs: # Upload the SARIF file generated in the previous step - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 + uses: github/codeql-action/upload-sarif@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 with: sarif_file: results.sarif diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index a875964ff..9f0fa2b44 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -168,7 +168,7 @@ jobs: - name: Upload thisbuild artifacts - apps if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildAppsArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/Apps/' @@ -177,7 +177,7 @@ jobs: - name: Upload thisbuild artifacts - dependencies if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildDependenciesArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/Dependencies/' @@ -186,7 +186,7 @@ jobs: - name: Upload thisbuild artifacts - test apps if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildTestAppsArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/TestApps/' @@ -194,7 +194,7 @@ jobs: retention-days: 1 - name: Publish artifacts - apps - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.AppsArtifactsName }} @@ -202,7 +202,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - dependencies - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts && env.generateDependencyArtifact == 'True' with: name: ${{ steps.calculateArtifactsNames.outputs.DependenciesArtifactsName }} @@ -210,7 +210,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - test apps - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.TestAppsArtifactsName }} @@ -218,7 +218,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - build output - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/BuildOutput.txt',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.BuildOutputArtifactsName }} @@ -226,7 +226,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - container event log - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (failure()) && (hashFiles(format('{0}/ContainerEventLog.evtx',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.ContainerEventLogArtifactsName }} @@ -234,7 +234,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - test results - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/TestResults.xml',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.TestResultsArtifactsName }} @@ -242,7 +242,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - bcpt test results - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/bcptTestResults.json',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.BcptTestResultsArtifactsName }} diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index a875964ff..9f0fa2b44 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -168,7 +168,7 @@ jobs: - name: Upload thisbuild artifacts - apps if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildAppsArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/Apps/' @@ -177,7 +177,7 @@ jobs: - name: Upload thisbuild artifacts - dependencies if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildDependenciesArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/Dependencies/' @@ -186,7 +186,7 @@ jobs: - name: Upload thisbuild artifacts - test apps if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildTestAppsArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/TestApps/' @@ -194,7 +194,7 @@ jobs: retention-days: 1 - name: Publish artifacts - apps - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.AppsArtifactsName }} @@ -202,7 +202,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - dependencies - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts && env.generateDependencyArtifact == 'True' with: name: ${{ steps.calculateArtifactsNames.outputs.DependenciesArtifactsName }} @@ -210,7 +210,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - test apps - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.TestAppsArtifactsName }} @@ -218,7 +218,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - build output - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/BuildOutput.txt',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.BuildOutputArtifactsName }} @@ -226,7 +226,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - container event log - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (failure()) && (hashFiles(format('{0}/ContainerEventLog.evtx',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.ContainerEventLogArtifactsName }} @@ -234,7 +234,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - test results - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/TestResults.xml',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.TestResultsArtifactsName }} @@ -242,7 +242,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - bcpt test results - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/bcptTestResults.json',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.BcptTestResultsArtifactsName }} diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml index bcf6e5479..d49bc9f3e 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml @@ -92,7 +92,7 @@ jobs: suffix: ${{ inputs.artifactsNameSuffix }} - name: Publish artifacts - Power Platform Solution - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.PowerPlatformSolutionArtifactsName }} From e599e3873415ceae163632b930d07c5b033e9bc1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:20:54 +0200 Subject: [PATCH 080/215] update --- .github/workflows/powershell.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/powershell.yaml b/.github/workflows/powershell.yaml index 5a7883ad4..04885b056 100644 --- a/.github/workflows/powershell.yaml +++ b/.github/workflows/powershell.yaml @@ -31,6 +31,6 @@ jobs: # Upload the SARIF file generated in the previous step - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 + uses: github/codeql-action/upload-sarif@429e1977040da7a23b6822b13c129cd1ba93dbb2 # v3.26.2 with: sarif_file: results.sarif From 5cc070b3c4b671586bf20aaf7a0ef2e382addfa2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:27:39 +0200 Subject: [PATCH 081/215] update release notes --- Actions/ReadSecrets/ReadSecretsHelper.psm1 | 11 ++++- RELEASENOTES.md | 57 ++++++++++++---------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index caa10acd0..4f7b48044 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -26,7 +26,6 @@ function GetAzureCredentials { return $null } - function MaskValue { Param( [string] $key, @@ -119,6 +118,16 @@ function GetKeyVaultCredentials { $creds = $null } } + if ($creds) { + try { + # check that we have access to get secrets from the keyvault by trying to get a dummy secret + GetKeyVaultSecret -secretName 'algodummysecret' -keyVaultCredentials $creds -encrypted | Out-Null + } + catch { + Write-Host "Unable to get secrets from Azure Key Vault. Error was $($_.Exception.Message). Using Github secrets instead." + $creds = $null + } + } return $creds } diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 9abaac27e..55490330f 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,36 @@ ### Issues +### New Settings + +- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: + + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) + - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Add custom steps to the AL-Go build workflow + +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Indirect AL-Go template repository + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + +## v5.3 + +### Issues + - Issue 1105 Increment Version Number - repoVersion in .github/AL-Go-Settings.json is not updated - Issue 1073 Publish to AppSource - Automated validation: failure - Issue 980 Allow Scope to be PTE in continuousDeployment for PTE extensions in Sandbox (enhancement request) @@ -10,6 +41,7 @@ - Fix issue with github ref when running reusable workflows - Issue 1098 Support for specifying the name of the AZURE_CREDENTIALS secret by adding a AZURE_CREDENTIALSSecretName setting - Fix placeholder syntax for git ref in PullRequestHandler.yaml +- Issue 1164 Getting secrets from Azure key vault fails in Preview ### Dependencies to PowerShell modules @@ -47,34 +79,9 @@ AL-Go for GitHub now includes a new telemetry module. For detailed information o - **NumberOfSqlStmtsWarning** - a warning is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 5) - **NumberOfSqlStmtsError** - an error is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 10) -- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: - - - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) - - **Source** (mandatory) - URL to a either a single file or a .zip file containing - - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. - - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) - > \[!NOTE\] > Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. -### Add custom jobs to AL-Go workflows - -It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. - -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Add custom steps to the AL-Go build workflow - -It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. - -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Indirect AL-Go template repository - -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. - -Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. - ## v5.2 ### Issues From cde82c9ce5a7d89b52d4ef1b81f08b2d8871f806 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:29:03 +0200 Subject: [PATCH 082/215] rev --- RELEASENOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 55490330f..74551a3ba 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -212,7 +212,7 @@ If false, the templateSha repository setting is used to download specific AL-Go ### New Settings -- `[alDoc](https://aka.ms/algosettings#aldoc)` : JSON object with properties for the ALDoc reference document generation +- `alDoc` : JSON object with properties for the ALDoc reference document generation - **continuousDeployment** = Determines if reference documentation will be deployed continuously as part of CI/CD. You can run the **Deploy Reference Documentation** workflow to deploy manually or on a schedule. (Default false) - **deployToGitHubPages** = Determines whether or not the reference documentation site should be deployed to GitHub Pages for the repository. In order to deploy to GitHub Pages, GitHub Pages must be enabled and set to GitHub Actions. (Default true) - **maxReleases** = Maximum number of releases to include in the reference documentation. (Default 3) From d60be053389f36bcc8e76cf17c444f670253ab44 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:40:21 +0200 Subject: [PATCH 083/215] psa --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 11a29212b..ea51d358c 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -494,7 +494,7 @@ function GetCustomALGoSystemFiles { [string] $destination, [string[]] $projects ) - + if ($destination -like ".AL-Go$([IO.Path]::DirectorySeparatorChar)*") { $destinations = $projects | ForEach-Object { Join-Path $_ $destination } } From fa4a71bb17f27e5e59ac56b1adbd393802941b2c Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:43:19 +0200 Subject: [PATCH 084/215] remove --- RELEASENOTES.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 3fce26636..74551a3ba 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,3 @@ -## v5.3 - ### Issues ### New Settings From 6685835c9676489effbb9c9dd4fb68c7361e5178 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:46:15 +0200 Subject: [PATCH 085/215] move --- .../.github/workflows/UpdateGitHubGoSystemFiles.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 9ded4de35..bfdf73edd 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -65,10 +65,10 @@ jobs: - name: Override templateUrl env: - templateUrl: ${{ github.event.inputs.templateUrl }} + newTemplateUrl: ${{ github.event.inputs.templateUrl }} run: | $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 - $templateUrl = $ENV:templateUrl + $templateUrl = $ENV:newTemplateUrl if ($templateUrl) { Write-Host "Using Template Url: $templateUrl" Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "templateUrl=$templateUrl" From 99265e6e4dec11ac8e44f1abe13fd9eb7f448b37 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:48:14 +0200 Subject: [PATCH 086/215] remove fake button --- Internal/Deploy.ps1 | 8 -------- Templates/AppSource App/README.md | 2 -- Templates/Per Tenant Extension/README.md | 2 -- 3 files changed, 12 deletions(-) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 122cab4ee..c6a9e05d6 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -252,14 +252,6 @@ try { } Set-Content -Path (Join-Path "./.github" "RELEASENOTES.copy.md") -Value $releaseNotes -Encoding utf8 } - - # Replace template_owner in README.md - $readmeFile = './README.md' - $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" - $readme = $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") - Set-Content -Path $readmeFile -Encoding utf8 -Value $readme - - # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit } diff --git a/Templates/AppSource App/README.md b/Templates/AppSource App/README.md index ab1063e63..ea984de77 100644 --- a/Templates/AppSource App/README.md +++ b/Templates/AppSource App/README.md @@ -2,8 +2,6 @@ This template repository can be used for managing AppSource Apps for Business Central. -[![Use this template](https://github.com/microsoft/AL-Go/assets/10775043/ca1ecc85-2fd3-4ab5-a866-bd2e7e80259d)](https://github.com/new?template_name=AL-Go-AppSource&template_owner=microsoft) - Please go to https://aka.ms/AL-Go to learn more. ## Contributing diff --git a/Templates/Per Tenant Extension/README.md b/Templates/Per Tenant Extension/README.md index 81dd89505..45baaa734 100644 --- a/Templates/Per Tenant Extension/README.md +++ b/Templates/Per Tenant Extension/README.md @@ -2,8 +2,6 @@ This template repository can be used for managing AppSource Apps for Business Central. -[![Use this template](https://github.com/microsoft/AL-Go/assets/10775043/ca1ecc85-2fd3-4ab5-a866-bd2e7e80259d)](https://github.com/new?template_name=AL-Go-PTE&template_owner=microsoft) - Please go to https://aka.ms/AL-Go to learn more. ## Contributing From 0969502bf96c62df795b65b21dbcea4ce6f25182 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:50:16 +0200 Subject: [PATCH 087/215] use preview --- Actions/AL-Go-Helper.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 73c731ad9..5f98c53e8 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -18,7 +18,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent -$defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/archive/refs/heads/runpipeline.zip" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" +$defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" $notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","EnvironmentName") $runAlPipelineOverrides = @( From a2dbcfdcdde03e6827e037a98fd984ee1c408bb7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:55:05 +0200 Subject: [PATCH 088/215] precommit --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- Scenarios/CustomizingALGoForGitHub.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ea51d358c..b6f8a68c0 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -583,4 +583,4 @@ function GetCustomALGoSystemFiles { Remove-Item -Path $tempFolder -Recurse -Force } } -} \ No newline at end of file +} diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 3aa08539f..f6095e957 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -14,7 +14,7 @@ Microsoft will continuously develop and maintain AL-Go for GitHub and ensure tha Keeping your repositories up-to-date can be done manually or on a schedule (like Windows update really). You will be notified when an update is available and we recommend that you keep your repositories up-to-date at all time. If you make modifications to the AL-Go System Files (scripts and workflows) in your repository, in other ways than described in this document, these changes will be removed with the next AL-Go update. -> \[!TIP] +> \[!TIP\] > If for some reason the updated version of AL-Go for GitHub doesn't work for you, we recommend that you file an issue [here](https://github.com/microsoft/AL-Go/issues) with a detailed description of the problem and full logs of the failing workflows. You can then revert back to the prior version of AL-Go for GitHub until the issue is resolved. > > It is important to get back to the mainstream version of AL-Go for GitHub as soon as the issue is resolved. @@ -222,13 +222,13 @@ At this time, the anchor-points where you can add custom steps are: | Workflow | Job | Step | Before or AFter | | :-- | :-- | :-- | :-: | -| _BuildALGoProject.yaml | BuildALGoProject | Read settings | After | +| \_BuildALGoProject.yaml | BuildALGoProject | Read settings | After | | | | Read secrets | After | | | | Build | Before | | | | Read secrets | After | | | | Cleanup | Before | -The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic +The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic ```yaml - name: CustomStep-ModifySettings From a01e57addc0fe9b24aceb40f8b223d1e568241a2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:10:06 +0200 Subject: [PATCH 089/215] dump token length --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index b1bf85328..5e1a27208 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -28,6 +28,7 @@ if (-not $token) { } else { $token = $env:GITHUB_TOKEN + Write-Host $token.Length } } else { From 46da82c8433ad268924088bc4c369c3429dfaad9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:21:30 +0200 Subject: [PATCH 090/215] add ghTokenWorkflow parameter --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 18 +++++++----------- Actions/CheckForUpdates/action.yaml | 4 ++++ .../AppSource App/.github/workflows/CICD.yaml | 2 +- .../.github/workflows/CreateRelease.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- .../.github/workflows/CICD.yaml | 2 +- .../.github/workflows/CreateRelease.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 5e1a27208..43e33e90e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -3,6 +3,8 @@ [string] $actor, [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] [string] $token, + [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] + [string] $ghTokenWorkflow, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", [Parameter(HelpMessage = "Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update)", Mandatory = $true)] @@ -22,18 +24,12 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -if (-not $token) { - if ($update -eq 'Y') { - throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." - } - else { - $token = $env:GITHUB_TOKEN - Write-Host $token.Length - } -} -else { +if ($ghTokenWorkflow) { # Specified token is GhTokenWorkflow secret - decode from base 64 - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ghTokenWorkflow)) +} +elseif ($update -eq 'Y') { + throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } # Use Authenticated API request to avoid the 60 API calls per hour limit diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 28ea8e24e..18ff727fb 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -16,6 +16,10 @@ inputs: token: description: The GitHub token running the action required: false + default: ${{ github.token }} + ghTokenWorkflow: + description: Base64 encoded GhTokenWorkflow secret + required: false default: '' templateUrl: description: URL of the template repository (default is the template repository used to create the repository) diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index 851f956ed..cf4c0882a 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -146,7 +146,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index c3d74ff85..7d0410947 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -112,7 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index bfdf73edd..9c2f857f7 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -96,7 +96,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ env.templateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 1ee6dc260..c7f158d0a 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -146,7 +146,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index b84620364..c10a981b6 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -112,7 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index bfdf73edd..9c2f857f7 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -96,7 +96,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ env.templateUrl }} From fa5a2d43f34e7401ecf94b73346ab798be816cf5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:25:09 +0200 Subject: [PATCH 091/215] add ght --- Actions/CheckForUpdates/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 18ff727fb..e799558ad 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -48,6 +48,7 @@ runs: env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} + _ghTokenWorkflow: ${{ inputs.ghTokenWorkflow }} _templateUrl: ${{ inputs.templateUrl }} _downloadLatest: ${{ inputs.downloadLatest }} _update: ${{ inputs.update }} @@ -55,7 +56,7 @@ runs: _directCommit: ${{ inputs.directCommit }} run: | ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "CheckForUpdates" -Action { - ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') + ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -ghTokenWorkflow $ENV:_ghTokenWorkflow -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') } branding: icon: terminal From 5c40c1516a89abf4a1e318c2fd48f8f5a5c2e6ea Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:37:30 +0200 Subject: [PATCH 092/215] use token --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 19 ++++++++++++------- Actions/CheckForUpdates/README.md | 3 ++- Actions/CheckForUpdates/action.yaml | 12 ++++++------ .../AppSource App/.github/workflows/CICD.yaml | 2 +- .../.github/workflows/CreateRelease.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- .../.github/workflows/CICD.yaml | 2 +- .../.github/workflows/CreateRelease.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- 9 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 43e33e90e..952a2dfbc 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -1,10 +1,10 @@ Param( [Parameter(HelpMessage = "The GitHub actor running the action", Mandatory = $false)] [string] $actor, - [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] - [string] $token, [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] - [string] $ghTokenWorkflow, + [string] $token, + [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] + [string] $githubToken, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", [Parameter(HelpMessage = "Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update)", Mandatory = $true)] @@ -24,12 +24,17 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -if ($ghTokenWorkflow) { +if ($token) { # Specified token is GhTokenWorkflow secret - decode from base 64 - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ghTokenWorkflow)) + $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } -elseif ($update -eq 'Y') { - throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." +else { + if ($update -eq 'Y') { + throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." + } + else { + $token = $githubToken + } } # Use Authenticated API request to avoid the 60 API calls per hour limit diff --git a/Actions/CheckForUpdates/README.md b/Actions/CheckForUpdates/README.md index 554ca9db5..c3db39db0 100644 --- a/Actions/CheckForUpdates/README.md +++ b/Actions/CheckForUpdates/README.md @@ -14,7 +14,8 @@ none | :-- | :-: | :-- | :-- | | shell | | The shell (powershell or pwsh) in which the PowerShell script in this action should run | powershell | | actor | | The GitHub actor running the action | github.actor | -| token | | The GitHub token running the action | github.token | +| token | | Base64 encoded GhTokenWorkflow secret | | +| githubToken | | The GitHub token running the action | github.token | | templateUrl | | URL of the template repository (default is the template repository used to create the repository) | default | | downloadLatest | Yes | Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update) | | | update | | Set this input to Y in order to update AL-Go System Files if needed | N | diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index e799558ad..b1dafdad6 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -14,13 +14,13 @@ inputs: required: false default: ${{ github.actor }} token: - description: The GitHub token running the action - required: false - default: ${{ github.token }} - ghTokenWorkflow: description: Base64 encoded GhTokenWorkflow secret required: false default: '' + githubToken: + description: The GitHub token running the action + required: false + default: ${{ github.token }} templateUrl: description: URL of the template repository (default is the template repository used to create the repository) required: false @@ -48,7 +48,7 @@ runs: env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} - _ghTokenWorkflow: ${{ inputs.ghTokenWorkflow }} + _githubToken: ${{ inputs.githubToken }} _templateUrl: ${{ inputs.templateUrl }} _downloadLatest: ${{ inputs.downloadLatest }} _update: ${{ inputs.update }} @@ -56,7 +56,7 @@ runs: _directCommit: ${{ inputs.directCommit }} run: | ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "CheckForUpdates" -Action { - ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -ghTokenWorkflow $ENV:_ghTokenWorkflow -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') + ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -githubToken $ENV:_githubToken -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') } branding: icon: terminal diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index cf4c0882a..851f956ed 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -146,7 +146,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index 7d0410947..c3d74ff85 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -112,7 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 9c2f857f7..bfdf73edd 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -96,7 +96,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ env.templateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index c7f158d0a..1ee6dc260 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -146,7 +146,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index c10a981b6..b84620364 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -112,7 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 9c2f857f7..bfdf73edd 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -96,7 +96,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ env.templateUrl }} From 000ae79c8bf638d58320efcfb323485c402c1e4f Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:47:43 +0200 Subject: [PATCH 093/215] dump with token is used --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 952a2dfbc..d6068148e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -26,6 +26,7 @@ DownloadAndImportBcContainerHelper if ($token) { # Specified token is GhTokenWorkflow secret - decode from base 64 + Write-Host "Using ghTokenWorkflow secret" $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } else { @@ -33,6 +34,7 @@ else { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } else { + Write-Host "Using GitHub Token" $token = $githubToken } } From 26149ded3e120e4f3b95e460fec1b8e92d8a3d17 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 13:00:29 +0200 Subject: [PATCH 094/215] check if permissions --- Actions/CheckForUpdates/yamlclass.ps1 | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index d8ae1016d..a109ed574 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -443,10 +443,14 @@ class Yaml { } # Merge permissions Write-host "Merge permissions" - $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) - $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) - if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { - $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + $srcPermissionsObj = $srcYaml.Get('permissions:/') + $yamlPermissionsObj = $yaml.Get('permissions:/') + if ($srcPermissionsObj -and $yamlPermissionsObj) { + $srcPermissions = [Yaml]::GetPermissionsFromArray($srcPermissionsObj.content) + $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) + if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { + $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + } } # Apply cystom steps From 0e6e7c31a1b16cf3587e35dbb3f83efa121dc834 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 2 Sep 2024 08:30:37 +0200 Subject: [PATCH 095/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index f6095e957..a1946081c 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -2,7 +2,7 @@ AL-Go for GitHub is a plug-and-play DevOps solution, intended to support 100% of the functionality needed by 90% of the people developing applications for Microsoft Dynamics 365 Business Central out-of-the-box. -If AL-Go functionality out-of-the-box doesn't match your needs, you really three options: +If AL-Go functionality out-of-the-box doesn't match your needs, you really have three options: 1. Customize AL-Go for GitHub to fit your needs 1. Select another managed DevOps solution From 858865e95f94ba930813d7ea845d256bdd8c35f7 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 2 Sep 2024 08:30:52 +0200 Subject: [PATCH 096/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index a1946081c..6403cbe9b 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -23,7 +23,7 @@ There are three ways you can customize AL-Go for GitHub to fit your needs. You c 1. customize the repository with custom scripts, workflows, jobs or steps following the guidelines below 1. create a customized repository and use this as your template repository (indirect template) -1. fork the AL-Go for GitHub and create your "own" version +1. fork the AL-Go for GitHub and create your "own" version (not recommended) > \[!CAUTION\] > The more you customize AL-Go for GitHub, the more likely you are to be broken by future updates to AL-Go for GitHub, meaning that you will have to update your customizations to match the changes in AL-Go for GitHub. From f252f296cd6b19946ca9b245a867422d3a3bda74 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 3 Sep 2024 14:25:13 +0200 Subject: [PATCH 097/215] empty line --- RELEASENOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index abd73e308..e7ec62523 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,4 +1,5 @@ ### Issues + - Issue 1184 Publish to Environment fails on 'Permission Denied' ### New Settings From 2e7722ad04af521d257d172890f44ac730b976c2 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 9 Sep 2024 14:14:29 +0200 Subject: [PATCH 098/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 6403cbe9b..5dfad600c 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -172,7 +172,7 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings | :-- | :-- | :-: | :-- | | Destination | Path in which the file should be placed. Can include the filename if the source doesn't point to a .zip file, must include a terminating / or \\ if a filename is not included. | Yes | | | Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | -| FileSpec | If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | +| FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | | Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | This setting will cause AL-Go for GitHub to include these files during the next update. From a76dc7deaff8862038506c76d8174215144080a1 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 9 Sep 2024 14:14:46 +0200 Subject: [PATCH 099/215] Update Scenarios/settings.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 524688c6e..8a582b9af 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -187,7 +187,7 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza # Expert level -The settings and functionality in the expert section might requires knowledge about GitHub Workflows/Actions, yaml, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. +The settings and functionality in the expert section might require knowledge about GitHub Workflows/Actions, YAML, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. Please read the release notes carefully when installing new versions of AL-Go for GitHub. From ddef716f38001eee63b1457f711c4684eac3dbd2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 13 Sep 2024 12:49:48 -0400 Subject: [PATCH 100/215] prop --- Actions/AL-Go-Helper.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 65335006c..e85599238 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -465,6 +465,10 @@ function MergeCustomObjectIntoOrderedDictionary { if ($srcPropType -eq "PSCustomObject" -and $dstPropType -eq "OrderedDictionary") { MergeCustomObjectIntoOrderedDictionary -dst $dst."$prop" -src $srcProp } + elseif ($dstProp -is [String] -and $srcProp -is [Object[]]) { + # For properties like "runs-on", which is a string, you can specify an array in settings, which gets joined with a comma + $dst."$prop" = $srcProp -join ', ' + } elseif ($dstPropType -ne $srcPropType -and !($srcPropType -eq "Int64" -and $dstPropType -eq "Int32")) { # Under Linux, the Int fields read from the .json file will be Int64, while the settings defaults will be Int32 # This is not seen as an error and will not throw an error From 2ca9c43b14a14316b7cddfcb3775b421140ebf64 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 13 Oct 2024 08:25:51 +0200 Subject: [PATCH 101/215] merge --- .../AppSource App/.github/workflows/_BuildALGoProject.yaml | 2 +- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index ba5187c9e..c5ba272db 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -106,7 +106,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS,cicdAuthContext' + getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,cicdAuthContext' - name: Determine ArtifactUrl uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index ba5187c9e..c5ba272db 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -106,7 +106,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS,cicdAuthContext' + getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,cicdAuthContext' - name: Determine ArtifactUrl uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main From dc4580f19b9668ea5f9dc05df6917531c07e762b Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 13 Oct 2024 08:27:18 +0200 Subject: [PATCH 102/215] merge --- .github/workflows/jekyll-gh-pages.yml | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml new file mode 100644 index 000000000..10ed10e90 --- /dev/null +++ b/.github/workflows/jekyll-gh-pages.yml @@ -0,0 +1,49 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy Jekyll with GitHub Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Setup Pages + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 + with: + source: . + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 + with: + preview: true From 2cf539e33eb1eb58177b474f1b3efcf5b3000a53 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 29 Oct 2024 07:50:50 +0100 Subject: [PATCH 103/215] remove --- .github/workflows/jekyll-gh-pages.yml | 49 --------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml deleted file mode 100644 index 10ed10e90..000000000 --- a/.github/workflows/jekyll-gh-pages.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll with GitHub Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 - with: - source: . - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 - with: - preview: true From 3ad4cbe6bdd0a5575a3de1040985afc53700048b Mon Sep 17 00:00:00 2001 From: Patrick Schiefer <35697862+PatrickSchiefer@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:59:48 +0100 Subject: [PATCH 104/215] feat: add needsContext as env to build process (#206) --- Templates/AppSource App/.github/workflows/CICD.yaml | 1 + .../AppSource App/.github/workflows/_BuildALGoProject.yaml | 5 +++++ Templates/Per Tenant Extension/.github/workflows/CICD.yaml | 1 + .../.github/workflows/_BuildALGoProject.yaml | 5 +++++ 4 files changed, 12 insertions(+) diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index 74fc6fac8..796d68a29 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -172,6 +172,7 @@ jobs: publishArtifacts: ${{ github.ref_name == 'main' || startswith(github.ref_name, 'release/') || startswith(github.ref_name, 'releases/') || needs.Initialization.outputs.deliveryTargetsJson != '[]' || needs.Initialization.outputs.environmentCount > 0 }} signArtifacts: true useArtifactCache: true + needsContext: ${{ toJson(needs) }} DeployALDoc: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index a8f4017fd..91b3fc308 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -67,6 +67,10 @@ on: description: Flag determining whether to use the Artifacts Cache type: boolean default: false + needsContext: + description: JSON formated needs context + type: string + default: false permissions: actions: read @@ -139,6 +143,7 @@ jobs: env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} + NeedsContext: ${{ inputs.needsContext }} with: shell: ${{ inputs.shell }} artifact: ${{ env.artifact }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 307dca38e..a7068a0dd 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -172,6 +172,7 @@ jobs: publishArtifacts: ${{ github.ref_name == 'main' || startswith(github.ref_name, 'release/') || startswith(github.ref_name, 'releases/') || needs.Initialization.outputs.deliveryTargetsJson != '[]' || needs.Initialization.outputs.environmentCount > 0 }} signArtifacts: true useArtifactCache: true + needsContext: ${{ toJson(needs) }} BuildPP: needs: [ Initialization ] diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index a8f4017fd..91b3fc308 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -67,6 +67,10 @@ on: description: Flag determining whether to use the Artifacts Cache type: boolean default: false + needsContext: + description: JSON formated needs context + type: string + default: false permissions: actions: read @@ -139,6 +143,7 @@ jobs: env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} + NeedsContext: ${{ inputs.needsContext }} with: shell: ${{ inputs.shell }} artifact: ${{ env.artifact }} From 1355bcaf593cb93b354ad0ab34e8662b64bd2080 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 29 Oct 2024 09:02:04 +0100 Subject: [PATCH 105/215] modify default --- .../AppSource App/.github/workflows/_BuildALGoProject.yaml | 4 ++-- .../.github/workflows/_BuildALGoProject.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index 91b3fc308..b2bf7ab39 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -68,9 +68,9 @@ on: type: boolean default: false needsContext: - description: JSON formated needs context + description: JSON formatted needs context type: string - default: false + default: '{}' permissions: actions: read diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index 91b3fc308..b2bf7ab39 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -68,9 +68,9 @@ on: type: boolean default: false needsContext: - description: JSON formated needs context + description: JSON formatted needs context type: string - default: false + default: '{}' permissions: actions: read From 3fa79d6ce8d58947cb3213c0e22e7fccbab3abdb Mon Sep 17 00:00:00 2001 From: Patrick Schiefer <35697862+PatrickSchiefer@users.noreply.github.com> Date: Tue, 29 Oct 2024 20:17:38 +0100 Subject: [PATCH 106/215] feat: add needContext to Build job (#208) --- Templates/AppSource App/.github/workflows/Current.yaml | 1 + Templates/AppSource App/.github/workflows/NextMajor.yaml | 1 + Templates/AppSource App/.github/workflows/NextMinor.yaml | 1 + .../AppSource App/.github/workflows/PullRequestHandler.yaml | 1 + Templates/Per Tenant Extension/.github/workflows/Current.yaml | 1 + Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml | 1 + Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml | 1 + .../.github/workflows/PullRequestHandler.yaml | 1 + 8 files changed, 8 insertions(+) diff --git a/Templates/AppSource App/.github/workflows/Current.yaml b/Templates/AppSource App/.github/workflows/Current.yaml index 9226401b1..1d8c4ee26 100644 --- a/Templates/AppSource App/.github/workflows/Current.yaml +++ b/Templates/AppSource App/.github/workflows/Current.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'Current' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/NextMajor.yaml b/Templates/AppSource App/.github/workflows/NextMajor.yaml index ec3bade7f..0d0f77b53 100644 --- a/Templates/AppSource App/.github/workflows/NextMajor.yaml +++ b/Templates/AppSource App/.github/workflows/NextMajor.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMajor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/NextMinor.yaml b/Templates/AppSource App/.github/workflows/NextMinor.yaml index 79ad8e15e..4dff645ca 100644 --- a/Templates/AppSource App/.github/workflows/NextMinor.yaml +++ b/Templates/AppSource App/.github/workflows/NextMinor.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMinor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml b/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml index 2d452db89..15482ee49 100644 --- a/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml +++ b/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml @@ -99,6 +99,7 @@ jobs: secrets: 'licenseFileUrl,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'PR${{ github.event.number }}' + needsContext: ${{ toJson(needs) }} StatusCheck: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/Current.yaml b/Templates/Per Tenant Extension/.github/workflows/Current.yaml index 9226401b1..1d8c4ee26 100644 --- a/Templates/Per Tenant Extension/.github/workflows/Current.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/Current.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'Current' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml b/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml index ec3bade7f..0d0f77b53 100644 --- a/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMajor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml b/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml index 79ad8e15e..4dff645ca 100644 --- a/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMinor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml b/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml index 2d452db89..15482ee49 100644 --- a/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml @@ -99,6 +99,7 @@ jobs: secrets: 'licenseFileUrl,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'PR${{ github.event.number }}' + needsContext: ${{ toJson(needs) }} StatusCheck: needs: [ Initialization, Build ] From bbd9cd2121f7771656d0b85d851cd9079717f905 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 18 Dec 2024 23:01:07 +0100 Subject: [PATCH 107/215] move --- RELEASENOTES.md | 54 ++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1cd003ba7..56da4fd55 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,6 +2,33 @@ - It is now possible to skip the modification of dependency version numbers when running the Increment Version number workflow or the Create Release workflow +### New Settings + +- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: + + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) + - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Add custom steps to the AL-Go build workflow + +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Indirect AL-Go template repository + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + ### New Versioning Strategy Setting versioning strategy to 3 will allow 3 segments of the version number to be defined in app.json and repoVersion. Only the 4th segment (Revision) will be defined by the GitHub [run_number](https://go.microsoft.com/fwlink/?linkid=2217416&clcid=0x409) for the CI/CD workflow. Increment version number and Create Release now also supports the ability to set a third segment to the RepoVersion and appversion in app.json. @@ -82,33 +109,6 @@ Page Scripting tests are now supported as part of CI/CD. By specifying pageScrip With this release the implementation for delivering to NuGet packages (by adding the NuGetContext secret), is similar to the functionality behind delivering to GitHub packages and the implementation is no longer in preview. -### New Settings - -- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: - - - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) - - **Source** (mandatory) - URL to a either a single file or a .zip file containing - - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. - - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) - -### Add custom jobs to AL-Go workflows - -It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. - -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Add custom steps to the AL-Go build workflow - -It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. - -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Indirect AL-Go template repository - -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. - -Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. - ### Allow GitHubRunner and GitHubRunnerShell as project settings Previously, AL-Go required the GitHubRunner and GitHubRunnerShell settings to be set on repository level. This has now been changed such that they can be set on project level. From 11a3f6785af383b32691259b3793241a5302bbaa Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 25 Dec 2024 12:27:12 -0800 Subject: [PATCH 108/215] or --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 32a298a2b..245a006ab 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -566,7 +566,7 @@ function GetCustomALGoSystemFiles { if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." } - if (!($destination.EndsWith([IO.Path]::DirectorySeparatorChar))) { + if (!($destination.EndsWith('/') -or $destination.EndsWith('\'))) { throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." } Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName From d52953807217ed50f795bd1aec14a77e386c7a1d Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 11 Jan 2025 06:15:04 +0100 Subject: [PATCH 109/215] precommit --- Scenarios/CustomizingALGoForGitHub.md | 42 +++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 5dfad600c..24db42256 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -14,7 +14,7 @@ Microsoft will continuously develop and maintain AL-Go for GitHub and ensure tha Keeping your repositories up-to-date can be done manually or on a schedule (like Windows update really). You will be notified when an update is available and we recommend that you keep your repositories up-to-date at all time. If you make modifications to the AL-Go System Files (scripts and workflows) in your repository, in other ways than described in this document, these changes will be removed with the next AL-Go update. -> \[!TIP\] +> [!TIP] > If for some reason the updated version of AL-Go for GitHub doesn't work for you, we recommend that you file an issue [here](https://github.com/microsoft/AL-Go/issues) with a detailed description of the problem and full logs of the failing workflows. You can then revert back to the prior version of AL-Go for GitHub until the issue is resolved. > > It is important to get back to the mainstream version of AL-Go for GitHub as soon as the issue is resolved. @@ -25,7 +25,7 @@ There are three ways you can customize AL-Go for GitHub to fit your needs. You c 1. create a customized repository and use this as your template repository (indirect template) 1. fork the AL-Go for GitHub and create your "own" version (not recommended) -> \[!CAUTION\] +> [!CAUTION] > The more you customize AL-Go for GitHub, the more likely you are to be broken by future updates to AL-Go for GitHub, meaning that you will have to update your customizations to match the changes in AL-Go for GitHub. ## Customizing your repository @@ -120,7 +120,7 @@ jobs: It is recommended to prefix your workflows with `my`, `our`, your name or your organization name in order to avoid that the workflow suddenly gets overridden by a new workflow in AL-Go for GitHub. The above workflow is a real example from [here](https://github.com/microsoft/BCApps/blob/main/.github/workflows/CreateBuildTag.yaml). -> \[!CAUTION\] +> [!CAUTION] > This workflow gets triggered when the CI/CD workflow has completed. Note that the name of the CI/CD workflow currently is prefixed with a space, this space will very likely be removed in the future, which is why we specify both names in this example. Obviously this workflow would break if we decide to rename the CI/CD workflow to something different. ### Adding custom scripts @@ -138,7 +138,7 @@ $script = Join-Path $PSScriptRoot "../../../scripts/NewBcContainer.ps1" -Resolve Which basically launches a script located in the script folder in the repository for creating the build container needed for building and testing the System Application. -> \[!CAUTION\] +> [!CAUTION] > Script overrides will almost certainly be broken in the future. The current script overrides is very much tied to the current implementation of the `Run-AlPipeline` function in BcContainerHelper. In the future, we will move this functionality to GitHub actions and no longer depend on BcContainerHelper and Run-AlPipeline. At that time, these script overrides will have to be changed to follow the new implementation. @@ -171,16 +171,16 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings | Property | Description | Mandatory | Default | | :-- | :-- | :-: | :-- | | Destination | Path in which the file should be placed. Can include the filename if the source doesn't point to a .zip file, must include a terminating / or \\ if a filename is not included. | Yes | | -| Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | -| FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | +| Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | +| FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | | Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | This setting will cause AL-Go for GitHub to include these files during the next update. -> \[!WARNING\] +> [!WARNING] > You can override existing AL-Go for GitHub system files this way, please prefix files in your repository with `my` or your organization name (except for DeployTo and DeliverTo) in order to avoid overriding future workflows from AL-Go for GitHub. -> \[!NOTE\] +> [!NOTE] > If the destination is in the .AL-Go folder, the file(s) will be copied to all .AL-Go folders in multi-project repositories. ### Adding custom jobs @@ -208,10 +208,10 @@ You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Adding a custom job like this, will cause this job to run simultaneously with the deploy and the deliver jobs. -> \[!NOTE\] +> [!NOTE] > All custom jobs will be moved to the tail of the yaml file when running Update AL-Go System Files, but dependencies to/from the custom jobs will be maintained. -> \[!CAUTION\] +> [!CAUTION] > Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. ### Adding custom steps @@ -239,10 +239,10 @@ The custom step needs to be named `CustomStep` and if inserted in any Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "artifact=$($settings.artifact)" ``` -> \[!TIP\] +> [!TIP] > Create a feature request [here](https://github.com/microsoft/AL-Go/issues/new?assignees=&labels=enhancement&projects=&template=enhancement.yaml&title=%5BEnhancement%5D%3A+) with a description on where you would like additional anchor-points and what you want to use it for. -> \[!CAUTION\] +> [!CAUTION] > Please be aware that changes to AL-Go for GitHub might break with future versions of AL-Go for GitHub. We will of course try to keep these breaking changes to a minimum, but the only way you can be sure to NOT be broken is by NOT customizing AL-Go for GitHub. ### Modifying workflow permissions @@ -255,45 +255,45 @@ If any of your custom steps require permissions, which exceeds the permissions a If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. -> \[!NOTE\] +> [!NOTE] > Indirect templates can be public or private. If you are using a private indirect template, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. **UnusedALGoSystemFiles** and **CustomALGoSystemFiles** will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. -> \[!TIP\] +> [!TIP] > The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a **GhTokenWorkflow** secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. -> \[!NOTE\] +> [!NOTE] > If you use the indirect template as a GitHub template for creating the repository, by clicking use this template in your indirect template - then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. Repositories based on your indirect template will notify you that changes are available for your AL-Go System Files when you update the indirect template only. You will not be notified when new versions of AL-Go for GitHub is released in every repository - only in the indirect template repository. -> \[!WARNING\] +> [!WARNING] > You should ensure that your indirect template repository is kept up-to-date with the latest changes in AL-Go for GitHub. -> \[!TIP\] +> [!TIP] > You can setup the Update AL-Go System Files workflow to run on a schedule to uptake new releases of AL-Go for GitHub regularly. ## Forking AL-Go for GitHub and making your "own" **public** version Using a fork of AL-Go for GitHub to have your "own" public version of AL-Go for GitHub gives you the maximum customization capabilities. It does however also come with the most work. -> \[!NOTE\] +> [!NOTE] > When customizing AL-Go for GitHub using a fork, your customizations are public and will be visible to everyone. For more information, [read this](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-permissions-and-visibility-of-forks). There are two ways of forking AL-Go for GitHub. You can fork the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository or you can fork the template repositories [AL-Go PTE](https://github.com/microsoft/AL-Go-PTE) and/or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource). For simple changes to the templates, you can fork the template repositories and make the changes directly in your fork. Note that we do not accept any pull requests to the template repositories as they are deployed from the main AL-Go repository. We do not actually develop anything in the template repositories ourself. In the template repositories you will find a branch for every version of AL-Go we have shipped. The main branch is the latest version and the preview branch is the next version. You can customize the preview branch and/or the main branch and then use your fork as the template repository when running Update AL-Go System Files from your app repositories. -> \[!NOTE\] +> [!NOTE] > We do NOT accept pull requests to the template repositories. You need to follow the guidelines [here](Contribute.md) in order to contribute to AL-Go development. -> \[!TIP\] +> [!TIP] > When forking the template repositories, you should include all branches in order to be able to use either the latest version of AL-Go or the preview version of AL-Go. When forking the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository, you are basically developing AL-Go in the same way as we are doing in Microsoft. Please follow the guidelines [here](Contribute.md) on how to develop. This gives you maximum customization capabilities, but if your changes are not being contributed to AL-Go, then you will have to merge our changes all the time. -> \[!CAUTION\] +> [!CAUTION] > We strongly suggest that you keep your changes to a minimum and that you keep your fork up-to-date with the latest changes of AL-Go for GitHub at all time. ______________________________________________________________________ From 38fd7bf8dd4bb70e6db628cd9feceb75901b3399 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 11 Jan 2025 06:25:27 +0100 Subject: [PATCH 110/215] precommit --- RELEASENOTES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 09da19d95..c224d2382 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -6,10 +6,12 @@ - It is now possible to skip the modification of dependency version numbers when running the Increment Version number workflow or the Create Release workflow -### New Settings +### New Repository Settings - [`shortLivedArtifactsRetentionDays`](https://aka.ms/algosettings#shortLivedArtifactsRetentionDays) determines the number of days to keep short lived build artifacts (f.ex build artifacts from pull request builds, next minor or next major builds). 1 is default. 0 means use GitHub default. + - [`preProcessorSymbols`](https://aka.ms/algosettings#preProcessorSymbols) is a list of preprocessor symbols to use when building the apps. This setting can be specified in [workflow specific settings files](https://aka.ms/algosettings#where-are-the-settings-located) or in [conditional settings](https://aka.ms/algosettings#conditional-settings). + - `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) From 969e23b3a17fbc61e4c4103d4a32e4051c0676cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20St=C3=B6hr?= <52424849+ps610@users.noreply.github.com> Date: Thu, 30 Jan 2025 10:59:24 +0100 Subject: [PATCH 111/215] fix: existing custom jobs are dropped during updates, don't duplicate existing custom jobs (#210) * fix * Revert "fix" This reverts commit 2bdb909b4dd42b90cd9edb574f786eb6bf402647. * Enhance ModifyBuildWorkflows function to retrieve and include existing job dependencies * fix or * Fix handling of existing job dependencies in ModifyBuildWorkflows function * revert * fix order * select unique * fix --- Actions/CheckForUpdates/yamlclass.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index a109ed574..95d83d5d6 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -275,10 +275,6 @@ class Yaml { $existingJobs = $this.GetNextLevel('jobs:/').Trim(':') Write-Host "Adding New Jobs" foreach($customJob in $customJobs) { - if ($existingJobs -contains $customJob.Name) { - Write-Host "Job $($customJob.Name) already exists" - continue - } Write-Host "$($customJob.Name) has dependencies from $($customJob.NeedsThis -join ',')" foreach($needsthis in $customJob.NeedsThis) { if ($needsthis -eq 'Build') { @@ -289,10 +285,14 @@ class Yaml { } elseif ($existingJobs -contains $needsthis) { # Add dependency to job - $needs = @(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) | Where-Object { $_ }) -join ', ' + $needs = @(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) | Where-Object { $_ } | Select-Object -Unique) -join ', ' $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $needs ]") } } + if ($existingJobs -contains $customJob.Name) { + Write-Host "Job $($customJob.Name) already exists" + continue + } $this.content += @('') + @($customJob.content | ForEach-Object { " $_" }) } } From 4329866de87734bd29cac4d929f7318d168da1c6 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 3 Mar 2025 06:23:35 +0100 Subject: [PATCH 112/215] remove githubtoken --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 -- Actions/CheckForUpdates/README.md | 1 - Actions/CheckForUpdates/action.yaml | 7 +------ 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 0141af7e6..ba805233e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -3,8 +3,6 @@ [string] $actor, [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] [string] $token, - [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] - [string] $githubToken, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", [Parameter(HelpMessage = "Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update)", Mandatory = $true)] diff --git a/Actions/CheckForUpdates/README.md b/Actions/CheckForUpdates/README.md index c3db39db0..c7208f425 100644 --- a/Actions/CheckForUpdates/README.md +++ b/Actions/CheckForUpdates/README.md @@ -15,7 +15,6 @@ none | shell | | The shell (powershell or pwsh) in which the PowerShell script in this action should run | powershell | | actor | | The GitHub actor running the action | github.actor | | token | | Base64 encoded GhTokenWorkflow secret | | -| githubToken | | The GitHub token running the action | github.token | | templateUrl | | URL of the template repository (default is the template repository used to create the repository) | default | | downloadLatest | Yes | Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update) | | | update | | Set this input to Y in order to update AL-Go System Files if needed | N | diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index b1dafdad6..9373ef37c 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -17,10 +17,6 @@ inputs: description: Base64 encoded GhTokenWorkflow secret required: false default: '' - githubToken: - description: The GitHub token running the action - required: false - default: ${{ github.token }} templateUrl: description: URL of the template repository (default is the template repository used to create the repository) required: false @@ -48,7 +44,6 @@ runs: env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} - _githubToken: ${{ inputs.githubToken }} _templateUrl: ${{ inputs.templateUrl }} _downloadLatest: ${{ inputs.downloadLatest }} _update: ${{ inputs.update }} @@ -56,7 +51,7 @@ runs: _directCommit: ${{ inputs.directCommit }} run: | ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "CheckForUpdates" -Action { - ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -githubToken $ENV:_githubToken -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') + ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') } branding: icon: terminal From 78d6a2f452eb4844958100dd69d7f611b4608755 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 3 Mar 2025 06:30:00 +0100 Subject: [PATCH 113/215] one token --- Templates/AppSource App/.github/workflows/CICD.yaml | 1 - Templates/AppSource App/.github/workflows/CreateRelease.yaml | 1 - Templates/Per Tenant Extension/.github/workflows/CICD.yaml | 1 - .../Per Tenant Extension/.github/workflows/CreateRelease.yaml | 1 - 4 files changed, 4 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index c66b81afa..dfae6dd7a 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -166,7 +166,6 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index 2670f2b6a..e664b5a7e 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -117,7 +117,6 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).TokenForPush }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 6a971adaf..1a546972b 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -166,7 +166,6 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index 2670f2b6a..e664b5a7e 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -117,7 +117,6 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).TokenForPush }} downloadLatest: true From 75a7fb56a2ea49b8bd1e81b3b1b4958fc72f039c Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 22 Mar 2025 17:54:34 -0700 Subject: [PATCH 114/215] template settings --- Actions/AL-Go-Helper.ps1 | 10 +++ .../CheckForUpdates.HelperFunctions.ps1 | 17 +---- Actions/CheckForUpdates/CheckForUpdates.ps1 | 62 +++++++++---------- 3 files changed, 40 insertions(+), 49 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index ad6c40ef5..e8a98c45e 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -13,6 +13,8 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S $ALGoFolderName = '.AL-Go' $ALGoSettingsFile = Join-Path '.AL-Go' 'settings.json' $RepoSettingsFile = Join-Path '.github' 'AL-Go-Settings.json' +$TemplateRepoSettingsFile = Join-Path '.github' 'templateRepoSettings.json' +$TemplateProjectSettingsFile = Join-Path '.github' 'templateProjectSettings.json' [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPushBranches', Justification = 'False positive.')] $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] @@ -535,8 +537,10 @@ function MergeCustomObjectIntoOrderedDictionary { # Read settings from the settings files # Settings are read from the following files: # - ALGoOrgSettings (github Variable) = Organization settings variable +# - .github/templateRepoSettings.json = Repository settings from indirect template # - .github/AL-Go-Settings.json = Repository Settings file # - ALGoRepoSettings (github Variable) = Repository settings variable +# - .github/templateProjectSettings.json = Project settings from indirect template # - /.AL-Go/settings.json = Project settings file # - .github/.settings.json = Workflow settings file # - /.AL-Go/.settings.json = Project workflow settings file @@ -734,6 +738,9 @@ function ReadSettings { $settingsObjects += @($orgSettingsVariableObject) } # Read settings from repository settings file + $templateRepoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $TemplateRepoSettingsFile) + $settingsObjects += @($templateRepoSettingsObject) + # Read settings from repository settings file $repoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $RepoSettingsFile) $settingsObjects += @($repoSettingsObject) # Read settings from repository settings variable (parameter) @@ -742,6 +749,9 @@ function ReadSettings { $settingsObjects += @($repoSettingsVariableObject) } if ($project) { + # Read settings from repository settings file + $templateProjectSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $TemplateProjectSettingsFile) + $settingsObjects += @($templateProjectSettingsObject) # Read settings from project settings file $projectFolder = Join-Path $baseFolder $project -Resolve $projectSettingsObject = GetSettingsObject -Path (Join-Path $projectFolder $ALGoSettingsFile) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index b2a610410..5035ed475 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -502,8 +502,7 @@ function GetSrcFolder { function UpdateSettingsFile { Param( [string] $settingsFile, - [hashtable] $updateSettings, - [hashtable] $indirectTemplateSettings = @{} + [hashtable] $updateSettings ) $modified = $false @@ -528,20 +527,6 @@ function UpdateSettingsFile { $modified = $true } } - # Grab settings from indirectTemplateSettings if they are not already in settings - foreach($key in $indirectTemplateSettings.Keys) { - # CustomALGoSystemFiles will not be copied from the indirect template settings - they will be applied to the indirect template - # UnusedALGoSystemFiles will not be copied from the indirect template settings - they will be used during the update process - if (@('customALGoSystemFiles','unusedALGoSystemFiles') -contains $key) { - continue - } - if (!($settings.PSObject.Properties.Name -eq $key)) { - # Add the property if it doesn't exist - $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $indirectTemplateSettings."$key" - $modified = $true - } - } - if ($modified) { # Save the file with LF line endings and UTF8 encoding $settings | Set-JsonContentLF -path $settingsFile diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 48670edf7..0ffda567d 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -85,18 +85,18 @@ $templateBranch = $templateUrl.Split('@')[1] $templateOwner = $templateUrl.Split('/')[3] $templateInfo = "$templateOwner/$($templateUrl.Split('/')[4])" -$indirectTemplateRepoSettings = @{} -$indirectTemplateProjectSettings = @{} - $isDirectALGo = IsDirectALGo -templateUrl $templateUrl if (-not $isDirectALGo) { - $myRepoSettingsFile = Join-Path $templateFolder "*/$RepoSettingsFile" - if (Test-Path -Path $myRepoSettingsFile -PathType Leaf) { - $templateRepoSettings = Get-Content $myRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + $templateRepoSettingsFile = Join-Path $templateFolder "*/$RepoSettingsFile" + if (Test-Path -Path $templateRepoSettingsFile -PathType Leaf) { + $templateRepoSettings = Get-Content $templateRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { # The template repository is a url to another AL-Go repository (an indirect template repository) # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template reposotiry points to the "real" template repository # Copy files and folders from the indirect template repository, but grab the unmodified file from the "real" template repository if it exists and apply customizations + # Copy .github/AL-Go-Settings.json to .github/templateRepoSettings.json (will be read before .github/AL-Go-Settings.json in the final repo) + # Copy .AL-Go/settings.json to .github/templateProjectSettings.json (will be read before .AL-Go/settings.json in the final repo) + Write-Host "Indirect AL-Go template repository detected, downloading the 'real' template repository" $realTemplateUrl = $templateRepoSettings.templateUrl if ($templateRepoSettings.Keys -contains "templateSha") { @@ -105,6 +105,7 @@ if (-not $isDirectALGo) { else { $realTemplateSha = "" } + # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository $realTemplateFolder = DownloadTemplateRepository -headers $headers -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') Write-Host "Real Template Folder: $realTemplateFolder" @@ -114,15 +115,9 @@ if (-not $isDirectALGo) { $templateBranch = $realTemplateUrl.Split('@')[1] $templateOwner = $realTemplateUrl.Split('/')[3] - $indirectTemplateRepoSettings = $templateRepoSettings # If the indirect template contains unusedALGoSystemFiles, we need to remove them from the current repository - if ($indirectTemplateRepoSettings.ContainsKey('unusedALGoSystemFiles')) { - $unusedALGoSystemFiles += $indirectTemplateRepoSettings.unusedALGoSystemFiles - } - $myALGoSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" - if (Test-Path $myALGoSettingsFile -PathType Leaf) { - Write-Host "Read project settings from indirect template repository" - $indirectTemplateProjectSettings = Get-Content $myALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + if ($templateRepoSettings.ContainsKey('unusedALGoSystemFiles')) { + $unusedALGoSystemFiles += $templateRepoSettings.unusedALGoSystemFiles } } } @@ -138,17 +133,24 @@ if (-not $isDirectALGo) { # - All files in .github that ends with .copy.md # - All PowerShell scripts in .AL-Go folders (all projects) $checkfiles = @( - @{ 'dstPath' = Join-Path '.github' 'workflows'; 'srcPath' = Join-Path '.github' 'workflows'; 'pattern' = '*'; 'type' = 'workflow' }, - @{ 'dstPath' = '.github'; 'srcPath' = '.github'; 'pattern' = '*.copy.md'; 'type' = 'releasenotes' } + @{ 'dstPath' = (Join-Path '.github' 'workflows'); 'newname' = ''; 'srcPath' = Join-Path '.github' 'workflows'; 'pattern' = '*'; 'type' = 'workflow' }, + @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.copy.md'; 'type' = 'releasenotes' } ) +if ($realTemplateFolder) { + $checkfiles += @( + @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($TemplateRepoSettingsFile); 'newname' = [system.IO.Path]::GetFileName($TemplateRepoSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($RepoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($RepoSettingsFile); 'type' = 'template repo settings' } + @{ "DstFile" = [system.IO.Path]::GetDirectoryName($TemplateProjectSettingsFile); 'newname' = [system.IO.Path]::GetFileName($TemplateProjectSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($ALGoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($ALGoSettingsFile); ; 'type' = 'template project settings' } + ) +} + # Get the list of projects in the current repository $baseFolder = $ENV:GITHUB_WORKSPACE $projects = @(GetProjectsFromRepository -baseFolder $baseFolder -projectsFromSettings $repoSettings.projects) Write-Host "Projects found: $($projects.Count)" foreach($project in $projects) { Write-Host "- $project" - $checkfiles += @(@{ 'dstPath' = Join-Path $project '.AL-Go'; 'srcPath' = '.AL-Go'; 'pattern' = '*.ps1'; 'type' = 'script' }) + $checkfiles += @(@{ 'dstPath' = Join-Path $project '.AL-Go'; 'newname' = ''; 'srcPath' = '.AL-Go'; 'pattern' = '*.ps1'; 'type' = 'script' }) } # $updateFiles will hold an array of files, which needs to be updated @@ -175,21 +177,13 @@ foreach($checkfile in $checkfiles) { $dstFolder = Join-Path $baseFolder $dstPath $srcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath $realSrcFolder = $null - if ($realTemplateFolder) { + if ($realTemplateFolder -and $type -notlike 'template*settings') { + # Get Real source folder except for template settings - these are applied from the indirect temoplate´repository $realSrcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath } if ($srcFolder) { Push-Location -Path $srcFolder try { - if ($srcPath -eq '.AL-Go' -and $type -eq "script" -and $realSrcFolder) { - Write-Host "Update Project Settings" - # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) - $projectSettingsFile = Join-Path $dstFolder "settings.json" - if (UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -indirectTemplateSettings $indirectTemplateProjectSettings) { - $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } - } - } - # Remove unused AL-Go system files $unusedALGoSystemFiles | ForEach-Object { if (Test-Path -Path (Join-Path $dstFolder $_) -PathType Leaf) { @@ -202,9 +196,14 @@ foreach($checkfile in $checkfiles) { Get-ChildItem -Path $srcFolder -Filter $checkfile.pattern | ForEach-Object { # Read the template file and modify it based on the settings # Compare the modified file with the file in the current repository - $fileName = $_.Name + if ($checkfile.newname) { + $filename = $checkfile.newname + } + else { + $filename = $_.Name + } Write-Host "- $filename" - $dstFile = Join-Path $dstFolder $fileName + $dstFile = Join-Path $dstFolder $filename $srcFile = $_.FullName $realSrcFile = $srcFile $isFileDirectALGo = $isDirectALGo @@ -312,10 +311,7 @@ else { invoke-git status - $repoSettingsFile = Join-Path ".github" "AL-Go-Settings.json" - if (UpdateSettingsFile -settingsFile $repoSettingsFile -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -indirectTemplateSettings $indirectTemplateRepoSettings) { - $updateFiles += @{ "DstFile" = $repoSettingsFile; "content" = (Get-Content -Path $repoSettingsFile -Encoding UTF8 -Raw) } - } + UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } # Update the files # Calculate the release notes, while updating From 941d95f48469551e49bb7427ef6fa9d62dd61963 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 22 Mar 2025 18:07:07 -0700 Subject: [PATCH 115/215] dstpath --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 0ffda567d..a80d2ead6 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -140,7 +140,7 @@ $checkfiles = @( if ($realTemplateFolder) { $checkfiles += @( @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($TemplateRepoSettingsFile); 'newname' = [system.IO.Path]::GetFileName($TemplateRepoSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($RepoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($RepoSettingsFile); 'type' = 'template repo settings' } - @{ "DstFile" = [system.IO.Path]::GetDirectoryName($TemplateProjectSettingsFile); 'newname' = [system.IO.Path]::GetFileName($TemplateProjectSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($ALGoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($ALGoSettingsFile); ; 'type' = 'template project settings' } + @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($TemplateProjectSettingsFile); 'newname' = [system.IO.Path]::GetFileName($TemplateProjectSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($ALGoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($ALGoSettingsFile); ; 'type' = 'template project settings' } ) } From a3ab9cd476b07a12f8dcda9ded8b61866d7132f7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 05:06:37 -0700 Subject: [PATCH 116/215] indirect --- Actions/AL-Go-Helper.ps1 | 4 ++-- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index e8a98c45e..a7f18bd27 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -13,8 +13,8 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S $ALGoFolderName = '.AL-Go' $ALGoSettingsFile = Join-Path '.AL-Go' 'settings.json' $RepoSettingsFile = Join-Path '.github' 'AL-Go-Settings.json' -$TemplateRepoSettingsFile = Join-Path '.github' 'templateRepoSettings.json' -$TemplateProjectSettingsFile = Join-Path '.github' 'templateProjectSettings.json' +$IndirectTemplateRepoSettingsFile = Join-Path '.github' 'templateRepoSettings.json' +$IndirectTemplateProjectSettingsFile = Join-Path '.github' 'templateProjectSettings.json' [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPushBranches', Justification = 'False positive.')] $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index a80d2ead6..992dba8ba 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -139,8 +139,8 @@ $checkfiles = @( if ($realTemplateFolder) { $checkfiles += @( - @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($TemplateRepoSettingsFile); 'newname' = [system.IO.Path]::GetFileName($TemplateRepoSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($RepoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($RepoSettingsFile); 'type' = 'template repo settings' } - @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($TemplateProjectSettingsFile); 'newname' = [system.IO.Path]::GetFileName($TemplateProjectSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($ALGoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($ALGoSettingsFile); ; 'type' = 'template project settings' } + @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($IndirectTemplateRepoSettingsFile); 'newname' = [system.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($RepoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($RepoSettingsFile); 'type' = 'template repo settings' } + @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($IndirectTemplateProjectSettingsFile); 'newname' = [system.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($ALGoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($ALGoSettingsFile); ; 'type' = 'template project settings' } ) } From e22005b1af619f42374f68ef049cd49b3094b908 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 05:08:55 -0700 Subject: [PATCH 117/215] use indirect --- Actions/AL-Go-Helper.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index a7f18bd27..c9285a2a0 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -738,8 +738,8 @@ function ReadSettings { $settingsObjects += @($orgSettingsVariableObject) } # Read settings from repository settings file - $templateRepoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $TemplateRepoSettingsFile) - $settingsObjects += @($templateRepoSettingsObject) + $indirectTemplateRepoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $IndirectTemplateRepoSettingsFile) + $settingsObjects += @($indirectTemplateRepoSettingsObject) # Read settings from repository settings file $repoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $RepoSettingsFile) $settingsObjects += @($repoSettingsObject) @@ -750,8 +750,8 @@ function ReadSettings { } if ($project) { # Read settings from repository settings file - $templateProjectSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $TemplateProjectSettingsFile) - $settingsObjects += @($templateProjectSettingsObject) + $indirectTemplateProjectSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $IndirectTemplateProjectSettingsFile) + $settingsObjects += @($indirectTemplateProjectSettingsObject) # Read settings from project settings file $projectFolder = Join-Path $baseFolder $project -Resolve $projectSettingsObject = GetSettingsObject -Path (Join-Path $projectFolder $ALGoSettingsFile) From b1e6a233cf03f96de596749844d7990d9459eee6 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 05:39:24 -0700 Subject: [PATCH 118/215] test correct type --- Actions/AL-Go-TestRepoHelper.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Actions/AL-Go-TestRepoHelper.ps1 b/Actions/AL-Go-TestRepoHelper.ps1 index 07a8179bb..22e7cf413 100644 --- a/Actions/AL-Go-TestRepoHelper.ps1 +++ b/Actions/AL-Go-TestRepoHelper.ps1 @@ -1,4 +1,6 @@ -function Test-Property { +. (Join-Path $PSScriptRoot "AL-Go-Helper.ps1") + +function Test-Property { Param( [HashTable] $json, [string] $settingsDescription, @@ -178,9 +180,12 @@ function TestALGoRepository { Test-JsonFile -jsonFile $_.FullName -baseFolder $baseFolder -type 'Project' } elseif ($_.Directory.Name -eq '.github' -and $_.BaseName -like '*ettings') { - if ($_.BaseName -eq 'AL-Go-Settings') { + if ($_.BaseName -eq [System.IO.Path]::GetFileName($RepoSettingsFile) -or $_.BaseName -eq [System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile)) { $type = 'Repo' } + elseif ($_.BaseName -eq [System.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile)) { + $type = 'Project' + } else { $type = 'Workflow' } From ab8fb22d241707625b95d22b03db8dafaa594ad8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 05:49:52 -0700 Subject: [PATCH 119/215] test --- Actions/AL-Go-TestRepoHelper.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/AL-Go-TestRepoHelper.ps1 b/Actions/AL-Go-TestRepoHelper.ps1 index 22e7cf413..23e4ed43a 100644 --- a/Actions/AL-Go-TestRepoHelper.ps1 +++ b/Actions/AL-Go-TestRepoHelper.ps1 @@ -180,6 +180,8 @@ function TestALGoRepository { Test-JsonFile -jsonFile $_.FullName -baseFolder $baseFolder -type 'Project' } elseif ($_.Directory.Name -eq '.github' -and $_.BaseName -like '*ettings') { + Write-Host $_.BaseName + Write-Host [System.IO.Path]::GetFileName($RepoSettingsFile) if ($_.BaseName -eq [System.IO.Path]::GetFileName($RepoSettingsFile) -or $_.BaseName -eq [System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile)) { $type = 'Repo' } From ffd2164fdfdf97219e94d473d2924491948349cf Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 05:53:56 -0700 Subject: [PATCH 120/215] () --- Actions/AL-Go-TestRepoHelper.ps1 | 6 +++--- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Actions/AL-Go-TestRepoHelper.ps1 b/Actions/AL-Go-TestRepoHelper.ps1 index 23e4ed43a..e8887a1f7 100644 --- a/Actions/AL-Go-TestRepoHelper.ps1 +++ b/Actions/AL-Go-TestRepoHelper.ps1 @@ -181,11 +181,11 @@ function TestALGoRepository { } elseif ($_.Directory.Name -eq '.github' -and $_.BaseName -like '*ettings') { Write-Host $_.BaseName - Write-Host [System.IO.Path]::GetFileName($RepoSettingsFile) - if ($_.BaseName -eq [System.IO.Path]::GetFileName($RepoSettingsFile) -or $_.BaseName -eq [System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile)) { + Write-Host ([System.IO.Path]::GetFileName($RepoSettingsFile)) + if ($_.BaseName -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.BaseName -eq ([System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile))) { $type = 'Repo' } - elseif ($_.BaseName -eq [System.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile)) { + elseif ($_.BaseName -eq ([System.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile))) { $type = 'Project' } else { diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 992dba8ba..8380db633 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -139,8 +139,8 @@ $checkfiles = @( if ($realTemplateFolder) { $checkfiles += @( - @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($IndirectTemplateRepoSettingsFile); 'newname' = [system.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($RepoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($RepoSettingsFile); 'type' = 'template repo settings' } - @{ 'dstPath' = [system.IO.Path]::GetDirectoryName($IndirectTemplateProjectSettingsFile); 'newname' = [system.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile); 'SrcPath' = [system.IO.Path]::GetDirectoryName($ALGoSettingsFile); 'pattern' = [system.IO.Path]::GetFileName($ALGoSettingsFile); ; 'type' = 'template project settings' } + @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($IndirectTemplateRepoSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } + @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($IndirectTemplateProjectSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } ) } From 6f4cc989798a7c69b30628648fb64b784c7357a2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 05:56:17 -0700 Subject: [PATCH 121/215] name --- Actions/AL-Go-TestRepoHelper.ps1 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Actions/AL-Go-TestRepoHelper.ps1 b/Actions/AL-Go-TestRepoHelper.ps1 index e8887a1f7..53c5a3eef 100644 --- a/Actions/AL-Go-TestRepoHelper.ps1 +++ b/Actions/AL-Go-TestRepoHelper.ps1 @@ -180,12 +180,10 @@ function TestALGoRepository { Test-JsonFile -jsonFile $_.FullName -baseFolder $baseFolder -type 'Project' } elseif ($_.Directory.Name -eq '.github' -and $_.BaseName -like '*ettings') { - Write-Host $_.BaseName - Write-Host ([System.IO.Path]::GetFileName($RepoSettingsFile)) - if ($_.BaseName -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.BaseName -eq ([System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile))) { + if ($_.Name -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.Name -eq ([System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile))) { $type = 'Repo' } - elseif ($_.BaseName -eq ([System.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile))) { + elseif ($_.Name -eq ([System.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile))) { $type = 'Project' } else { From 950a27e22c099c28aa5a5698ce1a6d7eb890bc41 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 05:58:32 -0700 Subject: [PATCH 122/215] use --- Actions/AL-Go-TestRepoHelper.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/AL-Go-TestRepoHelper.ps1 b/Actions/AL-Go-TestRepoHelper.ps1 index 53c5a3eef..665d36082 100644 --- a/Actions/AL-Go-TestRepoHelper.ps1 +++ b/Actions/AL-Go-TestRepoHelper.ps1 @@ -176,10 +176,10 @@ function TestALGoRepository { # Test .json files are formatted correctly # Get-ChildItem needs -force to include folders starting with . (e.x. .github / .AL-Go) on Linux Get-ChildItem -Path $baseFolder -Filter '*.json' -Recurse -Force | ForEach-Object { - if ($_.Directory.Name -eq '.AL-Go' -and $_.BaseName -eq 'settings') { + if ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($ALGoSettingsFile) -and $_.Name -eq ([System.IO.Path]::GetFileName($ALGoSettingsFile)) { Test-JsonFile -jsonFile $_.FullName -baseFolder $baseFolder -type 'Project' } - elseif ($_.Directory.Name -eq '.github' -and $_.BaseName -like '*ettings') { + elseif ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($RepoSettingsFile) -and $_.BaseName -like '*ettings') { if ($_.Name -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.Name -eq ([System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile))) { $type = 'Repo' } From 83cb25990c88f247da7724e40f67dfbf3de97114 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 05:59:50 -0700 Subject: [PATCH 123/215] ) --- Actions/AL-Go-TestRepoHelper.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/AL-Go-TestRepoHelper.ps1 b/Actions/AL-Go-TestRepoHelper.ps1 index 665d36082..47f41791a 100644 --- a/Actions/AL-Go-TestRepoHelper.ps1 +++ b/Actions/AL-Go-TestRepoHelper.ps1 @@ -176,10 +176,10 @@ function TestALGoRepository { # Test .json files are formatted correctly # Get-ChildItem needs -force to include folders starting with . (e.x. .github / .AL-Go) on Linux Get-ChildItem -Path $baseFolder -Filter '*.json' -Recurse -Force | ForEach-Object { - if ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($ALGoSettingsFile) -and $_.Name -eq ([System.IO.Path]::GetFileName($ALGoSettingsFile)) { + if ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($ALGoSettingsFile)) -and $_.Name -eq ([System.IO.Path]::GetFileName($ALGoSettingsFile))) { Test-JsonFile -jsonFile $_.FullName -baseFolder $baseFolder -type 'Project' } - elseif ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($RepoSettingsFile) -and $_.BaseName -like '*ettings') { + elseif ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($RepoSettingsFile)) -and $_.BaseName -like '*ettings') { if ($_.Name -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.Name -eq ([System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile))) { $type = 'Repo' } From a4bb65c6b47885808fb429f50047307e3338af6d Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 06:53:46 -0700 Subject: [PATCH 124/215] document cicd authcontext --- Scenarios/secrets.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Scenarios/secrets.md b/Scenarios/secrets.md index 92a044c87..847e71265 100644 --- a/Scenarios/secrets.md +++ b/Scenarios/secrets.md @@ -64,7 +64,7 @@ By creating a secret called Azure_Credentials you can give your GitHub repositor > [!NOTE] > In order to use a KeyVault for signing apps, it needs to be a premium SKU KeyVault. You can use this command to modify an existing KeyVault: `az keyvault update --set properties.sku.name=premium --name --resource-group ` -n Azure Key Vault can be set up for two different security models: Role Based Access Control (RBAC) (recommended) and Vault Access Policy. In order for AL-Go for GitHub to use the Key Vault, the following roles/permissions need to be assigned to the app registration or Managed Identity, on which the authentication is performed: +An Azure Key Vault can be set up for two different security models: Role Based Access Control (RBAC) (recommended) and Vault Access Policy. In order for AL-Go for GitHub to use the Key Vault, the following roles/permissions need to be assigned to the app registration or Managed Identity, on which the authentication is performed: | Security Model | Read Secrets | Sign Apps | | :-- | :-- | :-- | @@ -90,10 +90,15 @@ Example: `{"keyVaultName":"MyKeyVault","clientId":"","clientSecret":"< With this setup, you can create a setting called `keyVaultCodesignCertificateName` containing the name of the imported certificate in your Key Vault in order for AL-Go for GitHub to sign your apps. + -## **AuthContext** -> Deploy to an environment +## **AuthContext** -> Connect to online environments -Whenever AL-Go for GitHub is doing to deploy to an environment, it will need an AuthContext secret. The AuthContext secret can be provided underneath the environment in GitHub. If you are using a private repository in the free GitHub plan, you do not have environments. Then you can create an AuthContext secret in the repository. If you have multiple environments, you can create different AuthContext secrets by using the environment name followed by an underscore and AuthContext (f.ex. **QA_AuthContext**). +Whenever AL-Go for GitHub is going to connect to an online environment, it will need an AuthContext secret. + +For publishing to an environment, the AuthContext secret can be provided underneath the environment in GitHub. If you are using a private repository in the free GitHub plan, you do not have environments. Then you can create an AuthContext secret in the repository. If you have multiple environments, you can create different AuthContext secrets by using the environment name followed by an underscore and AuthContext (f.ex. **QA_AuthContext**). + +For using online environments during CI/CD (for running tests etc.) you need to provide a secret called **CICDAuthContext**. If a CICDAuthContext secret exists, the online environment pointed out by this authentication context, will be used when running CI/CD for symbols, upgrade testing and test running. ### Managed identity From 71d5ce4f8bf2fd2e1be325b094da9d905b8213b2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 07:56:08 -0700 Subject: [PATCH 125/215] merge or add --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- Actions/CheckForUpdates/yamlclass.ps1 | 32 ++++++++++++++++----- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 4d7c55b1b..9fd932740 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -245,7 +245,7 @@ foreach($checkfile in $checkfiles) { if (Test-Path -Path $dstFile -PathType Leaf) { if ($type -eq 'workflow') { Write-Host "Apply customizations from my repository: $dstFile" - [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $customizationAnchors) + [Yaml]::ApplyCustomizations([ref] $srcContent, $dstFile, $customizationAnchors) } # file exists, compare and add to $updateFiles if different $dstContent = Get-ContentLF -Path $dstFile diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 00098358c..594c91174 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -450,15 +450,33 @@ class Yaml { catch { return } + # Merge permissions Write-host "Merge permissions" - $srcPermissionsObj = $srcYaml.Get('permissions:/') - $yamlPermissionsObj = $yaml.Get('permissions:/') - if ($srcPermissionsObj -and $yamlPermissionsObj) { - $srcPermissions = [Yaml]::GetPermissionsFromArray($srcPermissionsObj.content) - $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) - if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { - $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + $allJobs = $srcYaml.GetNextLevel('jobs:/') + $permissionPaths = @("permissions:/") + @($allJobs | ForEach-Object { "jobs:/$_/permissions:/" }) + foreach($permissionPath in $permissionPaths) { + $srcPermissionsObj = $srcYaml.Get($permissionPath) + $yamlPermissionsObj = $yaml.Get($permissionPath) + if ($yamlPermissionsObj) { + # permissions exist and should be merged/added + if ($srcPermissionsObj) { + # Merge permissions + $srcPermissions = [Yaml]::GetPermissionsFromArray($srcPermissionsObj.content) + $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) + if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { + $srcYaml.Replace($permissionPath, [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + } + } + else { + # Add permissions + [int]$start = 0 + [int]$count = 0 + if ($srcYaml.Find($permissionPath.SubString($permissionPath.LastIndexOf('/')+1), [ref] $start, [ref] $count)) { + $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) + $srcYaml.Insert($start + 1, $yamlPermissions) + } + } } } From 03aa39619ed34939e78fba6d34ad24f8f607cfe4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 08:03:33 -0700 Subject: [PATCH 126/215] perm --- Actions/CheckForUpdates/yamlclass.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 594c91174..84f7a0898 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -472,9 +472,10 @@ class Yaml { # Add permissions [int]$start = 0 [int]$count = 0 - if ($srcYaml.Find($permissionPath.SubString($permissionPath.LastIndexOf('/')+1), [ref] $start, [ref] $count)) { + $parentPath = $permissionPath.SubString(0,$permissionPath.SubString(0,$permissionPath.Length-1).LastIndexOf('/')+1) + if ($srcYaml.Find($parentPath), [ref] $start, [ref] $count)) { $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) - $srcYaml.Insert($start + 1, $yamlPermissions) + $srcYaml.Insert($start + 1, [Yaml]::GetPermissionsArray($yamlPermissions)) } } } From a1c2d3ad21529b8da3afc5efe17ecb697ce71fbb Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 08:07:02 -0700 Subject: [PATCH 127/215] syntax --- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 84f7a0898..504381532 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -473,7 +473,7 @@ class Yaml { [int]$start = 0 [int]$count = 0 $parentPath = $permissionPath.SubString(0,$permissionPath.SubString(0,$permissionPath.Length-1).LastIndexOf('/')+1) - if ($srcYaml.Find($parentPath), [ref] $start, [ref] $count)) { + if ($srcYaml.Find($parentPath, [ref] $start, [ref] $count)) { $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) $srcYaml.Insert($start + 1, [Yaml]::GetPermissionsArray($yamlPermissions)) } From 67dd1846f0a009e8a763779c1e813e2b1c184f38 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 12:52:15 -0700 Subject: [PATCH 128/215] insert --- Actions/CheckForUpdates/yamlclass.ps1 | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 504381532..05fa3dfbd 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -470,13 +470,8 @@ class Yaml { } else { # Add permissions - [int]$start = 0 - [int]$count = 0 - $parentPath = $permissionPath.SubString(0,$permissionPath.SubString(0,$permissionPath.Length-1).LastIndexOf('/')+1) - if ($srcYaml.Find($parentPath, [ref] $start, [ref] $count)) { - $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) - $srcYaml.Insert($start + 1, [Yaml]::GetPermissionsArray($yamlPermissions)) - } + $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) + $srcYaml.Replace("$($permissionPath)steps:", @("permissions:") + @([Yaml]::GetPermissionsArray($yamlPermissions) | ForEach-Object { " $_" }) + $srcYaml.Get("$($permissionPath)steps:").content) } } } From cfdaac92dbd39dbd2f2dc68bceb410cf9fa03dd4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Mar 2025 21:20:56 -0700 Subject: [PATCH 129/215] parent --- Actions/CheckForUpdates/yamlclass.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 05fa3dfbd..9bbaffb4e 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -470,8 +470,9 @@ class Yaml { } else { # Add permissions + $parentPath = $permissionPath.Substring(0, $permissionPath.TrimEnd('/').LastIndexOf('/')) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) - $srcYaml.Replace("$($permissionPath)steps:", @("permissions:") + @([Yaml]::GetPermissionsArray($yamlPermissions) | ForEach-Object { " $_" }) + $srcYaml.Get("$($permissionPath)steps:").content) + $srcYaml.Replace("$parentPath/steps:", @("permissions:") + @([Yaml]::GetPermissionsArray($yamlPermissions) | ForEach-Object { " $_" }) + $srcYaml.Get("$parentPath/steps:").content) } } } From 65f779884444c9bfeb0947f7f44d79623499ce01 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Mar 2025 04:57:35 -0700 Subject: [PATCH 130/215] support global permissions --- Actions/CheckForUpdates/yamlclass.ps1 | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 9bbaffb4e..25d08e35d 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -470,9 +470,17 @@ class Yaml { } else { # Add permissions - $parentPath = $permissionPath.Substring(0, $permissionPath.TrimEnd('/').LastIndexOf('/')) + $parentPath = $permissionPath.Substring(0, $permissionPath.TrimEnd('/').LastIndexOf('/')+1) + if ($parentPath) { + # Job permissions are inserted before steps + $beforePath = "$($parentPath)steps:" + } + else { + # Global permissions are inserted before jobs + $beforePath = "jobs:" + } $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) - $srcYaml.Replace("$parentPath/steps:", @("permissions:") + @([Yaml]::GetPermissionsArray($yamlPermissions) | ForEach-Object { " $_" }) + $srcYaml.Get("$parentPath/steps:").content) + $srcYaml.Replace($beforePath, @("permissions:") + @([Yaml]::GetPermissionsArray($yamlPermissions) | ForEach-Object { " $_" }) + $srcYaml.Get($beforePath).content) } } } From 87bb4e38da5ec0d4beff8cd318fa2040470f6523 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 25 Mar 2025 03:54:03 -0700 Subject: [PATCH 131/215] read secret --- .../CheckForUpdates.HelperFunctions.ps1 | 3 --- Actions/ReadSecrets/ReadSecrets.ps1 | 26 +++++++++++++++++++ .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 5035ed475..fe7f226f2 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -569,9 +569,6 @@ function GetCustomALGoSystemFiles { } } - if ($settings.customALGoSystemFiles -isnot [Array]) { - throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." - } foreach($customspec in $settings.customALGoSystemFiles) { if ($customspec -isnot [Hashtable]) { throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index caa401b37..e7409ba3d 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -28,6 +28,7 @@ try { $settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable $keyVaultCredentials = GetKeyVaultCredentials $getAppDependencySecrets = $false + $getCustomALGoSystemFilesSecrets = $false $getTokenForPush = $false [System.Collections.ArrayList]$secretsCollection = @() foreach($secret in ($getSecrets.Split(',') | Select-Object -Unique)) { @@ -41,6 +42,9 @@ try { if ($secret -eq 'AppDependencySecrets') { $getAppDependencySecrets = $true } + elseif ($secret -eq 'CustomALGoSystemFilesSecrets') { + $getCustomALGoSystemFilesSecrets = $true + } else { $secretName = $secret if ($settings.Keys -contains $secretNameProperty) { @@ -77,6 +81,7 @@ try { } } } + # Look through installApps and installTestApps for secrets and add them to the collection of secrets to get foreach($installSettingsKey in @('installApps','installTestApps')) { if ($settings.Keys -contains $installSettingsKey) { @@ -94,6 +99,27 @@ try { } } + if ($getCustomALGoSystemFilesSecrets) { + # Look through custominstallApps and installTestApps for secrets and add them to the collection of secrets to get + foreach($customspec in $settings.customALGoSystemFiles) { + if ($customspec.PSObject.Properties.name -eq "AuthTokenSecret") { + if ($secretsCollection -notcontains $customspec.authTokenSecret) { + $secretsCollection += $customspec.authTokenSecret + } + } + if ($customspec.PSObject.Properties.name -eq "Source") { + # If the Source URLs contains '${{SECRETNAME}}' we need to get the secret + $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*' + if ($_ -match $pattern) { + $secretName = $matches[2] + if ($secretsCollection -notcontains $secretName) { + $secretsCollection += $secretName + } + } + } + } + } + # Loop through secrets (use @() to allow us to remove items from the collection while looping) foreach($secret in @($secretsCollection)) { $secretSplit = $secret.Split('=') diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index a08778009..e3ef90cb9 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -109,7 +109,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'ghTokenWorkflow' + getSecrets: 'ghTokenWorkflow,CustomALGoSystemFilesSecrets' - name: Calculate Commit Options env: diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index a08778009..e3ef90cb9 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -109,7 +109,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'ghTokenWorkflow' + getSecrets: 'ghTokenWorkflow,CustomALGoSystemFilesSecrets' - name: Calculate Commit Options env: From cacf9e35ff16ec4d50488fc6bd6ad319cdd57728 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Tue, 25 Mar 2025 03:55:36 -0700 Subject: [PATCH 132/215] Update Scenarios/Contribute.md Co-authored-by: Johannes Wikman --- Scenarios/Contribute.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/Contribute.md b/Scenarios/Contribute.md index b05e62e05..0e337c577 100644 --- a/Scenarios/Contribute.md +++ b/Scenarios/Contribute.md @@ -5,7 +5,7 @@ This section describes how to contribute to AL-Go. How to set up your own enviro You can do this in two ways: - Use a fork of AL-Go for GitHub in your own **personal GitHub account** in development mode -- Use 2 public repositories in your own **personal GitHub account** (AL-Go-PTE, AL-Go-AppSource and AL-Go-Actions, much like in production) +- Use 2 public repositories in your own **personal GitHub account** (AL-Go-PTE and AL-Go-AppSource, much like in production) ## Use a fork of AL-Go for GitHub in "development mode" From 7fa9ccb33bfa67ebe3ffbac811170d276bd58b4b Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Tue, 25 Mar 2025 03:56:38 -0700 Subject: [PATCH 133/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Johannes Wikman --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 24db42256..86d4cb41e 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -220,7 +220,7 @@ You can also add custom steps to AL-Go for GitHub Workflows, but only in pre-def At this time, the anchor-points where you can add custom steps are: -| Workflow | Job | Step | Before or AFter | +| Workflow | Job | Step | Before or After | | :-- | :-- | :-- | :-: | | \_BuildALGoProject.yaml | BuildALGoProject | Read settings | After | | | | Read secrets | After | From ad90d28f80f5a3758fd8caa7930d9e11749eb1de Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Tue, 25 Mar 2025 03:56:50 -0700 Subject: [PATCH 134/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Johannes Wikman --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 86d4cb41e..10598fad3 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -235,7 +235,7 @@ The custom step needs to be named `CustomStep` and if inserted in any run: | $settings = $env:Settings | ConvertFrom-Json $settings.artifact = Invoke-RestMethod -Method GET -UseBasicParsing -Uri "https://bca-url-proxy.azurewebsites.net/bca-url/sandbox/us?select=weekly&doNotRedirect=true" - Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "Settings=$($$settings | ConvertTo-Json -Depth 99 -Compress)" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "Settings=$($settings | ConvertTo-Json -Depth 99 -Compress)" Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "artifact=$($settings.artifact)" ``` From e5d60caa761b37e29903579c6fb21b88c4fcecea Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Tue, 25 Mar 2025 03:57:49 -0700 Subject: [PATCH 135/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Johannes Wikman --- Scenarios/CustomizingALGoForGitHub.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 10598fad3..d36d6b705 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -223,10 +223,10 @@ At this time, the anchor-points where you can add custom steps are: | Workflow | Job | Step | Before or After | | :-- | :-- | :-- | :-: | | \_BuildALGoProject.yaml | BuildALGoProject | Read settings | After | -| | | Read secrets | After | -| | | Build | Before | -| | | Read secrets | After | -| | | Cleanup | Before | +| \_BuildALGoProject.yaml | BuildALGoProject | Read secrets | After | +| \_BuildALGoProject.yaml | BuildALGoProject | Build | Before | +| \_BuildALGoProject.yaml | BuildALGoProject | Build | After | +| \_BuildALGoProject.yaml | BuildALGoProject | Cleanup | Before | The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic From 2451a21827ee33f712668b5638bbb5b95db449b9 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Tue, 25 Mar 2025 03:58:01 -0700 Subject: [PATCH 136/215] Update Actions/CheckForUpdates/CheckForUpdates.ps1 Co-authored-by: Johannes Wikman --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 9fd932740..0bb0fe447 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -92,7 +92,7 @@ if (-not $isDirectALGo) { $templateRepoSettings = Get-Content $templateRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { # The template repository is a url to another AL-Go repository (an indirect template repository) - # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template reposotiry points to the "real" template repository + # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template repository points to the "real" template repository # Copy files and folders from the indirect template repository, but grab the unmodified file from the "real" template repository if it exists and apply customizations # Copy .github/AL-Go-Settings.json to .github/templateRepoSettings.json (will be read before .github/AL-Go-Settings.json in the final repo) # Copy .AL-Go/settings.json to .github/templateProjectSettings.json (will be read before .AL-Go/settings.json in the final repo) From f079b07179d88a5dcbd38e5d7aa06e48847ef544 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 25 Mar 2025 03:58:36 -0700 Subject: [PATCH 137/215] remove spaces --- Actions/ReadSecrets/ReadSecrets.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index e7409ba3d..ccc8ff4b2 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -119,7 +119,7 @@ try { } } } - + # Loop through secrets (use @() to allow us to remove items from the collection while looping) foreach($secret in @($secretsCollection)) { $secretSplit = $secret.Split('=') From 8d9c6e3eccfa46593953a6fb3238bc3c0d6b91b1 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Thu, 17 Apr 2025 17:09:41 +0200 Subject: [PATCH 138/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Johannes Wikman --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index d36d6b705..b6f669aaa 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -125,7 +125,7 @@ It is recommended to prefix your workflows with `my`, `our`, your name or your o ### Adding custom scripts -You can add custom powershell scripts under the .github folder for repository scoped scripts or in the .AL-Go folder for project scoped scripts. Specially named scripts in the .AL-Go folder can override standard functionality in AL-Go for GitHub workflows. A list of these script overrides can be found [here](https://aka.ms/algosettings#customdeployment). Scripts under the .github folder can be used in custom workflows instead of using inline scripts inside the workflow. +You can add custom powershell scripts under the .github folder for repository scoped scripts or in the .AL-Go folder for project scoped scripts. Specially named scripts in the .AL-Go folder can override standard functionality in AL-Go for GitHub workflows. A list of these script overrides can be found [here](https://aka.ms/algosettings#scriptoverrides). Scripts under the .github folder can be used in custom workflows instead of using inline scripts inside the workflow. One example of a script override is the NewBcContainer override used in the System Application project in BCApps (can be found [here](https://github.com/microsoft/BCApps/blob/main/build/projects/System%20Application/.AL-Go/NewBcContainer.ps1)). This override looks like: From 4e88c46a0f52e8867d039c49842321ba2eadd9cf Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 17 Apr 2025 17:16:51 +0200 Subject: [PATCH 139/215] docs --- Scenarios/CustomizingALGoForGitHub.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index d36d6b705..a9f6c3922 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -281,12 +281,9 @@ Using a fork of AL-Go for GitHub to have your "own" public version of AL-Go for > [!NOTE] > When customizing AL-Go for GitHub using a fork, your customizations are public and will be visible to everyone. For more information, [read this](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-permissions-and-visibility-of-forks). -There are two ways of forking AL-Go for GitHub. You can fork the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository or you can fork the template repositories [AL-Go PTE](https://github.com/microsoft/AL-Go-PTE) and/or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource). +There are two ways of forking AL-Go for GitHub. You can fork the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository and develop AL-Go for GitHub like we do in Microsoft, or you can fork the template repositories [AL-Go PTE](https://github.com/microsoft/AL-Go-PTE) and/or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource). -For simple changes to the templates, you can fork the template repositories and make the changes directly in your fork. Note that we do not accept any pull requests to the template repositories as they are deployed from the main AL-Go repository. We do not actually develop anything in the template repositories ourself. In the template repositories you will find a branch for every version of AL-Go we have shipped. The main branch is the latest version and the preview branch is the next version. You can customize the preview branch and/or the main branch and then use your fork as the template repository when running Update AL-Go System Files from your app repositories. - -> [!NOTE] -> We do NOT accept pull requests to the template repositories. You need to follow the guidelines [here](Contribute.md) in order to contribute to AL-Go development. +While we don't recommend forking the template repositories, we realize that it is possible for simple changes to the templates. You can fork the template repositories and make the changes directly in your fork. Note that we do not accept any pull requests to the template repositories as they are deployed from the main AL-Go repository. We do not actually develop anything in the template repositories ourself. In the template repositories you will find a branch for every version of AL-Go we have shipped. The main branch is the latest version and the preview branch is the next version. You can customize the preview branch and/or the main branch and then use your fork as the template repository when running Update AL-Go System Files from your app repositories. > [!TIP] > When forking the template repositories, you should include all branches in order to be able to use either the latest version of AL-Go or the preview version of AL-Go. From add7a0b4b3b410601b61b8e7fa200aafb8e3f8e9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 17 Apr 2025 17:55:13 +0200 Subject: [PATCH 140/215] Only use harden_runner on microsoft --- .github/workflows/CI.yaml | 1 + .github/workflows/CleanupTempRepos.yaml | 1 + .github/workflows/Deploy.yaml | 3 +++ .github/workflows/E2E.yaml | 8 ++++++++ .github/workflows/powershell.yaml | 1 + .github/workflows/pre-commit.yml | 1 + .github/workflows/scorecard-analysis.yml | 1 + 7 files changed, 16 insertions(+) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index b6d87aa0e..f705834d0 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -19,6 +19,7 @@ jobs: runs-on: [ ubuntu-latest ] steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit diff --git a/.github/workflows/CleanupTempRepos.yaml b/.github/workflows/CleanupTempRepos.yaml index 91802dadb..b4a7aa944 100644 --- a/.github/workflows/CleanupTempRepos.yaml +++ b/.github/workflows/CleanupTempRepos.yaml @@ -26,6 +26,7 @@ jobs: name: Cleanup Temp Repos steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit diff --git a/.github/workflows/Deploy.yaml b/.github/workflows/Deploy.yaml index 211d260d1..4fdb5fedc 100644 --- a/.github/workflows/Deploy.yaml +++ b/.github/workflows/Deploy.yaml @@ -53,6 +53,7 @@ jobs: defaultBcContainerHelperVersion: ${{ steps.CreateInputs.outputs.defaultBcContainerHelperVersion }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -85,6 +86,7 @@ jobs: needs: [ Inputs ] steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -126,6 +128,7 @@ jobs: contents: write steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit diff --git a/.github/workflows/E2E.yaml b/.github/workflows/E2E.yaml index 6a1527a3b..a77a94938 100644 --- a/.github/workflows/E2E.yaml +++ b/.github/workflows/E2E.yaml @@ -52,6 +52,7 @@ jobs: githubOwner: ${{ steps.check.outputs.githubOwner }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -105,6 +106,7 @@ jobs: appSourceAppRepo: ${{ steps.setup.outputs.appSourceAppRepo }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -132,6 +134,7 @@ jobs: scenarios: ${{ steps.Analyze.outputs.scenarios }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -213,6 +216,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.scenarios) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -249,6 +253,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.scenarios) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -285,6 +290,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.publictestruns) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -333,6 +339,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.privatetestruns) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit @@ -381,6 +388,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.releases) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit diff --git a/.github/workflows/powershell.yaml b/.github/workflows/powershell.yaml index 34bc0e6c4..de17f8627 100644 --- a/.github/workflows/powershell.yaml +++ b/.github/workflows/powershell.yaml @@ -21,6 +21,7 @@ jobs: security-events: write # for github/codeql-action/upload-sarif to upload SARIF results steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 292a55e1f..709d719db 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,6 +14,7 @@ jobs: runs-on: windows-latest steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit diff --git a/.github/workflows/scorecard-analysis.yml b/.github/workflows/scorecard-analysis.yml index 549a6e44d..210183990 100644 --- a/.github/workflows/scorecard-analysis.yml +++ b/.github/workflows/scorecard-analysis.yml @@ -18,6 +18,7 @@ jobs: steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1 with: egress-policy: audit From 22abae7e01e5fd07befafd27c83672aafe6ae04a Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 22 Apr 2025 15:46:36 +0200 Subject: [PATCH 141/215] Asess token in downloadTemplateRepo instead of early on --- .../CheckForUpdates.HelperFunctions.ps1 | 22 ++++++++++++++++--- Actions/CheckForUpdates/CheckForUpdates.ps1 | 19 ++-------------- Actions/Github-Helper.psm1 | 5 +++-- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index e720b528d..ed5859704 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -15,8 +15,8 @@ function GetCustomizationAnchors { <# .SYNOPSIS Downloads a template repository and returns the path to the downloaded folder -.PARAMETER headers -The headers to use when calling the GitHub API +.PARAMETER token +The GitHub token / PAT to use for authentication (if the template repository is private/internal) .PARAMETER templateUrl The URL to the template repository .PARAMETER templateSha @@ -26,12 +26,28 @@ If true, the latest SHA of the template repository will be downloaded #> function DownloadTemplateRepository { Param( - [hashtable] $headers, + [string] $token, [string] $templateUrl, [ref] $templateSha, [bool] $downloadLatest ) + $templateRepositoryUrl = $templateUrl.Split('@')[0] + $templateRepository = $templateRepositoryUrl.Split('/')[-2..-1] -join '/' + + if ([string]::IsNullOrEmpty($ENV:GITHUB_TOKEN)) { + throw "GITHUB_TOKEN is not set." + } + # Use Authenticated API request if possible to avoid the 60 API calls per hour limit + $headers = GetHeaders -token $ENV:GITHUB_TOKEN -repository $templateRepository + $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue + if (-not $response -or $response.StatusCode -ne 200) { + # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal + # Get token with read permissions for the template repository + # NOTE that the GitHub app needs to be installed in the template repository for this to work + $headers = GetHeaders -token $token -repository $templateRepository + } + # Construct API URL $apiUrl = $templateUrl.Split('@')[0] -replace "^(https:\/\/github\.com\/)(.*)$", "$ENV:GITHUB_API_URL/repos/`$2" diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 75f42d54b..6fc840fb4 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -45,21 +45,6 @@ if ($token) { $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } -# Use Authenticated API request if possible to avoid the 60 API calls per hour limit -$headers = GetHeaders -token $ENV:GITHUB_TOKEN -$templateRepositoryUrl = $templateUrl.Split('@')[0] -$response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue -if (-not $response -or $response.StatusCode -ne 200) { - # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal - # Get token with read permissions for the template repository - # NOTE that the GitHub app needs to be installed in the template repository for this to work - $templateRepository = $templateRepositoryUrl.Split('/')[-2..-1] -join '/' - $templateReadToken = GetAccessToken -token $token -permissions @{"actions"="read";"contents"="read";"metadata"="read"} -repository $templateRepository - - # Use read token for authenticated API request - $headers = GetHeaders -token $templateReadToken -} - # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository # CheckForUpdates will apply changes to the AL-Go System files based on AL-Go repo settings, such as "runs-on" etc. # if $update is set to Y, CheckForUpdates will also update the AL-Go System files in the current repository using a PR or a direct commit (if $directCommit is set to true) @@ -82,7 +67,7 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { } $realTemplateFolder = $null -$templateFolder = DownloadTemplateRepository -headers $headers -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest +$templateFolder = DownloadTemplateRepository -token $token -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" $templateBranch = $templateUrl.Split('@')[1] @@ -111,7 +96,7 @@ if (-not $isDirectALGo) { } # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository - $realTemplateFolder = DownloadTemplateRepository -headers $headers -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') + $realTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') Write-Host "Real Template Folder: $realTemplateFolder" # Set TemplateBranch and TemplateOwner diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index f39f536c6..d93dcc426 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -656,14 +656,15 @@ function GetHeaders { [string] $accept = "application/vnd.github+json", [string] $apiVersion = "2022-11-28", [string] $api_url = $ENV:GITHUB_API_URL, - [string] $repository = $ENV:GITHUB_REPOSITORY + [string] $repository = $ENV:GITHUB_REPOSITORY, + [hashtable] $permissions = @{"contents"="read";"metadata"="read";"actions"="read"} ) $headers = @{ "Accept" = $accept "X-GitHub-Api-Version" = $apiVersion } if (![string]::IsNullOrEmpty($token)) { - $accessToken = GetAccessToken -token $token -api_url $api_url -repository $repository -permissions @{"contents"="read";"metadata"="read";"actions"="read"} + $accessToken = GetAccessToken -token $token -api_url $api_url -repository $repository -permissions $permissions $headers["Authorization"] = "token $accessToken" } return $headers From 73665c317983c326437ed0dadea7dab6b31f067b Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 22 Apr 2025 17:10:19 +0200 Subject: [PATCH 142/215] rename tokens --- .../CheckForUpdates.HelperFunctions.ps1 | 5 +++-- Actions/CheckForUpdates/CheckForUpdates.ps1 | 22 ++++++++++++++----- Actions/CheckForUpdates/action.yaml | 11 +++++----- .../AppSource App/.github/workflows/CICD.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- .../.github/workflows/CICD.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- 7 files changed, 29 insertions(+), 17 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ed5859704..8e30c9b23 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -27,6 +27,7 @@ If true, the latest SHA of the template repository will be downloaded function DownloadTemplateRepository { Param( [string] $token, + [string] $ghTokenWorkflow, [string] $templateUrl, [ref] $templateSha, [bool] $downloadLatest @@ -39,13 +40,13 @@ function DownloadTemplateRepository { throw "GITHUB_TOKEN is not set." } # Use Authenticated API request if possible to avoid the 60 API calls per hour limit - $headers = GetHeaders -token $ENV:GITHUB_TOKEN -repository $templateRepository + $headers = GetHeaders -token $token -repository $templateRepository $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue if (-not $response -or $response.StatusCode -ne 200) { # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal # Get token with read permissions for the template repository # NOTE that the GitHub app needs to be installed in the template repository for this to work - $headers = GetHeaders -token $token -repository $templateRepository + $headers = GetHeaders -token $ghTokenWorkflow -repository $templateRepository } # Construct API URL diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 6fc840fb4..4a9b41e59 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -1,8 +1,10 @@ Param( [Parameter(HelpMessage = "The GitHub actor running the action", Mandatory = $false)] [string] $actor, - [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] + [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] [string] $token, + [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] + [string] $ghTokenWorkflow, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", [Parameter(HelpMessage = "Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update)", Mandatory = $true)] @@ -40,9 +42,17 @@ if ($update -eq 'Y') { } } -if ($token) { +if ($ghTokenWorkflow) { # token comes from a secret, base 64 encoded - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + $ghTokenWorkflow = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ghTokenWorkflow)) +} +elseif ($token) { + try { + # if running Direct AL-Go development, token might be the GhTokenWorkflow secret + $ghTokenWorkflow = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + $token = $null + } + catch {} } # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository @@ -67,7 +77,7 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { } $realTemplateFolder = $null -$templateFolder = DownloadTemplateRepository -token $token -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest +$templateFolder = DownloadTemplateRepository -token $token -ghTokenWorkflow $ghTokenWorkflow -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" $templateBranch = $templateUrl.Split('@')[1] @@ -96,7 +106,7 @@ if (-not $isDirectALGo) { } # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository - $realTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') + $realTemplateFolder = DownloadTemplateRepository -token $token -ghTokenWorkflow $ghTokenWorkflow -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') Write-Host "Real Template Folder: $realTemplateFolder" # Set TemplateBranch and TemplateOwner @@ -286,7 +296,7 @@ else { $commitMessage = "[$($updateBranch)@$($branchSHA.SubString(0,7))] Update AL-Go System Files from $templateInfo - $($templateSha.SubString(0,7))" # Get Token with permissions to modify workflows in this repository - $repoWriteToken = GetAccessToken -token $token -permissions @{"actions"="read";"contents"="write";"pull_requests"="write";"workflows"="write"} + $repoWriteToken = GetAccessToken -token $ghTokenWorkflow -permissions @{"actions"="read";"contents"="write";"pull_requests"="write";"workflows"="write"} $env:GH_TOKEN = $repoWriteToken $existingPullRequest = (gh api --paginate "/repos/$env:GITHUB_REPOSITORY/pulls?base=$updateBranch" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" | ConvertFrom-Json) | Where-Object { $_.title -eq $commitMessage } | Select-Object -First 1 diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 9373ef37c..ff52ea6f4 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -1,9 +1,5 @@ name: Check For Updates author: Microsoft Corporation -permissions: - contents: write - pull-requests: write - workflows: write inputs: shell: description: Shell in which you want to run the action (powershell or pwsh) @@ -14,6 +10,10 @@ inputs: required: false default: ${{ github.actor }} token: + description: The GitHub token running the action + required: false + default: ${{ github.token }} + ghTokenWorkflow: description: Base64 encoded GhTokenWorkflow secret required: false default: '' @@ -44,6 +44,7 @@ runs: env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} + _ghTokenWorkflow: ${{ inputs.ghTokenWorkflow }} _templateUrl: ${{ inputs.templateUrl }} _downloadLatest: ${{ inputs.downloadLatest }} _update: ${{ inputs.update }} @@ -51,7 +52,7 @@ runs: _directCommit: ${{ inputs.directCommit }} run: | ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "CheckForUpdates" -Action { - ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') + ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -ghTokenWorkflow $ENV:_ghTokenWorkflow -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') } branding: icon: terminal diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index 1ca9fa53d..f2f780e7b 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -167,7 +167,7 @@ jobs: with: shell: powershell templateUrl: ${{ env.templateUrl }} - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: true Build: diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index e3ef90cb9..f8e632966 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -135,7 +135,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ needs.Initialize.outputs.TemplateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index a6a0842ac..9531fee55 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -167,7 +167,7 @@ jobs: with: shell: powershell templateUrl: ${{ env.templateUrl }} - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: true Build: diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index e3ef90cb9..f8e632966 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -135,7 +135,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ needs.Initialize.outputs.TemplateUrl }} From 53421d9b361b1e49d166e087985fb5d7da253421 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 22 Apr 2025 17:11:02 +0200 Subject: [PATCH 143/215] warn --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 8e30c9b23..0829558a8 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -37,7 +37,7 @@ function DownloadTemplateRepository { $templateRepository = $templateRepositoryUrl.Split('/')[-2..-1] -join '/' if ([string]::IsNullOrEmpty($ENV:GITHUB_TOKEN)) { - throw "GITHUB_TOKEN is not set." + Write-Host "::WARNING::GITHUB_TOKEN is not set." } # Use Authenticated API request if possible to avoid the 60 API calls per hour limit $headers = GetHeaders -token $token -repository $templateRepository From 87cafa466b79a2453cd205f43290fdb89a336c90 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 22 Apr 2025 17:22:21 +0200 Subject: [PATCH 144/215] use token --- .../CheckForUpdates.HelperFunctions.ps1 | 5 ++--- Actions/CheckForUpdates/CheckForUpdates.ps1 | 22 +++++-------------- Actions/CheckForUpdates/action.yaml | 7 +----- .../AppSource App/.github/workflows/CICD.yaml | 4 +++- .../workflows/UpdateGitHubGoSystemFiles.yaml | 4 +++- .../.github/workflows/CICD.yaml | 4 +++- .../workflows/UpdateGitHubGoSystemFiles.yaml | 4 +++- 7 files changed, 21 insertions(+), 29 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 0829558a8..7c9a8d5a1 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -27,7 +27,6 @@ If true, the latest SHA of the template repository will be downloaded function DownloadTemplateRepository { Param( [string] $token, - [string] $ghTokenWorkflow, [string] $templateUrl, [ref] $templateSha, [bool] $downloadLatest @@ -40,13 +39,13 @@ function DownloadTemplateRepository { Write-Host "::WARNING::GITHUB_TOKEN is not set." } # Use Authenticated API request if possible to avoid the 60 API calls per hour limit - $headers = GetHeaders -token $token -repository $templateRepository + $headers = GetHeaders -token $env:GITHUB_TOKEN -repository $templateRepository $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue if (-not $response -or $response.StatusCode -ne 200) { # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal # Get token with read permissions for the template repository # NOTE that the GitHub app needs to be installed in the template repository for this to work - $headers = GetHeaders -token $ghTokenWorkflow -repository $templateRepository + $headers = GetHeaders -token $token -repository $templateRepository } # Construct API URL diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 4a9b41e59..6fc840fb4 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -1,10 +1,8 @@ Param( [Parameter(HelpMessage = "The GitHub actor running the action", Mandatory = $false)] [string] $actor, - [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] - [string] $token, [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] - [string] $ghTokenWorkflow, + [string] $token, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", [Parameter(HelpMessage = "Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update)", Mandatory = $true)] @@ -42,17 +40,9 @@ if ($update -eq 'Y') { } } -if ($ghTokenWorkflow) { +if ($token) { # token comes from a secret, base 64 encoded - $ghTokenWorkflow = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ghTokenWorkflow)) -} -elseif ($token) { - try { - # if running Direct AL-Go development, token might be the GhTokenWorkflow secret - $ghTokenWorkflow = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) - $token = $null - } - catch {} + $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository @@ -77,7 +67,7 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { } $realTemplateFolder = $null -$templateFolder = DownloadTemplateRepository -token $token -ghTokenWorkflow $ghTokenWorkflow -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest +$templateFolder = DownloadTemplateRepository -token $token -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" $templateBranch = $templateUrl.Split('@')[1] @@ -106,7 +96,7 @@ if (-not $isDirectALGo) { } # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository - $realTemplateFolder = DownloadTemplateRepository -token $token -ghTokenWorkflow $ghTokenWorkflow -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') + $realTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') Write-Host "Real Template Folder: $realTemplateFolder" # Set TemplateBranch and TemplateOwner @@ -296,7 +286,7 @@ else { $commitMessage = "[$($updateBranch)@$($branchSHA.SubString(0,7))] Update AL-Go System Files from $templateInfo - $($templateSha.SubString(0,7))" # Get Token with permissions to modify workflows in this repository - $repoWriteToken = GetAccessToken -token $ghTokenWorkflow -permissions @{"actions"="read";"contents"="write";"pull_requests"="write";"workflows"="write"} + $repoWriteToken = GetAccessToken -token $token -permissions @{"actions"="read";"contents"="write";"pull_requests"="write";"workflows"="write"} $env:GH_TOKEN = $repoWriteToken $existingPullRequest = (gh api --paginate "/repos/$env:GITHUB_REPOSITORY/pulls?base=$updateBranch" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" | ConvertFrom-Json) | Where-Object { $_.title -eq $commitMessage } | Select-Object -First 1 diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index ff52ea6f4..38c936c9b 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -10,10 +10,6 @@ inputs: required: false default: ${{ github.actor }} token: - description: The GitHub token running the action - required: false - default: ${{ github.token }} - ghTokenWorkflow: description: Base64 encoded GhTokenWorkflow secret required: false default: '' @@ -44,7 +40,6 @@ runs: env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} - _ghTokenWorkflow: ${{ inputs.ghTokenWorkflow }} _templateUrl: ${{ inputs.templateUrl }} _downloadLatest: ${{ inputs.downloadLatest }} _update: ${{ inputs.update }} @@ -52,7 +47,7 @@ runs: _directCommit: ${{ inputs.directCommit }} run: | ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "CheckForUpdates" -Action { - ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -ghTokenWorkflow $ENV:_ghTokenWorkflow -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') + ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') } branding: icon: terminal diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index f2f780e7b..92fa1004d 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -164,10 +164,12 @@ jobs: - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main + env: + GITHUB_TOKEN: ${{ github.token }} with: shell: powershell templateUrl: ${{ env.templateUrl }} - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: true Build: diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index f8e632966..975536e3a 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -133,9 +133,11 @@ jobs: - name: Update AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main + env: + GITHUB_TOKEN: ${{ github.token }} with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ needs.Initialize.outputs.TemplateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 9531fee55..c2e93d363 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -164,10 +164,12 @@ jobs: - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main + env: + GITHUB_TOKEN: ${{ github.token }} with: shell: powershell templateUrl: ${{ env.templateUrl }} - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: true Build: diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index f8e632966..975536e3a 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -133,9 +133,11 @@ jobs: - name: Update AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main + env: + GITHUB_TOKEN: ${{ github.token }} with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ needs.Initialize.outputs.TemplateUrl }} From 9d2349e66c1c81c50019b19490d5062c65b30e19 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Tue, 22 Apr 2025 21:14:49 +0200 Subject: [PATCH 145/215] Update Actions/CheckForUpdates/CheckForUpdates.ps1 Co-authored-by: Johannes Wikman --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 6fc840fb4..d2a7aa107 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -130,6 +130,7 @@ if ($realTemplateFolder) { $checkfiles += @( @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($IndirectTemplateRepoSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($IndirectTemplateProjectSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } + @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.ps1'; 'type' = 'script' } ) } From 7e01871867890fd8b75cb2d01fbba0473005f8f3 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Tue, 22 Apr 2025 22:14:34 +0200 Subject: [PATCH 146/215] Update Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 Co-authored-by: Johannes Wikman --- .../CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 7c9a8d5a1..85c52ea87 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -40,7 +40,13 @@ function DownloadTemplateRepository { } # Use Authenticated API request if possible to avoid the 60 API calls per hour limit $headers = GetHeaders -token $env:GITHUB_TOKEN -repository $templateRepository - $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue + $response = $null + try { + $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue + } + catch { + # Ignore error, $response will be null + } if (-not $response -or $response.StatusCode -ne 200) { # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal # Get token with read permissions for the template repository From b7725caf9340a1ea036a4cc95a9520af0ee6dbef Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 22 Apr 2025 22:24:50 +0200 Subject: [PATCH 147/215] rename internal file --- Actions/AL-Go-Helper.ps1 | 4 ++-- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 6 +++--- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 608e3e9c5..6754b1bd1 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -13,8 +13,8 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S $ALGoFolderName = '.AL-Go' $ALGoSettingsFile = Join-Path '.AL-Go' 'settings.json' $RepoSettingsFile = Join-Path '.github' 'AL-Go-Settings.json' -$IndirectTemplateRepoSettingsFile = Join-Path '.github' 'templateRepoSettings.json' -$IndirectTemplateProjectSettingsFile = Join-Path '.github' 'templateProjectSettings.json' +$IndirectTemplateRepoSettingsFile = Join-Path '.github' 'templateRepoSettings.doNotEdit.json' +$IndirectTemplateProjectSettingsFile = Join-Path '.github' 'templateProjectSettings.doNotEdit.json' [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPushBranches', Justification = 'False positive.')] $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 85c52ea87..475078896 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -40,12 +40,12 @@ function DownloadTemplateRepository { } # Use Authenticated API request if possible to avoid the 60 API calls per hour limit $headers = GetHeaders -token $env:GITHUB_TOKEN -repository $templateRepository - $response = $null try { - $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue + $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl } catch { - # Ignore error, $response will be null + # Ignore error + $response = $null } if (-not $response -or $response.StatusCode -ne 200) { # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index d2a7aa107..4fecbcce3 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -124,13 +124,13 @@ if (-not $isDirectALGo) { $checkfiles = @( @{ 'dstPath' = (Join-Path '.github' 'workflows'); 'newname' = ''; 'srcPath' = Join-Path '.github' 'workflows'; 'pattern' = '*'; 'type' = 'workflow' }, @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.copy.md'; 'type' = 'releasenotes' } + @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.ps1'; 'type' = 'script' } ) if ($realTemplateFolder) { $checkfiles += @( @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($IndirectTemplateRepoSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($IndirectTemplateProjectSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } - @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.ps1'; 'type' = 'script' } ) } From f1afbffeabbae92440feb5db0fc130b789449666 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 22 Apr 2025 22:29:12 +0200 Subject: [PATCH 148/215] add docs --- Scenarios/settings.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index ffab63942..91e74edeb 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -14,10 +14,14 @@ When running a workflow or a local script, the settings are applied by reading s 1. `ALGoOrgSettings` is a **GitHub variable**, which can be defined on an **organizational level** and will apply to **all AL-Go repositories** in this organization. +1. `.github/templateRepoSettings.doNotEdit.json` is the organizational settings from an indirect template repository (if applicable) + 1. `.github/AL-Go-settings.json` is the **repository settings file**. This settings file contains settings that are relevant for all projects in the repository. **Special note:** The repository settings file can also contains `BcContainerHelper` settings, which will be applied when loading `BcContainerHelper` in a workflow - the GitHub variables are not considered for BcContainerHelper settings. (see expert section). 1. `ALGoRepoSettings` is a **GitHub variable**, which can be defined on an **repository level** and can contain settings that are relevant for **all projects** in the repository. +1. `.github/templateProjectSettings.doNotEdit.json` is the project settings from an indirect template repository (if applicable) + 1. `.AL-Go/settings.json` is the **project settings file**. If the repository is a single project, the .AL-Go folder is in the root folder of the repository. If the repository contains multiple projects, there will be a .AL-Go folder in each project folder (like `project/.AL-Go/settings.json`) 1. `.github/.settings.json` is the **workflow-specific settings file** for **all projects**. This option is used for the Current, NextMinor and NextMajor workflows to determine artifacts and build numbers when running these workflows. From 7ca8ac27c32e068e707673d3fe3bfaa41839f6c8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 22 Apr 2025 22:42:12 +0200 Subject: [PATCH 149/215] fix ci test --- Tests/CheckForUpdates.Action.Test.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/Tests/CheckForUpdates.Action.Test.ps1 b/Tests/CheckForUpdates.Action.Test.ps1 index 393b1a7aa..8e14454c2 100644 --- a/Tests/CheckForUpdates.Action.Test.ps1 +++ b/Tests/CheckForUpdates.Action.Test.ps1 @@ -16,9 +16,6 @@ Describe "CheckForUpdates Action Tests" { It 'Test action.yaml matches script' { $permissions = [ordered]@{ - "contents" = "write" - "pull-requests" = "write" - "workflows" = "write" } $outputs = [ordered]@{ } From e53180b95a40b7784206d0b6f9f53ef22f0f35db Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 06:25:47 +0200 Subject: [PATCH 150/215] remove cicdauthcontext --- Actions/RunPipeline/RunPipeline.ps1 | 15 +-------------- Scenarios/secrets.md | 9 ++------- .../.github/workflows/_BuildALGoProject.yaml | 2 +- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index 05a3f4a0e..7a7822248 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -86,7 +86,7 @@ try { $appBuild = $settings.appBuild $appRevision = $settings.appRevision - 'licenseFileUrl','codeSignCertificateUrl','codeSignCertificatePassword','keyVaultCertificateUrl','*keyVaultCertificatePassword','keyVaultClientId','gitHubPackagesContext','applicationInsightsConnectionString','cicdAuthContext' | ForEach-Object { + 'licenseFileUrl','codeSignCertificateUrl','codeSignCertificatePassword','keyVaultCertificateUrl','*keyVaultCertificatePassword','keyVaultClientId','gitHubPackagesContext','applicationInsightsConnectionString' | ForEach-Object { # Secrets might not be read during Pull Request runs if ($secrets.Keys -contains $_) { $value = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$_")) @@ -281,19 +281,6 @@ try { } $authContext = $null $environmentName = "" - if ($cicdAuthContext) { - $authContext = $cicdAuthContext | ConvertFrom-Json | ConvertTo-HashTable - if ($authContext.ContainsKey('environmentName')) { - $environmentName = $authContext.environmentName - $authContext.Remove('environmentName') - if ($environmentName -notlike 'https://*') { - $authContext = New-BcAuthContext @authContext - } - } - else { - Write-Host "::WARNING::CI/CD AuthContext is missing environmentName, ignoring cicdAuthContext secret." - } - } $CreateRuntimePackages = $false if ($settings.versioningStrategy -eq -1) { diff --git a/Scenarios/secrets.md b/Scenarios/secrets.md index 51878e467..02a6d8a33 100644 --- a/Scenarios/secrets.md +++ b/Scenarios/secrets.md @@ -104,15 +104,10 @@ Example: `{"keyVaultName":"MyKeyVault","clientId":"","clientSecret":"< With this setup, you can create a setting called `keyVaultCodesignCertificateName` containing the name of the imported certificate in your Key Vault in order for AL-Go for GitHub to sign your apps. - -## **AuthContext** -> Connect to online environments +## **AuthContext** -> Deploy to an environment -Whenever AL-Go for GitHub is going to connect to an online environment, it will need an AuthContext secret. - -For publishing to an environment, the AuthContext secret can be provided underneath the environment in GitHub. If you are using a private repository in the free GitHub plan, you do not have environments. Then you can create an AuthContext secret in the repository. If you have multiple environments, you can create different AuthContext secrets by using the environment name followed by an underscore and AuthContext (f.ex. **QA_AuthContext**). - -For using online environments during CI/CD (for running tests etc.) you need to provide a secret called **CICDAuthContext**. If a CICDAuthContext secret exists, the online environment pointed out by this authentication context, will be used when running CI/CD for symbols, upgrade testing and test running. +Whenever AL-Go for GitHub is doing to deploy to an environment, it will need an AuthContext secret. The AuthContext secret can be provided underneath the environment in GitHub. If you are using a private repository in the free GitHub plan, you do not have environments. Then you can create an AuthContext secret in the repository. If you have multiple environments, you can create different AuthContext secrets by using the environment name followed by an underscore and AuthContext (f.ex. **QA_AuthContext**). ### Managed identity diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index d77bf32c1..08f1b241b 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -126,7 +126,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken,cicdAuthContext' + getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken' - name: Checkout Submodules if: env.useGitSubmodules != 'false' && env.useGitSubmodules != '' diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index d77bf32c1..08f1b241b 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -126,7 +126,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken,cicdAuthContext' + getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken' - name: Checkout Submodules if: env.useGitSubmodules != 'false' && env.useGitSubmodules != '' From 6349cd529a9d06ce45bb106ae47ee6ed4d2d8f32 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 06:26:59 +0200 Subject: [PATCH 151/215] revert --- Scenarios/secrets.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/secrets.md b/Scenarios/secrets.md index 02a6d8a33..caaf90f26 100644 --- a/Scenarios/secrets.md +++ b/Scenarios/secrets.md @@ -78,7 +78,7 @@ By creating a secret called Azure_Credentials you can give your GitHub repositor > [!NOTE] > In order to use a KeyVault for signing apps, it needs to be a premium SKU KeyVault. You can use this command to modify an existing KeyVault: `az keyvault update --set properties.sku.name=premium --name --resource-group ` -An Azure Key Vault can be set up for two different security models: Role Based Access Control (RBAC) (recommended) and Vault Access Policy. In order for AL-Go for GitHub to use the Key Vault, the following roles/permissions need to be assigned to the app registration or Managed Identity, on which the authentication is performed: +n Azure Key Vault can be set up for two different security models: Role Based Access Control (RBAC) (recommended) and Vault Access Policy. In order for AL-Go for GitHub to use the Key Vault, the following roles/permissions need to be assigned to the app registration or Managed Identity, on which the authentication is performed: | Security Model | Read Secrets | Sign Apps | | :-- | :-- | :-- | From 1aaf7946971c80c379974834bdc3eac419c45310 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 06:37:54 +0200 Subject: [PATCH 152/215] Remove custom steps --- .../CheckForUpdates.HelperFunctions.ps1 | 14 --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 5 +- Actions/CheckForUpdates/yamlclass.ps1 | 104 +----------------- RELEASENOTES.md | 47 ++++---- Scenarios/CustomizingALGoForGitHub.md | 31 ------ Scenarios/settings.md | 26 ----- Tests/CheckForUpdates.Action.Test.ps1 | 11 -- Tests/CustomizedYamlSnippet.txt | 12 -- Workshop/Index.md | 1 - .../scenarios/IndirectTemplate/runtest.ps1 | 63 ----------- 10 files changed, 24 insertions(+), 290 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 475078896..a23b878ef 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -1,17 +1,3 @@ -function GetCustomizationAnchors { - return @{ - "_BuildALGoProject.yaml" = @{ - "BuildALGoProject" = @( - @{ "Step" = 'Read settings'; "Before" = $false } - @{ "Step" = 'Read secrets'; "Before" = $false } - @{ "Step" = 'Build'; "Before" = $true } - @{ "Step" = 'Build'; "Before" = $false } - @{ "Step" = 'Cleanup'; "Before" = $true } - ) - } - } -} - <# .SYNOPSIS Downloads a template repository and returns the path to the downloaded folder diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 4fecbcce3..61a2e4690 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -224,18 +224,17 @@ foreach($checkfile in $checkfiles) { ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } - $customizationAnchors = GetCustomizationAnchors if ($type -eq 'workflow' -and $realSrcFile -ne $srcFile) { # Apply customizations from indirect template repository Write-Host "Apply customizations from indirect template repository: $srcFile" - [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $customizationAnchors) + [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile) } if ($unusedALGoSystemFiles -notcontains $fileName) { if (Test-Path -Path $dstFile -PathType Leaf) { if ($type -eq 'workflow') { Write-Host "Apply customizations from my repository: $dstFile" - [Yaml]::ApplyCustomizations([ref] $srcContent, $dstFile, $customizationAnchors) + [Yaml]::ApplyCustomizations([ref] $srcContent, $dstFile) } # file exists, compare and add to $updateFiles if different $dstContent = Get-ContentLF -Path $dstFile diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 25d08e35d..f2eeafd45 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -315,95 +315,6 @@ class Yaml { return $steps } - [hashtable[]] GetCustomStepsFromAnchor([string] $job, [string] $anchorStep, [bool] $before) { - $steps = $this.GetStepsFromJob($job) - $anchorIdx = $steps.IndexOf($anchorStep) - if ($anchorIdx -lt 0) { - Write-Host "Cannot find anchor step '$anchorStep' in job '$job'" - return @() - } - $idx = $anchorIdx - $customSteps = @() - if ($before) { - while ($idx -gt 0 -and $steps[$idx-1] -like 'CustomStep*') { - $idx-- - } - if ($idx -ne $anchorIdx) { - $customSteps = @($steps[$idx..($anchorIdx-1)]) - # Reverse the order of the custom steps in order to apply in correct order from the anchor step - [array]::Reverse($customSteps) - } - } - else { - while ($idx -lt $steps.Count-1 -and $steps[$idx+1] -like 'CustomStep*') { - $idx++ - } - if ($idx -ne $anchorIdx) { - $customSteps = @($steps[($anchorIdx+1)..$idx]) - } - } - $result = @() - foreach($customStep in $customSteps) { - $stepContent = $this.Get("Jobs:/$($job):/steps:/- name: $($customStep)").content - $result += @(@{"Name" = $customStep; "Content" = $stepContent; "AnchorStep" = $anchorStep; "Before" = $before }) - } - return $result - } - - [hashtable[]] GetCustomStepsFromYaml([string] $job, [hashtable[]] $anchors) { - $steps = $this.GetStepsFromJob($job) - $result = @() - foreach($anchor in $anchors) { - $result += $this.GetCustomStepsFromAnchor($job, $anchor.Step, $anchor.Before) - } - foreach($step in $steps) { - if ($step -like 'CustomStep*') { - if (-not ($result | Where-Object { $_.Name -eq $step })) { - Write-Host "Custom step '$step' does not belong to a supported anchor" - } - } - } - return $result - } - - [void] AddCustomStepsToAnchor([string] $job, [hashtable[]] $customSteps, [string] $anchorStep, [bool] $before) { - $steps = $this.GetStepsFromJob($job) - if (!$steps) { - Write-Host "::Warning::Cannot find job '$job'" - return - } - $anchorIdx = $steps.IndexOf($anchorStep) - if ($anchorIdx -lt 0) { - Write-Host "::Warning::Cannot find anchor step '$anchorStep' in job '$job'" - return - } - foreach($customStep in $customSteps | Where-Object { $_.AnchorStep -eq $anchorStep -and $_.Before -eq $before }) { - if ($steps -contains $customStep.Name) { - Write-Host "Custom step '$($customStep.Name)' already exists in job '$job'" - } - else { - $anchorStart = 0 - $anchorCount = 0 - if ($this.Find("Jobs:/$($job):/steps:/- name: $($anchorStep)", [ref] $anchorStart, [ref] $anchorCount)) { - if ($before) { - $this.Insert($anchorStart-1, @('') + @($customStep.Content | ForEach-Object { " $_" })) - } - else { - $this.Insert($anchorStart+$anchorCount, @('') + @($customStep.Content | ForEach-Object { " $_" })) - } - } - } - # Use added step as anchor for next step - $anchorStep = $customStep.Name - } - } - - [void] AddCustomStepsToYaml([string] $job, [hashtable[]] $customSteps, [hashtable[]] $anchors) { - foreach($anchor in $anchors) { - $this.AddCustomStepsToAnchor($job, $customSteps, $anchor.Step, $anchor.Before) - } - } - static [PSCustomObject] GetPermissionsFromArray([string[]] $permissionsArray) { $permissions = [PSCustomObject]@{} $permissionsArray | ForEach-Object { @@ -442,7 +353,7 @@ class Yaml { return $permissions } - static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile, [hashtable] $anchors) { + static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile) { $srcYaml = [Yaml]::new($srcContent.Value.Split("`n")) try { $yaml = [Yaml]::Load($yamlFile) @@ -485,19 +396,6 @@ class Yaml { } } - # Apply cystom steps - Write-Host "Apply custom steps" - $filename = [System.IO.Path]::GetFileName($yamlFile) - if ($anchors.ContainsKey($filename)) { - $fileAnchors = $anchors."$filename" - foreach($job in $fileAnchors.Keys) { - # Locate custom steps in destination YAML - $customSteps = $yaml.GetCustomStepsFromYaml($job, $fileAnchors."$job") - if ($customSteps) { - $srcYaml.AddCustomStepsToYaml($job, $customSteps, $fileAnchors."$job") - } - } - } # Locate custom jobs in destination YAML Write-Host "Apply custom jobs" $customJobs = @($yaml.GetCustomJobsFromYaml('CustomJob*')) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index ab8724cb9..186730340 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -11,6 +11,27 @@ - Issue 1644 Support for AppAuth when using a private Template repository from another organization - Issue 1478 Rate Limit Exceeded when running Update AL-Go System files +### New Repository settings + +- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: + + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) + - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Indirect AL-Go template repository + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + ## v7.0 ### Issues @@ -116,34 +137,8 @@ Which would hide the secret part of your URL instead of exposing it in clear tex ### New Repository Settings - [`shortLivedArtifactsRetentionDays`](https://aka.ms/algosettings#shortLivedArtifactsRetentionDays) determines the number of days to keep short lived build artifacts (f.ex build artifacts from pull request builds, next minor or next major builds). 1 is default. 0 means use GitHub default. - - [`preProcessorSymbols`](https://aka.ms/algosettings#preProcessorSymbols) is a list of preprocessor symbols to use when building the apps. This setting can be specified in [workflow specific settings files](https://aka.ms/algosettings#where-are-the-settings-located) or in [conditional settings](https://aka.ms/algosettings#conditional-settings). -- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: - - - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) - - **Source** (mandatory) - URL to a either a single file or a .zip file containing - - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. - - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) - -### Add custom jobs to AL-Go workflows - -It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. - -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Add custom steps to the AL-Go build workflow - -It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. - -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Indirect AL-Go template repository - -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. - -Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. - ### New Versioning Strategy Setting versioning strategy to 3 will allow 3 segments of the version number to be defined in app.json and repoVersion. Only the 4th segment (Revision) will be defined by the GitHub [run_number](https://go.microsoft.com/fwlink/?linkid=2217416&clcid=0x409) for the CI/CD workflow. Increment version number and Create Release now also supports the ability to set a third segment to the RepoVersion and appversion in app.json. diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 0a676fe17..a7303a6a1 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -214,37 +214,6 @@ Adding a custom job like this, will cause this job to run simultaneously with th > [!CAUTION] > Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. -### Adding custom steps - -You can also add custom steps to AL-Go for GitHub Workflows, but only in pre-defined anchor-points. The reason for only allowing custom steps at pre-defined anchor-points is that we want to limit the number of places where steps can be added in order to have some level of freedom to refactor, develop and maintain the AL-Go for GitHub workflows, without breaking customizations constantly. - -At this time, the anchor-points where you can add custom steps are: - -| Workflow | Job | Step | Before or After | -| :-- | :-- | :-- | :-: | -| \_BuildALGoProject.yaml | BuildALGoProject | Read settings | After | -| \_BuildALGoProject.yaml | BuildALGoProject | Read secrets | After | -| \_BuildALGoProject.yaml | BuildALGoProject | Build | Before | -| \_BuildALGoProject.yaml | BuildALGoProject | Build | After | -| \_BuildALGoProject.yaml | BuildALGoProject | Cleanup | Before | - -The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic - -```yaml - - name: CustomStep-ModifySettings - run: | - $settings = $env:Settings | ConvertFrom-Json - $settings.artifact = Invoke-RestMethod -Method GET -UseBasicParsing -Uri "https://bca-url-proxy.azurewebsites.net/bca-url/sandbox/us?select=weekly&doNotRedirect=true" - Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "Settings=$($settings | ConvertTo-Json -Depth 99 -Compress)" - Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "artifact=$($settings.artifact)" -``` - -> [!TIP] -> Create a feature request [here](https://github.com/microsoft/AL-Go/issues/new?assignees=&labels=enhancement&projects=&template=enhancement.yaml&title=%5BEnhancement%5D%3A+) with a description on where you would like additional anchor-points and what you want to use it for. - -> [!CAUTION] -> Please be aware that changes to AL-Go for GitHub might break with future versions of AL-Go for GitHub. We will of course try to keep these breaking changes to a minimum, but the only way you can be sure to NOT be broken is by NOT customizing AL-Go for GitHub. - ### Modifying workflow permissions If any of your custom jobs require permissions, which exceeds the permissions already assigned in the workflow, then these permissions can be specified directly on the custom job. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 91e74edeb..93f27f1e6 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -413,32 +413,6 @@ Custom jobs will be preserved when running Update AL-Go System Files. **Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). - - -## Custom steps in the \_BuildALGoProject workflow - -Adding a custom step is done by adding a step with the name `CustomStep` to the \_BuildALGoProject.yaml workflow at one of these anchor points: - -- Before Read Settings -- Before Read Secrets -- Before or After Build -- Before Cleanup - -Example, insert the following step before the Build step: - -``` - - name: CustomStep that will run before the Build step - run: | - Write-Host "before build" - - - name: Build - uses: ... -``` - -Custom steps will be preserved when running Update AL-Go System Files. - -**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). - ## Indirect template repositories diff --git a/Tests/CheckForUpdates.Action.Test.ps1 b/Tests/CheckForUpdates.Action.Test.ps1 index 46804e248..85410ab3a 100644 --- a/Tests/CheckForUpdates.Action.Test.ps1 +++ b/Tests/CheckForUpdates.Action.Test.ps1 @@ -127,19 +127,8 @@ Describe "CheckForUpdates Action Tests" { $customJobs | Should -Not -BeNullOrEmpty $customJobs.Count | Should -be 1 - # Get Custom steps from yaml - $customStep1 = $customizedYaml.GetCustomStepsFromAnchor('Initialization', 'Read settings', $true) - $customStep1 | Should -Not -BeNullOrEmpty - $customStep1.Count | Should -be 2 - - $customStep2 = $customizedYaml.GetCustomStepsFromAnchor('Initialization', 'Read settings', $false) - $customStep2 | Should -Not -BeNullOrEmpty - $customStep2.Count | Should -be 1 - # Apply Custom jobs and steps to yaml $yaml.AddCustomJobsToYaml($customJobs) - $yaml.AddCustomStepsToAnchor('Initialization', $customStep1, 'Read settings', $true) - $yaml.AddCustomStepsToAnchor('Initialization', $customStep2, 'Read settings', $false) # Check if new yaml content is equal to customized yaml content ($yaml.content -join "`r`n") | Should -be ($customizedYaml.content -join "`r`n") diff --git a/Tests/CustomizedYamlSnippet.txt b/Tests/CustomizedYamlSnippet.txt index 0ccb32cb5..efee10311 100644 --- a/Tests/CustomizedYamlSnippet.txt +++ b/Tests/CustomizedYamlSnippet.txt @@ -46,24 +46,12 @@ jobs: with: shell: powershell - - name: CustomStep-MyStep1 - run: | - Write-Host 'My own step1!' - - - name: CustomStep-MyStepX - run: | - Write-Host 'Some stepX!' - - name: Read settings id: ReadSettings uses: microsoft/AL-Go-Actions/ReadSettings@main with: shell: powershell - - name: CustomStep-MyStep2 - run: | - Write-Host 'My own step2!' - CheckForUpdates: runs-on: [ windows-latest ] needs: [ Initialization, CustomJob-MyJob ] diff --git a/Workshop/Index.md b/Workshop/Index.md index 3c702fc63..57ab48882 100644 --- a/Workshop/Index.md +++ b/Workshop/Index.md @@ -27,7 +27,6 @@ This workshop shows you how to take advantage of the functionality, which is pro 1. [Custom Delivery](CustomDelivery.md) - *FUTURE TOPIC: Setting up custom delivery to f.ex. a Teams channel.* 1. [Custom Delivery](CustomDeployment.md) - *FUTURE TOPIC: Setting up custom deployment to f.ex. an on-premises environment.* 1. [Custom Jobs](CustomJobs.md) - *FUTURE TOPIC: Adding a custom job to an AL-Go for GitHub workflows.* -1. [Custom Steps](CustomSteps.md) - *FUTURE TOPIC: Adding a custom step to the BuildALGoProject workflow.* 1. [Using Indirect Templates](IndirectTemplates.md) - *FUTURE TOPIC: Using indirect templates to ensure that all repositories are using the same customizations* ## Additional Future topics diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 index 432e5d9ab..600175ee2 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -25,12 +25,10 @@ Write-Host -ForegroundColor Yellow @' # - Create a new repository based on the PTE template with no apps (this will be the "indirect" template repository) # - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) # - Run Update AL-Go System Files in final repo (using indirect repo as template) -# - Add CustomStep to indirect repo # - Run Update AL-Go System files in indirect repo # - Validate that custom step is present in indirect repo # - Run Update AL-Go System files in final repo # - Validate that custom step is present in final repo -# - Add CustomStep in final repo # - Run Update AL-Go System files in final repo # - Validate that both custom steps is present in final repo # @@ -94,24 +92,6 @@ $buildYaml | Should -Not -BeNullOrEmpty # Modify the permissions $buildYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) - -# Add customization steps -$customizationAnchors = GetCustomizationAnchors -$idx = 0 -foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { - $idx++ - $customStep = @{ - "Name" = "CustomStep-Template$idx" - "Content" = @( - "- name: CustomStep-Template$idx" - " run: |" - " Write-Host 'CustomStep-Template$idx was here!'" - ) - "AnchorStep" = $anchor.Step - "Before" = $anchor.Before - } - $buildYaml.AddCustomStepsToAnchor('BuildALGoProject', $customStep, $anchor.Step, $anchor.Before) -} $buildYaml.Save($buildALGoProjectWorkflow) # Add Custom Jobs to CICD.yaml @@ -167,24 +147,6 @@ $buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoPro $buildYaml = [yaml]::Load($buildALGoProjectWorkflow) $buildYaml | Should -Not -BeNullOrEmpty -# Add customization steps -$customizationAnchors = GetCustomizationAnchors -$idx = 0 -foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { - $idx++ - $customStep = @{ - "Name" = "CustomStep-Final$idx" - "Content" = @( - "- name: CustomStep-Final$idx" - " run: |" - " Write-Host 'CustomStep-Final$idx was here!'" - ) - "AnchorStep" = $anchor.Step - "Before" = $anchor.Before - } - $buildYaml.AddCustomStepsToAnchor('BuildALGoProject', $customStep, $anchor.Step, $anchor.Before) -} - # save $buildYaml.Save($buildALGoProjectWorkflow) @@ -247,31 +209,6 @@ Pull # Run CICD $run = RunCICD -repository $repository -branch $branch -wait -# Check Custom Steps -1..$idx | ForEach-Object { - Test-LogContainsFromRun -runid $run.id -jobName 'Build . (Default) . (Default)' -stepName "CustomStep-Template$_" -expectedText "CustomStep-Template$_ was here!" -} -1..$idx | ForEach-Object { - Test-LogContainsFromRun -runid $run.id -jobName 'Build . (Default) . (Default)' -stepName "CustomStep-Final$_" -expectedText "CustomStep-Final$_ was here!" -} - -# Check correct order of custom steps -DownloadWorkflowLog -repository $repository -runid $run.id -path 'logs' -$logcontent = Get-Content -Path 'logs/0_Build . (Default) . (Default).txt' -Encoding utf8 -Raw -Remove-Item -Path 'logs' -Recurse -Force -$idx = 0 -foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { - $idx++ - $templateStepIdx = $logcontent.IndexOf("CustomStep-Template$idx was here!") - $finalStepIdx = $logcontent.IndexOf("CustomStep-Final$idx was here!") - if ($anchor.Before) { - $finalStepIdx | Should -BeGreaterThan $templateStepIdx -Because "CustomStep-Final$idx should be after CustomStep-Template$idx" - } - else { - $finalStepIdx | Should -BeLessThan $templateStepIdx -Because "CustomStep-Final$idx should be before CustomStep-Template$idx" - } -} - # Check Custom Jobs Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateInit' -stepName 'Init' -expectedText 'CustomJob-TemplateInit was here!' Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateDeploy' -stepName 'Deploy' -expectedText 'CustomJob-TemplateDeploy was here!' From 3c7a9a788cb9c1735724411e1d86fd9104ffd58b Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 07:11:55 +0200 Subject: [PATCH 153/215] add authentication for custom files --- .../CheckForUpdates.HelperFunctions.ps1 | 23 ++++++++++++++----- RELEASENOTES.md | 7 +++--- .../AppSource App/.github/workflows/CICD.yaml | 3 ++- .../workflows/UpdateGitHubGoSystemFiles.yaml | 1 + .../.github/workflows/CICD.yaml | 3 ++- .../workflows/UpdateGitHubGoSystemFiles.yaml | 1 + 6 files changed, 27 insertions(+), 11 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index a23b878ef..b53bf45f5 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -21,9 +21,6 @@ function DownloadTemplateRepository { $templateRepositoryUrl = $templateUrl.Split('@')[0] $templateRepository = $templateRepositoryUrl.Split('/')[-2..-1] -join '/' - if ([string]::IsNullOrEmpty($ENV:GITHUB_TOKEN)) { - Write-Host "::WARNING::GITHUB_TOKEN is not set." - } # Use Authenticated API request if possible to avoid the 60 API calls per hour limit $headers = GetHeaders -token $env:GITHUB_TOKEN -repository $templateRepository try { @@ -565,14 +562,28 @@ function GetCustomALGoSystemFiles { if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." } + $source = $customspec.Source + # Replace embedded secrets in the source URL with the value of the secret + $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*' + if ($customspec.Source -match $pattern) { + $source = $source.Replace($matches[1],[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($matches[2])"))) + } + $destination = $customSpec.Destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) if ($destination -isnot [string] -or $destination -eq '') { throw "customALGoSystemFiles setting is wrongly formatted, Destination must be a string, which isn't blank. See https://aka.ms/algosettings#customalgosystemfiles." } if ($source -isnot [string] -or $source -notlike 'https://*' -or (-not [System.Uri]::IsWellFormedUriString($source,1))) { - throw "customALGoSystemFiles setting is wrongly formatted, Source must secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." + throw "customALGoSystemFiles setting is wrongly formatted, Source must be a secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." + } + + $authToken = $null + if ($customspec.PSObject.Properties.Name -eq 'AuthTokenSecret') { + $authTokenSecret = $customspec.AuthTokenSecret + $authToken = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($authTokenSecret)")) } + $headers = Get-Headers -token $authToken -repository $source $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) New-Item -Path $tempFolder -ItemType Directory | Out-Null @@ -589,7 +600,7 @@ function GetCustomALGoSystemFiles { if (!($destination.EndsWith('/') -or $destination.EndsWith('\'))) { throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." } - Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName + Invoke-RestMethod -UseBasicParsing -Method Get -Headers $headers -Uri $source -OutFile $zipName Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve Push-Location -Path $subFolder @@ -614,7 +625,7 @@ function GetCustomALGoSystemFiles { } Write-Host "$($destination):" $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) - Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $tempFilename + Invoke-RestMethod -UseBasicParsing -Method Get -Headers $headers -Uri $source -OutFile $tempFilename YieldItem -baseFolder $baseFolder -source $tempFilename -destination $destination -projects $projects } } diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 186730340..5a81e2e96 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -15,10 +15,11 @@ - `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: - - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) - - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file). + - **Source** (mandatory) - URL to a either a single file or a .zip file containing. Note that the source URL can contain embedded secrets using ${{SECRETNAME}} as part of the URL to hide SAS tokens or like. - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. - - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec). + - **AuthTokenSecret** (optional) - the name of a secret (like ghTokenWorkflow) which contains a Personal Access Token or GitHub App specification with access to the source files. ### Add custom jobs to AL-Go workflows diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index 92fa1004d..6da8c158a 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -160,12 +160,13 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'ghTokenWorkflow' + getSecrets: 'ghTokenWorkflow,CustomALGoSystemFilesSecrets' - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main env: GITHUB_TOKEN: ${{ github.token }} + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' with: shell: powershell templateUrl: ${{ env.templateUrl }} diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 975536e3a..3b887ba28 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -135,6 +135,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main env: GITHUB_TOKEN: ${{ github.token }} + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' with: shell: powershell token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index c2e93d363..4a37274fc 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -160,12 +160,13 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'ghTokenWorkflow' + getSecrets: 'ghTokenWorkflow,CustomALGoSystemFilesSecrets' - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main env: GITHUB_TOKEN: ${{ github.token }} + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' with: shell: powershell templateUrl: ${{ env.templateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 975536e3a..3b887ba28 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -135,6 +135,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main env: GITHUB_TOKEN: ${{ github.token }} + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' with: shell: powershell token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} From fde3d9d01228f2ccc5c8a99b3813cc0803a01252 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 07:31:02 +0200 Subject: [PATCH 154/215] use source --- Actions/ReadSecrets/ReadSecrets.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index 8185be55e..6f08006a1 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -110,7 +110,7 @@ try { if ($customspec.PSObject.Properties.name -eq "Source") { # If the Source URLs contains '${{SECRETNAME}}' we need to get the secret $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*' - if ($_ -match $pattern) { + if ($customspec.Source -match $pattern) { $secretName = $matches[2] if ($secretsCollection -notcontains $secretName) { $secretsCollection += $secretName From 8fd6fdc987ca3c9d20dc30a344bf9fcfacffd3b6 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 07:31:54 +0200 Subject: [PATCH 155/215] comment --- Actions/ReadSecrets/ReadSecrets.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index 6f08006a1..a6d7dcecb 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -100,7 +100,7 @@ try { } if ($getCustomALGoSystemFilesSecrets) { - # Look through custominstallApps and installTestApps for secrets and add them to the collection of secrets to get + # Look through customALGoSystemFiles for secrets and add them to the collection of secrets to get foreach($customspec in $settings.customALGoSystemFiles) { if ($customspec.PSObject.Properties.name -eq "AuthTokenSecret") { if ($secretsCollection -notcontains $customspec.authTokenSecret) { From dc5f89fb07c923a80d539233e163ff288185ba73 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 07:34:05 +0200 Subject: [PATCH 156/215] getheaders --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index b53bf45f5..6ace5b88e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -583,7 +583,7 @@ function GetCustomALGoSystemFiles { $authTokenSecret = $customspec.AuthTokenSecret $authToken = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($authTokenSecret)")) } - $headers = Get-Headers -token $authToken -repository $source + $headers = GetHeaders -token $authToken -repository $source $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) New-Item -Path $tempFolder -ItemType Directory | Out-Null From 5215c9f56c569cbec847478e6b368d4008924ec7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 07:40:36 +0200 Subject: [PATCH 157/215] read secrets --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 ++ Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 6ace5b88e..eabd0739d 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -581,7 +581,9 @@ function GetCustomALGoSystemFiles { $authToken = $null if ($customspec.PSObject.Properties.Name -eq 'AuthTokenSecret') { $authTokenSecret = $customspec.AuthTokenSecret + Write-Host "Using secret $authTokenSecret" $authToken = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($authTokenSecret)")) + Write-Host "$authToken.Substring(0, 5)..." } $headers = GetHeaders -token $authToken -repository $source diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 61a2e4690..96a4af014 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -45,6 +45,8 @@ if ($token) { $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } +$secrets = $env:Secrets | ConvertFrom-Json | ConvertTo-HashTable + # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository # CheckForUpdates will apply changes to the AL-Go System files based on AL-Go repo settings, such as "runs-on" etc. # if $update is set to Y, CheckForUpdates will also update the AL-Go System files in the current repository using a PR or a direct commit (if $directCommit is set to true) From 4e7e4192c837a6237f86a9f0e7c55feed026cdd1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 07:43:48 +0200 Subject: [PATCH 158/215] dump properties --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index eabd0739d..ffcd077c9 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -579,6 +579,7 @@ function GetCustomALGoSystemFiles { } $authToken = $null + $customspec.PSObject.Properties.Name | Out-Host if ($customspec.PSObject.Properties.Name -eq 'AuthTokenSecret') { $authTokenSecret = $customspec.AuthTokenSecret Write-Host "Using secret $authTokenSecret" From 8ee29627bf271458da3e47c9522580ee97ae18a3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 07:45:27 +0200 Subject: [PATCH 159/215] ht --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ffcd077c9..b3522b2a2 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -579,8 +579,7 @@ function GetCustomALGoSystemFiles { } $authToken = $null - $customspec.PSObject.Properties.Name | Out-Host - if ($customspec.PSObject.Properties.Name -eq 'AuthTokenSecret') { + if ($customspec.ContainsKey('AuthTokenSecret')) { $authTokenSecret = $customspec.AuthTokenSecret Write-Host "Using secret $authTokenSecret" $authToken = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($authTokenSecret)")) From d5da27c8487e60db394f45c9b656c64949ab5827 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 08:25:18 +0200 Subject: [PATCH 160/215] get repo name --- .../CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index b3522b2a2..8b03c6fcf 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -579,13 +579,16 @@ function GetCustomALGoSystemFiles { } $authToken = $null + $repository = $null if ($customspec.ContainsKey('AuthTokenSecret')) { $authTokenSecret = $customspec.AuthTokenSecret Write-Host "Using secret $authTokenSecret" $authToken = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($authTokenSecret)")) - Write-Host "$authToken.Substring(0, 5)..." + Write-Host "$($authToken.Substring(0, 5))..." + # if the AuthToken is a GitHub App specification, we need to get the repository name from the URL + $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[0..1] -join '/' } - $headers = GetHeaders -token $authToken -repository $source + $headers = GetHeaders -token $authToken -repository $repository $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) New-Item -Path $tempFolder -ItemType Directory | Out-Null From ddb2dbf17636911b22c722431db4fed1e1518c90 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 08:27:53 +0200 Subject: [PATCH 161/215] dumps --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 8b03c6fcf..1a3ddfede 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -605,6 +605,8 @@ function GetCustomALGoSystemFiles { if (!($destination.EndsWith('/') -or $destination.EndsWith('\'))) { throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." } + Write-Host "Downloading $source to $zipName" + $headers | Out-Host Invoke-RestMethod -UseBasicParsing -Method Get -Headers $headers -Uri $source -OutFile $zipName Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve From e109d671eb72db04b351cfc9d5a36e9c66f4b05d Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 08:36:43 +0200 Subject: [PATCH 162/215] default zip --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 1a3ddfede..e4557d64b 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -593,6 +593,9 @@ function GetCustomALGoSystemFiles { $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) New-Item -Path $tempFolder -ItemType Directory | Out-Null $ext = [System.IO.Path]::GetExtension($source) + if ($ext -eq '') { + $ext = '.zip' + } $zipName = "$tempFolder$ext" try { if ($ext -eq '.zip') { From f1e019d2aea8e1c9591cd5319c6ed1f14f8fd28e Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 08:41:59 +0200 Subject: [PATCH 163/215] support api.gi... --- .../CheckForUpdates.HelperFunctions.ps1 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index e4557d64b..d3e303204 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -578,24 +578,26 @@ function GetCustomALGoSystemFiles { throw "customALGoSystemFiles setting is wrongly formatted, Source must be a secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." } + $ext = [System.IO.Path]::GetExtension($source) $authToken = $null $repository = $null if ($customspec.ContainsKey('AuthTokenSecret')) { $authTokenSecret = $customspec.AuthTokenSecret Write-Host "Using secret $authTokenSecret" $authToken = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($authTokenSecret)")) - Write-Host "$($authToken.Substring(0, 5))..." # if the AuthToken is a GitHub App specification, we need to get the repository name from the URL - $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[0..1] -join '/' + if ($source -like 'https://api.github.com/repos/*/*/zipball/*') { + $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[1..2] -join '/' + $ext = '.zip' + } + else { + $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[0..1] -join '/' + } } $headers = GetHeaders -token $authToken -repository $repository $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) New-Item -Path $tempFolder -ItemType Directory | Out-Null - $ext = [System.IO.Path]::GetExtension($source) - if ($ext -eq '') { - $ext = '.zip' - } $zipName = "$tempFolder$ext" try { if ($ext -eq '.zip') { From 365dc51e0318b0300d497f03a7a3c5fadf07bf72 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 08:48:22 +0200 Subject: [PATCH 164/215] add filespec --- .../CheckForUpdates.HelperFunctions.ps1 | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index d3e303204..4c3468f34 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -569,13 +569,23 @@ function GetCustomALGoSystemFiles { if ($customspec.Source -match $pattern) { $source = $source.Replace($matches[1],[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($matches[2])"))) } + if ($source -isnot [string] -or $source -notlike 'https://*' -or (-not [System.Uri]::IsWellFormedUriString($source,1))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source must be a secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." + } $destination = $customSpec.Destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) if ($destination -isnot [string] -or $destination -eq '') { throw "customALGoSystemFiles setting is wrongly formatted, Destination must be a string, which isn't blank. See https://aka.ms/algosettings#customalgosystemfiles." } - if ($source -isnot [string] -or $source -notlike 'https://*' -or (-not [System.Uri]::IsWellFormedUriString($source,1))) { - throw "customALGoSystemFiles setting is wrongly formatted, Source must be a secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." + + if ($customSpec.ContainsKey('FileSpec')) { $fileSpec = $customSpec.FileSpec } else { $fileSpec = '*' } + if ($fileSpec -isnot [string]) { + throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string. See https://aka.ms/algosettings#customalgosystemfiles." + } + + if ($customSpec.ContainsKey('Recurse')) { $recurse = $customSpec.Recurse } else { $recurse = $true } + if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { + throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." } $ext = [System.IO.Path]::GetExtension($source) @@ -589,6 +599,8 @@ function GetCustomALGoSystemFiles { if ($source -like 'https://api.github.com/repos/*/*/zipball/*') { $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[1..2] -join '/' $ext = '.zip' + # When downloading a zipball, the content is inside a subfolder with the repository name and the sha + $filespec = "*/$filespec" } else { $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[0..1] -join '/' @@ -601,12 +613,6 @@ function GetCustomALGoSystemFiles { $zipName = "$tempFolder$ext" try { if ($ext -eq '.zip') { - Write-Host "$($destination):" - if ($customSpec.ContainsKey('FileSpec')) { $fileSpec = $customSpec.FileSpec } else { $fileSpec = '*' } - if ($customSpec.ContainsKey('Recurse')) { $recurse = $customSpec.Recurse } else { $recurse = $true } - if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { - throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." - } if (!($destination.EndsWith('/') -or $destination.EndsWith('\'))) { throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." } From 3f43611c1a651d4b326a2746931ce77181bd133b Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 09:09:52 +0200 Subject: [PATCH 165/215] add force --- .../CheckForUpdates.HelperFunctions.ps1 | 4 +--- Scenarios/CustomizingALGoForGitHub.md | 10 +++++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 4c3468f34..5d58c3e3a 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -599,8 +599,6 @@ function GetCustomALGoSystemFiles { if ($source -like 'https://api.github.com/repos/*/*/zipball/*') { $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[1..2] -join '/' $ext = '.zip' - # When downloading a zipball, the content is inside a subfolder with the repository name and the sha - $filespec = "*/$filespec" } else { $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[0..1] -join '/' @@ -623,7 +621,7 @@ function GetCustomALGoSystemFiles { $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve Push-Location -Path $subFolder try { - Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File -Force | ForEach-Object { $destRelativeFileName = Resolve-Path $_.FullName -Relative $destFileName = Join-Path $destination $destRelativeFileName $destFileName = $destFileName.TrimStart('\/') diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index a7303a6a1..a011271ef 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -145,7 +145,7 @@ Which basically launches a script located in the script folder in the repository ### Adding custom workflows and/or scripts using a URL -By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings#customalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these files should be included in the update. Example: +By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings#customalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these files should be included in the update. Examples: ``` "customALGoSystemFiles": [ @@ -162,6 +162,13 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings "Source": "https://github.com/freddydk/CustomALGoSystemFiles/archive/refs/heads/main.zip", "FileSpec": "*/.github/*", "Recurse": true + }, + { + "Destination": ".github/", + "Source": "https://api.github.com/repos/BusinessCentralApps/CustomALGoSystemFiles/zipball/main", + "AuthTokenSecret": "SECRETNAME", + "FileSpec": "*/.github/*", + "Recurse": true } ] ``` @@ -174,6 +181,7 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings | Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | | FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | | Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | +| AuthTokenSecret | The name of a secret (like ghTokenWorkflow) which contains a Personal Access Token or GitHub App specification with access to the files. | This setting will cause AL-Go for GitHub to include these files during the next update. From 4786ae7ebbfe1784ad5f82bd194a79325fb25d06 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 09:18:28 +0200 Subject: [PATCH 166/215] docs --- Scenarios/CustomizingALGoForGitHub.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index a011271ef..2f6477a59 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -157,6 +157,10 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings "Destination": ".AL-Go/NewBcContainer.ps1", "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1" }, + { + "Destination": ".AL-Go/NewBcContainer.ps1", + "Source": "https://myfiles.blob.core.windows.net/myalgosystemfiles/NewBcContainer.ps1?${{SECRETNAME}}" + }, { "Destination": ".github/", "Source": "https://github.com/freddydk/CustomALGoSystemFiles/archive/refs/heads/main.zip", @@ -178,8 +182,8 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings | Property | Description | Mandatory | Default | | :-- | :-- | :-: | :-- | | Destination | Path in which the file should be placed. Can include the filename if the source doesn't point to a .zip file, must include a terminating / or \\ if a filename is not included. | Yes | | -| Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | -| FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | +| Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. The source URL can contain embedded secrets using the `${{SECRETNAME}}` construct. | Yes | | +| FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include subfolders inside the .zip file, and must include a file name pattern. | No | * | | Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | | AuthTokenSecret | The name of a secret (like ghTokenWorkflow) which contains a Personal Access Token or GitHub App specification with access to the files. | From 43605b2025956d7eba8c5d424d678a4b241119fc Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 24 Apr 2025 09:28:17 +0200 Subject: [PATCH 167/215] remove --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- Scenarios/CustomizingALGoForGitHub.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 5d58c3e3a..de2af6302 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -646,8 +646,8 @@ function GetCustomALGoSystemFiles { } } finally { - if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } Remove-Item -Path $tempFolder -Recurse -Force + if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } } } } diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 2f6477a59..9ca07cbef 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -195,6 +195,9 @@ This setting will cause AL-Go for GitHub to include these files during the next > [!NOTE] > If the destination is in the .AL-Go folder, the file(s) will be copied to all .AL-Go folders in multi-project repositories. +> [!NOTE] +> If the source is a private or internal GitHub repository and you are using a GitHub app for authentication, you should use https://api.github.com/repos/OWNER/REPO/zipball/BRANCH instead of https://github.com/OWNER/REPO/archive/refs/heads/BRANCH.zip. + ### Adding custom jobs You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Custom jobs can depend on other jobs and other jobs can made to depend on custom jobs. Custom jobs needs to be named `CustomJob`, but can specify another name to be shown in the UI. Example: From 15d54e258edcc2b12e3c00321b382e287598a59e Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 25 Apr 2025 08:56:52 +0200 Subject: [PATCH 168/215] optional secrets --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 96a4af014..905fb1dc6 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -45,7 +45,10 @@ if ($token) { $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } -$secrets = $env:Secrets | ConvertFrom-Json | ConvertTo-HashTable +$secrets = @{} +if ($env:secrets) { + $secrets = $env:Secrets | ConvertFrom-Json | ConvertTo-HashTable +} # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository # CheckForUpdates will apply changes to the AL-Go System files based on AL-Go repo settings, such as "runs-on" etc. From cc12c096466d971df2e040eb109b6e1044856d27 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 25 Apr 2025 08:57:52 +0200 Subject: [PATCH 169/215] S --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 905fb1dc6..db7273be1 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -46,7 +46,7 @@ if ($token) { } $secrets = @{} -if ($env:secrets) { +if ($env:Secrets) { $secrets = $env:Secrets | ConvertFrom-Json | ConvertTo-HashTable } From 35ca22a5c496a5d7d892e7bbc632df96c1b00b8a Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 28 Apr 2025 07:31:16 +0200 Subject: [PATCH 170/215] remove permissions merge (as we only support custom jobs) --- Actions/CheckForUpdates/yamlclass.ps1 | 81 --------------------------- 1 file changed, 81 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index f2eeafd45..fa498d3b2 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -306,53 +306,6 @@ class Yaml { } } - [string[]] GetStepsFromJob([string] $job) { - $steps = $this.GetNextLevel("Jobs:/$($job):/steps:/") | Where-Object { $_ -like '- name: *' } | ForEach-Object { $_.Substring(8).Trim() } - if ($steps | Group-Object | Where-Object { $_.Count -gt 1 }) { - Write-Host "Duplicate step names in job '$job'" - return @() - } - return $steps - } - - static [PSCustomObject] GetPermissionsFromArray([string[]] $permissionsArray) { - $permissions = [PSCustomObject]@{} - $permissionsArray | ForEach-Object { - $permissions | Add-Member -MemberType NoteProperty -Name $_.Split(':')[0].Trim() -Value $_.Split(':')[1].Trim() - } - return $permissions - } - - static [string[]] GetPermissionsArray([PSCustomObject] $permissions) { - $permissionsArray = @() - $permissions.PSObject.Properties.Name | ForEach-Object { - $permissionsArray += "$($_): $($permissions."$_")" - } - return $permissionsArray - } - - static [PSCustomObject] MergePermissions([PSCustomObject] $permissions, [PSCustomObject] $permissions2) { - $permissions2.PSObject.Properties.Name | ForEach-Object { - if ($permissions.PSObject.Properties.Name -eq $_) { - $permission = $permissions."$_" - $permission2 = $permissions2."$_" - if ($permission -eq 'write' -or $permission2 -eq 'write') { - $permissions."$_" = 'write' - } - elseif ($permission -eq 'read' -or $permission2 -eq 'read') { - $permissions."$_" = 'read' - } - else { - $permissions."$_" = 'none' - } - } - else { - $permissions | Add-Member -MemberType NoteProperty -Name $_ -Value $permissions2."$_" - } - } - return $permissions - } - static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile) { $srcYaml = [Yaml]::new($srcContent.Value.Split("`n")) try { @@ -362,40 +315,6 @@ class Yaml { return } - # Merge permissions - Write-host "Merge permissions" - $allJobs = $srcYaml.GetNextLevel('jobs:/') - $permissionPaths = @("permissions:/") + @($allJobs | ForEach-Object { "jobs:/$_/permissions:/" }) - foreach($permissionPath in $permissionPaths) { - $srcPermissionsObj = $srcYaml.Get($permissionPath) - $yamlPermissionsObj = $yaml.Get($permissionPath) - if ($yamlPermissionsObj) { - # permissions exist and should be merged/added - if ($srcPermissionsObj) { - # Merge permissions - $srcPermissions = [Yaml]::GetPermissionsFromArray($srcPermissionsObj.content) - $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) - if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { - $srcYaml.Replace($permissionPath, [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) - } - } - else { - # Add permissions - $parentPath = $permissionPath.Substring(0, $permissionPath.TrimEnd('/').LastIndexOf('/')+1) - if ($parentPath) { - # Job permissions are inserted before steps - $beforePath = "$($parentPath)steps:" - } - else { - # Global permissions are inserted before jobs - $beforePath = "jobs:" - } - $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) - $srcYaml.Replace($beforePath, @("permissions:") + @([Yaml]::GetPermissionsArray($yamlPermissions) | ForEach-Object { " $_" }) + $srcYaml.Get($beforePath).content) - } - } - } - # Locate custom jobs in destination YAML Write-Host "Apply custom jobs" $customJobs = @($yaml.GetCustomJobsFromYaml('CustomJob*')) From b5ed00e060f00a48d8ae1a9888f1ca0dde187094 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 28 Apr 2025 08:15:50 +0200 Subject: [PATCH 171/215] modify e2e --- .../scenarios/IndirectTemplate/runtest.ps1 | 79 +++++++++++-------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 index 600175ee2..32084398c 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -26,6 +26,10 @@ Write-Host -ForegroundColor Yellow @' # - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) # - Run Update AL-Go System Files in final repo (using indirect repo as template) # - Run Update AL-Go System files in indirect repo + +# TODO: describe the scenario + + # - Validate that custom step is present in indirect repo # - Run Update AL-Go System files in final repo # - Validate that custom step is present in final repo @@ -65,7 +69,7 @@ Set-Location $prevLocation $appName = 'MyApp' $publisherName = 'Contoso' -# Create repository +# Create final repository CreateAlGoRepository ` -github:$github ` -linux:$linux ` @@ -76,9 +80,9 @@ CreateAlGoRepository ` Param([string] $path) $null = CreateNewAppInFolder -folder $path -name $appName -publisher $publisherName } -$repoPath = (Get-Location).Path +$finalRepoPath = (Get-Location).Path -# Update AL-Go System Files to uptake UseProjectDependencies setting +# Update AL-Go System Files to use template repository RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null Set-Location $templateRepoPath @@ -86,13 +90,6 @@ Set-Location $templateRepoPath Pull # Make modifications to the template repository -$buildALGoProjectWorkflow = Join-Path $templateRepoPath '.github/workflows/_BuildALGoProject.yaml' -$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) -$buildYaml | Should -Not -BeNullOrEmpty - -# Modify the permissions -$buildYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) -$buildYaml.Save($buildALGoProjectWorkflow) # Add Custom Jobs to CICD.yaml $cicdWorkflow = Join-Path $templateRepoPath '.github/workflows/CICD.yaml' @@ -100,7 +97,6 @@ $cicdYaml = [yaml]::Load($cicdWorkflow) $cicdYaml | Should -Not -BeNullOrEmpty # Modify the permissions -$cicdYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) $customJobs = @( @{ "Name" = "CustomJob-TemplateInit" @@ -127,6 +123,19 @@ $customJobs = @( ) "NeedsThis" = @( 'PostProcess' ) } + @{ + "Name" = "JustSomeTemplateJob" + "Content" = @( + "JustSomeTemplateJob:" + " needs: [ PostProcess ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: JustSomeTemplateStep" + " run: |" + " Write-Host 'JustSomeTemplateJob was here!'" + ) + "NeedsThis" = @( ) + } ) # Add custom Jobs $cicdYaml.AddCustomJobsToYaml($customJobs) @@ -139,26 +148,30 @@ CommitAndPush -commitMessage 'Add template customizations' CancelAllWorkflows -repository $templateRepository # Add local customizations to the final repository -Set-Location $repoPath +Set-Location $finalRepoPath Pull -# Make modifications to the template repository -$buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoProject.yaml' -$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) -$buildYaml | Should -Not -BeNullOrEmpty - -# save -$buildYaml.Save($buildALGoProjectWorkflow) +# Make modifications to the final repository # Add Custom Jobs to CICD.yaml -$cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' +$cicdWorkflow = Join-Path $finalRepoPath '.github/workflows/CICD.yaml' $cicdYaml = [yaml]::Load($cicdWorkflow) $cicdYaml | Should -Not -BeNullOrEmpty -# Modify the permissions -$cicdYaml.Replace('permissions:/contents: read', @('contents: read', 'issues: write')) - $customJobs = @( + @{ + "Name" = "JustSomeJob" + "Content" = @( + "JustSomeJob:" + " needs: [ Initialization ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: JustSomeStep" + " run: |" + " Write-Host 'JustSomeJob was here!'" + ) + "NeedsThis" = @( 'Build' ) + } @{ "Name" = "CustomJob-PreDeploy" "Content" = @( @@ -206,6 +219,8 @@ CancelAllWorkflows -repository $repository # Pull changes Pull +# TODO: Check that the settings from the indirect template repository was copied to $IndirectTemplateRepoSettingsFile and $IndirectTemplateProjectSettingsFile + # Run CICD $run = RunCICD -repository $repository -branch $branch -wait @@ -214,22 +229,16 @@ Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateInit' -stepNa Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateDeploy' -stepName 'Deploy' -expectedText 'CustomJob-TemplateDeploy was here!' Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PreDeploy' -stepName 'PreDeploy' -expectedText 'CustomJob-PreDeploy was here!' Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PostDeploy' -stepName 'PostDeploy' -expectedText 'CustomJob-PostDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeJob' -stepName 'JustSomeStep' -expectedText 'JustSomeJob was here!' | Should -Throw +Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeTemplateJob' -stepName 'JustSomeTemplateStep' -expectedText 'JustSomeTemplateJob was here!' | Should -Throw -# Check Permissions -# TODO: check issues: write in cicd.yaml (from final) and issues: read in _buildALGoProject.yaml (from template) -$buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoProject.yaml' -$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) -$buildYaml | Should -Not -BeNullOrEmpty -$buildYaml.get('Permissions:/issues:').content | Should -Be 'issues: read' +Set-Location $prevLocation -$cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' -$cicdYaml = [yaml]::Load($cicdWorkflow) -$cicdYaml | Should -Not -BeNullOrEmpty -$cicdYaml.get('Permissions:/issues:').content | Should -Be 'issues: write' +# TODO: Modify settings in the template repository and re-run Update AL-Go System Files in the final repository to check that the settings are copied to the final repository -Set-Location $prevLocation +# TODO: Add tests for CustomALGoSystemFiles (with and without security) Read-Host "Press Enter to continue" -RemoveRepository -repository $repository -path $repoPath +RemoveRepository -repository $repository -path $finalRepoPath RemoveRepository -repository $templateRepository -path $templateRepoPath From 5a7a2a2e6e3be841ebfa94e1b5cf2486b35a9e1f Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 29 Apr 2025 09:10:48 +0200 Subject: [PATCH 172/215] indent --- .github/workflows/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d7c900d43..de67e2ec1 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,7 +14,7 @@ jobs: runs-on: windows-latest steps: - name: Harden Runner - if: github.repository_owner == 'microsoft' + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit From e2ac67f9137415e3888290424ce012f992c139b1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 1 May 2025 13:26:27 +0200 Subject: [PATCH 173/215] move --- RELEASENOTES.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6b74e3738..bd57de793 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,23 +2,6 @@ - Add top-level permissions for _Increment Version Number_ workflow -## v7.1 - -### Issues - -- Issue 1678 Test summary is showing too many status icons -- Issue 1640 AL1040 error due to app folder within the artifacts cache being incorrectly recognized as an app folder -- Issue 1630 Error when downloading a release, when the destination folder already exists. -- Issue 1540 and 1649 Apps with dependencies to Microsft\_\_EXCLUDE\_ apps fails deployment -- Issue 1547 Dependencies will be installed even if DependencyInstallMode is ignore, but dependencies will never be installed on production environments -- Issue 1654 GithubPackageContext does not work together with private trustedNuGetFeeds -- Issue 1627 AL-Go should throw an error or a warning if you create a release, which is older than the latest release -- Issue 1657 When no files modified on Git, deployment fails -- Issue 1530 Dependency Field Service Integration does not get published in container while Installing apps -- Issue 1644 Support for AppAuth when using a private Template repository from another organization -- Issue 1669 GitHub App authentication to download dependencies from private repositories -- Issue 1478 Rate Limit Exceeded when running Update AL-Go System files - ### New Repository settings - `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: @@ -41,6 +24,23 @@ Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https:// Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. +## v7.1 + +### Issues + +- Issue 1678 Test summary is showing too many status icons +- Issue 1640 AL1040 error due to app folder within the artifacts cache being incorrectly recognized as an app folder +- Issue 1630 Error when downloading a release, when the destination folder already exists. +- Issue 1540 and 1649 Apps with dependencies to Microsft\_\_EXCLUDE\_ apps fails deployment +- Issue 1547 Dependencies will be installed even if DependencyInstallMode is ignore, but dependencies will never be installed on production environments +- Issue 1654 GithubPackageContext does not work together with private trustedNuGetFeeds +- Issue 1627 AL-Go should throw an error or a warning if you create a release, which is older than the latest release +- Issue 1657 When no files modified on Git, deployment fails +- Issue 1530 Dependency Field Service Integration does not get published in container while Installing apps +- Issue 1644 Support for AppAuth when using a private Template repository from another organization +- Issue 1669 GitHub App authentication to download dependencies from private repositories +- Issue 1478 Rate Limit Exceeded when running Update AL-Go System files + ## v7.0 ### Issues From 817f7eebb7936e963766083a63ffc4bfe317ffc3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 8 May 2025 16:04:18 +0700 Subject: [PATCH 174/215] get secrets --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 9e31a1c8e..5934d1def 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -63,6 +63,12 @@ if ($token) { $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } +# Get secrets for Custom AL-Go System Files +$secrets = @{} +if ($env:Secrets) { + $secrets = $env:Secrets | ConvertFrom-Json | ConvertTo-HashTable +} + # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository # CheckForUpdates will apply changes to the AL-Go System files based on AL-Go repo settings, such as "runs-on" etc. # if $update is set to Y, CheckForUpdates will also update the AL-Go System files in the current repository using a PR or a direct commit (if $directCommit is set to true) From d8523dd27245742e0fffda49710084ca7e0e3673 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 9 May 2025 00:23:46 +0700 Subject: [PATCH 175/215] dumop --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 5934d1def..567025a15 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -24,6 +24,7 @@ function GetHeadersForTemplateUrl([string] $templateUrl, [string] $token) { # Use Authenticated API request if possible to avoid the 60 API calls per hour limit $headers = GetHeaders -token $ENV:GITHUB_TOKEN $templateRepositoryUrl = $templateUrl.Split('@')[0] + Write-Host "Template Repository URL: $templateRepositoryUrl" $response = Invoke-WebRequest -UseBasicParsing -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue if (-not $response -or $response.StatusCode -ne 200) { # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal From c97fd3377e8b4fd796813947a8daeab376f3a539 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 14 May 2025 07:00:56 +0200 Subject: [PATCH 176/215] add customALGoSystemFiles --- Actions/settings.schema.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Actions/settings.schema.json b/Actions/settings.schema.json index 9cbf8f026..b5367c5ec 100644 --- a/Actions/settings.schema.json +++ b/Actions/settings.schema.json @@ -16,6 +16,13 @@ "description": "An array of files to be excluded from AL-Go for GitHub updates. See https://aka.ms/ALGoSettings#unusedALGoSystemFiles" } }, + "customALGoSystemFiles": { + "type": "array", + "items": { + "type": "string", + "description": "An array of custom AL-Go System File specifications, which will be included in Update AL-Go System Files. See https://aka.ms/ALGoSettings#customALGoSystemFiles" + } + }, "projects": { "type": "array", "items": { From 94c657397312d967204d8d3d28b051f32d3b6a45 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 15 May 2025 03:34:22 +0200 Subject: [PATCH 177/215] read info from realSrcFile --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 0302774c7..4bdda7b67 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -256,7 +256,7 @@ foreach($checkfile in $checkfiles) { } "settings" { # For settings files, we need to modify the file based on the settings - $srcContent = GetModifiedSettingsContent -srcSettingsFile $srcFile -dstSettingsFile $dstFile + $srcContent = GetModifiedSettingsContent -srcSettingsFile $realSrcFile -dstSettingsFile $dstFile } Default { # For non-workflow files, just read the file content From e2ee4cd713838a7da86a12e499e035564d49bf4b Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 15 May 2025 07:20:31 +0200 Subject: [PATCH 178/215] do not fail on missing schema --- .../CheckForUpdates.HelperFunctions.ps1 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 50b908a17..85b40ac1e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -502,12 +502,14 @@ function GetModifiedSettingsContent { } $schemaKey = '$schema' - $schemaValue = $srcSettings."$schemaKey" + if ($srcSettings.PSObject.Properties.Name -eq $schemaKey) { + $schemaValue = $srcSettings."$schemaKey" - $dstSettings | Add-Member -MemberType NoteProperty -Name "$schemaKey" -Value $schemaValue -Force + $dstSettings | Add-Member -MemberType NoteProperty -Name "$schemaKey" -Value $schemaValue -Force - # Make sure the $schema property is the first property in the object - $dstSettings = $dstSettings | Select-Object @{ Name = '$schema'; Expression = { $_.'$schema' } }, * -ExcludeProperty '$schema' + # Make sure the $schema property is the first property in the object + $dstSettings = $dstSettings | Select-Object @{ Name = '$schema'; Expression = { $_.'$schema' } }, * -ExcludeProperty '$schema' + } return $dstSettings | ConvertTo-JsonLF } From 829ef73fb5b7ed66d5a71880118c2fad176714a1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 May 2025 07:19:47 +0200 Subject: [PATCH 179/215] Fix merge err --- .../CheckForUpdates.HelperFunctions.ps1 | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 8e8f6fd3c..c74aa048d 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -505,16 +505,14 @@ function GetModifiedSettingsContent { else { # Change the $schema property to be the same as the source settings file (add it if it doesn't exist) $schemaKey = '$schema' - $schemaValue = $srcSettings."$schemaKey" + if ($srcSettings.PSObject.Properties.Name -eq $schemaKey) { + $schemaValue = $srcSettings."$schemaKey" - $schemaKey = '$schema' - if ($srcSettings.PSObject.Properties.Name -eq $schemaKey) { - $schemaValue = $srcSettings."$schemaKey" + $dstSettings | Add-Member -MemberType NoteProperty -Name "$schemaKey" -Value $schemaValue -Force - $dstSettings | Add-Member -MemberType NoteProperty -Name "$schemaKey" -Value $schemaValue -Force - - # Make sure the $schema property is the first property in the object - $dstSettings = $dstSettings | Select-Object @{ Name = '$schema'; Expression = { $_.'$schema' } }, * -ExcludeProperty '$schema' + # Make sure the $schema property is the first property in the object + $dstSettings = $dstSettings | Select-Object @{ Name = '$schema'; Expression = { $_.'$schema' } }, * -ExcludeProperty '$schema' + } } return $dstSettings | ConvertTo-JsonLF From be455b5eec7b09ed6685daf54f66195e64487fe1 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Thu, 29 May 2025 07:25:43 +0200 Subject: [PATCH 180/215] Update Scenarios/settings.md Co-authored-by: Alexander Holstrup <117829001+aholstrup1@users.noreply.github.com> --- Scenarios/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 49d747816..b4c70a810 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -14,7 +14,7 @@ When running a workflow or a local script, the settings are applied by reading s 1. `ALGoOrgSettings` is a **GitHub variable**, which can be defined on an **organizational level** and will apply to **all AL-Go repositories** in this organization. -1. `.github/templateRepoSettings.doNotEdit.json` is the organizational settings from an indirect template repository (if applicable) +1. `.github/templateRepoSettings.doNotEdit.json` is the repository settings from an indirect template repository (if applicable) 1. `.github/AL-Go-settings.json` is the **repository settings file**. This settings file contains settings that are relevant for all projects in the repository. **Special note:** The repository settings file can also contains `BcContainerHelper` settings, which will be applied when loading `BcContainerHelper` in a workflow - the GitHub variables are not considered for BcContainerHelper settings. (see expert section). From 1c45ca4dcd6316978e799f7716e1350283ea90cf Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Thu, 29 May 2025 07:43:47 +0200 Subject: [PATCH 181/215] Update Actions/CheckForUpdates/CheckForUpdates.ps1 Co-authored-by: Alexander Holstrup <117829001+aholstrup1@users.noreply.github.com> --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 4bdda7b67..1fc98e020 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -123,7 +123,7 @@ if (-not $isDirectALGo) { # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository $realTemplateHeaders = GetHeadersForTemplateUrl -templateUrl $realTemplateUrl -token $token - $realTemplateFolder = DownloadTemplateRepository -headers $realTemplateHeaders -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') + $realTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') Write-Host "Real Template Folder: $realTemplateFolder" # Set TemplateBranch and TemplateOwner From 0e27b40556f96d366bcc985f4085923ba543df9d Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Thu, 29 May 2025 07:44:10 +0200 Subject: [PATCH 182/215] Update Actions/CheckForUpdates/CheckForUpdates.ps1 Co-authored-by: Alexander Holstrup <117829001+aholstrup1@users.noreply.github.com> --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 1fc98e020..95baa5f05 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -93,7 +93,7 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { $realTemplateFolder = $null $templateHeaders = GetHeadersForTemplateUrl -templateUrl $templateUrl -token $token -$templateFolder = DownloadTemplateRepository -headers $templateHeaders -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest +$templateFolder = DownloadTemplateRepository -token $token -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" $templateBranch = $templateUrl.Split('@')[1] From 4ddc87b9f55b90262cff3e15c62f1872ec2c2d52 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 29 May 2025 07:50:42 +0200 Subject: [PATCH 183/215] review --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 1 + Scenarios/CustomizingALGoForGitHub.md | 10 ++++++---- e2eTests/scenarios/IndirectTemplate/runtest.ps1 | 1 - 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 4bdda7b67..1feae00e6 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -318,6 +318,7 @@ if ($update -ne 'Y') { } if ($removeFiles) { + $removeFiles = @($removeFiles | Select-Object -Unique) Write-Host "Removed files:" $removeFiles | ForEach-Object { Write-Host "- $_" } } diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 9ca07cbef..211deeaaf 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -2,7 +2,11 @@ AL-Go for GitHub is a plug-and-play DevOps solution, intended to support 100% of the functionality needed by 90% of the people developing applications for Microsoft Dynamics 365 Business Central out-of-the-box. -If AL-Go functionality out-of-the-box doesn't match your needs, you really have three options: +If AL-Go functionality out-of-the-box doesn't match your needs, you should always start by creating a feature suggestion [here](https://github.com/microsoft/AL-Go/discussions) and see whether your needs are met by other mechanisms or required by other partners and should be part of AL-Go for GitHub? + +If your feature should be part of AL-Go for GitHub, you can select to [contribute](Contribute.md) to AL-Go for GitHub yourself or wait for Microsoft or other partners to pickup your feature suggestion. + +If your feature suggestion isn't accepted, you really have three options: 1. Customize AL-Go for GitHub to fit your needs 1. Select another managed DevOps solution @@ -229,12 +233,10 @@ Adding a custom job like this, will cause this job to run simultaneously with th > [!CAUTION] > Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. -### Modifying workflow permissions +### Custom job permissions If any of your custom jobs require permissions, which exceeds the permissions already assigned in the workflow, then these permissions can be specified directly on the custom job. -If any of your custom steps require permissions, which exceeds the permissions already assigned in the workflow, you can modify the permissions of the workflow and assign additional permissions. AL-Go for GitHub will not allow you to remove permissions, which might be needed in other steps/jobs, but additional permissions will be included when running Update AL-Go System Files. - ## Using indirect templates If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 index 32084398c..a7bd1bf81 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -96,7 +96,6 @@ $cicdWorkflow = Join-Path $templateRepoPath '.github/workflows/CICD.yaml' $cicdYaml = [yaml]::Load($cicdWorkflow) $cicdYaml | Should -Not -BeNullOrEmpty -# Modify the permissions $customJobs = @( @{ "Name" = "CustomJob-TemplateInit" From b91717f14ec51e8f4e3206740fdf26f3696baa04 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 29 May 2025 07:57:47 +0200 Subject: [PATCH 184/215] review --- Actions/CheckForUpdates/yamlclass.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index fa498d3b2..8665fc189 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -259,15 +259,18 @@ class Yaml { Write-Host "Custom Jobs:" foreach($customJob in $customJobs) { Write-Host "- $customJob" - $jobsWithDependency = $nativeJobs | Where-Object { $this.GetPropertyArray("jobs:/$($_):/needs:") | Where-Object { $_ -eq $customJob } } + $jobsWithDependency = @($nativeJobs | Where-Object { $this.GetPropertyArray("jobs:/$($_):/needs:") | Where-Object { $_ -eq $customJob } }) # If any Build Job has a dependency on this CustomJob, add will be added to all build jobs later if ($jobsWithDependency | Where-Object { $_ -like 'Build*' }) { $jobsWithDependency = @($jobsWithDependency | Where-Object { $_ -notlike 'Build*' }) + @('Build') } if ($jobsWithDependency) { Write-Host " - Jobs with dependency: $($jobsWithDependency -join ', ')" - $result += @(@{ "Name" = $customJob; "Content" = @($this.Get("jobs:/$($customJob):").content); "NeedsThis" = @($jobsWithDependency) }) } + else { + Write-Host " - No jobs with dependency on this" + } + $result += @(@{ "Name" = $customJob; "Content" = @($this.Get("jobs:/$($customJob):").content); "NeedsThis" = $jobsWithDependency }) } } return $result From e107d09506016baa28ea3cc0cd358c51086fd16f Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 29 May 2025 08:00:26 +0200 Subject: [PATCH 185/215] remove templateheaders - is handled in download function --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index a64ae5d32..d7385ba3f 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -20,25 +20,6 @@ Import-Module (Join-Path -Path $PSScriptRoot -ChildPath '..\TelemetryHelper.psm1 . (Join-Path -Path $PSScriptRoot -ChildPath "yamlclass.ps1") . (Join-Path -Path $PSScriptRoot -ChildPath "CheckForUpdates.HelperFunctions.ps1") -function GetHeadersForTemplateUrl([string] $templateUrl, [string] $token) { - # Use Authenticated API request if possible to avoid the 60 API calls per hour limit - $headers = GetHeaders -token $ENV:GITHUB_TOKEN - $templateRepositoryUrl = $templateUrl.Split('@')[0] - Write-Host "Template Repository URL: $templateRepositoryUrl" - $response = Invoke-WebRequest -UseBasicParsing -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue - if (-not $response -or $response.StatusCode -ne 200) { - # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal - # Get token with read permissions for the template repository - # NOTE that the GitHub app needs to be installed in the template repository for this to work - $templateRepository = $templateRepositoryUrl.Split('/')[-2..-1] -join '/' - $templateReadToken = GetAccessToken -token $token -permissions @{"actions"="read";"contents"="read";"metadata"="read"} -repository $templateRepository - - # Use read token for authenticated API request - $headers = GetHeaders -token $templateReadToken - } - return $headers -} - # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper @@ -92,7 +73,6 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { } $realTemplateFolder = $null -$templateHeaders = GetHeadersForTemplateUrl -templateUrl $templateUrl -token $token $templateFolder = DownloadTemplateRepository -token $token -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" @@ -122,7 +102,6 @@ if (-not $isDirectALGo) { } # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository - $realTemplateHeaders = GetHeadersForTemplateUrl -templateUrl $realTemplateUrl -token $token $realTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') Write-Host "Real Template Folder: $realTemplateFolder" From 4ed5800d277644fb6dcafefb64e7cb2acca3dc00 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Sun, 1 Jun 2025 22:35:41 +0200 Subject: [PATCH 186/215] Update Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index c74aa048d..41e17fa8b 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -31,7 +31,9 @@ function DownloadTemplateRepository { $response = $null } if (-not $response -or $response.StatusCode -ne 200) { - # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal + # GITHUB_TOKEN doesn't have access to template repository, must be private/internal + +`` # Get token with read permissions for the template repository # NOTE that the GitHub app needs to be installed in the template repository for this to work $headers = GetHeaders -token $token -repository $templateRepository From 33bdb53f9465823844525510fb13d57c50b8708b Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Sun, 1 Jun 2025 22:36:09 +0200 Subject: [PATCH 187/215] Update Actions/CheckForUpdates/CheckForUpdates.ps1 Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index d7385ba3f..0abb66b03 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -246,7 +246,7 @@ foreach($checkfile in $checkfiles) { $srcContent = $srcContent.Replace('{TEMPLATEURL}', $templateUrl) if ($isFileDirectALGo) { - # If we are using direct AL-Go repo, we need to change the owner to the remplateOwner, the repo names to AL-Go and AL-Go/Actions and the branch to templateBranch + # If we are using direct AL-Go repo, we need to change the owner to the templateOwner, the repo names to AL-Go and AL-Go/Actions and the branch to templateBranch ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } From 5be373f03930fa027c0671e11a8c2db62808b320 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Sun, 1 Jun 2025 22:37:10 +0200 Subject: [PATCH 188/215] Update Actions/CheckForUpdates/CheckForUpdates.ps1 Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 0abb66b03..d2a996aba 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -258,7 +258,7 @@ foreach($checkfile in $checkfiles) { if ($dstFileExists) { if ($type -eq 'workflow') { - Write-Host "Apply customizations from my repository: $dstFile" + Write-Host "Apply customizations from current repository: $dstFile" [Yaml]::ApplyCustomizations([ref] $srcContent, $dstFile) } From 89dd776f102571dfcbe78f5f2ee24291e73d26f6 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 1 Jun 2025 22:53:14 +0200 Subject: [PATCH 189/215] Remove CustomALGoSystemFiles and review comments --- Actions/AL-Go-Helper.ps1 | 23 ++- .../CheckForUpdates.HelperFunctions.ps1 | 132 ------------------ Actions/CheckForUpdates/CheckForUpdates.ps1 | 3 - Actions/ReadSecrets/ReadSecrets.ps1 | 25 ---- Actions/settings.schema.json | 7 - RELEASENOTES.md | 10 -- Scenarios/CustomizingALGoForGitHub.md | 58 -------- Scenarios/settings.md | 1 - .../AppSource App/.github/workflows/CICD.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- .../.github/workflows/CICD.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- .../scenarios/IndirectTemplate/runtest.ps1 | 9 +- 13 files changed, 17 insertions(+), 259 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index ae068de20..cee9cc727 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -13,8 +13,8 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S $ALGoFolderName = '.AL-Go' $ALGoSettingsFile = Join-Path '.AL-Go' 'settings.json' $RepoSettingsFile = Join-Path '.github' 'AL-Go-Settings.json' -$IndirectTemplateRepoSettingsFile = Join-Path '.github' 'templateRepoSettings.doNotEdit.json' -$IndirectTemplateProjectSettingsFile = Join-Path '.github' 'templateProjectSettings.doNotEdit.json' +$IndirectTemplateRepoSettingsFile = Join-Path '.github' 'AL-Go-TemplateRepoSettings.doNotEdit.json' +$IndirectTemplateProjectSettingsFile = Join-Path '.github' 'AL-Go-TemplateProjectSettings.doNotEdit.json' [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPushBranches', Justification = 'False positive.')] $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] @@ -544,7 +544,6 @@ function GetDefaultSettings return [ordered]@{ "type" = "PTE" "unusedALGoSystemFiles" = @() - "customALGoSystemFiles" = @() "projects" = @() "powerPlatformSolutionFolder" = "" "country" = "us" @@ -681,15 +680,15 @@ function GetDefaultSettings # Read settings from the settings files # Settings are read from the following files: -# - ALGoOrgSettings (github Variable) = Organization settings variable -# - .github/templateRepoSettings.json = Repository settings from indirect template -# - .github/AL-Go-Settings.json = Repository Settings file -# - ALGoRepoSettings (github Variable) = Repository settings variable -# - .github/templateProjectSettings.json = Project settings from indirect template -# - /.AL-Go/settings.json = Project settings file -# - .github/.settings.json = Workflow settings file -# - /.AL-Go/.settings.json = Project workflow settings file -# - /.AL-Go/.settings.json = User settings file +# - ALGoOrgSettings (github Variable) = Organization settings variable +# - .github/AL-Go-TemplateRepoSettings.doNotEdit.json = Repository settings from indirect template +# - .github/AL-Go-Settings.json = Repository Settings file +# - ALGoRepoSettings (github Variable) = Repository settings variable +# - .github/AL-Go-TemplateProjectSettings.doNotEdit.json = Project settings from indirect template +# - /.AL-Go/settings.json = Project settings file +# - .github/.settings.json = Workflow settings file +# - /.AL-Go/.settings.json = Project workflow settings file +# - /.AL-Go/.settings.json = User settings file function ReadSettings { Param( [string] $baseFolder = "$ENV:GITHUB_WORKSPACE", diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 41e17fa8b..566ef659f 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -554,135 +554,3 @@ function UpdateSettingsFile { } return $modified } - -function GetCustomALGoSystemFiles { - Param( - [string] $baseFolder, - [hashtable] $settings, - [string[]] $projects - ) - - function YieldItem{ - Param( - [string] $baseFolder, - [string] $source, - [string] $destination, - [string[]] $projects - ) - - if ($destination -like ".AL-Go$([IO.Path]::DirectorySeparatorChar)*") { - $destinations = $projects | ForEach-Object { Join-Path $_ $destination } - } - else { - $destinations = @($destination) - } - $destinations | ForEach-Object { - Write-Host "- $_" - $content = Get-ContentLF -Path $source - $existingFile = Join-Path $baseFolder $_ - $existingContent = '' - if (Test-Path -Path $existingFile) { - $existingContent = Get-ContentLF -Path $existingFile - } - if ($content -ne $existingContent) { - Write-Output @{ "DstFile" = $_; "content" = $content } - } - } - } - - foreach($customspec in $settings.customALGoSystemFiles) { - if ($customspec -isnot [Hashtable]) { - throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." - } - if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { - throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." - } - - $source = $customspec.Source - # Replace embedded secrets in the source URL with the value of the secret - $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*' - if ($customspec.Source -match $pattern) { - $source = $source.Replace($matches[1],[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($matches[2])"))) - } - if ($source -isnot [string] -or $source -notlike 'https://*' -or (-not [System.Uri]::IsWellFormedUriString($source,1))) { - throw "customALGoSystemFiles setting is wrongly formatted, Source must be a secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." - } - - $destination = $customSpec.Destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) - if ($destination -isnot [string] -or $destination -eq '') { - throw "customALGoSystemFiles setting is wrongly formatted, Destination must be a string, which isn't blank. See https://aka.ms/algosettings#customalgosystemfiles." - } - - if ($customSpec.ContainsKey('FileSpec')) { $fileSpec = $customSpec.FileSpec } else { $fileSpec = '*' } - if ($fileSpec -isnot [string]) { - throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string. See https://aka.ms/algosettings#customalgosystemfiles." - } - - if ($customSpec.ContainsKey('Recurse')) { $recurse = $customSpec.Recurse } else { $recurse = $true } - if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { - throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." - } - - $ext = [System.IO.Path]::GetExtension($source) - $authToken = $null - $repository = $null - if ($customspec.ContainsKey('AuthTokenSecret')) { - $authTokenSecret = $customspec.AuthTokenSecret - Write-Host "Using secret $authTokenSecret" - $authToken = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($authTokenSecret)")) - # if the AuthToken is a GitHub App specification, we need to get the repository name from the URL - if ($source -like 'https://api.github.com/repos/*/*/zipball/*') { - $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[1..2] -join '/' - $ext = '.zip' - } - else { - $repository = ([Uri]$source).AbsolutePath.TrimStart('/').Split('/')[0..1] -join '/' - } - } - $headers = GetHeaders -token $authToken -repository $repository - - $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - New-Item -Path $tempFolder -ItemType Directory | Out-Null - $zipName = "$tempFolder$ext" - try { - if ($ext -eq '.zip') { - if (!($destination.EndsWith('/') -or $destination.EndsWith('\'))) { - throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." - } - Write-Host "Downloading $source to $zipName" - $headers | Out-Host - Invoke-RestMethod -UseBasicParsing -Method Get -Headers $headers -Uri $source -OutFile $zipName - Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force - $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve - Push-Location -Path $subFolder - try { - Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File -Force | ForEach-Object { - $destRelativeFileName = Resolve-Path $_.FullName -Relative - $destFileName = Join-Path $destination $destRelativeFileName - $destFileName = $destFileName.TrimStart('\/') - YieldItem -baseFolder $baseFolder -source $_.FullName -destination $destFileName -projects $projects - } - } - finally { - Pop-Location - } - } - else { - if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { - throw "customALGoSystemFiles setting is wrongly formatted, FileSpec and Recurse are only allowed with .zip files. See https://aka.ms/algosettings#customalgosystemfiles." - } - if ($destination.endsWith([IO.Path]::DirectorySeparatorChar)) { - $destination = Join-Path $destination ([System.IO.Path]::GetFileName($source)) - } - Write-Host "$($destination):" - $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) - Invoke-RestMethod -UseBasicParsing -Method Get -Headers $headers -Uri $source -OutFile $tempFilename - YieldItem -baseFolder $baseFolder -source $tempFilename -destination $destination -projects $projects - } - } - finally { - Remove-Item -Path $tempFolder -Recurse -Force - if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } - } - } -} diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index d2a996aba..5d11f6aa9 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -285,9 +285,6 @@ foreach($checkfile in $checkfiles) { } } -# Apply Custom AL-Go System Files from settings -$updateFiles += @(GetCustomALGoSystemFiles -baseFolder $baseFolder -settings $repoSettings -projects $projects) - if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index a6d7dcecb..76be8b09b 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -28,7 +28,6 @@ try { $settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable $keyVaultCredentials = GetKeyVaultCredentials $getAppDependencySecrets = $false - $getCustomALGoSystemFilesSecrets = $false $getTokenForPush = $false [System.Collections.ArrayList]$secretsCollection = @() foreach($secret in ($getSecrets.Split(',') | Select-Object -Unique)) { @@ -42,9 +41,6 @@ try { if ($secret -eq 'AppDependencySecrets') { $getAppDependencySecrets = $true } - elseif ($secret -eq 'CustomALGoSystemFilesSecrets') { - $getCustomALGoSystemFilesSecrets = $true - } else { $secretName = $secret if ($settings.Keys -contains $secretNameProperty) { @@ -99,27 +95,6 @@ try { } } - if ($getCustomALGoSystemFilesSecrets) { - # Look through customALGoSystemFiles for secrets and add them to the collection of secrets to get - foreach($customspec in $settings.customALGoSystemFiles) { - if ($customspec.PSObject.Properties.name -eq "AuthTokenSecret") { - if ($secretsCollection -notcontains $customspec.authTokenSecret) { - $secretsCollection += $customspec.authTokenSecret - } - } - if ($customspec.PSObject.Properties.name -eq "Source") { - # If the Source URLs contains '${{SECRETNAME}}' we need to get the secret - $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*' - if ($customspec.Source -match $pattern) { - $secretName = $matches[2] - if ($secretsCollection -notcontains $secretName) { - $secretsCollection += $secretName - } - } - } - } - } - # Loop through secrets (use @() to allow us to remove items from the collection while looping) foreach($secret in @($secretsCollection)) { $secretSplit = $secret.Split('=') diff --git a/Actions/settings.schema.json b/Actions/settings.schema.json index 77fa2ef42..b23cd8ac4 100644 --- a/Actions/settings.schema.json +++ b/Actions/settings.schema.json @@ -16,13 +16,6 @@ "description": "An array of files to be excluded from AL-Go for GitHub updates. See https://aka.ms/ALGoSettings#unusedALGoSystemFiles" } }, - "customALGoSystemFiles": { - "type": "array", - "items": { - "type": "string", - "description": "An array of custom AL-Go System File specifications, which will be included in Update AL-Go System Files. See https://aka.ms/ALGoSettings#customALGoSystemFiles" - } - }, "projects": { "type": "array", "items": { diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1c4f9265e..cf9f09acf 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -15,16 +15,6 @@ As stated in [AL-Go Deprecations](https://aka.ms/algodeprecations#cleanModePrepr Some workflow inputs are now validated early in order to avoid workflows to make modifications like creating a release, when we already should know that an error will occur later. -### New Repository settings - -- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: - - - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file). - - **Source** (mandatory) - URL to a either a single file or a .zip file containing. Note that the source URL can contain embedded secrets using ${{SECRETNAME}} as part of the URL to hide SAS tokens or like. - - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. - - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec). - - **AuthTokenSecret** (optional) - the name of a secret (like ghTokenWorkflow) which contains a Personal Access Token or GitHub App specification with access to the source files. - ### Add custom jobs to AL-Go workflows It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 211deeaaf..c0ea9a7cf 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -145,62 +145,6 @@ Which basically launches a script located in the script folder in the repository > [!CAUTION] > Script overrides will almost certainly be broken in the future. The current script overrides is very much tied to the current implementation of the `Run-AlPipeline` function in BcContainerHelper. In the future, we will move this functionality to GitHub actions and no longer depend on BcContainerHelper and Run-AlPipeline. At that time, these script overrides will have to be changed to follow the new implementation. - - -### Adding custom workflows and/or scripts using a URL - -By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings#customalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these files should be included in the update. Examples: - -``` - "customALGoSystemFiles": [ - { - "Destination": ".AL-Go/", - "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1" - }, - { - "Destination": ".AL-Go/NewBcContainer.ps1", - "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1" - }, - { - "Destination": ".AL-Go/NewBcContainer.ps1", - "Source": "https://myfiles.blob.core.windows.net/myalgosystemfiles/NewBcContainer.ps1?${{SECRETNAME}}" - }, - { - "Destination": ".github/", - "Source": "https://github.com/freddydk/CustomALGoSystemFiles/archive/refs/heads/main.zip", - "FileSpec": "*/.github/*", - "Recurse": true - }, - { - "Destination": ".github/", - "Source": "https://api.github.com/repos/BusinessCentralApps/CustomALGoSystemFiles/zipball/main", - "AuthTokenSecret": "SECRETNAME", - "FileSpec": "*/.github/*", - "Recurse": true - } - ] -``` - -`customALGoSystemFiles` is an array of objects, which currently can have 4 properties: - -| Property | Description | Mandatory | Default | -| :-- | :-- | :-: | :-- | -| Destination | Path in which the file should be placed. Can include the filename if the source doesn't point to a .zip file, must include a terminating / or \\ if a filename is not included. | Yes | | -| Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. The source URL can contain embedded secrets using the `${{SECRETNAME}}` construct. | Yes | | -| FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include subfolders inside the .zip file, and must include a file name pattern. | No | * | -| Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | -| AuthTokenSecret | The name of a secret (like ghTokenWorkflow) which contains a Personal Access Token or GitHub App specification with access to the files. | - -This setting will cause AL-Go for GitHub to include these files during the next update. - -> [!WARNING] -> You can override existing AL-Go for GitHub system files this way, please prefix files in your repository with `my` or your organization name (except for DeployTo and DeliverTo) in order to avoid overriding future workflows from AL-Go for GitHub. - -> [!NOTE] -> If the destination is in the .AL-Go folder, the file(s) will be copied to all .AL-Go folders in multi-project repositories. - -> [!NOTE] -> If the source is a private or internal GitHub repository and you are using a GitHub app for authentication, you should use https://api.github.com/repos/OWNER/REPO/zipball/BRANCH instead of https://github.com/OWNER/REPO/archive/refs/heads/BRANCH.zip. ### Adding custom jobs @@ -244,8 +188,6 @@ If you have have customizations you want to apply to multiple repositories, you > [!NOTE] > Indirect templates can be public or private. If you are using a private indirect template, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. -Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. **UnusedALGoSystemFiles** and **CustomALGoSystemFiles** will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. - > [!TIP] > The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a **GhTokenWorkflow** secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index b4c70a810..3e046e953 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -230,7 +230,6 @@ Please read the release notes carefully when installing new versions of AL-Go fo | memoryLimit | Specifies the memory limit for the build container. By default, this is left to BcContainerHelper to handle and will currently be set to 8G | 8G | | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | [ ] | -| customALGoSystemFiles | An array of objects, specifying custom AL-Go System Files. Each object must contain **Destination** and **Source** url and if the source url points to a .zip file, you can also add **FileSpec** and **Recurse**. See more [here](./CustomizingALGoForGitHub.md#customALGoSystemFiles) | [ ] | diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index 53feaae58..ed75d2e87 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -160,7 +160,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'ghTokenWorkflow,CustomALGoSystemFilesSecrets' + getSecrets: 'ghTokenWorkflow' - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 3c77aaa8c..4cc5cb564 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -109,7 +109,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'ghTokenWorkflow,CustomALGoSystemFilesSecrets' + getSecrets: 'ghTokenWorkflow' - name: Calculate Commit Options env: diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 70bfbd69d..1b8bf1f78 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -160,7 +160,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'ghTokenWorkflow,CustomALGoSystemFilesSecrets' + getSecrets: 'ghTokenWorkflow' - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 3c77aaa8c..4cc5cb564 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -109,7 +109,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'ghTokenWorkflow,CustomALGoSystemFilesSecrets' + getSecrets: 'ghTokenWorkflow' - name: Calculate Commit Options env: diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 index a7bd1bf81..e4df2afe5 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -218,7 +218,8 @@ CancelAllWorkflows -repository $repository # Pull changes Pull -# TODO: Check that the settings from the indirect template repository was copied to $IndirectTemplateRepoSettingsFile and $IndirectTemplateProjectSettingsFile +(Join-Path (Get-Location) $IndirectTemplateRepoSettingsFile) | Should -Exist +(Join-Path (Get-Location) $IndirectTemplateProjectSettingsFile) | Should -Exist # Run CICD $run = RunCICD -repository $repository -branch $branch -wait @@ -233,11 +234,5 @@ Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeTemplateJob' -stepName Set-Location $prevLocation -# TODO: Modify settings in the template repository and re-run Update AL-Go System Files in the final repository to check that the settings are copied to the final repository - -# TODO: Add tests for CustomALGoSystemFiles (with and without security) - -Read-Host "Press Enter to continue" - RemoveRepository -repository $repository -path $finalRepoPath RemoveRepository -repository $templateRepository -path $templateRepoPath From 2093b31fb152d4fc6d7c078dfb41f9b7a7c66d07 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Sun, 1 Jun 2025 23:02:55 +0200 Subject: [PATCH 190/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index c0ea9a7cf..0a4ee5836 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -2,7 +2,7 @@ AL-Go for GitHub is a plug-and-play DevOps solution, intended to support 100% of the functionality needed by 90% of the people developing applications for Microsoft Dynamics 365 Business Central out-of-the-box. -If AL-Go functionality out-of-the-box doesn't match your needs, you should always start by creating a feature suggestion [here](https://github.com/microsoft/AL-Go/discussions) and see whether your needs are met by other mechanisms or required by other partners and should be part of AL-Go for GitHub? +If AL-Go functionality out-of-the-box doesn't match your needs, you should always start by creating a feature suggestion [here](https://github.com/microsoft/AL-Go/discussions) and see whether your needs are met by other mechanisms or required by other partners and should be part of AL-Go for GitHub. If your feature should be part of AL-Go for GitHub, you can select to [contribute](Contribute.md) to AL-Go for GitHub yourself or wait for Microsoft or other partners to pickup your feature suggestion. From f5889393f7ea9c109b265e8a585ebd543281fc22 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 2 Jun 2025 05:36:57 +0200 Subject: [PATCH 191/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 0a4ee5836..1e8a5ba56 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -25,7 +25,7 @@ Keeping your repositories up-to-date can be done manually or on a schedule (like There are three ways you can customize AL-Go for GitHub to fit your needs. You can -1. customize the repository with custom scripts, workflows, jobs or steps following the guidelines below +1. customize the repository with custom scripts, workflows or jobs following the guidelines below 1. create a customized repository and use this as your template repository (indirect template) 1. fork the AL-Go for GitHub and create your "own" version (not recommended) From 4c9742a743a6e05a7f1433bc9c3581b7efefa195 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 2 Jun 2025 05:37:17 +0200 Subject: [PATCH 192/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 1e8a5ba56..623fb5395 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -97,7 +97,7 @@ In this example the AuthContext secret is assumed to contain a Token property. R ### Adding custom workflows -If you add new workflows to the `.github/workflows` folder, which is unknown to AL-Go for GitHub, AL-Go will leave them un-touched. These workflows needs to follow standard GitHub Actions schema (yaml) and can be triggered as any other workflows. Example: +If you add new workflows to the `.github/workflows` folder, which is unknown to AL-Go for GitHub, AL-Go will leave them untouched. These workflows need to follow standard GitHub Actions schema (yaml) and can be triggered as any other workflows. Example: ```yaml name: 'Create Build Tag' From 50928d9fb0c36936621f4dbf661c2f2b164d01bc Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 2 Jun 2025 05:37:35 +0200 Subject: [PATCH 193/215] Update Actions/Github-Helper.psm1 Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Actions/Github-Helper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index 39e4a9ee9..96653675b 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -788,7 +788,7 @@ function Set-ContentLF { Param( [parameter(Mandatory = $true, ValueFromPipeline = $false)] [string] $path, - [parameter(mandatory = $true, ValueFromPipeline = $true)] + [parameter(Mandatory = $true, ValueFromPipeline = $true)] [string] $content ) From e99e8530d2a94373434f354468aba23db0f4f57d Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 2 Jun 2025 05:38:29 +0200 Subject: [PATCH 194/215] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 623fb5395..392a72e2c 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -148,7 +148,7 @@ Which basically launches a script located in the script folder in the repository ### Adding custom jobs -You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Custom jobs can depend on other jobs and other jobs can made to depend on custom jobs. Custom jobs needs to be named `CustomJob`, but can specify another name to be shown in the UI. Example: +You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Custom jobs can depend on other jobs and other jobs can be made to depend on custom jobs. Custom jobs need to be named `CustomJob`, but can specify another name to be shown in the UI. Example: ```yaml CustomJob-CreateBuildTag: From 42c8204f43d4f2ec6ee94502ff32cae8817bb156 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 2 Jun 2025 05:41:41 +0200 Subject: [PATCH 195/215] review --- Actions/CheckForUpdates/yamlclass.ps1 | 1 + e2eTests/scenarios/IndirectTemplate/runtest.ps1 | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 8665fc189..49087e379 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -315,6 +315,7 @@ class Yaml { $yaml = [Yaml]::Load($yamlFile) } catch { + Write-Host "Unable to read YAML file $yamlFile. Skipping custom jobs." return } diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 index e4df2afe5..b2f65edd7 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -26,10 +26,6 @@ Write-Host -ForegroundColor Yellow @' # - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) # - Run Update AL-Go System Files in final repo (using indirect repo as template) # - Run Update AL-Go System files in indirect repo - -# TODO: describe the scenario - - # - Validate that custom step is present in indirect repo # - Run Update AL-Go System files in final repo # - Validate that custom step is present in final repo From 9801efc30a175668faa57e47660cca3145255f4e Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 2 Jun 2025 06:00:01 +0200 Subject: [PATCH 196/215] rename to custom template --- Actions/AL-Go-Helper.ps1 | 16 +++++------ Actions/AL-Go-TestRepoHelper.ps1 | 4 +-- Actions/CheckForUpdates/CheckForUpdates.ps1 | 26 ++++++++--------- RELEASENOTES.md | 7 +++-- Scenarios/CustomizingALGoForGitHub.md | 16 +++++------ Scenarios/settings.md | 20 ++++++------- Workshop/Index.md | 2 +- .../scenarios/IndirectTemplate/runtest.ps1 | 28 +++++++++---------- 8 files changed, 60 insertions(+), 59 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index cee9cc727..a9982e024 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -13,8 +13,8 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S $ALGoFolderName = '.AL-Go' $ALGoSettingsFile = Join-Path '.AL-Go' 'settings.json' $RepoSettingsFile = Join-Path '.github' 'AL-Go-Settings.json' -$IndirectTemplateRepoSettingsFile = Join-Path '.github' 'AL-Go-TemplateRepoSettings.doNotEdit.json' -$IndirectTemplateProjectSettingsFile = Join-Path '.github' 'AL-Go-TemplateProjectSettings.doNotEdit.json' +$CustomTemplateRepoSettingsFile = Join-Path '.github' 'AL-Go-TemplateRepoSettings.doNotEdit.json' +$CustomTemplateProjectSettingsFile = Join-Path '.github' 'AL-Go-TemplateProjectSettings.doNotEdit.json' [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPushBranches', Justification = 'False positive.')] $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] @@ -681,10 +681,10 @@ function GetDefaultSettings # Read settings from the settings files # Settings are read from the following files: # - ALGoOrgSettings (github Variable) = Organization settings variable -# - .github/AL-Go-TemplateRepoSettings.doNotEdit.json = Repository settings from indirect template +# - .github/AL-Go-TemplateRepoSettings.doNotEdit.json = Repository settings from custom template # - .github/AL-Go-Settings.json = Repository Settings file # - ALGoRepoSettings (github Variable) = Repository settings variable -# - .github/AL-Go-TemplateProjectSettings.doNotEdit.json = Project settings from indirect template +# - .github/AL-Go-TemplateProjectSettings.doNotEdit.json = Project settings from custom template # - /.AL-Go/settings.json = Project settings file # - .github/.settings.json = Workflow settings file # - /.AL-Go/.settings.json = Project workflow settings file @@ -748,8 +748,8 @@ function ReadSettings { } # Read settings from repository settings file - $indirectTemplateRepoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $IndirectTemplateRepoSettingsFile) - $settingsObjects += @($indirectTemplateRepoSettingsObject) + $customTemplateRepoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $CustomTemplateRepoSettingsFile) + $settingsObjects += @($customTemplateRepoSettingsObject) # Read settings from repository settings file $repoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $RepoSettingsFile) @@ -763,8 +763,8 @@ function ReadSettings { if ($project) { # Read settings from repository settings file - $indirectTemplateProjectSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $IndirectTemplateProjectSettingsFile) - $settingsObjects += @($indirectTemplateProjectSettingsObject) + $customTemplateProjectSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $CustomTemplateProjectSettingsFile) + $settingsObjects += @($customTemplateProjectSettingsObject) # Read settings from project settings file $projectFolder = Join-Path $baseFolder $project -Resolve $projectSettingsObject = GetSettingsObject -Path (Join-Path $projectFolder $ALGoSettingsFile) diff --git a/Actions/AL-Go-TestRepoHelper.ps1 b/Actions/AL-Go-TestRepoHelper.ps1 index 2c9fc7759..f8187bd02 100644 --- a/Actions/AL-Go-TestRepoHelper.ps1 +++ b/Actions/AL-Go-TestRepoHelper.ps1 @@ -175,10 +175,10 @@ function TestALGoRepository { Test-JsonFile -jsonFile $_.FullName -baseFolder $baseFolder -type 'Project' } elseif ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($RepoSettingsFile)) -and $_.BaseName -like '*ettings') { - if ($_.Name -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.Name -eq ([System.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile))) { + if ($_.Name -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.Name -eq ([System.IO.Path]::GetFileName($CustomTemplateRepoSettingsFile))) { $type = 'Repo' } - elseif ($_.Name -eq ([System.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile))) { + elseif ($_.Name -eq ([System.IO.Path]::GetFileName($CustomTemplateProjectSettingsFile))) { $type = 'Project' } else { diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 5d11f6aa9..508f56cc5 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -86,13 +86,13 @@ if (-not $isDirectALGo) { if (Test-Path -Path $templateRepoSettingsFile -PathType Leaf) { $templateRepoSettings = Get-Content $templateRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { - # The template repository is a url to another AL-Go repository (an indirect template repository) - # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template repository points to the "real" template repository - # Copy files and folders from the indirect template repository, but grab the unmodified file from the "real" template repository if it exists and apply customizations + # The template repository is a url to another AL-Go repository (a custom template repository) + # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the custom template repository points to the "real" template repository + # Copy files and folders from the custom template repository, but grab the unmodified file from the "real" template repository if it exists and apply customizations # Copy .github/AL-Go-Settings.json to .github/templateRepoSettings.json (will be read before .github/AL-Go-Settings.json in the final repo) # Copy .AL-Go/settings.json to .github/templateProjectSettings.json (will be read before .AL-Go/settings.json in the final repo) - Write-Host "Indirect AL-Go template repository detected, downloading the 'real' template repository" + Write-Host "Custom AL-Go template repository detected, downloading the 'real' template repository" $realTemplateUrl = $templateRepoSettings.templateUrl if ($templateRepoSettings.Keys -contains "templateSha") { $realTemplateSha = $templateRepoSettings.templateSha @@ -101,16 +101,16 @@ if (-not $isDirectALGo) { $realTemplateSha = "" } - # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository + # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the custom template repository $realTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') Write-Host "Real Template Folder: $realTemplateFolder" # Set TemplateBranch and TemplateOwner - # Keep TemplateUrl and TemplateSha pointing to the indirect template repository + # Keep TemplateUrl and TemplateSha pointing to the custom template repository $templateBranch = $realTemplateUrl.Split('@')[1] $templateOwner = $realTemplateUrl.Split('/')[3] - # If the indirect template contains unusedALGoSystemFiles, we need to remove them from the current repository + # If the custom template contains unusedALGoSystemFiles, we need to remove them from the current repository if ($templateRepoSettings.ContainsKey('unusedALGoSystemFiles')) { $unusedALGoSystemFiles += $templateRepoSettings.unusedALGoSystemFiles } @@ -137,8 +137,8 @@ $checkfiles = @( if ($realTemplateFolder) { $checkfiles += @( - @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($IndirectTemplateRepoSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($IndirectTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } - @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($IndirectTemplateProjectSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($IndirectTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } + @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateRepoSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($CustomTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } + @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateProjectSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($CustomTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } ) } @@ -179,7 +179,7 @@ foreach($checkfile in $checkfiles) { $srcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath $realSrcFolder = $null if ($realTemplateFolder -and $type -notlike 'template*settings') { - # Get Real source folder except for template settings - these are applied from the indirect temoplate´repository + # Get Real source folder except for template settings - these are applied from the custom temoplate´repository $realSrcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath } if ($srcFolder) { @@ -210,7 +210,7 @@ foreach($checkfile in $checkfiles) { $isFileDirectALGo = $isDirectALGo Write-Host "SrcFolder: $srcFolder" if ($realSrcFolder) { - # if SrcFile is an indirect template repository, we need to find the file in the "real" template repository + # if SrcFile is a custom template repository, we need to find the file in the "real" template repository $fname = Join-Path $realSrcFolder (Resolve-Path $srcFile -Relative) if (Test-Path -Path $fname -PathType Leaf) { Write-Host "File is available in the 'real' template repository" @@ -251,8 +251,8 @@ foreach($checkfile in $checkfiles) { } if ($type -eq 'workflow' -and $realSrcFile -ne $srcFile) { - # Apply customizations from indirect template repository - Write-Host "Apply customizations from indirect template repository: $srcFile" + # Apply customizations from custom template repository + Write-Host "Apply customizations from custom template repository: $srcFile" [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile) } diff --git a/RELEASENOTES.md b/RELEASENOTES.md index cf9f09acf..56af12edb 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -21,11 +21,12 @@ It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs t Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. -### Indirect AL-Go template repository +### Support for Custom AL-Go template repositories -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using custom template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#customtemplate](https://aka.ms/algosettings#customtemplate) for details. -Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. +> [!NOTE] +> Customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your custom template repositories. ### Test settings against a JSON schema diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 392a72e2c..3d909e943 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -26,7 +26,7 @@ Keeping your repositories up-to-date can be done manually or on a schedule (like There are three ways you can customize AL-Go for GitHub to fit your needs. You can 1. customize the repository with custom scripts, workflows or jobs following the guidelines below -1. create a customized repository and use this as your template repository (indirect template) +1. create a customized repository and use this as your custom template repository 1. fork the AL-Go for GitHub and create your "own" version (not recommended) > [!CAUTION] @@ -181,23 +181,23 @@ Adding a custom job like this, will cause this job to run simultaneously with th If any of your custom jobs require permissions, which exceeds the permissions already assigned in the workflow, then these permissions can be specified directly on the custom job. -## Using indirect templates +## Using custom template repositories -If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. +If you have have customizations you want to apply to multiple repositories, you might want to consider using a custom template. A custom template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. > [!NOTE] -> Indirect templates can be public or private. If you are using a private indirect template, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. +> Custom templates can be public or private. If you are using a private custom template repository, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. > [!TIP] -> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a **GhTokenWorkflow** secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. +> The recommended way to create a new repository based on your custom AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a **GhTokenWorkflow** secret and then run the `Update AL-Go System Files` workflow with your custom template specified. > [!NOTE] -> If you use the indirect template as a GitHub template for creating the repository, by clicking use this template in your indirect template - then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. +> If you use the custom template as a GitHub template for creating the repository, by clicking use this template in your custom template - then you need to re-specify the custom Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the custom template as it's template repository. -Repositories based on your indirect template will notify you that changes are available for your AL-Go System Files when you update the indirect template only. You will not be notified when new versions of AL-Go for GitHub is released in every repository - only in the indirect template repository. +Repositories based on your custom template will notify you that changes are available for your AL-Go System Files when you update the custom template only. You will not be notified when new versions of AL-Go for GitHub is released in every repository - only in the custom template repository. > [!WARNING] -> You should ensure that your indirect template repository is kept up-to-date with the latest changes in AL-Go for GitHub. +> You should ensure that your custom template repository is kept up-to-date with the latest changes in AL-Go for GitHub. > [!TIP] > You can setup the Update AL-Go System Files workflow to run on a schedule to uptake new releases of AL-Go for GitHub regularly. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 3e046e953..eb5c9c661 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -14,13 +14,13 @@ When running a workflow or a local script, the settings are applied by reading s 1. `ALGoOrgSettings` is a **GitHub variable**, which can be defined on an **organizational level** and will apply to **all AL-Go repositories** in this organization. -1. `.github/templateRepoSettings.doNotEdit.json` is the repository settings from an indirect template repository (if applicable) +1. `.github/AL-Go-TemplateRepoSettings.doNotEdit.json` is the repository settings from a custom template repository (if applicable) 1. `.github/AL-Go-settings.json` is the **repository settings file**. This settings file contains settings that are relevant for all projects in the repository. **Special note:** The repository settings file can also contains `BcContainerHelper` settings, which will be applied when loading `BcContainerHelper` in a workflow - the GitHub variables are not considered for BcContainerHelper settings. (see expert section). 1. `ALGoRepoSettings` is a **GitHub variable**, which can be defined on an **repository level** and can contain settings that are relevant for **all projects** in the repository. -1. `.github/templateProjectSettings.doNotEdit.json` is the project settings from an indirect template repository (if applicable) +1. `.github/AL-Go-TemplateProjectSettings.doNotEdit.json` is the project settings from a custom template repository (if applicable) 1. `.AL-Go/settings.json` is the **project settings file**. If the repository is a single project, the .AL-Go folder is in the root folder of the repository. If the repository contains multiple projects, there will be a .AL-Go folder in each project folder (like `project/.AL-Go/settings.json`) @@ -412,27 +412,27 @@ Custom jobs will be preserved when running Update AL-Go System Files. **Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). - + -## Indirect template repositories +## Custom template repositories -If you are utilizing script overrides, custom jobs, custom steps, custom delivery or like in many repositories, you might want to take advantage of the indirect template repository feature. +If you are utilizing script overrides, custom jobs, custom steps, custom delivery or like in many repositories, you might want to take advantage of the custom template repository feature. -An indirect template repository is an AL-Go for GitHub repository (without any apps), which is used as a template for the remaining AL-Go for GitHub repositories. As an example, if you are using a custom delivery script, which you want to have in all your repositories, you can create an empty AL-Go for GitHub repository, place the delivery script in the .github folder and use that repository as a template when running Update AL-Go system files in your other repositories. +A custom template repository is an AL-Go for GitHub repository (without any apps), which is used as a template for the remaining AL-Go for GitHub repositories. As an example, if you are using a custom delivery script, which you want to have in all your repositories, you can create an empty AL-Go for GitHub repository, place the delivery script in the .github folder and use that repository as a template when running Update AL-Go system files in your other repositories. This would make sure that all repositories would have this script (and updated versions of the script) in the future. -The items, which are currently supported from indirect template repositories are: +The items, which are currently supported from custom template repositories are: - Repository script overrides in the .github folder - Project script overrides in the .AL-Go folder - Custom workflows in the .github/workflows folder - Custom jobs in any AL-Go for GitHub workflow - Custom steps in the \_BuildALGoProject workflow -- New repository settings -- New project settings +- Changes to repository settings in .github/AL-Go-settings.json +- Changes to project settings in .AL-Go/settings.json -**Note** that an AL-Go for GitHub indirect template repository can be private or public. +**Note** that an AL-Go for GitHub custom template repository can be private or public. ## Your own version of AL-Go for GitHub diff --git a/Workshop/Index.md b/Workshop/Index.md index c8daf6b22..b3171b33e 100644 --- a/Workshop/Index.md +++ b/Workshop/Index.md @@ -27,7 +27,7 @@ This workshop shows you how to take advantage of the functionality, which is pro 1. [Custom Delivery](CustomDelivery.md) - *FUTURE TOPIC: Setting up custom delivery to f.ex. a Teams channel.* 1. [Custom Delivery](CustomDeployment.md) - *FUTURE TOPIC: Setting up custom deployment to f.ex. an on-premises environment.* 1. [Custom Jobs](CustomJobs.md) - *FUTURE TOPIC: Adding a custom job to an AL-Go for GitHub workflows.* -1. [Using Indirect Templates](IndirectTemplates.md) - *FUTURE TOPIC: Using indirect templates to ensure that all repositories are using the same customizations* +1. [Using Custom Templates](CustomTemplates.md) - *FUTURE TOPIC: Using custom templates to ensure that all repositories are using the same customizations* ## Additional Future topics diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 index b2f65edd7..9b1843881 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -12,21 +12,21 @@ Param( ) Write-Host -ForegroundColor Yellow @' -# _____ _ _ _ _______ _ _ -# |_ _| | (_) | | |__ __| | | | | -# | | _ __ __| |_ _ __ ___ ___| |_ | | ___ _ __ ___ _ __ | | __ _| |_ ___ -# | | | '_ \ / _` | | '__/ _ \/ __| __| | |/ _ \ '_ ` _ \| '_ \| |/ _` | __/ _ \ -# _| |_| | | | (_| | | | | __/ (__| |_ | | __/ | | | | | |_) | | (_| | || __/ -# |_____|_| |_|\__,_|_|_| \___|\___|\__| |_|\___|_| |_| |_| .__/|_|\__,_|\__\___| -# | | -# |_| +# _____ _ _______ _ _ +# / ____| | | |__ __| | | | | +# | | _ _ ___| |_ ___ _ __ ___ | | ___ _ __ ___ _ __ | | __ _| |_ ___ +# | | | | | / __| __/ _ \| '_ ` _ \ | |/ _ \ '_ ` _ \| '_ \| |/ _` | __/ _ \ +# | |___| |_| \__ \ || (_) | | | | | | | | __/ | | | | | |_) | | (_| | || __/ +# \_____\__,_|___/\__\___/|_| |_| |_| |_|\___|_| |_| |_| .__/|_|\__,_|\__\___| +# | | +# |_| # This test tests the following scenario: # -# - Create a new repository based on the PTE template with no apps (this will be the "indirect" template repository) +# - Create a new repository based on the PTE template with no apps (this will be the custom template repository) # - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) -# - Run Update AL-Go System Files in final repo (using indirect repo as template) -# - Run Update AL-Go System files in indirect repo -# - Validate that custom step is present in indirect repo +# - Run Update AL-Go System Files in final repo (using custom template repository as template) +# - Run Update AL-Go System files in custom template repository +# - Validate that custom step is present in custom template repository # - Run Update AL-Go System files in final repo # - Validate that custom step is present in final repo # - Run Update AL-Go System files in final repo @@ -214,8 +214,8 @@ CancelAllWorkflows -repository $repository # Pull changes Pull -(Join-Path (Get-Location) $IndirectTemplateRepoSettingsFile) | Should -Exist -(Join-Path (Get-Location) $IndirectTemplateProjectSettingsFile) | Should -Exist +(Join-Path (Get-Location) $CustomTemplateRepoSettingsFile) | Should -Exist +(Join-Path (Get-Location) $CustomTemplateProjectSettingsFile) | Should -Exist # Run CICD $run = RunCICD -repository $repository -branch $branch -wait From 79211b608e60fa187106d8c7d94d3fa2b5a6b64b Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 2 Jun 2025 16:08:02 +0200 Subject: [PATCH 197/215] review --- Scenarios/CustomizingALGoForGitHub.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 3d909e943..65f11adb9 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -131,7 +131,7 @@ It is recommended to prefix your workflows with `my`, `our`, your name or your o You can add custom powershell scripts under the .github folder for repository scoped scripts or in the .AL-Go folder for project scoped scripts. Specially named scripts in the .AL-Go folder can override standard functionality in AL-Go for GitHub workflows. A list of these script overrides can be found [here](https://aka.ms/algosettings#scriptoverrides). Scripts under the .github folder can be used in custom workflows instead of using inline scripts inside the workflow. -One example of a script override is the NewBcContainer override used in the System Application project in BCApps (can be found [here](https://github.com/microsoft/BCApps/blob/main/build/projects/System%20Application/.AL-Go/NewBcContainer.ps1)). This override looks like: +One example of a script override is the NewBcContainer override used in the System Application project in BCApps (can be found [here](https://github.com/microsoft/BCApps/blob/647efdacac0c0d13d726e14c89180a32cbb55cf2/build/projects/System%20Application/.AL-Go/NewBcContainer.ps1)). This override looks like: ```powershell Param([Hashtable] $parameters) @@ -140,12 +140,11 @@ $script = Join-Path $PSScriptRoot "../../../scripts/NewBcContainer.ps1" -Resolve . $script -parameters $parameters ``` -Which basically launches a script located in the script folder in the repository for creating the build container needed for building and testing the System Application. +Which basically launches a script located in the script folder in the repository for creating the build container needed for building and testing the System Application. That script can be found [here](https://github.com/microsoft/BCApps/blob/647efdacac0c0d13d726e14c89180a32cbb55cf2/build/scripts/NewBcContainer.ps1). > [!CAUTION] > Script overrides will almost certainly be broken in the future. The current script overrides is very much tied to the current implementation of the `Run-AlPipeline` function in BcContainerHelper. In the future, we will move this functionality to GitHub actions and no longer depend on BcContainerHelper and Run-AlPipeline. At that time, these script overrides will have to be changed to follow the new implementation. - ### Adding custom jobs You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Custom jobs can depend on other jobs and other jobs can be made to depend on custom jobs. Custom jobs need to be named `CustomJob`, but can specify another name to be shown in the UI. Example: From f2e931a8b745fab460983cfe66139eadc50b5161 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Thu, 5 Jun 2025 06:08:47 +0200 Subject: [PATCH 198/215] Update Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 Co-authored-by: Alexander Holstrup <117829001+aholstrup1@users.noreply.github.com> --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 566ef659f..e11dc6b30 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -32,8 +32,6 @@ function DownloadTemplateRepository { } if (-not $response -or $response.StatusCode -ne 200) { # GITHUB_TOKEN doesn't have access to template repository, must be private/internal - -`` # Get token with read permissions for the template repository # NOTE that the GitHub app needs to be installed in the template repository for this to work $headers = GetHeaders -token $token -repository $templateRepository From c601c398acd645f4ab5416a584e6167cc7d0dae2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 5 Jun 2025 06:16:53 +0200 Subject: [PATCH 199/215] review --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 60 ++++++++++----------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 508f56cc5..98934aed7 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -72,7 +72,7 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { $downloadLatest = $true } -$realTemplateFolder = $null +$originalTemplateFolder = $null $templateFolder = DownloadTemplateRepository -token $token -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" @@ -87,28 +87,28 @@ if (-not $isDirectALGo) { $templateRepoSettings = Get-Content $templateRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { # The template repository is a url to another AL-Go repository (a custom template repository) - # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the custom template repository points to the "real" template repository - # Copy files and folders from the custom template repository, but grab the unmodified file from the "real" template repository if it exists and apply customizations - # Copy .github/AL-Go-Settings.json to .github/templateRepoSettings.json (will be read before .github/AL-Go-Settings.json in the final repo) - # Copy .AL-Go/settings.json to .github/templateProjectSettings.json (will be read before .AL-Go/settings.json in the final repo) + # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the custom template repository points to the "original" template repository + # Copy files and folders from the custom template repository, but grab the unmodified file from the "original" template repository if it exists and apply customizations + # Copy .github/AL-Go-Settings.json to .github/templateRepoSettings.doNotEdit.json (will be read before .github/AL-Go-Settings.json in the final repo) + # Copy .AL-Go/settings.json to .github/templateProjectSettings.doNotEdit.json (will be read before .AL-Go/settings.json in the final repo) - Write-Host "Custom AL-Go template repository detected, downloading the 'real' template repository" - $realTemplateUrl = $templateRepoSettings.templateUrl + Write-Host "Custom AL-Go template repository detected, downloading the 'original' template repository" + $originalTemplateUrl = $templateRepoSettings.templateUrl if ($templateRepoSettings.Keys -contains "templateSha") { - $realTemplateSha = $templateRepoSettings.templateSha + $originalTemplateSha = $templateRepoSettings.templateSha } else { - $realTemplateSha = "" + $originalTemplateSha = "" } - # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the custom template repository - $realTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') - Write-Host "Real Template Folder: $realTemplateFolder" + # Download the "original" template repository - use downloadLatest if no TemplateSha is specified in the custom template repository + $originalTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $originalTemplateUrl -templateSha ([ref]$originalTemplateSha) -downloadLatest ($originalTemplateSha -eq '') + Write-Host "Original Template Folder: $originalTemplateFolder" # Set TemplateBranch and TemplateOwner # Keep TemplateUrl and TemplateSha pointing to the custom template repository - $templateBranch = $realTemplateUrl.Split('@')[1] - $templateOwner = $realTemplateUrl.Split('/')[3] + $templateBranch = $originalTemplateUrl.Split('@')[1] + $templateOwner = $originalTemplateUrl.Split('/')[3] # If the custom template contains unusedALGoSystemFiles, we need to remove them from the current repository if ($templateRepoSettings.ContainsKey('unusedALGoSystemFiles')) { @@ -135,7 +135,7 @@ $checkfiles = @( @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.settings.json'; 'type' = 'settings' } ) -if ($realTemplateFolder) { +if ($originalTemplateFolder) { $checkfiles += @( @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateRepoSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($CustomTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateProjectSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($CustomTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } @@ -177,10 +177,10 @@ foreach($checkfile in $checkfiles) { $dstPath = $checkfile.dstPath $dstFolder = Join-Path $baseFolder $dstPath $srcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath - $realSrcFolder = $null - if ($realTemplateFolder -and $type -notlike 'template*settings') { - # Get Real source folder except for template settings - these are applied from the custom temoplate´repository - $realSrcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath + $originalSrcFolder = $null + if ($originalTemplateFolder -and $type -notlike 'template*settings') { + # Get Original source folder except for template settings - these are applied from the custom template´repository + $originalSrcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $originalTemplateUrl -templateFolder $originalTemplateFolder -srcPath $srcPath } if ($srcFolder) { Push-Location -Path $srcFolder @@ -206,16 +206,16 @@ foreach($checkfile in $checkfiles) { Write-Host "- $filename" $dstFile = Join-Path $dstFolder $filename $srcFile = $_.FullName - $realSrcFile = $srcFile + $originalSrcFile = $srcFile $isFileDirectALGo = $isDirectALGo Write-Host "SrcFolder: $srcFolder" - if ($realSrcFolder) { - # if SrcFile is a custom template repository, we need to find the file in the "real" template repository - $fname = Join-Path $realSrcFolder (Resolve-Path $srcFile -Relative) + if ($originalSrcFolder) { + # if SrcFile is a custom template repository, we need to find the file in the "original" template repository + $fname = Join-Path $originalSrcFolder (Resolve-Path $srcFile -Relative) if (Test-Path -Path $fname -PathType Leaf) { - Write-Host "File is available in the 'real' template repository" - $realSrcFile = $fname - $isFileDirectALGo = IsDirectALGo -templateUrl $realTemplateUrl + Write-Host "File is available in the 'original' template repository" + $originalSrcFile = $fname + $isFileDirectALGo = IsDirectALGo -templateUrl $originalTemplateUrl } } $dstFileExists = Test-Path -Path $dstFile -PathType Leaf @@ -231,15 +231,15 @@ foreach($checkfile in $checkfiles) { switch ($type) { "workflow" { # For workflow files, we might need to modify the file based on the settings - $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $realsrcFile -repoSettings $repoSettings -depth $depth -includeBuildPP $includeBuildPP + $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $originalsrcFile -repoSettings $repoSettings -depth $depth -includeBuildPP $includeBuildPP } "settings" { # For settings files, we need to modify the file based on the settings - $srcContent = GetModifiedSettingsContent -srcSettingsFile $realSrcFile -dstSettingsFile $dstFile + $srcContent = GetModifiedSettingsContent -srcSettingsFile $originalSrcFile -dstSettingsFile $dstFile } Default { # For non-workflow files, just read the file content - $srcContent = Get-ContentLF -Path $realSrcFile + $srcContent = Get-ContentLF -Path $originalSrcFile } } # Replace static placeholders @@ -250,7 +250,7 @@ foreach($checkfile in $checkfiles) { ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } - if ($type -eq 'workflow' -and $realSrcFile -ne $srcFile) { + if ($type -eq 'workflow' -and $originalSrcFile -ne $srcFile) { # Apply customizations from custom template repository Write-Host "Apply customizations from custom template repository: $srcFile" [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile) From 9895d9ff9e1ae8bebaaef0372879e5196bafba88 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 5 Jun 2025 06:18:45 +0200 Subject: [PATCH 200/215] move target --- RELEASENOTES.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index ff072af02..65b291a99 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,3 +1,16 @@ +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Support for Custom AL-Go template repositories + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using custom template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#customtemplate](https://aka.ms/algosettings#customtemplate) for details. + +> [!NOTE] +> Customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your custom template repositories. + ## v7.2 ### Removed functionality @@ -18,19 +31,6 @@ As stated in [AL-Go Deprecations](https://aka.ms/algodeprecations#cleanModePrepr Some workflow inputs are now validated early in order to avoid workflows to make modifications like creating a release, when we already should know that an error will occur later. -### Add custom jobs to AL-Go workflows - -It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. - -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Support for Custom AL-Go template repositories - -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using custom template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#customtemplate](https://aka.ms/algosettings#customtemplate) for details. - -> [!NOTE] -> Customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your custom template repositories. - ### Test settings against a JSON schema AL-Go for GitHub settings now has a schema. The following line is added at the beginning to any AL-Go settings files to utilize the schema: From f3b6bf3103545e436b807641f600c0b969160bd9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 5 Jun 2025 06:51:08 +0200 Subject: [PATCH 201/215] remove duplicates always --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 98934aed7..e8a6dbc6c 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -284,6 +284,7 @@ foreach($checkfile in $checkfiles) { } } } +$removeFiles = @($removeFiles | Select-Object -Unique) if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available @@ -294,7 +295,6 @@ if ($update -ne 'Y') { } if ($removeFiles) { - $removeFiles = @($removeFiles | Select-Object -Unique) Write-Host "Removed files:" $removeFiles | ForEach-Object { Write-Host "- $_" } } From fe30ec4552b6f0374afe084d822db18d31811186 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 5 Jun 2025 07:15:48 +0200 Subject: [PATCH 202/215] fix issue with private/internal template repo --- Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 | 15 ++++++++++++--- Templates/AppSource App/.AL-Go/localDevEnv.ps1 | 15 ++++++++++++--- .../Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 | 15 ++++++++++++--- .../Per Tenant Extension/.AL-Go/localDevEnv.ps1 | 15 ++++++++++++--- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 b/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 index 70e8eb4f5..a436ce474 100644 --- a/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 +++ b/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 @@ -15,14 +15,23 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S function DownloadHelperFile { param( [string] $url, - [string] $folder + [string] $folder, + [switch] $first ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' $name = [System.IO.Path]::GetFileName($url) Write-Host "Downloading $name from $url" $path = Join-Path $folder $name - Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + try { + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + } + catch { + if ($first) { + Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" + } + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } + } $ProgressPreference = $prevProgressPreference return $path } @@ -42,7 +51,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -first $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/AppSource App/.AL-Go/localDevEnv.ps1 b/Templates/AppSource App/.AL-Go/localDevEnv.ps1 index ce06cb2ca..92c03bb58 100644 --- a/Templates/AppSource App/.AL-Go/localDevEnv.ps1 +++ b/Templates/AppSource App/.AL-Go/localDevEnv.ps1 @@ -19,14 +19,23 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S function DownloadHelperFile { param( [string] $url, - [string] $folder + [string] $folder, + [switch] $first ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' $name = [System.IO.Path]::GetFileName($url) Write-Host "Downloading $name from $url" $path = Join-Path $folder $name - Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + try { + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + } + catch { + if ($first) { + Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" + } + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } + } $ProgressPreference = $prevProgressPreference return $path } @@ -46,7 +55,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -first $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 b/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 index 70e8eb4f5..a436ce474 100644 --- a/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 +++ b/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 @@ -15,14 +15,23 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S function DownloadHelperFile { param( [string] $url, - [string] $folder + [string] $folder, + [switch] $first ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' $name = [System.IO.Path]::GetFileName($url) Write-Host "Downloading $name from $url" $path = Join-Path $folder $name - Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + try { + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + } + catch { + if ($first) { + Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" + } + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } + } $ProgressPreference = $prevProgressPreference return $path } @@ -42,7 +51,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -first $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 b/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 index ce06cb2ca..92c03bb58 100644 --- a/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 +++ b/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 @@ -19,14 +19,23 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S function DownloadHelperFile { param( [string] $url, - [string] $folder + [string] $folder, + [switch] $first ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' $name = [System.IO.Path]::GetFileName($url) Write-Host "Downloading $name from $url" $path = Join-Path $folder $name - Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + try { + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + } + catch { + if ($first) { + Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" + } + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } + } $ProgressPreference = $prevProgressPreference return $path } @@ -46,7 +55,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -first $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null From 6323421a6ad5114a0b7e5e3ecd421b8912b995ef Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 5 Jun 2025 07:20:53 +0200 Subject: [PATCH 203/215] update parameters --- .../runtest.ps1 | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) rename e2eTests/scenarios/{IndirectTemplate => CustomTemplate}/runtest.ps1 (91%) diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/CustomTemplate/runtest.ps1 similarity index 91% rename from e2eTests/scenarios/IndirectTemplate/runtest.ps1 rename to e2eTests/scenarios/CustomTemplate/runtest.ps1 index 9b1843881..e31df2869 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/CustomTemplate/runtest.ps1 @@ -1,14 +1,19 @@ [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidGlobalVars', '', Justification = 'Global vars used for local test execution only.')] [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'All scenario tests have equal parameter set.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'Secrets are transferred as plain text.')] Param( [switch] $github, [switch] $linux, [string] $githubOwner = $global:E2EgithubOwner, [string] $repoName = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName()), - [string] $token = ($Global:SecureE2EPAT | Get-PlainText), + [string] $e2eAppId, + [string] $e2eAppKey, + [string] $algoauthapp = ($global:SecureALGOAUTHAPP | Get-PlainText), [string] $pteTemplate = $global:pteTemplate, [string] $appSourceTemplate = $global:appSourceTemplate, - [string] $adminCenterApiToken = ($global:SecureAdminCenterApiToken | Get-PlainText) + [string] $adminCenterApiCredentials = ($global:SecureadminCenterApiCredentials | Get-PlainText), + [string] $azureCredentials = ($global:SecureAzureCredentials | Get-PlainText), + [string] $githubPackagesToken = ($global:SecureGitHubPackagesToken | Get-PlainText) ) Write-Host -ForegroundColor Yellow @' @@ -49,7 +54,7 @@ $branch = "main" $template = "https://github.com/$pteTemplate" # Login -SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository +SetTokenAndRepository -github:$github -githubOwner $githubOwner -appId $e2eAppId -appKey $e2eAppKey -repository $repository # Create tempolate repository CreateAlGoRepository ` @@ -79,7 +84,7 @@ CreateAlGoRepository ` $finalRepoPath = (Get-Location).Path # Update AL-Go System Files to use template repository -RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $algoauthapp -repository $repository -branch $branch | Out-Null Set-Location $templateRepoPath @@ -206,7 +211,7 @@ $cicdYaml.Save($cicdWorkflow) CommitAndPush -commitMessage 'Add final repo customizations' # Update AL-Go System Files to uptake UseProjectDependencies setting -RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $algoauthapp -repository $repository -branch $branch | Out-Null # Stop all currently running workflows and run a new CI/CD workflow CancelAllWorkflows -repository $repository From 7f4b3807694c2d4c896240ef2ea52d9edcb3e5ab Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 5 Jun 2025 07:43:52 +0200 Subject: [PATCH 204/215] add al-go-helper --- e2eTests/scenarios/CustomTemplate/runtest.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2eTests/scenarios/CustomTemplate/runtest.ps1 b/e2eTests/scenarios/CustomTemplate/runtest.ps1 index e31df2869..1937bed3c 100644 --- a/e2eTests/scenarios/CustomTemplate/runtest.ps1 +++ b/e2eTests/scenarios/CustomTemplate/runtest.ps1 @@ -31,9 +31,9 @@ Write-Host -ForegroundColor Yellow @' # - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) # - Run Update AL-Go System Files in final repo (using custom template repository as template) # - Run Update AL-Go System files in custom template repository -# - Validate that custom step is present in custom template repository +# - Validate that custom job is present in custom template repository # - Run Update AL-Go System files in final repo -# - Validate that custom step is present in final repo +# - Validate that custom job is present in final repo # - Run Update AL-Go System files in final repo # - Validate that both custom steps is present in final repo # @@ -44,6 +44,7 @@ $prevLocation = Get-Location Remove-Module e2eTestHelper -ErrorAction SilentlyContinue Import-Module (Join-Path $PSScriptRoot "..\..\e2eTestHelper.psm1") -DisableNameChecking +. (Join-Path -Path $PSScriptRoot -ChildPath '..\..\..\Actions\AL-Go-Helper.ps1') . (Join-Path -Path $PSScriptRoot -ChildPath '..\..\..\Actions\CheckForUpdates\yamlclass.ps1') . (Join-Path -Path $PSScriptRoot -ChildPath "..\..\..\Actions\CheckForUpdates\CheckForUpdates.HelperFunctions.ps1") From 9cedb34b27ce17c38d3f9a3e4a4585a5e6e7b149 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 6 Jun 2025 08:56:11 +0200 Subject: [PATCH 205/215] fix end 2 end --- e2eTests/scenarios/CustomTemplate/runtest.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2eTests/scenarios/CustomTemplate/runtest.ps1 b/e2eTests/scenarios/CustomTemplate/runtest.ps1 index 1937bed3c..1cf91294b 100644 --- a/e2eTests/scenarios/CustomTemplate/runtest.ps1 +++ b/e2eTests/scenarios/CustomTemplate/runtest.ps1 @@ -231,8 +231,8 @@ Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateInit' -stepNa Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateDeploy' -stepName 'Deploy' -expectedText 'CustomJob-TemplateDeploy was here!' Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PreDeploy' -stepName 'PreDeploy' -expectedText 'CustomJob-PreDeploy was here!' Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PostDeploy' -stepName 'PostDeploy' -expectedText 'CustomJob-PostDeploy was here!' -Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeJob' -stepName 'JustSomeStep' -expectedText 'JustSomeJob was here!' | Should -Throw -Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeTemplateJob' -stepName 'JustSomeTemplateStep' -expectedText 'JustSomeTemplateJob was here!' | Should -Throw +{ Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeJob' -stepName 'JustSomeStep' -expectedText 'JustSomeJob was here!' } | Should -Throw +{ Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeTemplateJob' -stepName 'JustSomeTemplateStep' -expectedText 'JustSomeTemplateJob was here!' } | Should -Throw Set-Location $prevLocation From dc2928a59c21bbbe49d8f2f9a5c948c5ba6f3472 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Fri, 13 Jun 2025 07:29:43 +0200 Subject: [PATCH 206/215] Update CustomizingALGoForGitHub.md Co-authored-by: Alexander Holstrup <117829001+aholstrup1@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 65f11adb9..beb2029ed 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -174,7 +174,7 @@ Adding a custom job like this, will cause this job to run simultaneously with th > All custom jobs will be moved to the tail of the yaml file when running Update AL-Go System Files, but dependencies to/from the custom jobs will be maintained. > [!CAUTION] -> Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. +> Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. Therefore, please make sure to review the changes in AL-Go workflows when running Update AL-Go System Files. ### Custom job permissions From 28a378c745930b4fc27d5d410b8ab8027ed4fbd6 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Fri, 13 Jun 2025 07:30:08 +0200 Subject: [PATCH 207/215] Update Index.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Workshop/Index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Workshop/Index.md b/Workshop/Index.md index b3171b33e..638afe540 100644 --- a/Workshop/Index.md +++ b/Workshop/Index.md @@ -25,7 +25,7 @@ This workshop shows you how to take advantage of the functionality, which is pro ## Expert level 1. [Custom Delivery](CustomDelivery.md) - *FUTURE TOPIC: Setting up custom delivery to f.ex. a Teams channel.* -1. [Custom Delivery](CustomDeployment.md) - *FUTURE TOPIC: Setting up custom deployment to f.ex. an on-premises environment.* +1. [Custom Deployment](CustomDeployment.md) - *FUTURE TOPIC: Setting up custom deployment to f.ex. an on-premises environment.* 1. [Custom Jobs](CustomJobs.md) - *FUTURE TOPIC: Adding a custom job to an AL-Go for GitHub workflows.* 1. [Using Custom Templates](CustomTemplates.md) - *FUTURE TOPIC: Using custom templates to ensure that all repositories are using the same customizations* From f0a6adc02680ce934196019b24bbb7395eccbbab Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 14 Jun 2025 00:37:27 +0200 Subject: [PATCH 208/215] review --- RELEASENOTES.md | 2 +- Scenarios/settings.md | 5 ++--- e2eTests/scenarios/CustomTemplate/runtest.ps1 | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index a9d257641..1592a6279 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -14,7 +14,7 @@ Note that custom jobs might break by future changes to AL-Go for GitHub workflow ### Support for Custom AL-Go template repositories -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using custom template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#customtemplate](https://aka.ms/algosettings#customtemplate) for details. +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs and/or settings to this repository and then use that repository as the template repository for other repositories. Using custom template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#customtemplate](https://aka.ms/algosettings#customtemplate) for details. > [!NOTE] > Customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your custom template repositories. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 3fdcf92e2..564ba37fe 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -410,13 +410,13 @@ In the `needs` property, you specify which jobs should be complete before this j Custom jobs will be preserved when running Update AL-Go System Files. -**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). +**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). ## Custom template repositories -If you are utilizing script overrides, custom jobs, custom steps, custom delivery or like in many repositories, you might want to take advantage of the custom template repository feature. +If you are utilizing script overrides, custom jobs, custom delivery or like in many repositories, you might want to take advantage of the custom template repository feature. A custom template repository is an AL-Go for GitHub repository (without any apps), which is used as a template for the remaining AL-Go for GitHub repositories. As an example, if you are using a custom delivery script, which you want to have in all your repositories, you can create an empty AL-Go for GitHub repository, place the delivery script in the .github folder and use that repository as a template when running Update AL-Go system files in your other repositories. @@ -428,7 +428,6 @@ The items, which are currently supported from custom template repositories are: - Project script overrides in the .AL-Go folder - Custom workflows in the .github/workflows folder - Custom jobs in any AL-Go for GitHub workflow -- Custom steps in the \_BuildALGoProject workflow - Changes to repository settings in .github/AL-Go-settings.json - Changes to project settings in .AL-Go/settings.json diff --git a/e2eTests/scenarios/CustomTemplate/runtest.ps1 b/e2eTests/scenarios/CustomTemplate/runtest.ps1 index 1cf91294b..7968a81bb 100644 --- a/e2eTests/scenarios/CustomTemplate/runtest.ps1 +++ b/e2eTests/scenarios/CustomTemplate/runtest.ps1 @@ -34,8 +34,6 @@ Write-Host -ForegroundColor Yellow @' # - Validate that custom job is present in custom template repository # - Run Update AL-Go System files in final repo # - Validate that custom job is present in final repo -# - Run Update AL-Go System files in final repo -# - Validate that both custom steps is present in final repo # '@ From 85d79730f1786dece4343d5f8b5ddedae90840e2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 14 Jun 2025 00:39:35 +0200 Subject: [PATCH 209/215] review --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index e8a6dbc6c..00d4bd7f6 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -45,12 +45,6 @@ if ($token) { $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } -# Get secrets for Custom AL-Go System Files -$secrets = @{} -if ($env:Secrets) { - $secrets = $env:Secrets | ConvertFrom-Json | ConvertTo-HashTable -} - # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository # CheckForUpdates will apply changes to the AL-Go System files based on AL-Go repo settings, such as "runs-on" etc. # if $update is set to Y, CheckForUpdates will also update the AL-Go System files in the current repository using a PR or a direct commit (if $directCommit is set to true) From fa7e33eeb3103d68ab89b19920af0c99516020d4 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Sat, 14 Jun 2025 00:43:21 +0200 Subject: [PATCH 210/215] Update RELEASENOTES.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- RELEASENOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1592a6279..b620ce357 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -17,7 +17,7 @@ Note that custom jobs might break by future changes to AL-Go for GitHub workflow Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs and/or settings to this repository and then use that repository as the template repository for other repositories. Using custom template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#customtemplate](https://aka.ms/algosettings#customtemplate) for details. > [!NOTE] -> Customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your custom template repositories. +> Customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always double-check the pull request when updating AL-Go system files in your custom template repositories. ## v7.2 From 6372d4d698e8b8247a3e2a47990ff780a49be354 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 14 Jun 2025 00:48:00 +0200 Subject: [PATCH 211/215] review --- Templates/AppSource App/.github/workflows/CICD.yaml | 1 - Templates/AppSource App/.github/workflows/CreateRelease.yaml | 2 ++ .../.github/workflows/UpdateGitHubGoSystemFiles.yaml | 1 - Templates/Per Tenant Extension/.github/workflows/CICD.yaml | 1 - .../Per Tenant Extension/.github/workflows/CreateRelease.yaml | 2 ++ .../.github/workflows/UpdateGitHubGoSystemFiles.yaml | 1 - 6 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index ed75d2e87..1a5e2d96f 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -166,7 +166,6 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main env: GITHUB_TOKEN: ${{ github.token }} - Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' with: shell: powershell templateUrl: ${{ env.templateUrl }} diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index a7e1ff6c7..e538db408 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -121,6 +121,8 @@ jobs: - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main + env: + GITHUB_TOKEN: ${{ github.token }} with: shell: powershell templateUrl: ${{ env.templateUrl }} diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 4cc5cb564..013b02414 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -135,7 +135,6 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main env: GITHUB_TOKEN: ${{ github.token }} - Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' with: shell: powershell token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 1b8bf1f78..d28aefcf7 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -166,7 +166,6 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main env: GITHUB_TOKEN: ${{ github.token }} - Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' with: shell: powershell templateUrl: ${{ env.templateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index a7e1ff6c7..e538db408 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -121,6 +121,8 @@ jobs: - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main + env: + GITHUB_TOKEN: ${{ github.token }} with: shell: powershell templateUrl: ${{ env.templateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 4cc5cb564..013b02414 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -135,7 +135,6 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main env: GITHUB_TOKEN: ${{ github.token }} - Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' with: shell: powershell token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} From 9867aec63bd71c7018aae1698101617e52c6cebd Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 14 Jun 2025 00:52:52 +0200 Subject: [PATCH 212/215] rename first --- Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 | 6 +++--- Templates/AppSource App/.AL-Go/localDevEnv.ps1 | 6 +++--- Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 | 6 +++--- Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 b/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 index a436ce474..39e13e4ea 100644 --- a/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 +++ b/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 @@ -16,7 +16,7 @@ function DownloadHelperFile { param( [string] $url, [string] $folder, - [switch] $first + [switch] $notifyAuthenticatedAttempt ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' @@ -27,7 +27,7 @@ function DownloadHelperFile { Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path } catch { - if ($first) { + if ($notifyAuthenticatedAttempt) { Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" } Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } @@ -51,7 +51,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -first +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -notifyAuthenticatedAttempt $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/AppSource App/.AL-Go/localDevEnv.ps1 b/Templates/AppSource App/.AL-Go/localDevEnv.ps1 index 92c03bb58..aeaaedb73 100644 --- a/Templates/AppSource App/.AL-Go/localDevEnv.ps1 +++ b/Templates/AppSource App/.AL-Go/localDevEnv.ps1 @@ -20,7 +20,7 @@ function DownloadHelperFile { param( [string] $url, [string] $folder, - [switch] $first + [switch] $notifyAuthenticatedAttempt ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' @@ -31,7 +31,7 @@ function DownloadHelperFile { Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path } catch { - if ($first) { + if ($notifyAuthenticatedAttempt) { Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" } Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } @@ -55,7 +55,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -first +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -notifyAuthenticatedAttempt $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 b/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 index a436ce474..39e13e4ea 100644 --- a/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 +++ b/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 @@ -16,7 +16,7 @@ function DownloadHelperFile { param( [string] $url, [string] $folder, - [switch] $first + [switch] $notifyAuthenticatedAttempt ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' @@ -27,7 +27,7 @@ function DownloadHelperFile { Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path } catch { - if ($first) { + if ($notifyAuthenticatedAttempt) { Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" } Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } @@ -51,7 +51,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -first +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -notifyAuthenticatedAttempt $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 b/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 index 92c03bb58..aeaaedb73 100644 --- a/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 +++ b/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 @@ -20,7 +20,7 @@ function DownloadHelperFile { param( [string] $url, [string] $folder, - [switch] $first + [switch] $notifyAuthenticatedAttempt ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' @@ -31,7 +31,7 @@ function DownloadHelperFile { Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path } catch { - if ($first) { + if ($notifyAuthenticatedAttempt) { Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" } Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } @@ -55,7 +55,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -first +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -notifyAuthenticatedAttempt $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null From cbc9cc50dce692cc409345fa592856c9b132e52b Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 14 Jun 2025 01:03:20 +0200 Subject: [PATCH 213/215] rename --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 00d4bd7f6..945e7bc09 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -122,17 +122,17 @@ if (-not $isDirectALGo) { # - All files in .github that ends with .copy.md # - All PowerShell scripts in .AL-Go folders (all projects) $checkfiles = @( - @{ 'dstPath' = (Join-Path '.github' 'workflows'); 'newname' = ''; 'srcPath' = (Join-Path '.github' 'workflows'); 'pattern' = '*'; 'type' = 'workflow' }, - @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.copy.md'; 'type' = 'releasenotes' } - @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.ps1'; 'type' = 'script' } - @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = 'AL-Go-Settings.json'; 'type' = 'settings' }, - @{ 'dstPath' = '.github'; 'newname' = ''; 'srcPath' = '.github'; 'pattern' = '*.settings.json'; 'type' = 'settings' } + @{ 'dstPath' = (Join-Path '.github' 'workflows'); 'dstName' = ''; 'srcPath' = (Join-Path '.github' 'workflows'); 'pattern' = '*'; 'type' = 'workflow' }, + @{ 'dstPath' = '.github'; 'dstName' = ''; 'srcPath' = '.github'; 'pattern' = '*.copy.md'; 'type' = 'releasenotes' } + @{ 'dstPath' = '.github'; 'dstName' = ''; 'srcPath' = '.github'; 'pattern' = '*.ps1'; 'type' = 'script' } + @{ 'dstPath' = '.github'; 'dstName' = ''; 'srcPath' = '.github'; 'pattern' = 'AL-Go-Settings.json'; 'type' = 'settings' }, + @{ 'dstPath' = '.github'; 'dstName' = ''; 'srcPath' = '.github'; 'pattern' = '*.settings.json'; 'type' = 'settings' } ) if ($originalTemplateFolder) { $checkfiles += @( - @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateRepoSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($CustomTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } - @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateProjectSettingsFile)); 'newname' = ([system.IO.Path]::GetFileName($CustomTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } + @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateRepoSettingsFile)); 'dstName' = ([system.IO.Path]::GetFileName($CustomTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } + @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateProjectSettingsFile)); 'dstName' = ([system.IO.Path]::GetFileName($CustomTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } ) } @@ -143,8 +143,8 @@ Write-Host "Projects found: $($projects.Count)" foreach($project in $projects) { Write-Host "- $project" $checkfiles += @( - @{ 'dstPath' = Join-Path $project '.AL-Go'; 'newname' = ''; 'srcPath' = '.AL-Go'; 'pattern' = '*.ps1'; 'type' = 'script' }, - @{ 'dstPath' = Join-Path $project '.AL-Go'; 'newname' = ''; 'srcPath' = '.AL-Go'; 'pattern' = 'settings.json'; 'type' = 'settings' } + @{ 'dstPath' = Join-Path $project '.AL-Go'; 'dstName' = ''; 'srcPath' = '.AL-Go'; 'pattern' = '*.ps1'; 'type' = 'script' }, + @{ 'dstPath' = Join-Path $project '.AL-Go'; 'dstName' = ''; 'srcPath' = '.AL-Go'; 'pattern' = 'settings.json'; 'type' = 'settings' } ) } @@ -191,8 +191,8 @@ foreach($checkfile in $checkfiles) { Get-ChildItem -Path $srcFolder -Filter $checkfile.pattern | ForEach-Object { # Read the template file and modify it based on the settings # Compare the modified file with the file in the current repository - if ($checkfile.newname) { - $filename = $checkfile.newname + if ($checkfile.dstName) { + $filename = $checkfile.dstName } else { $filename = $_.Name From 1106eb1e17ddb3d096099e4885a24cdb768b64d5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 14 Jun 2025 01:17:59 +0200 Subject: [PATCH 214/215] add telemetry --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 945e7bc09..739c84876 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -81,6 +81,8 @@ if (-not $isDirectALGo) { $templateRepoSettings = Get-Content $templateRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { # The template repository is a url to another AL-Go repository (a custom template repository) + Trace-Information -Message "Using custom AL-Go template repository: $($templateRepoSettings.templateUrl)" + # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the custom template repository points to the "original" template repository # Copy files and folders from the custom template repository, but grab the unmodified file from the "original" template repository if it exists and apply customizations # Copy .github/AL-Go-Settings.json to .github/templateRepoSettings.doNotEdit.json (will be read before .github/AL-Go-Settings.json in the final repo) From b74f3a37995e3334ffedcff344bd7dbd8d3f23da Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Thu, 19 Jun 2025 05:52:12 +0200 Subject: [PATCH 215/215] Update Actions/CheckForUpdates/CheckForUpdates.ps1 Co-authored-by: Alexander Holstrup <117829001+aholstrup1@users.noreply.github.com> --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 739c84876..01335d0ef 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -81,7 +81,7 @@ if (-not $isDirectALGo) { $templateRepoSettings = Get-Content $templateRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { # The template repository is a url to another AL-Go repository (a custom template repository) - Trace-Information -Message "Using custom AL-Go template repository: $($templateRepoSettings.templateUrl)" + Trace-Information -Message "Using custom AL-Go template repository" # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the custom template repository points to the "original" template repository # Copy files and folders from the custom template repository, but grab the unmodified file from the "original" template repository if it exists and apply customizations