From 42f8f138cdd8b0f322a822d799bc17aa1f9b013e Mon Sep 17 00:00:00 2001 From: david22swan Date: Thu, 31 Jul 2025 15:51:00 +0100 Subject: [PATCH 1/3] (CAT-2433) Adds option to return latest agent build Adds a command-line option to specify whether the latest agent build should be included in the matrix. This is due to the Mac and Windows puppet_agent install tasks requiring a specific version when installing. --- exe/matrix_from_metadata_v3 | 29 ++++++++++++++++++++---- spec/exe/matrix_from_metadata_v3_spec.rb | 16 ++++++------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/exe/matrix_from_metadata_v3 b/exe/matrix_from_metadata_v3 index 063ba13..7943d03 100755 --- a/exe/matrix_from_metadata_v3 +++ b/exe/matrix_from_metadata_v3 @@ -163,6 +163,7 @@ begin opt.on('--provision-exclude NAME', String, "Filter provisioner (default: #{default_options[:'provision-exclude'] || 'none'})") { |o| options.provision_exclude.push(*o.split(',')) } opt.on('--nightly', TrueClass, 'Enable nightly builds') { |o| options.nightly = o } + opt.on('--latest-agent', TrueClass, 'Return the latest agent build as part of the matrix') { |o| options.latest_agent = o } end.parse! Action.config(debug: true) if options[:debug] @@ -262,13 +263,31 @@ options[:metadata]['requirements']&.each do |req| # This assumes that such a boundary will always allow the latest actually existing puppet version of a release stream, trading off simplicity vs accuracy here. next unless gem_req.satisfied_by?(Gem::Version.new("#{collection['puppet'].to_i}.9999")) + # # If requested, return the latest agent version for the collection's puppet version + # # If not requested, return the value latest + agent_version = if options[:latest_agent] && collection['puppet'] != 'nightly' + require 'net/http' + require 'json' + require 'uri' + + uri = URI('https://forgeapi.puppet.com/private/versions/puppet-agent') + response = Net::HTTP.get_response(uri) + json_data = JSON.parse(response.body) + all_versions = json_data.keys + versionx = all_versions.select { |v| v.start_with?("#{collection['puppet'].to_i}.") } + versionx.sort_by { |v| Gem::Version.new(v) }.last # rubocop:disable Style/RedundantSort + else + 'latest' + end + version = collection['puppet'].to_i prefix = 'puppetcore' - matrix[:collection] << if options[:nightly] - "#{prefix}#{version}-nightly" - else - "#{prefix}#{version}" - end + group = if options[:nightly] + "#{prefix}#{version}-nightly" + else + "#{prefix}#{version}" + end + matrix[:collection] << { collection: group, version: agent_version } spec_matrix[:include] << { puppet_version: "~> #{collection['puppet']}", diff --git a/spec/exe/matrix_from_metadata_v3_spec.rb b/spec/exe/matrix_from_metadata_v3_spec.rb index 43c6885..12dc3e1 100644 --- a/spec/exe/matrix_from_metadata_v3_spec.rb +++ b/spec/exe/matrix_from_metadata_v3_spec.rb @@ -30,7 +30,7 @@ '{"label":"Ubuntu-22.04","provider":"docker","arch":"x86_64","image":"litmusimage/ubuntu:22.04","runner":"ubuntu-latest"}', '],', '"collection":[', - '"puppetcore8"', + '{"collection":"puppetcore8","version":"latest"}', ']', '}' ].join @@ -63,7 +63,7 @@ '{"label":"Ubuntu-22.04-arm","provider":"provision_service","arch":"arm","image":"ubuntu-2204-lts-arm64","runner":"ubuntu-latest"}', '],', '"collection":[', - '"puppetcore8"', + '{"collection":"puppetcore8","version":"latest"}', ']', '}' ].join @@ -103,7 +103,7 @@ '{"label":"Ubuntu-22.04-arm","provider":"provision_service","arch":"arm","image":"ubuntu-2204-lts-arm64","runner":"ubuntu-latest"}', '],', '"collection":[', - '"puppetcore8"', + '{"collection":"puppetcore8","version":"latest"}', ']', '}' ].join @@ -142,7 +142,7 @@ '{"label":"Ubuntu-22.04-arm","provider":"provision_service","arch":"arm","image":"ubuntu-2204-lts-arm64","runner":"ubuntu-latest"}', '],', '"collection":[', - '"puppetcore8"', + '{"collection":"puppetcore8","version":"latest"}', ']', '}' ].join @@ -175,7 +175,7 @@ '"platforms":[', '],', '"collection":[', - '"puppetcore8"', + '{"collection":"puppetcore8","version":"latest"}', ']', '}' ].join @@ -215,7 +215,7 @@ '{"label":"Ubuntu-22.04","provider":"docker","arch":"x86_64","image":"litmusimage/ubuntu:22.04","runner":"ubuntu-latest"}', '],', '"collection":[', - '"puppetcore8"', + '{"collection":"puppetcore8","version":"latest"}', ']', '}' ].join @@ -233,7 +233,7 @@ '::group::spec_matrix' ) expect(github_output_content).to include( - '"collection":["2023.8.4-puppet_enterprise","2021.7.9-puppet_enterprise","puppetcore8"' + '"collection":["2023.8.4-puppet_enterprise","2021.7.9-puppet_enterprise",{"collection":"puppetcore8","version":"latest"}' ) expect(github_output_content).to include( 'spec_matrix={"include":[{"puppet_version":"~> 8.0","ruby_version":3.2}]}' @@ -257,7 +257,7 @@ '{"label":"Ubuntu-22.04-arm","provider":"provision_service","arch":"arm","image":"ubuntu-2204-lts-arm64","runner":"ubuntu-latest"}', '],', '"collection":[', - '"puppetcore8-nightly"', + '{"collection":"puppetcore8-nightly","version":"latest"}', ']', '}' ].join From 81bc355f7fd707993cbebec56dda66b41d53e8fd Mon Sep 17 00:00:00 2001 From: david22swan Date: Wed, 6 Aug 2025 11:53:15 +0100 Subject: [PATCH 2/3] (CAT-2344) Updates matrix creation logic Updates matrix creation logic to only include the version if requested. Introduces a check to ensure the `latest-agent` and `pe-include` options are not used simultaneously. --- exe/matrix_from_metadata_v3 | 20 +++++++++++-- spec/exe/matrix_from_metadata_v3_spec.rb | 37 +++++++++++++++++++----- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/exe/matrix_from_metadata_v3 b/exe/matrix_from_metadata_v3 index 7943d03..b21074c 100755 --- a/exe/matrix_from_metadata_v3 +++ b/exe/matrix_from_metadata_v3 @@ -169,6 +169,8 @@ begin Action.config(debug: true) if options[:debug] Action.config(notice: false) if options[:quiet] && !options[:debug] + raise OptionParser::InvalidArgument, 'latest-agent and pe-include are mutually exclusive options' if options[:latest_agent] && options[:pe_include] + # validate provisioners options[:provision_include].select! do |p| options[:matrix]['provisioners'].key?(p) or raise OptionParser::InvalidArgument, "--provision-include '#{p}' not found in provisioners" @@ -276,8 +278,6 @@ options[:metadata]['requirements']&.each do |req| all_versions = json_data.keys versionx = all_versions.select { |v| v.start_with?("#{collection['puppet'].to_i}.") } versionx.sort_by { |v| Gem::Version.new(v) }.last # rubocop:disable Style/RedundantSort - else - 'latest' end version = collection['puppet'].to_i @@ -287,7 +287,21 @@ options[:metadata]['requirements']&.each do |req| else "#{prefix}#{version}" end - matrix[:collection] << { collection: group, version: agent_version } + matrix[:collection] << if options[:latest_agent] && collection['puppet'] != 'nightly' + require 'net/http' + require 'json' + require 'uri' + + uri = URI('https://forgeapi.puppet.com/private/versions/puppet-agent') + response = Net::HTTP.get_response(uri) + json_data = JSON.parse(response.body) + all_versions = json_data.keys + versionx = all_versions.select { |v| v.start_with?("#{collection['puppet'].to_i}.") } + versionx.sort_by { |v| Gem::Version.new(v) }.last # rubocop:disable Style/RedundantSort + { collection: group, version: agent_version } + else + group + end spec_matrix[:include] << { puppet_version: "~> #{collection['puppet']}", diff --git a/spec/exe/matrix_from_metadata_v3_spec.rb b/spec/exe/matrix_from_metadata_v3_spec.rb index 12dc3e1..304dcf3 100644 --- a/spec/exe/matrix_from_metadata_v3_spec.rb +++ b/spec/exe/matrix_from_metadata_v3_spec.rb @@ -30,7 +30,7 @@ '{"label":"Ubuntu-22.04","provider":"docker","arch":"x86_64","image":"litmusimage/ubuntu:22.04","runner":"ubuntu-latest"}', '],', '"collection":[', - '{"collection":"puppetcore8","version":"latest"}', + '"puppetcore8"', ']', '}' ].join @@ -63,7 +63,7 @@ '{"label":"Ubuntu-22.04-arm","provider":"provision_service","arch":"arm","image":"ubuntu-2204-lts-arm64","runner":"ubuntu-latest"}', '],', '"collection":[', - '{"collection":"puppetcore8","version":"latest"}', + '"puppetcore8"', ']', '}' ].join @@ -103,7 +103,7 @@ '{"label":"Ubuntu-22.04-arm","provider":"provision_service","arch":"arm","image":"ubuntu-2204-lts-arm64","runner":"ubuntu-latest"}', '],', '"collection":[', - '{"collection":"puppetcore8","version":"latest"}', + '"puppetcore8"', ']', '}' ].join @@ -142,7 +142,7 @@ '{"label":"Ubuntu-22.04-arm","provider":"provision_service","arch":"arm","image":"ubuntu-2204-lts-arm64","runner":"ubuntu-latest"}', '],', '"collection":[', - '{"collection":"puppetcore8","version":"latest"}', + '"puppetcore8"', ']', '}' ].join @@ -175,7 +175,7 @@ '"platforms":[', '],', '"collection":[', - '{"collection":"puppetcore8","version":"latest"}', + '"puppetcore8"', ']', '}' ].join @@ -215,7 +215,7 @@ '{"label":"Ubuntu-22.04","provider":"docker","arch":"x86_64","image":"litmusimage/ubuntu:22.04","runner":"ubuntu-latest"}', '],', '"collection":[', - '{"collection":"puppetcore8","version":"latest"}', + '"puppetcore8"', ']', '}' ].join @@ -233,7 +233,7 @@ '::group::spec_matrix' ) expect(github_output_content).to include( - '"collection":["2023.8.4-puppet_enterprise","2021.7.9-puppet_enterprise",{"collection":"puppetcore8","version":"latest"}' + '"collection":["2023.8.4-puppet_enterprise","2021.7.9-puppet_enterprise","puppetcore8"' ) expect(github_output_content).to include( 'spec_matrix={"include":[{"puppet_version":"~> 8.0","ruby_version":3.2}]}' @@ -257,7 +257,7 @@ '{"label":"Ubuntu-22.04-arm","provider":"provision_service","arch":"arm","image":"ubuntu-2204-lts-arm64","runner":"ubuntu-latest"}', '],', '"collection":[', - '{"collection":"puppetcore8-nightly","version":"latest"}', + '"puppetcore8-nightly"', ']', '}' ].join @@ -280,4 +280,25 @@ ) end end + + context 'with argument --latest-agent' do + let(:result) { run_matrix_from_metadata_v3(['--puppetlabs', '--latest-agent']) } + + it 'run successfully' do + expect(result.status_code).to eq 0 + end + + it 'generates the matrix' do + expect(result.stdout).to include( + '::warning::CentOS-6 no provisioner found', + '::warning::Ubuntu-14.04 no provisioner found', + '::group::matrix', + '::group::spec_matrix' + ) + expect(github_output_content).to match(/{"collection":"puppetcore8","version":"\d+\.\d+\.\d+"}/) + expect(github_output_content).to include( + 'spec_matrix={"include":[{"puppet_version":"~> 8.0","ruby_version":3.2}]}' + ) + end + end end From 369561a33de6755638c79b6a450b1da326075cb3 Mon Sep 17 00:00:00 2001 From: david22swan Date: Wed, 6 Aug 2025 16:26:13 +0100 Subject: [PATCH 3/3] (CAT-2344) Update the README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 991a529..2652d32 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ in the project module root directory run `bundle exec matrix_from_metadata_v3` | --provision-include | NAME | all | Select provisioner | | --provision-exclude | NAME | provision_service | Filter provisioner | | --nightly | | | Install from the nightly puppetcore images | +| --latest-agent | | | Return the latest agent version as part of the matrix. Note: This changes how the collection is returned | > Refer to the [built-in matrix.json](https://github.com/puppetlabs/puppet_litmus/blob/main/exe/matrix.json) for a list of supported collection, provisioners, and platforms.