Use Aspire as distributed service orchestrator, start on integration tests (#1664)

* Initial infra work for assembler integration tests

* Move to Aspire

* Move integration tests over to aspire

* Touchup logging

* ensure we log startup in case of failed tests

* LlmGatwewayAskAiGateway needs to be registered as scoped since it depends on scoped options

* Ensure we set default branch on CI to main

To limit noise from clone-all

* Support building only public repositories

* update test names

* format

* ensure we only mutate navigation if --skip-private-repositories is actually specified

* formatting

* comment

* Separate integration tests

* update free disk pace

* keep dotnet

* Disable integration tests temporarily

Author: Mpdreamz

.github/actions/bootstrap/action.yml

@@ -18,6 +18,12 @@ runs:
       run: |
         git fetch --prune --unshallow --tags
         git tag --list
+
+    - name: Git config
+      shell: bash
+      run: |
+        git config --global init.defaultBranch main
+
     - uses: actions/setup-dotnet@v4
       with:
         global-json-file: global.json
@@ -37,4 +43,3 @@ runs:
         cache: npm
         cache-dependency-path: src/Elastic.Documentation.Site/package-lock.json
         node-version-file: .nvmrc
-      

.github/workflows/ci.yml

@@ -84,11 +84,36 @@ jobs:
         id: bootstrap
         uses: ./.github/actions/bootstrap
 
+      - name: Install Aspire workload
+        run: dotnet workload install aspire
+
       - name: Build
         run: dotnet run --project build -c release
 
       - name: Test
-        run: dotnet run --project build -c release -- test
+        run: dotnet run --project build -c release -- unit-test
 
       - name: Publish AOT
         run: dotnet run --project build -c release -- publishbinaries
+
+  integration:
+    if: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        with:
+          tool-cache: false
+          dotnet: true
+
+      - name: Bootstrap Action Workspace
+        id: bootstrap
+        uses: ./.github/actions/bootstrap
+
+      - name: Install Aspire workload
+        run: dotnet workload install aspire
+
+      - name: Integration Tests
+        run: dotnet run --project build -c release -- integrate

Directory.Packages.props

@@ -3,6 +3,8 @@
     <!-- Enable central package management, https://learn.microsoft.com/en-us/nuget/consume-packages/Central-Package-Management -->
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
     <MinVerSkip Condition="'$(Configuration)' == 'Debug'">true</MinVerSkip>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
     <GlobalPackageReference Include="MinVer" Version="6.0.0" PrivateAssets="All" />
@@ -15,14 +17,21 @@
     <PackageVersion Include="Amazon.Lambda.S3Events" Version="3.1.0" />
     <PackageVersion Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4" />
     <PackageVersion Include="Amazon.Lambda.SQSEvents" Version="2.2.0" />
+    <PackageVersion Include="Aspire.Hosting" Version="9.4.0" />
+    <PackageVersion Include="Aspire.Hosting.Testing" Version="9.4.0" />
     <PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
+    <PackageVersion Include="Elastic.Aspire.Hosting.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="FakeItEasy" Version="8.3.0" />
     <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.11.3" />
+    <PackageVersion Include="InMemoryLogger" Version="1.0.66" />
+    <PackageVersion Include="MartinCostello.Logging.XUnit.v3" Version="0.6.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.4" />
     <PackageVersion Include="Microsoft.OpenApi" Version="2.0.0-preview9" />
     <PackageVersion Include="System.Text.Json" Version="9.0.5" />
+    <PackageVersion Include="TUnit" Version="0.25.21" />
+    <PackageVersion Include="xunit.v3.extensibility.core" Version="2.0.2" />
   </ItemGroup>
   <!-- Build -->
   <ItemGroup>
@@ -40,8 +49,8 @@
     <PackageVersion Include="DotNet.Glob" Version="3.1.3" />
     <PackageVersion Include="Errata" Version="0.14.0" />
     <PackageVersion Include="Github.Actions.Core" Version="9.0.0" />
-    <PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.4" />
-    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="9.0.4" />
+    <PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.7" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="9.0.7" />
     <PackageVersion Include="Markdig" Version="0.41.1" />
     <PackageVersion Include="NetEscapades.EnumGenerators" Version="1.0.0-beta12" PrivateAssets="all" ExcludeAssets="runtime" />
     <PackageVersion Include="Proc" Version="0.9.1" />
@@ -58,6 +67,17 @@
   </ItemGroup>
   <!-- Test packages -->
   <ItemGroup>
+    <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="9.7.0" />
+    <PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="9.4.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.12.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.12.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.12.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.12.0" />
+  </ItemGroup>
+  <!-- Test packages -->
+  <ItemGroup>
+    <PackageVersion Include="Aspire.Hosting.AppHost" Version="9.4.0" />
     <PackageVersion Include="AngleSharp.Diffing" Version="1.0.0" />
     <PackageVersion Include="DiffPlex" Version="1.7.2" />
     <PackageVersion Include="FluentAssertions" Version="7.2.0" />

build/CommandLine.fs

@@ -9,13 +9,22 @@ open Microsoft.FSharp.Reflection
 open System
 open Bullseye
 
+type TestSuite = All | Unit | Integration
+    with
+    member this.SuitName =
+        match FSharpValue.GetUnionFields(this, typeof<TestSuite>) with
+        | case, _ -> case.Name.ToLowerInvariant()
+
 type Build =
     | [<CliPrefix(CliPrefix.None);SubCommand>] Clean
     | [<CliPrefix(CliPrefix.None);SubCommand>] Version
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] Compile
     | [<CliPrefix(CliPrefix.None);SubCommand>] Build
+
     | [<CliPrefix(CliPrefix.None);SubCommand>] Test
-    
+    | [<CliPrefix(CliPrefix.None);SubCommand>] Unit_Test
+    | [<CliPrefix(CliPrefix.None);SubCommand>] Integrate
+
     | [<CliPrefix(CliPrefix.None);SubCommand>] Format
     | [<CliPrefix(CliPrefix.None);SubCommand>] Watch
 
@@ -33,6 +42,7 @@ type Build =
     | [<Inherit;AltCommandLine("-s")>] Single_Target
     | [<Inherit>] Token of string 
     | [<Inherit;AltCommandLine("-c")>] Skip_Dirty_Check
+    | [<Inherit;EqualsAssignment>] Test_Suite of TestSuite
 with
     interface IArgParserTemplate with
         member this.Usage =
@@ -41,8 +51,11 @@ with
             | Clean -> "clean known output locations"
             | Version -> "print version information"
             | Build -> "Run build"
-            
-            | Test -> "runs a clean build and then runs all the tests "
+
+            | Unit_Test -> "alias to providing: test --test-suite=unit"
+            | Integrate -> "alias to providing: test --test-suite=integration"
+            | Test -> "runs a clean build and then runs all the tests unless --test-suite is provided"
+
             | Release -> "runs build, tests, and create and validates the packages shy of publishing them"
             | Publish -> "Publishes artifacts"
             | Format -> "runs dotnet format"
@@ -62,6 +75,7 @@ with
             | Single_Target -> "Runs the provided sub command without running their dependencies"
             | Token _ -> "Token to be used to authenticate with github"
             | Skip_Dirty_Check -> "Skip the clean checkout check that guards the release/publish targets"
+            | Test_Suite _ -> "Specify the test suite to run, defaults to all"
 
     member this.StepName =
         match FSharpValue.GetUnionFields(this, typeof<Build>) with
@@ -87,4 +101,4 @@ with
         let dependsOn = if singleTarget then [] else dependsOn
 
         let steps = dependsOn @ composedOf |> List.map _.StepName
-        Targets.Target(target.StepName, steps, Action(fun _ -> action parsed))
+        Targets.Target(target.StepName, steps, Action(fun _ -> action parsed))

build/Targets.fs

@@ -107,10 +107,17 @@ let private publishContainers _ =
     createImage "docs-builder"
     createImage "docs-assembler"
 
-let private runTests _ =
+let private runTests (testSuite: TestSuite) _ =
+    let testFilter =
+        match testSuite with
+        | All -> []
+        | Unit -> ["--filter"; "FullyQualifiedName~.Tests"]
+        | Integration -> ["--filter"; "FullyQualifiedName~.IntegrationTests"]
+
     exec {
         run "dotnet" (
             ["test"; "-c"; "release"; "--no-restore"; "--no-build"; "--logger"; "GitHubActions"]
+            @ testFilter
             @ ["--"; "RunConfiguration.CollectSourceInformation=true"]
         )
     }
@@ -132,8 +139,10 @@ let Setup (parsed:ParseResults<Build>) =
             Build.Cmd
                 [Clean; Lint; Compile] [] build
 
-        | Test -> Build.Cmd [Compile] [] runTests
-        
+        | Test -> Build.Cmd [Compile] [] <| runTests TestSuite.All
+        | Unit_Test -> Build.Cmd [Compile] [] <| runTests TestSuite.Unit
+        | Integrate -> Build.Cmd [Compile] [] <| runTests TestSuite.Integration
+
         | Release ->
             Build.Cmd 
                 [PristineCheck; Build]
@@ -159,6 +168,7 @@ let Setup (parsed:ParseResults<Build>) =
 
         // flags
         | Single_Target
+        | Test_Suite _
         | Token _
         | Skip_Dirty_Check -> Build.Ignore
 

config/assembler.yml

@@ -95,6 +95,7 @@ references:
   elastic-otel-python:
   elastic-serverless-forwarder:
   integration-docs:
+    private: true
   integrations:
   logstash-docs-md:
   opentelemetry:
@@ -105,7 +106,12 @@ references:
 
   # @elastic/admin-docs
   cloud-on-k8s:
-  cloud: *master
+  cloud:
+    current: master
+    next: master
+    edge: master
+    private: true
+
   curator: *master
   ecctl: *master
 

docs-builder.sln

@@ -25,6 +25,9 @@ EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = ".github", ".github\.github.csproj", "{1A8659C1-222A-4824-B562-ED8F88658C05}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{67B576EE-02FA-4F9B-94BC-3630BC09ECE5}"
+	ProjectSection(SolutionItems) = preProject
+		tests\Directory.Build.props = tests\Directory.Build.props
+	EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Markdown.Tests", "tests\Elastic.Markdown.Tests\Elastic.Markdown.Tests.csproj", "{B27C5107-128B-465A-B8F8-8985399E4CFB}"
 EndProject
@@ -119,6 +122,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{6FAB56
 		config\navigation.yml = config\navigation.yml
 	EndProjectSection
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests-integration", "tests-integration", "{BCAD38D5-6C83-46E2-8398-4BE463931098}"
+	ProjectSection(SolutionItems) = preProject
+		tests-integration\Directory.Build.props = tests-integration\Directory.Build.props
+	EndProjectSection
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Assembler.IntegrationTests", "tests-integration\Elastic.Assembler.IntegrationTests\Elastic.Assembler.IntegrationTests.csproj", "{A272D3EC-FAAF-4795-A796-302725382AFF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Aspire", "tests-integration\Elastic.Documentation.Aspire\Elastic.Documentation.Aspire.csproj", "{4DFECE72-4A1F-4B58-918E-DCD07B585231}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.ServiceDefaults", "src\Elastic.Documentation.ServiceDefaults\Elastic.Documentation.ServiceDefaults.csproj", "{2A83ED35-B631-4F02-8D4C-15611D0DB72C}"
+EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "api", "api", "{B042CC78-5060-4091-B95A-79C71BA3908A}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Api.Core", "src\api\Elastic.Documentation.Api.Core\Elastic.Documentation.Api.Core.csproj", "{F30B90AD-1A01-4A6F-9699-809FA6875B22}"
@@ -212,6 +226,18 @@ Global
 		{164F55EC-9412-4CD4-81AD-3598B57632A6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{164F55EC-9412-4CD4-81AD-3598B57632A6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{164F55EC-9412-4CD4-81AD-3598B57632A6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A272D3EC-FAAF-4795-A796-302725382AFF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A272D3EC-FAAF-4795-A796-302725382AFF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A272D3EC-FAAF-4795-A796-302725382AFF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A272D3EC-FAAF-4795-A796-302725382AFF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4DFECE72-4A1F-4B58-918E-DCD07B585231}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4DFECE72-4A1F-4B58-918E-DCD07B585231}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4DFECE72-4A1F-4B58-918E-DCD07B585231}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4DFECE72-4A1F-4B58-918E-DCD07B585231}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2A83ED35-B631-4F02-8D4C-15611D0DB72C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2A83ED35-B631-4F02-8D4C-15611D0DB72C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2A83ED35-B631-4F02-8D4C-15611D0DB72C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2A83ED35-B631-4F02-8D4C-15611D0DB72C}.Release|Any CPU.Build.0 = Release|Any CPU
 		{F30B90AD-1A01-4A6F-9699-809FA6875B22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{F30B90AD-1A01-4A6F-9699-809FA6875B22}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F30B90AD-1A01-4A6F-9699-809FA6875B22}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -254,6 +280,9 @@ Global
 		{89B83007-71E6-4B57-BA78-2544BFA476DB} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{111E7029-BB29-4039-9B45-04776798A8DD} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{164F55EC-9412-4CD4-81AD-3598B57632A6} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
+		{A272D3EC-FAAF-4795-A796-302725382AFF} = {BCAD38D5-6C83-46E2-8398-4BE463931098}
+		{4DFECE72-4A1F-4B58-918E-DCD07B585231} = {BCAD38D5-6C83-46E2-8398-4BE463931098}
+		{2A83ED35-B631-4F02-8D4C-15611D0DB72C} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{B042CC78-5060-4091-B95A-79C71BA3908A} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{F30B90AD-1A01-4A6F-9699-809FA6875B22} = {B042CC78-5060-4091-B95A-79C71BA3908A}
 		{AE3FC78E-167F-4B6E-88EC-84743EB748B7} = {B042CC78-5060-4091-B95A-79C71BA3908A}

src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs

@@ -4,7 +4,6 @@
 
 using System.Text.RegularExpressions;
 using Elastic.Documentation.Extensions;
-using YamlDotNet.RepresentationModel;
 using YamlDotNet.Serialization;
 using YamlStaticContext = Elastic.Documentation.Configuration.Serialization.YamlStaticContext;
 
@@ -13,9 +12,9 @@ namespace Elastic.Documentation.Configuration.Assembler;
 public record AssemblyConfiguration
 {
 	public static AssemblyConfiguration Create(ConfigurationFileProvider provider) =>
-		Deserialize(provider.AssemblerFile.ReadToEnd());
+		Deserialize(provider.AssemblerFile.ReadToEnd(), skipPrivateRepositories: provider.SkipPrivateRepositories);
 
-	public static AssemblyConfiguration Deserialize(string yaml)
+	public static AssemblyConfiguration Deserialize(string yaml, bool skipPrivateRepositories = false)
 	{
 		var input = new StringReader(yaml);
 
@@ -28,13 +27,28 @@ public static AssemblyConfiguration Deserialize(string yaml)
 			var config = deserializer.Deserialize<AssemblyConfiguration>(input);
 			foreach (var (name, r) in config.ReferenceRepositories)
 			{
+				if (name == "cloud")
+				{
+				}
 				var repository = RepositoryDefaults(r, name);
 				config.ReferenceRepositories[name] = repository;
 			}
+			var privateRepositories = config.ReferenceRepositories.Where(r => r.Value.Private).ToList();
+			foreach (var (name, _) in privateRepositories)
+			{
+				if (skipPrivateRepositories)
+					_ = config.ReferenceRepositories.Remove(name);
+			}
 
 			foreach (var (name, env) in config.Environments)
 				env.Name = name;
 			config.Narrative = RepositoryDefaults(config.Narrative, NarrativeRepository.RepositoryName);
+
+			config.AvailableRepositories = config.ReferenceRepositories.Values
+				.Where(r => !r.Skip)
+				.Concat([config.Narrative]).ToDictionary(kvp => kvp.Name, kvp => kvp);
+
+			config.PrivateRepositories = privateRepositories.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
 			return config;
 		}
 		catch (Exception e)
@@ -80,6 +94,15 @@ private static TRepository RepositoryDefaults<TRepository>(TRepository r, string
 	[YamlMember(Alias = "references")]
 	public Dictionary<string, Repository> ReferenceRepositories { get; set; } = [];
 
+	/// All available repositories, combines <see cref="ReferenceRepositories"/> and <see cref="Narrative"/> and will filter private repositories if `skip-private-repositories`
+	/// is specified
+	[YamlIgnore]
+	public IReadOnlyDictionary<string, Repository> AvailableRepositories { get; private set; } = new Dictionary<string, Repository>();
+
+	/// Repositories marked as private, these are listed under <see cref="AvailableRepositories"/> if `--skip-private-repositories` is not specified
+	[YamlIgnore]
+	public IReadOnlyDictionary<string, Repository> PrivateRepositories { get; private set; } = new Dictionary<string, Repository>();
+
 	[YamlMember(Alias = "environments")]
 	public Dictionary<string, PublishEnvironment> Environments { get; set; } = [];
 

src/Elastic.Documentation.Configuration/Assembler/Repository.cs

@@ -48,6 +48,9 @@ public record Repository
 	[YamlMember(Alias = "sparse_paths")]
 	public string[] SparsePaths { get; set; } = ["docs"];
 
+	[YamlMember(Alias = "private")]
+	public bool Private { get; set; }
+
 	public string GetBranch(ContentSource contentSource) => contentSource switch
 	{
 		ContentSource.Current => GitReferenceCurrent,