Add project template build tests + CG reporting

eng/pipelines/templates/BuildAndTest.yml

@@ -56,7 +56,25 @@ steps:
               --settings $(Build.SourcesDirectory)/eng/CodeCoverage.config
               --output ${{ parameters.repoTestResultsPath }}/$(Agent.JobName)_CodeCoverageResults/$(Agent.JobName)_cobertura.xml
               "${{ parameters.buildScript }} -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)"
-      displayName: Run tests
+      displayName: Run unit tests
+
+  - script: ${{ parameters.buildScript }}
+            -pack
+            -configuration ${{ parameters.buildConfig }}
+            -warnAsError 1
+            /bl:${{ parameters.repoLogPath }}/pack.binlog
+            /p:Restore=false /p:Build=false
+            $(_OfficialBuildIdArgs)
+    displayName: Pack
+
+  - ${{ if ne(parameters.skipTests, 'true') }}:
+    - script: ${{ parameters.buildScript }}
+              -integrationTest
+              -configuration ${{ parameters.buildConfig }}
+              -warnAsError 1
+              /bl:${{ parameters.repoLogPath }}/integration_tests.binlog
+              $(_OfficialBuildIdArgs)
+      displayName: Run integration tests
 
     - pwsh: |
         $SourcesDirectory = '$(Build.SourcesDirectory)';
@@ -151,12 +169,11 @@ steps:
       displayName: Build Azure DevOps plugin
 
     - script: ${{ parameters.buildScript }}
-              -pack
               -sign $(_SignArgs)
               -publish $(_PublishArgs)
               -configuration ${{ parameters.buildConfig }}
               -warnAsError 1
-              /bl:${{ parameters.repoLogPath }}/pack.binlog
+              /bl:${{ parameters.repoLogPath }}/publish.binlog
               /p:Restore=false /p:Build=false
               $(_OfficialBuildIdArgs)
-      displayName: Pack, sign, and publish
+      displayName: Sign and publish

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs

@@ -0,0 +1,76 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public class AIChatWebExecutionTests : TemplateExecutionTestBase<AIChatWebExecutionTests>, ITemplateExecutionTestConfigurationProvider
+{
+    public AIChatWebExecutionTests(TemplateExecutionTestFixture fixture, ITestOutputHelper outputHelper)
+        : base(fixture, outputHelper)
+    {
+    }
+
+    public static TemplateExecutionTestConfiguration Configuration { get; } = new()
+    {
+        TemplatePackageName = "Microsoft.Extensions.AI.Templates",
+        TestOutputFolderPrefix = "AIChatWeb"
+    };
+
+    [Theory]
+    [InlineData("AzureApp",
+        "--provider", "azureopenai",
+        "--vector-store", "azureaisearch")]
+    [InlineData("AzureAppManagedIdentity",
+        "--provider", "azureopenai",
+        "--vector-store", "azureaisearch",
+        "--managed-identity")]
+    [InlineData("GitHubModelsWithLocalStore",
+        "--provider", "githubmodels",
+        "--vector-store", "local")]
+    [InlineData("OllamaWithLocalStore",
+        "--provider", "ollama",
+        "--vector-store", "local")]
+    [InlineData("OpenAIWithLocalStore",
+        "--provider", "openai",
+        "--vector-store", "local")]
+    public async Task CreateRestoreAndBuild_BasicTemplate(string projectName, params string[] args)
+    {
+        var project = await Fixture.CreateProjectAsync(
+            templateName: "aichatweb",
+            projectName,
+            args);
+
+        await Fixture.RestoreProjectAsync(project);
+        await Fixture.BuildProjectAsync(project);
+    }
+
+    [Theory]
+    [InlineData("AzureApp",
+        "--provider", "azureopenai",
+        "--vector-store", "azureaisearch")]
+    [InlineData("GitHubModelsWithQdrant",
+        "--provider", "githubmodels",
+        "--vector-store", "qdrant")]
+    [InlineData("OllamaWithLocalStore",
+        "--provider", "ollama",
+        "--vector-store", "local")]
+    [InlineData("OpenAIWithLocalStore",
+        "--provider", "ollama",
+        "--vector-store", "local")]
+    public async Task CreateRestoreAndBuild_AspireTemplate(string projectName, params string[] args)
+    {
+        var project = await Fixture.CreateProjectAsync(
+            templateName: "aichatweb",
+            projectName,
+            args: ["--aspire", .. args]);
+
+        project.StartupProjectRelativePath = $"{projectName}.AppHost";
+
+        await Fixture.RestoreProjectAsync(project);
+        await Fixture.BuildProjectAsync(project);
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs → test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebSnapshotTests.cs

@@ -6,17 +6,16 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using Microsoft.Extensions.AI.Templates.IntegrationTests;
 using Microsoft.Extensions.AI.Templates.Tests;
 using Microsoft.Extensions.Logging;
 using Microsoft.TemplateEngine.Authoring.TemplateVerifier;
 using Microsoft.TemplateEngine.TestHelper;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Microsoft.Extensions.AI.Templates.InegrationTests;
+namespace Microsoft.Extensions.AI.Templates.Tests;
 
-public class AichatwebTemplatesTests : TestBase
+public class AIChatWebSnapshotTests
 {
     // Keep the exclude patterns below in sync with those in Microsoft.Extensions.AI.Templates.csproj.
     private static readonly string[] _verificationExcludePatterns = [
@@ -36,7 +35,7 @@ public class AichatwebTemplatesTests : TestBase
 
     private readonly ILogger _log;
 
-    public AichatwebTemplatesTests(ITestOutputHelper log)
+    public AIChatWebSnapshotTests(ITestOutputHelper log)
     {
 #pragma warning disable CA2000 // Dispose objects before losing scope
         _log = new XunitLoggerProvider(log).CreateLogger("TestRun");
@@ -67,7 +66,7 @@ private async Task TestTemplateCoreAsync(string scenarioName, IEnumerable<string
         string templateShortName = "aichatweb";
 
         // Get the template location
-        string templateLocation = Path.Combine(TemplateFeedLocation, "Microsoft.Extensions.AI.Templates", "src", "ChatWithCustomData");
+        string templateLocation = Path.Combine(WellKnownPaths.TemplateFeedLocation, "Microsoft.Extensions.AI.Templates", "src", "ChatWithCustomData");
 
         var verificationExcludePatterns = Path.DirectorySeparatorChar is '/'
             ? _verificationExcludePatterns

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public class DotNetCommand : TestCommand
+{
+    public DotNetCommand(params ReadOnlySpan<string> args)
+    {
+        FileName = WellKnownPaths.RepoDotNetExePath;
+
+        foreach (var arg in args)
+        {
+            Arguments.Add(arg);
+        }
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs

@@ -0,0 +1,39 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public sealed class DotNetNewCommand : DotNetCommand
+{
+    private bool _customHiveSpecified;
+
+    public DotNetNewCommand(params ReadOnlySpan<string> args)
+        : base(["new", .. args])
+    {
+    }
+
+    public DotNetNewCommand WithCustomHive(string path)
+    {
+        Arguments.Add("--debug:custom-hive");
+        Arguments.Add(path);
+        _customHiveSpecified = true;
+        return this;
+    }
+
+    public override Task<TestCommandResult> ExecuteAsync(ITestOutputHelper outputHelper)
+    {
+        if (!_customHiveSpecified)
+        {
+            // If this exception starts getting thrown in cases where a custom hive is
+            // legitimately undesirable, we can add a new 'WithoutCustomHive()' method that
+            // just sets '_customHiveSpecified' to 'true'.
+            throw new InvalidOperationException($"A {nameof(DotNetNewCommand)} should specify a custom hive with '{nameof(WithCustomHive)}()'.");
+        }
+
+        return base.ExecuteAsync(outputHelper);
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateExecutionTestConfigurationProvider.cs

@@ -0,0 +1,9 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public interface ITemplateExecutionTestConfigurationProvider
+{
+    static abstract TemplateExecutionTestConfiguration Configuration { get; }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs

@@ -0,0 +1,27 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public sealed class MessageSinkTestOutputHelper : ITestOutputHelper
+{
+    private readonly IMessageSink _messageSink;
+
+    public MessageSinkTestOutputHelper(IMessageSink messageSink)
+    {
+        _messageSink = messageSink;
+    }
+
+    public void WriteLine(string message)
+    {
+        _messageSink.OnMessage(new DiagnosticMessage(message));
+    }
+
+    public void WriteLine(string format, params object[] args)
+    {
+        _messageSink.OnMessage(new DiagnosticMessage(format, args));
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Diagnostics;
+
+public static class ProcessExtensions
+{
+    public static bool TryGetHasExited(this Process process)
+    {
+        try
+        {
+            return process.HasExited;
+        }
+        catch (InvalidOperationException ex) when (ex.Message.Contains("No process is associated with this object"))
+        {
+            return true;
+        }
+    }
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs

@@ -0,0 +1,37 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+public sealed class Project(string rootPath, string name)
+{
+    private string? _startupProjectRelativePath;
+    private string? _startupProjectFullPath;
+
+    public string RootPath => rootPath;
+
+    public string Name => name;
+
+    public string? StartupProjectRelativePath
+    {
+        get => _startupProjectRelativePath;
+        set
+        {
+            if (value is null)
+            {
+                _startupProjectRelativePath = null;
+                _startupProjectFullPath = null;
+            }
+            else if (!string.Equals(value, _startupProjectRelativePath, StringComparison.Ordinal))
+            {
+                _startupProjectRelativePath = value;
+                _startupProjectFullPath = Path.Combine(rootPath, _startupProjectRelativePath);
+            }
+        }
+    }
+
+    public string StartupProjectFullPath => _startupProjectFullPath ?? rootPath;
+}

test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs

@@ -0,0 +1,66 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.Extensions.AI.Templates.Tests;
+
+/// <summary>
+/// Represents a test that executes a project template (create, restore, build, and run).
+/// </summary>
+/// <typeparam name="TConfiguration">A type defining global test execution settings.</typeparam>
+[Collection(TemplateExecutionTestCollection.Name)]
+public abstract class TemplateExecutionTestBase<TConfiguration> : IClassFixture<TemplateExecutionTestBase<TConfiguration>.TemplateExecutionTestFixture>, IDisposable
+    where TConfiguration : ITemplateExecutionTestConfigurationProvider
+{
+    private bool _disposed;
+
+    protected TemplateExecutionTestFixture Fixture { get; }
+
+    protected ITestOutputHelper OutputHelper { get; }
+
+    protected TemplateExecutionTestBase(TemplateExecutionTestFixture fixture, ITestOutputHelper outputHelper)
+    {
+        Fixture = fixture;
+        Fixture.SetCurrentTestOutputHelper(outputHelper);
+
+        OutputHelper = outputHelper;
+    }
+
+    protected virtual void Dispose(bool disposing)
+    {
+        if (_disposed)
+        {
+            return;
+        }
+
+        _disposed = true;
+
+        if (disposing)
+        {
+            Fixture.SetCurrentTestOutputHelper(null);
+        }
+    }
+
+    public void Dispose()
+    {
+        Dispose(true);
+        GC.SuppressFinalize(this);
+    }
+
+    /// <summary>
+    /// An implementation of <see cref="TemplateExecutionTestClassFixtureBase"/> that utilizes
+    /// the configuration provided by <c>TConfiguration</c>.
+    /// </summary>
+    /// <remarks>
+    /// The configuration has to be provided "statically" because the lifetime of the class fixture
+    /// is longer than the lifetime of each test class instance. In other words, it's not possible for
+    /// an instance of the test class to configure to the fixture directly, as the test class instance
+    /// gets created after the fixture has a chance to perform global setup.
+    /// </remarks>
+    /// <param name="messageSink">The <see cref="IMessageSink"/>The <see cref="IMessageSink"/>.</param>
+    public sealed class TemplateExecutionTestFixture(IMessageSink messageSink)
+        : TemplateExecutionTestClassFixtureBase(TConfiguration.Configuration, messageSink);
+}