From 8cfa76d084f42252c770f31dfcf8ba9aa4ba10c3 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Tue, 29 Apr 2025 17:28:44 -0700 Subject: [PATCH 01/17] Started on template build tests. --- .../AichatwebTemplatesTests.cs | 3 +- .../BuildWebTemplateTests.cs | 49 +++++++ .../Infrastructure/DotNetCommand.cs | 12 ++ .../Infrastructure/DotNetNewCommand.cs | 44 ++++++ .../ITemplateConfigurationProvider.cs | 9 ++ .../MessageSinkTestOutputHelper.cs | 27 ++++ .../Infrastructure/ProcessExtensions.cs | 19 +++ .../Infrastructure/Project.cs | 13 ++ .../Infrastructure/TemplateConfiguration.cs | 11 ++ .../Infrastructure/TemplateTestBase.cs | 35 +++++ .../Infrastructure/TemplateTestFixture.cs | 129 ++++++++++++++++++ .../Infrastructure/TestCommand.cs | 125 +++++++++++++++++ .../Infrastructure/TestCommandExtensions.cs | 30 ++++ .../Infrastructure/TestCommandResult.cs | 18 +++ .../TestCommandResultExtensions.cs | 22 +++ .../TemplateSandbox/Directory.Build.props | 6 + .../TemplateSandbox/Directory.Build.targets | 6 + .../TemplateSandbox/README.md | 2 + .../nuget.template_install.config | 27 ++++ .../nuget.template_test.config | 28 ++++ .../TestBase.cs | 35 ++++- .../xunit.runner.json | 5 + 22 files changed, 652 insertions(+), 3 deletions(-) create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/BuildWebTemplateTests.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateConfigurationProvider.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateConfiguration.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandExtensions.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResult.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_install.config create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/xunit.runner.json diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs index db142ff68ff..d34de93103b 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs @@ -6,7 +6,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Templates.IntegrationTests; using Microsoft.Extensions.AI.Templates.Tests; using Microsoft.Extensions.Logging; using Microsoft.TemplateEngine.Authoring.TemplateVerifier; @@ -14,7 +13,7 @@ using Xunit; using Xunit.Abstractions; -namespace Microsoft.Extensions.AI.Templates.InegrationTests; +namespace Microsoft.Extensions.AI.Templates.Tests; public class AichatwebTemplatesTests : TestBase { diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/BuildWebTemplateTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/BuildWebTemplateTests.cs new file mode 100644 index 00000000000..7e7f1aefcfe --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/BuildWebTemplateTests.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public class BuildWebTemplateTests : TemplateTestBase, ITemplateConfigurationProvider +{ + public BuildWebTemplateTests(Fixture fixture, ITestOutputHelper outputHelper) + : base(fixture, outputHelper) + { + } + + public static TemplateConfiguration Configuration { get; } = new() + { + TemplatePackageName = "Microsoft.Extensions.AI.Templates", + TestOutputFolderPrefix = "BuildWebTemplate" + }; + + [Fact] + public async Task BuildBasicTemplate() + { + var fixture = GetFixture(); + var project = await fixture.CreateProjectAsync( + templateName: "aichatweb", + projectName: "BasicApp"); + + await fixture.RestoreProjectAsync(project); + await fixture.BuildProjectAsync(project); + } + + [Fact] + public async Task BuildAspireTemplate() + { + var fixture = GetFixture(); + var project = await fixture.CreateProjectAsync( + templateName: "aichatweb", + projectName: "AspireApp", + args: ["--aspire"]); + + project.StartupProjectRelativePath = "AspireApp.AppHost"; + + await fixture.RestoreProjectAsync(project); + await fixture.BuildProjectAsync(project); + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs new file mode 100644 index 00000000000..c6964bef23a --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public class DotNetCommand : TestCommand +{ + public DotNetCommand() + { + FileName = TestBase.RepoDotNetExePath; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs new file mode 100644 index 00000000000..22715263019 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading.Tasks; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class DotNetNewCommand : DotNetCommand +{ + private bool _customHiveSpecified; + + public DotNetNewCommand(params ReadOnlySpan args) + { + Arguments.Add("new"); + + foreach (var arg in args) + { + Arguments.Add(arg); + } + } + + public DotNetNewCommand WithCustomHive(string path) + { + Arguments.Add("--debug:custom-hive"); + Arguments.Add(path); + _customHiveSpecified = true; + return this; + } + + public override Task ExecuteAsync(ITestOutputHelper outputHelper) + { + if (!_customHiveSpecified) + { + // If this exception starts getting thrown in cases where a custom hive is + // legitimately undesirable, we can add a new 'WithoutCustomHive()' method that + // just sets '_customHiveSpecified' to 'true'. + throw new InvalidOperationException($"A {nameof(DotNetNewCommand)} should specify a custom hive with '{nameof(WithCustomHive)}()'."); + } + + return base.ExecuteAsync(outputHelper); + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateConfigurationProvider.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateConfigurationProvider.cs new file mode 100644 index 00000000000..d4121442a14 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateConfigurationProvider.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public interface ITemplateConfigurationProvider +{ + static abstract TemplateConfiguration Configuration { get; } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs new file mode 100644 index 00000000000..d81c1f7c434 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class MessageSinkTestOutputHelper : ITestOutputHelper +{ + private readonly IMessageSink _messageSink; + + public MessageSinkTestOutputHelper(IMessageSink messageSink) + { + _messageSink = messageSink; + } + + public void WriteLine(string message) + { + _messageSink.OnMessage(new DiagnosticMessage(message)); + } + + public void WriteLine(string format, params object[] args) + { + _messageSink.OnMessage(new DiagnosticMessage(format, args)); + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs new file mode 100644 index 00000000000..a20d390794d --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Diagnostics; + +public static class ProcessExtensions +{ + public static bool TryGetHasExited(this Process process) + { + try + { + return process.HasExited; + } + catch (InvalidOperationException ex) when (ex.Message.Contains("No process is associated with this object")) + { + return true; + } + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs new file mode 100644 index 00000000000..f1c4a47e560 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class Project(string rootPath, string name) +{ + public string RootPath => rootPath; + + public string Name => name; + + public string? StartupProjectRelativePath { get; set; } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateConfiguration.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateConfiguration.cs new file mode 100644 index 00000000000..3419b85ec86 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateConfiguration.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class TemplateConfiguration +{ + public required string TemplatePackageName { get; init; } + + public required string TestOutputFolderPrefix { get; init; } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs new file mode 100644 index 00000000000..4e4266d8538 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public abstract class TemplateTestBase : IClassFixture.Fixture>, IDisposable + where TSelf : TemplateTestBase, ITemplateConfigurationProvider +{ + private readonly Fixture _fixture; + + protected ITestOutputHelper OutputHelper { get; } + + protected TemplateTestBase(Fixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + fixture.SetCurrentTestOutputHelper(outputHelper); + + OutputHelper = outputHelper; + } + + public Fixture GetFixture() => _fixture; + + public void Dispose() + { + GC.SuppressFinalize(this); + _fixture.SetCurrentTestOutputHelper(null); + } + + public sealed class Fixture(IMessageSink messageSink) : TemplateTestFixture(TSelf.Configuration, messageSink); +} + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs new file mode 100644 index 00000000000..603b29bea15 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs @@ -0,0 +1,129 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public abstract class TemplateTestFixture : IAsyncLifetime +{ + private readonly TemplateConfiguration _configuration; + private readonly string _templateInstallNuGetConfigPath; + private readonly string _templateTestNuGetConfigPath; + private readonly string _localShippingPackagesPath; + private readonly string _templateTestOutputPath; + private readonly string _nuGetPackagesPath; + private readonly string _customHivePath; + private readonly MessageSinkTestOutputHelper _fallbackGlobalOutputHelper; + private ITestOutputHelper? _currentTestOutputHelper; + + private ITestOutputHelper OutputHelper => _currentTestOutputHelper ?? _fallbackGlobalOutputHelper; + + protected TemplateTestFixture(TemplateConfiguration configuration, IMessageSink messageSink) + { + _configuration = configuration; + _fallbackGlobalOutputHelper = new(messageSink); + + var templateSandboxRoot = Path.Combine(TestBase.TestProjectRoot, "TemplateSandbox"); + _templateInstallNuGetConfigPath = Path.Combine(templateSandboxRoot, "nuget.template_install.config"); + _templateTestNuGetConfigPath = Path.Combine(templateSandboxRoot, "nuget.template_test.config"); + + const string BuildConfigurationFolder = +#if DEBUG + "Debug"; +#else + "Release"; +#endif + _localShippingPackagesPath = Path.Combine(TestBase.CodeBaseRoot, "artifacts", "packages", BuildConfigurationFolder, "Shipping"); + + var outputFolderName = GetRandomizedFileName(prefix: _configuration.TestOutputFolderPrefix); + _templateTestOutputPath = Path.Combine(TestBase.TestProjectRoot, "TemplateSandbox", "output", outputFolderName); + _nuGetPackagesPath = Path.Combine(_templateTestOutputPath, "packages"); + _customHivePath = Path.Combine(_templateTestOutputPath, "hive"); + } + + private static string GetRandomizedFileName(string prefix) + => prefix + "_" + Guid.NewGuid().ToString("N").Substring(0, 10).ToLowerInvariant(); + + public async Task InitializeAsync() + { + Directory.CreateDirectory(_templateTestOutputPath); + + await InstallTemplatesAsync(); + + async Task InstallTemplatesAsync() + { + var installSandboxPath = Path.Combine(_templateTestOutputPath, "install"); + Directory.CreateDirectory(installSandboxPath); + + var installNuGetConfigPath = Path.Combine(installSandboxPath, "nuget.config"); + File.Copy(_templateInstallNuGetConfigPath, installNuGetConfigPath); + + var installResult = await new DotNetNewCommand("install", _configuration.TemplatePackageName) + .WithWorkingDirectory(installSandboxPath) + .WithEnvironmentVariable("LOCAL_SHIPPING_PATH", _localShippingPackagesPath) + .WithEnvironmentVariable("NUGET_PACKAGES", _nuGetPackagesPath) + .WithCustomHive(_customHivePath) + .ExecuteAsync(OutputHelper); + + installResult + .AssertZeroExitCode() + .AssertEmptyStandardError(); + } + } + + public async Task CreateProjectAsync(string templateName, string projectName, params string[] args) + { + var outputFolderName = GetRandomizedFileName(projectName); + var outputFolderPath = Path.Combine(_templateTestOutputPath, outputFolderName); + + ReadOnlySpan dotNetNewCommandArgs = [ + templateName, + "-o", outputFolderPath, + "-n", projectName, + "--no-update-check", + .. args + ]; + + var newProjectResult = await new DotNetNewCommand(dotNetNewCommandArgs) + .WithWorkingDirectory(_templateTestOutputPath) + .WithCustomHive(_customHivePath) + .ExecuteAsync(OutputHelper); + + newProjectResult + .AssertZeroExitCode() + .AssertEmptyStandardError(); + + var templateNuGetConfigPath = Path.Combine(outputFolderPath, "nuget.config"); + File.Copy(_templateTestNuGetConfigPath, templateNuGetConfigPath); + + return new Project(outputFolderPath, projectName); + } + + public Task RestoreProjectAsync(Project project) + { + // TODO + return Task.CompletedTask; + } + + public Task BuildProjectAsync(Project project) + { + // TODO + return Task.CompletedTask; + } + + public void SetCurrentTestOutputHelper(ITestOutputHelper? outputHelper) + { + _currentTestOutputHelper = outputHelper; + } + + public Task DisposeAsync() + { + Directory.Delete(_templateTestOutputPath, recursive: true); + return Task.CompletedTask; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs new file mode 100644 index 00000000000..b24d94d1323 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs @@ -0,0 +1,125 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public abstract class TestCommand +{ + public string? FileName { get; set; } + + public string? WorkingDirectory { get; set; } + + public TimeSpan? Timeout { get; set; } + + public List Arguments { get; } = []; + + public Dictionary EnvironmentVariables = []; + + public virtual async Task ExecuteAsync(ITestOutputHelper outputHelper) + { + if (string.IsNullOrEmpty(FileName)) + { + throw new InvalidOperationException($"The {nameof(TestCommand)} did not specify an executable file name."); + } + + var processStartInfo = new ProcessStartInfo(FileName, Arguments) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + UseShellExecute = false, + }; + + if (WorkingDirectory is not null) + { + processStartInfo.WorkingDirectory = WorkingDirectory; + } + + foreach (var (key, value) in EnvironmentVariables) + { + processStartInfo.EnvironmentVariables[key] = value; + } + + var exitedTcs = new TaskCompletionSource(); + var standardOutputBuilder = new StringBuilder(); + var standardErrorBuilder = new StringBuilder(); + + using var process = new Process + { + StartInfo = processStartInfo, + }; + + process.EnableRaisingEvents = true; + process.OutputDataReceived += MakeOnDataReceivedHandler(standardOutputBuilder); + process.ErrorDataReceived += MakeOnDataReceivedHandler(standardErrorBuilder); + process.Exited += (sender, args) => + { + exitedTcs.SetResult(); + }; + + DataReceivedEventHandler MakeOnDataReceivedHandler(StringBuilder outputBuilder) => (sender, args) => + { + if (args.Data is null) + { + return; + } + + lock (outputBuilder) + { + outputBuilder.AppendLine(args.Data); + } + + lock (outputHelper) + { + outputHelper.WriteLine(args.Data); + } + }; + + outputHelper.WriteLine($"Executing '{processStartInfo.FileName} {processStartInfo.Arguments}' in working directory '{processStartInfo.WorkingDirectory}'"); + + using var timeoutCts = new CancellationTokenSource(); + if (Timeout is { } timeout) + { + timeoutCts.CancelAfter(timeout); + } + + var startTimestamp = Stopwatch.GetTimestamp(); + + try + { + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await exitedTcs.Task.WaitAsync(timeoutCts.Token).ConfigureAwait(false); + await process.WaitForExitAsync(timeoutCts.Token).ConfigureAwait(false); + + var elapsedTime = Stopwatch.GetElapsedTime(startTimestamp); + outputHelper.WriteLine($"Process ran for {elapsedTime} seconds."); + + return new(standardOutputBuilder, standardErrorBuilder, process.ExitCode); + } + catch (Exception ex) + { + outputHelper.WriteLine($"An exception occurred: {ex}"); + throw; + } + finally + { + if (!process.TryGetHasExited()) + { + var elapsedTime = Stopwatch.GetElapsedTime(startTimestamp); + outputHelper.WriteLine($"The process has been running for {elapsedTime} seconds. Terminating the process."); + process.Kill(entireProcessTree: true); + } + } + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandExtensions.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandExtensions.cs new file mode 100644 index 00000000000..957c0efbb79 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandExtensions.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public static class TestCommandExtensions +{ + public static TCommand WithEnvironmentVariable(this TCommand command, string name, string value) + where TCommand : TestCommand + { + command.EnvironmentVariables[name] = value; + return command; + } + + public static TCommand WithWorkingDirectory(this TCommand command, string workingDirectory) + where TCommand : TestCommand + { + command.WorkingDirectory = workingDirectory; + return command; + } + + public static TCommand WithTimeout(this TCommand command, TimeSpan timeout) + where TCommand : TestCommand + { + command.Timeout = timeout; + return command; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResult.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResult.cs new file mode 100644 index 00000000000..09d09d50a1c --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResult.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class TestCommandResult(StringBuilder standardOutputBuilder, StringBuilder standardErrorBuilder, int exitCode) +{ + private string? _standardOutput; + private string? _standardError; + + public string StandardOutput => _standardOutput ??= standardOutputBuilder.ToString(); + + public string StandardError => _standardError ??= standardErrorBuilder.ToString(); + + public int ExitCode => exitCode; +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs new file mode 100644 index 00000000000..d31dbd136ba --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public static class TestCommandResultExtensions +{ + public static TestCommandResult AssertZeroExitCode(this TestCommandResult result) + { + Assert.True(result.ExitCode == 0, $"Expected an exit code of zero, got {result.ExitCode}"); + return result; + } + + public static TestCommandResult AssertEmptyStandardError(this TestCommandResult result) + { + var standardError = result.StandardError; + Assert.True(string.IsNullOrWhiteSpace(standardError), $"Standard error output was unexpectedly non-empty:\n{standardError}"); + return result; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props new file mode 100644 index 00000000000..4508b7d4139 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props @@ -0,0 +1,6 @@ + + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets new file mode 100644 index 00000000000..f0643c8ea7b --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets @@ -0,0 +1,6 @@ + + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md new file mode 100644 index 00000000000..a9024a24011 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md @@ -0,0 +1,2 @@ +This folder exists to serve as an isolated environment for templates to run in. +Generated projects will get placed here. diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_install.config b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_install.config new file mode 100644 index 00000000000..4d77494baee --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_install.config @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config new file mode 100644 index 00000000000..f1ff7f2ddcb --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs index 59946ab283f..6095e9256e4 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs @@ -3,8 +3,10 @@ using System; using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; -namespace Microsoft.Extensions.AI.Templates.IntegrationTests; +namespace Microsoft.Extensions.AI.Templates.Tests; /// /// The class contains the utils for unit and integration tests. @@ -13,6 +15,10 @@ public abstract class TestBase { internal static string CodeBaseRoot { get; } = GetCodeBaseRoot(); + internal static string RepoDotNetExePath { get; } = GetRepoDotNetExePath(); + + internal static string TestProjectRoot { get; } = GetTestProjectRoot(); + internal static string TemplateFeedLocation { get; } = Path.Combine(CodeBaseRoot, "src", "ProjectTemplates"); private static string GetCodeBaseRoot() @@ -34,4 +40,31 @@ private static string GetCodeBaseRoot() throw new InvalidOperationException("Failed to establish root of the repository"); } + + private static string GetTestProjectRoot([CallerFilePath] string callerFilePath = "") + { + if (Path.GetDirectoryName(callerFilePath) is not { Length: > 0 } testProjectRoot) + { + throw new InvalidOperationException("Could not determine the root of the test project."); + } + + return testProjectRoot; + } + + private static string GetRepoDotNetExePath() + { + var codeBaseRoot = CodeBaseRoot; + var dotNetExeName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? "dotnet.exe" + : "dotnet"; + + var dotNetExePath = Path.Combine(codeBaseRoot, ".dotnet", dotNetExeName); + + if (!File.Exists(dotNetExePath)) + { + throw new InvalidOperationException($"Expected to find '{dotNetExeName}' at '{dotNetExePath}', but it was not found."); + } + + return dotNetExePath; + } } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/xunit.runner.json b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/xunit.runner.json new file mode 100644 index 00000000000..9faab404ec0 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/xunit.runner.json @@ -0,0 +1,5 @@ +{ + "longRunningTestSeconds": 60, + "diagnosticMessages": true, + "maxParallelThreads": 1 +} From 90572ad1c736924c93a1532b397cd578252107cd Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 11:28:49 -0700 Subject: [PATCH 02/17] Working restore/build tests + helper script for debugging --- .../Infrastructure/DotNetCommand.cs | 9 +- .../Infrastructure/DotNetNewCommand.cs | 7 +- .../Infrastructure/Project.cs | 26 ++++- .../TemplateSandboxCollection.cs | 11 ++ .../Infrastructure/TemplateSandboxFixture.cs | 15 +++ .../Infrastructure/TemplateTestBase.cs | 1 + .../Infrastructure/TemplateTestFixture.cs | 37 +++--- .../TestCommandResultExtensions.cs | 5 + ...osoft.Extensions.AI.Templates.Tests.csproj | 4 +- .../TemplateSandbox/.gitignore | 2 + .../TemplateSandbox/activate.ps1 | 109 ++++++++++++++++++ .../nuget.template_test.config | 4 +- 12 files changed, 200 insertions(+), 30 deletions(-) create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxCollection.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxFixture.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.gitignore create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/activate.ps1 diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs index c6964bef23a..b11513796a1 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs @@ -1,12 +1,19 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; + namespace Microsoft.Extensions.AI.Templates.Tests; public class DotNetCommand : TestCommand { - public DotNetCommand() + public DotNetCommand(params ReadOnlySpan args) { FileName = TestBase.RepoDotNetExePath; + + foreach (var arg in args) + { + Arguments.Add(arg); + } } } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs index 22715263019..cdd6ab73f03 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs @@ -12,13 +12,8 @@ public sealed class DotNetNewCommand : DotNetCommand private bool _customHiveSpecified; public DotNetNewCommand(params ReadOnlySpan args) + : base(["new", .. args]) { - Arguments.Add("new"); - - foreach (var arg in args) - { - Arguments.Add(arg); - } } public DotNetNewCommand WithCustomHive(string path) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs index f1c4a47e560..38ced5b1867 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs @@ -1,13 +1,37 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.IO; + namespace Microsoft.Extensions.AI.Templates.Tests; public sealed class Project(string rootPath, string name) { + private string? _startupProjectRelativePath; + private string? _startupProjectFullPath; + public string RootPath => rootPath; public string Name => name; - public string? StartupProjectRelativePath { get; set; } + public string? StartupProjectRelativePath + { + get => _startupProjectRelativePath; + set + { + if (value is null) + { + _startupProjectRelativePath = null; + _startupProjectFullPath = null; + } + else if (!string.Equals(value, _startupProjectRelativePath, StringComparison.Ordinal)) + { + _startupProjectRelativePath = value; + _startupProjectFullPath = Path.Combine(rootPath, _startupProjectRelativePath); + } + } + } + + public string StartupProjectFullPath => _startupProjectFullPath ?? rootPath; } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxCollection.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxCollection.cs new file mode 100644 index 00000000000..e49082ee398 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxCollection.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.Extensions.AI.Templates.Tests.Infrastructure; + +[CollectionDefinition("Template sandbox")] +public sealed class TemplateSandboxCollection : ICollectionFixture +{ +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxFixture.cs new file mode 100644 index 00000000000..e11a6282455 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxFixture.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class TemplateSandboxFixture +{ + public TemplateSandboxFixture() + { + var templateSandboxOutputRoot = Path.Combine(TestBase.TestProjectRoot, "TemplateSandbox", "output"); + Directory.Delete(templateSandboxOutputRoot, recursive: true); + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs index 4e4266d8538..383a531d69b 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs @@ -7,6 +7,7 @@ namespace Microsoft.Extensions.AI.Templates.Tests; +[Collection("Template sandbox")] public abstract class TemplateTestBase : IClassFixture.Fixture>, IDisposable where TSelf : TemplateTestBase, ITemplateConfigurationProvider { diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs index 603b29bea15..84b83c41a8e 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs @@ -15,8 +15,8 @@ public abstract class TemplateTestFixture : IAsyncLifetime private readonly string _templateInstallNuGetConfigPath; private readonly string _templateTestNuGetConfigPath; private readonly string _localShippingPackagesPath; - private readonly string _templateTestOutputPath; private readonly string _nuGetPackagesPath; + private readonly string _templateTestOutputPath; private readonly string _customHivePath; private readonly MessageSinkTestOutputHelper _fallbackGlobalOutputHelper; private ITestOutputHelper? _currentTestOutputHelper; @@ -40,9 +40,10 @@ protected TemplateTestFixture(TemplateConfiguration configuration, IMessageSink #endif _localShippingPackagesPath = Path.Combine(TestBase.CodeBaseRoot, "artifacts", "packages", BuildConfigurationFolder, "Shipping"); + var templateSandboxOutputRoot = Path.Combine(templateSandboxRoot, "output"); + _nuGetPackagesPath = Path.Combine(templateSandboxOutputRoot, "packages"); var outputFolderName = GetRandomizedFileName(prefix: _configuration.TestOutputFolderPrefix); - _templateTestOutputPath = Path.Combine(TestBase.TestProjectRoot, "TemplateSandbox", "output", outputFolderName); - _nuGetPackagesPath = Path.Combine(_templateTestOutputPath, "packages"); + _templateTestOutputPath = Path.Combine(templateSandboxOutputRoot, outputFolderName); _customHivePath = Path.Combine(_templateTestOutputPath, "hive"); } @@ -69,10 +70,7 @@ async Task InstallTemplatesAsync() .WithEnvironmentVariable("NUGET_PACKAGES", _nuGetPackagesPath) .WithCustomHive(_customHivePath) .ExecuteAsync(OutputHelper); - - installResult - .AssertZeroExitCode() - .AssertEmptyStandardError(); + installResult.AssertSucceeded(); } } @@ -93,10 +91,7 @@ .. args .WithWorkingDirectory(_templateTestOutputPath) .WithCustomHive(_customHivePath) .ExecuteAsync(OutputHelper); - - newProjectResult - .AssertZeroExitCode() - .AssertEmptyStandardError(); + newProjectResult.AssertSucceeded(); var templateNuGetConfigPath = Path.Combine(outputFolderPath, "nuget.config"); File.Copy(_templateTestNuGetConfigPath, templateNuGetConfigPath); @@ -104,16 +99,22 @@ .. args return new Project(outputFolderPath, projectName); } - public Task RestoreProjectAsync(Project project) + public async Task RestoreProjectAsync(Project project) { - // TODO - return Task.CompletedTask; + var restoreResult = await new DotNetCommand("restore") + .WithWorkingDirectory(project.StartupProjectFullPath) + .WithEnvironmentVariable("LOCAL_SHIPPING_PATH", _localShippingPackagesPath) + .WithEnvironmentVariable("NUGET_PACKAGES", _nuGetPackagesPath) + .ExecuteAsync(OutputHelper); + restoreResult.AssertSucceeded(); } - public Task BuildProjectAsync(Project project) + public async Task BuildProjectAsync(Project project) { - // TODO - return Task.CompletedTask; + var buildResult = await new DotNetCommand("build", "--no-restore") + .WithWorkingDirectory(project.StartupProjectFullPath) + .ExecuteAsync(OutputHelper); + buildResult.AssertSucceeded(); } public void SetCurrentTestOutputHelper(ITestOutputHelper? outputHelper) @@ -123,7 +124,7 @@ public void SetCurrentTestOutputHelper(ITestOutputHelper? outputHelper) public Task DisposeAsync() { - Directory.Delete(_templateTestOutputPath, recursive: true); + // Only here to implement IAsyncLifetime. Not currently used. return Task.CompletedTask; } } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs index d31dbd136ba..867cc2303ac 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs @@ -19,4 +19,9 @@ public static TestCommandResult AssertEmptyStandardError(this TestCommandResult Assert.True(string.IsNullOrWhiteSpace(standardError), $"Standard error output was unexpectedly non-empty:\n{standardError}"); return result; } + + public static TestCommandResult AssertSucceeded(this TestCommandResult result) + => result + .AssertZeroExitCode() + .AssertEmptyStandardError(); } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj index 2c1c66e4d3e..cefb4946743 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj @@ -16,7 +16,9 @@ + + + - diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.gitignore b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.gitignore new file mode 100644 index 00000000000..ee80e74117d --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.gitignore @@ -0,0 +1,2 @@ +# Template test output +output/ diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/activate.ps1 b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/activate.ps1 new file mode 100644 index 00000000000..a3dea765dd5 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/activate.ps1 @@ -0,0 +1,109 @@ +# +# This file creates an environment similar to the one that the template tests use. +# This makes it convenient to restore, build, and run projects generated by the template tests +# to debug test failures. +# +# This file must be used by invoking ". .\activate.ps1" from the command line. +# You cannot run it directly. See https://learn.microsoft.com/powershell/module/microsoft.powershell.core/about/about_scripts#script-scope-and-dot-sourcing +# +# To exit from the environment this creates, execute the 'deactivate' function. +# + +[CmdletBinding(PositionalBinding=$false)] +Param( + [string][Alias('c')]$configuration = "Debug" +) + +if ($MyInvocation.CommandOrigin -eq 'runspace') { + $cwd = (Get-Location).Path + $scriptPath = $MyInvocation.MyCommand.Path + $relativePath = [System.IO.Path]::GetRelativePath($cwd, $scriptPath) + Write-Host -f Red "This script cannot be invoked directly." + Write-Host -f Red "To function correctly, this script file must be 'dot sourced' by calling `". .\$relativePath`" (notice the dot at the beginning)." + exit 1 +} + +function deactivate ([switch]$init) { + # reset old environment variables + if (Test-Path variable:_OLD_PATH) { + $env:PATH = $_OLD_PATH + Remove-Item variable:_OLD_PATH + } + + if (test-path function:_old_prompt) { + Set-Item Function:prompt -Value $function:_old_prompt -ea ignore + remove-item function:_old_prompt + } + + Remove-Item env:DOTNET_ROOT -ea Ignore + Remove-Item 'env:DOTNET_ROOT(x86)' -ea Ignore + Remove-Item env:DOTNET_MULTILEVEL_LOOKUP -ea Ignore + Remove-Item env:NUGET_PACKAGES -ea Ignore + Remove-Item env:LOCAL_SHIPPING_PATH -ea Ignore + if (-not $init) { + # Remove functions defined + Remove-Item function:deactivate + Remove-Item function:Get-RepoRoot + } +} + +# Cleanup the environment +deactivate -init + +function Get-RepoRoot { + $directory = $PSScriptRoot + + while ($directory) { + $gitPath = Join-Path $directory ".git" + + if (Test-Path $gitPath) { + return $directory + } + + $parent = Split-Path $directory -Parent + if ($parent -eq $directory) { + # We've reached the filesystem root + break + } + + $directory = $parent + } + + throw "Failed to establish root of the repository" +} + +# Find the root of the repository +$repoRoot = Get-RepoRoot + +$_OLD_PATH = $env:PATH +# Tell dotnet where to find itself +$env:DOTNET_ROOT = "$repoRoot\.dotnet" +${env:DOTNET_ROOT(x86)} = "$repoRoot\.dotnet\x86" +# Tell dotnet not to look beyond the DOTNET_ROOT folder for more dotnet things +$env:DOTNET_MULTILEVEL_LOOKUP = 0 +# Put dotnet first on PATH +$env:PATH = "${env:DOTNET_ROOT};${env:PATH}" +# Set NUGET_PACKAGES and LOCAL_SHIPPING_PATH +$env:NUGET_PACKAGES = "$PSScriptRoot\output\packages" +$env:LOCAL_SHIPPING_PATH = "$repoRoot\artifacts\packages\$configuration\Shipping\" + +# Set the shell prompt +$function:_old_prompt = $function:prompt +function dotnet_prompt { + # Add a prefix to the current prompt, but don't discard it. + write-host "($( split-path $PSScriptRoot -leaf )) " -nonewline + & $function:_old_prompt +} + +Set-Item Function:prompt -Value $function:dotnet_prompt -ea ignore + +Write-Host -f Magenta "Enabled the template testing environment. Execute 'deactivate' to exit." +Write-Host -f Magenta "Using the '$configuration' configuration. Use the -c option to specify a different configuration." +if (-not (Test-Path "${env:DOTNET_ROOT}\dotnet.exe")) { + Write-Host -f Yellow ".NET Core has not been installed yet. Run $repoRoot\build.cmd -restore to install it." +} +else { + Write-Host "dotnet = ${env:DOTNET_ROOT}\dotnet.exe" +} +Write-Host "NUGET_PACKAGES = ${env:NUGET_PACKAGES}" +Write-Host "LOCAL_SHIPPING_PATH = ${env:LOCAL_SHIPPING_PATH}" diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config index f1ff7f2ddcb..c98b9777011 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config @@ -13,9 +13,7 @@ - - - + From 504d6c36b310f0660a85c8e51702054784fa1363 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 13:04:31 -0700 Subject: [PATCH 03/17] Clean up --- .../AIChatWebExecutionTests.cs | 51 ++++++++++++ ...atesTests.cs => AIChatWebSnapshotTests.cs} | 6 +- .../BuildWebTemplateTests.cs | 49 ----------- .../Infrastructure/DotNetCommand.cs | 2 +- ...lateExecutionTestConfigurationProvider.cs} | 4 +- .../TemplateExecutionTestBase.cs | 52 ++++++++++++ ... TemplateExecutionTestClassFixtureBase.cs} | 56 ++++++------- .../TemplateExecutionTestCollection.cs | 12 +++ .../TemplateExecutionTestCollectionFixture.cs | 14 ++++ ... => TemplateExecutionTestConfiguration.cs} | 2 +- .../TemplateSandboxCollection.cs | 11 --- .../Infrastructure/TemplateSandboxFixture.cs | 15 ---- .../Infrastructure/TemplateTestBase.cs | 36 --------- .../Infrastructure/WellKnownPaths.cs | 81 +++++++++++++++++++ .../ProjectRootHelper.cs | 35 ++++++++ .../TestBase.cs | 70 ---------------- 16 files changed, 276 insertions(+), 220 deletions(-) create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs rename test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/{AichatwebTemplatesTests.cs => AIChatWebSnapshotTests.cs} (94%) delete mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/BuildWebTemplateTests.cs rename test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/{ITemplateConfigurationProvider.cs => ITemplateExecutionTestConfigurationProvider.cs} (58%) create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs rename test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/{TemplateTestFixture.cs => TemplateExecutionTestClassFixtureBase.cs} (65%) create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollection.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs rename test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/{TemplateConfiguration.cs => TemplateExecutionTestConfiguration.cs} (85%) delete mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxCollection.cs delete mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxFixture.cs delete mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/WellKnownPaths.cs create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/ProjectRootHelper.cs delete mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs new file mode 100644 index 00000000000..dc4d045e983 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public class AIChatWebExecutionTests : TemplateExecutionTestBase, ITemplateExecutionTestConfigurationProvider +{ + public AIChatWebExecutionTests(TemplateExecutionTestFixture fixture, ITestOutputHelper outputHelper) + : base(fixture, outputHelper) + { + } + + public static TemplateExecutionTestConfiguration Configuration { get; } = new() + { + TemplatePackageName = "Microsoft.Extensions.AI.Templates", + TestOutputFolderPrefix = "BuildWebTemplate" + }; + + [Theory] + [InlineData("BasicAppDefault")] + [InlineData("BasicAppWithAzureOpenAI", "--provider", "azureopenai")] + public async Task CreateRestoreAndBuild_BasicTemplate(string projectName, params string[] args) + { + var project = await Fixture.CreateProjectAsync( + templateName: "aichatweb", + projectName, + args); + + await Fixture.RestoreProjectAsync(project); + await Fixture.BuildProjectAsync(project); + } + + [Theory] + [InlineData("BasicAspireAppDefault")] + public async Task CreateRestoreAndBuild_AspireTemplate(string projectName, params string[] args) + { + var project = await Fixture.CreateProjectAsync( + templateName: "aichatweb", + projectName, + args: ["--aspire", .. args]); + + project.StartupProjectRelativePath = $"{projectName}.AppHost"; + + await Fixture.RestoreProjectAsync(project); + await Fixture.BuildProjectAsync(project); + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebSnapshotTests.cs similarity index 94% rename from test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs rename to test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebSnapshotTests.cs index d34de93103b..1e4cf0415f4 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebSnapshotTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Extensions.AI.Templates.Tests; -public class AichatwebTemplatesTests : TestBase +public class AIChatWebSnapshotTests { // Keep the exclude patterns below in sync with those in Microsoft.Extensions.AI.Templates.csproj. private static readonly string[] _verificationExcludePatterns = [ @@ -35,7 +35,7 @@ public class AichatwebTemplatesTests : TestBase private readonly ILogger _log; - public AichatwebTemplatesTests(ITestOutputHelper log) + public AIChatWebSnapshotTests(ITestOutputHelper log) { #pragma warning disable CA2000 // Dispose objects before losing scope _log = new XunitLoggerProvider(log).CreateLogger("TestRun"); @@ -66,7 +66,7 @@ private async Task TestTemplateCoreAsync(string scenarioName, IEnumerable, ITemplateConfigurationProvider -{ - public BuildWebTemplateTests(Fixture fixture, ITestOutputHelper outputHelper) - : base(fixture, outputHelper) - { - } - - public static TemplateConfiguration Configuration { get; } = new() - { - TemplatePackageName = "Microsoft.Extensions.AI.Templates", - TestOutputFolderPrefix = "BuildWebTemplate" - }; - - [Fact] - public async Task BuildBasicTemplate() - { - var fixture = GetFixture(); - var project = await fixture.CreateProjectAsync( - templateName: "aichatweb", - projectName: "BasicApp"); - - await fixture.RestoreProjectAsync(project); - await fixture.BuildProjectAsync(project); - } - - [Fact] - public async Task BuildAspireTemplate() - { - var fixture = GetFixture(); - var project = await fixture.CreateProjectAsync( - templateName: "aichatweb", - projectName: "AspireApp", - args: ["--aspire"]); - - project.StartupProjectRelativePath = "AspireApp.AppHost"; - - await fixture.RestoreProjectAsync(project); - await fixture.BuildProjectAsync(project); - } -} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs index b11513796a1..4758e14dc1f 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetCommand.cs @@ -9,7 +9,7 @@ public class DotNetCommand : TestCommand { public DotNetCommand(params ReadOnlySpan args) { - FileName = TestBase.RepoDotNetExePath; + FileName = WellKnownPaths.RepoDotNetExePath; foreach (var arg in args) { diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateConfigurationProvider.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateExecutionTestConfigurationProvider.cs similarity index 58% rename from test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateConfigurationProvider.cs rename to test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateExecutionTestConfigurationProvider.cs index d4121442a14..3a499013495 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateConfigurationProvider.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateExecutionTestConfigurationProvider.cs @@ -3,7 +3,7 @@ namespace Microsoft.Extensions.AI.Templates.Tests; -public interface ITemplateConfigurationProvider +public interface ITemplateExecutionTestConfigurationProvider { - static abstract TemplateConfiguration Configuration { get; } + static abstract TemplateExecutionTestConfiguration Configuration { get; } } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs new file mode 100644 index 00000000000..c919d2adc43 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +[Collection(TemplateExecutionTestCollection.Name)] +public abstract class TemplateExecutionTestBase : IClassFixture.TemplateExecutionTestFixture>, IDisposable + where TConfiguration : ITemplateExecutionTestConfigurationProvider +{ + private bool _disposed; + + protected TemplateExecutionTestFixture Fixture { get; } + + protected ITestOutputHelper OutputHelper { get; } + + protected TemplateExecutionTestBase(TemplateExecutionTestFixture fixture, ITestOutputHelper outputHelper) + { + Fixture = fixture; + Fixture.SetCurrentTestOutputHelper(outputHelper); + + OutputHelper = outputHelper; + } + + protected virtual void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (disposing) + { + Fixture.SetCurrentTestOutputHelper(null); + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + public sealed class TemplateExecutionTestFixture(IMessageSink messageSink) + : TemplateExecutionTestClassFixtureBase(TConfiguration.Configuration, messageSink); +} + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs similarity index 65% rename from test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs rename to test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs index 84b83c41a8e..dfe3b181a22 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestFixture.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs @@ -9,41 +9,28 @@ namespace Microsoft.Extensions.AI.Templates.Tests; -public abstract class TemplateTestFixture : IAsyncLifetime +public abstract class TemplateExecutionTestClassFixtureBase : IAsyncLifetime { - private readonly TemplateConfiguration _configuration; - private readonly string _templateInstallNuGetConfigPath; - private readonly string _templateTestNuGetConfigPath; - private readonly string _localShippingPackagesPath; - private readonly string _nuGetPackagesPath; + private readonly TemplateExecutionTestConfiguration _configuration; private readonly string _templateTestOutputPath; private readonly string _customHivePath; - private readonly MessageSinkTestOutputHelper _fallbackGlobalOutputHelper; + private readonly MessageSinkTestOutputHelper _messageSinkTestOutputHelper; private ITestOutputHelper? _currentTestOutputHelper; - private ITestOutputHelper OutputHelper => _currentTestOutputHelper ?? _fallbackGlobalOutputHelper; + /// + /// Gets the current preferred output helper. + /// If a test is underway, the output will be associated with that test. + /// Otherwise, the output will appear as a diagnostic message via . + /// + private ITestOutputHelper OutputHelper => _currentTestOutputHelper ?? _messageSinkTestOutputHelper; - protected TemplateTestFixture(TemplateConfiguration configuration, IMessageSink messageSink) + protected TemplateExecutionTestClassFixtureBase(TemplateExecutionTestConfiguration configuration, IMessageSink messageSink) { _configuration = configuration; - _fallbackGlobalOutputHelper = new(messageSink); - - var templateSandboxRoot = Path.Combine(TestBase.TestProjectRoot, "TemplateSandbox"); - _templateInstallNuGetConfigPath = Path.Combine(templateSandboxRoot, "nuget.template_install.config"); - _templateTestNuGetConfigPath = Path.Combine(templateSandboxRoot, "nuget.template_test.config"); - - const string BuildConfigurationFolder = -#if DEBUG - "Debug"; -#else - "Release"; -#endif - _localShippingPackagesPath = Path.Combine(TestBase.CodeBaseRoot, "artifacts", "packages", BuildConfigurationFolder, "Shipping"); - - var templateSandboxOutputRoot = Path.Combine(templateSandboxRoot, "output"); - _nuGetPackagesPath = Path.Combine(templateSandboxOutputRoot, "packages"); + _messageSinkTestOutputHelper = new(messageSink); + var outputFolderName = GetRandomizedFileName(prefix: _configuration.TestOutputFolderPrefix); - _templateTestOutputPath = Path.Combine(templateSandboxOutputRoot, outputFolderName); + _templateTestOutputPath = Path.Combine(WellKnownPaths.TemplateSandboxOutputRoot, outputFolderName); _customHivePath = Path.Combine(_templateTestOutputPath, "hive"); } @@ -62,12 +49,12 @@ async Task InstallTemplatesAsync() Directory.CreateDirectory(installSandboxPath); var installNuGetConfigPath = Path.Combine(installSandboxPath, "nuget.config"); - File.Copy(_templateInstallNuGetConfigPath, installNuGetConfigPath); + File.Copy(WellKnownPaths.TemplateInstallNuGetConfigPath, installNuGetConfigPath); var installResult = await new DotNetNewCommand("install", _configuration.TemplatePackageName) .WithWorkingDirectory(installSandboxPath) - .WithEnvironmentVariable("LOCAL_SHIPPING_PATH", _localShippingPackagesPath) - .WithEnvironmentVariable("NUGET_PACKAGES", _nuGetPackagesPath) + .WithEnvironmentVariable("LOCAL_SHIPPING_PATH", WellKnownPaths.LocalShippingPackagesPath) + .WithEnvironmentVariable("NUGET_PACKAGES", WellKnownPaths.NuGetPackagesPath) .WithCustomHive(_customHivePath) .ExecuteAsync(OutputHelper); installResult.AssertSucceeded(); @@ -94,7 +81,7 @@ .. args newProjectResult.AssertSucceeded(); var templateNuGetConfigPath = Path.Combine(outputFolderPath, "nuget.config"); - File.Copy(_templateTestNuGetConfigPath, templateNuGetConfigPath); + File.Copy(WellKnownPaths.TemplateTestNuGetConfigPath, templateNuGetConfigPath); return new Project(outputFolderPath, projectName); } @@ -103,8 +90,8 @@ public async Task RestoreProjectAsync(Project project) { var restoreResult = await new DotNetCommand("restore") .WithWorkingDirectory(project.StartupProjectFullPath) - .WithEnvironmentVariable("LOCAL_SHIPPING_PATH", _localShippingPackagesPath) - .WithEnvironmentVariable("NUGET_PACKAGES", _nuGetPackagesPath) + .WithEnvironmentVariable("LOCAL_SHIPPING_PATH", WellKnownPaths.LocalShippingPackagesPath) + .WithEnvironmentVariable("NUGET_PACKAGES", WellKnownPaths.NuGetPackagesPath) .ExecuteAsync(OutputHelper); restoreResult.AssertSucceeded(); } @@ -119,6 +106,11 @@ public async Task BuildProjectAsync(Project project) public void SetCurrentTestOutputHelper(ITestOutputHelper? outputHelper) { + if (_currentTestOutputHelper is not null && outputHelper is not null) + { + throw new InvalidOperationException(""); + } + _currentTestOutputHelper = outputHelper; } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollection.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollection.cs new file mode 100644 index 00000000000..9f10ffdf974 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollection.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +[CollectionDefinition(name: Name)] +public sealed class TemplateExecutionTestCollection : ICollectionFixture +{ + public const string Name = "Template execution test"; +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs new file mode 100644 index 00000000000..bf53540a8ae --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class TemplateExecutionTestCollectionFixture +{ + public TemplateExecutionTestCollectionFixture() + { + Directory.Delete(WellKnownPaths.TemplateSandboxOutputRoot, recursive: true); + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateConfiguration.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestConfiguration.cs similarity index 85% rename from test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateConfiguration.cs rename to test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestConfiguration.cs index 3419b85ec86..ce621e58528 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateConfiguration.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestConfiguration.cs @@ -3,7 +3,7 @@ namespace Microsoft.Extensions.AI.Templates.Tests; -public sealed class TemplateConfiguration +public sealed class TemplateExecutionTestConfiguration { public required string TemplatePackageName { get; init; } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxCollection.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxCollection.cs deleted file mode 100644 index e49082ee398..00000000000 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxCollection.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Xunit; - -namespace Microsoft.Extensions.AI.Templates.Tests.Infrastructure; - -[CollectionDefinition("Template sandbox")] -public sealed class TemplateSandboxCollection : ICollectionFixture -{ -} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxFixture.cs deleted file mode 100644 index e11a6282455..00000000000 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateSandboxFixture.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.IO; - -namespace Microsoft.Extensions.AI.Templates.Tests; - -public sealed class TemplateSandboxFixture -{ - public TemplateSandboxFixture() - { - var templateSandboxOutputRoot = Path.Combine(TestBase.TestProjectRoot, "TemplateSandbox", "output"); - Directory.Delete(templateSandboxOutputRoot, recursive: true); - } -} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs deleted file mode 100644 index 383a531d69b..00000000000 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateTestBase.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using Xunit; -using Xunit.Abstractions; - -namespace Microsoft.Extensions.AI.Templates.Tests; - -[Collection("Template sandbox")] -public abstract class TemplateTestBase : IClassFixture.Fixture>, IDisposable - where TSelf : TemplateTestBase, ITemplateConfigurationProvider -{ - private readonly Fixture _fixture; - - protected ITestOutputHelper OutputHelper { get; } - - protected TemplateTestBase(Fixture fixture, ITestOutputHelper outputHelper) - { - _fixture = fixture; - fixture.SetCurrentTestOutputHelper(outputHelper); - - OutputHelper = outputHelper; - } - - public Fixture GetFixture() => _fixture; - - public void Dispose() - { - GC.SuppressFinalize(this); - _fixture.SetCurrentTestOutputHelper(null); - } - - public sealed class Fixture(IMessageSink messageSink) : TemplateTestFixture(TSelf.Configuration, messageSink); -} - diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/WellKnownPaths.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/WellKnownPaths.cs new file mode 100644 index 00000000000..0d399dfcfe7 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/WellKnownPaths.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.InteropServices; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +internal static class WellKnownPaths +{ + public static readonly string RepoRoot; + public static readonly string RepoDotNetExePath; + public static readonly string ThisProjectRoot; + + public static readonly string TemplateFeedLocation; + public static readonly string TemplateSandboxRoot; + public static readonly string TemplateSandboxOutputRoot; + public static readonly string TemplateInstallNuGetConfigPath; + public static readonly string TemplateTestNuGetConfigPath; + public static readonly string LocalShippingPackagesPath; + public static readonly string NuGetPackagesPath; + + static WellKnownPaths() + { + RepoRoot = GetRepoRoot(); + RepoDotNetExePath = GetRepoDotNetExePath(); + ThisProjectRoot = ProjectRootHelper.GetThisProjectRoot(); + + TemplateFeedLocation = Path.Combine(RepoRoot, "src", "ProjectTemplates"); + TemplateSandboxRoot = Path.Combine(ThisProjectRoot, "TemplateSandbox"); + TemplateSandboxOutputRoot = Path.Combine(TemplateSandboxRoot, "output"); + TemplateInstallNuGetConfigPath = Path.Combine(TemplateSandboxRoot, "nuget.template_install.config"); + TemplateTestNuGetConfigPath = Path.Combine(TemplateSandboxRoot, "nuget.template_test.config"); + + const string BuildConfigurationFolder = +#if DEBUG + "Debug"; +#else + "Release"; +#endif + LocalShippingPackagesPath = Path.Combine(RepoRoot, "artifacts", "packages", BuildConfigurationFolder, "Shipping"); + NuGetPackagesPath = Path.Combine(TemplateSandboxOutputRoot, "packages"); + } + + private static string GetRepoRoot() + { + string? directory = AppContext.BaseDirectory; + + while (directory is not null) + { + var gitPath = Path.Combine(directory, ".git"); + if (Directory.Exists(gitPath) || File.Exists(gitPath)) + { + // Found the repo root, which should either have a .git folder or, if the repo + // is part of a Git worktree, a .git file. + return directory; + } + + directory = Directory.GetParent(directory)?.FullName; + } + + throw new InvalidOperationException("Failed to establish root of the repository"); + } + + private static string GetRepoDotNetExePath() + { + var dotNetExeName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? "dotnet.exe" + : "dotnet"; + + var dotNetExePath = Path.Combine(RepoRoot, ".dotnet", dotNetExeName); + + if (!File.Exists(dotNetExePath)) + { + throw new InvalidOperationException($"Expected to find '{dotNetExeName}' at '{dotNetExePath}', but it was not found."); + } + + return dotNetExePath; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/ProjectRootHelper.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/ProjectRootHelper.cs new file mode 100644 index 00000000000..3d076a438ad --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/ProjectRootHelper.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.CompilerServices; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +/// +/// Contains a helper for determining the disk location of the containing project folder. +/// +/// +/// It's important that this file resides in the root of the containing project, or the returned +/// project root path will be incorrect. +/// +internal static class ProjectRootHelper +{ + public static string GetThisProjectRoot() + => GetThisProjectRootCore(); + + // This helper method is defined separately from its public variant because it extracts the + // caller file path via the [CallerFilePath] attribute. + // Therefore, the caller must be in a known location, i.e., this source file, to produce + // a reliable result. + private static string GetThisProjectRootCore([CallerFilePath] string callerFilePath = "") + { + if (Path.GetDirectoryName(callerFilePath) is not { Length: > 0 } testProjectRoot) + { + throw new InvalidOperationException("Could not determine the root of the test project."); + } + + return testProjectRoot; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs deleted file mode 100644 index 6095e9256e4..00000000000 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace Microsoft.Extensions.AI.Templates.Tests; - -/// -/// The class contains the utils for unit and integration tests. -/// -public abstract class TestBase -{ - internal static string CodeBaseRoot { get; } = GetCodeBaseRoot(); - - internal static string RepoDotNetExePath { get; } = GetRepoDotNetExePath(); - - internal static string TestProjectRoot { get; } = GetTestProjectRoot(); - - internal static string TemplateFeedLocation { get; } = Path.Combine(CodeBaseRoot, "src", "ProjectTemplates"); - - private static string GetCodeBaseRoot() - { - string? directory = AppContext.BaseDirectory; - - while (directory is not null) - { - var gitPath = Path.Combine(directory, ".git"); - if (Directory.Exists(gitPath) || File.Exists(gitPath)) - { - // Found the repo root, which should either have a .git folder or, if the repo - // is part of a Git worktree, a .git file. - return directory; - } - - directory = Directory.GetParent(directory)?.FullName; - } - - throw new InvalidOperationException("Failed to establish root of the repository"); - } - - private static string GetTestProjectRoot([CallerFilePath] string callerFilePath = "") - { - if (Path.GetDirectoryName(callerFilePath) is not { Length: > 0 } testProjectRoot) - { - throw new InvalidOperationException("Could not determine the root of the test project."); - } - - return testProjectRoot; - } - - private static string GetRepoDotNetExePath() - { - var codeBaseRoot = CodeBaseRoot; - var dotNetExeName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? "dotnet.exe" - : "dotnet"; - - var dotNetExePath = Path.Combine(codeBaseRoot, ".dotnet", dotNetExeName); - - if (!File.Exists(dotNetExePath)) - { - throw new InvalidOperationException($"Expected to find '{dotNetExeName}' at '{dotNetExePath}', but it was not found."); - } - - return dotNetExePath; - } -} From 7876b037266c978ec54f320eddff14759343db54 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 14:45:09 -0700 Subject: [PATCH 04/17] Try updating CI pipeline --- eng/pipelines/templates/BuildAndTest.yml | 25 ++++++++++++++++--- ...osoft.Extensions.AI.Templates.Tests.csproj | 4 ++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/BuildAndTest.yml b/eng/pipelines/templates/BuildAndTest.yml index 92717160bf4..bd008ead075 100644 --- a/eng/pipelines/templates/BuildAndTest.yml +++ b/eng/pipelines/templates/BuildAndTest.yml @@ -56,7 +56,25 @@ steps: --settings $(Build.SourcesDirectory)/eng/CodeCoverage.config --output ${{ parameters.repoTestResultsPath }}/$(Agent.JobName)_CodeCoverageResults/$(Agent.JobName)_cobertura.xml "${{ parameters.buildScript }} -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)" - displayName: Run tests + displayName: Run unit tests + + - script: ${{ parameters.buildScript }} + -pack + -configuration ${{ parameters.buildConfig }} + -warnAsError 1 + /bl:${{ parameters.repoLogPath }}/pack.binlog + /p:Restore=false /p:Build=false + $(_OfficialBuildIdArgs) + displayName: Pack + + - ${{ if ne(parameters.skipTests, 'true') }}: + - script: ${{ parameters.buildScript }} + -integrationTest + -configuration ${{ parameters.buildConfig }} + -warnAsError 1 + /bl:${{ parameters.repoLogPath }}/integration_tests.binlog + $(_OfficialBuildIdArgs) + displayName: Run integration tests - pwsh: | $SourcesDirectory = '$(Build.SourcesDirectory)'; @@ -151,12 +169,11 @@ steps: displayName: Build Azure DevOps plugin - script: ${{ parameters.buildScript }} - -pack -sign $(_SignArgs) -publish $(_PublishArgs) -configuration ${{ parameters.buildConfig }} -warnAsError 1 - /bl:${{ parameters.repoLogPath }}/pack.binlog + /bl:${{ parameters.repoLogPath }}/publish.binlog /p:Restore=false /p:Build=false $(_OfficialBuildIdArgs) - displayName: Pack, sign, and publish + displayName: Sign and publish diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj index cefb4946743..4e92f2145d7 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj @@ -1,7 +1,9 @@  - Unit tests for Microsoft.Extensions.AI.Templates. + Tests for Microsoft.Extensions.AI.Templates. + false + true From d9ccb3525e26452aadad23625c6c7fa592faa184 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 15:04:25 -0700 Subject: [PATCH 05/17] Only clear test output if exists --- .../TemplateExecutionTestCollectionFixture.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs index bf53540a8ae..f8fd8126833 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs @@ -9,6 +9,10 @@ public sealed class TemplateExecutionTestCollectionFixture { public TemplateExecutionTestCollectionFixture() { - Directory.Delete(WellKnownPaths.TemplateSandboxOutputRoot, recursive: true); + // Clear output from previous test run, if it exists. + if (Directory.Exists(WellKnownPaths.TemplateSandboxOutputRoot)) + { + Directory.Delete(WellKnownPaths.TemplateSandboxOutputRoot, recursive: true); + } } } From 4a3ddca438418404a4d580a578a3456e246ce9c7 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 16:32:02 -0700 Subject: [PATCH 06/17] Add comments, update README, add more tests --- .../AIChatWebExecutionTests.cs | 33 ++++++++++++++++--- .../TemplateExecutionTestBase.cs | 15 +++++++++ .../TemplateExecutionTestClassFixtureBase.cs | 4 +++ .../TemplateExecutionTestCollectionFixture.cs | 4 +++ .../TemplateSandbox/README.md | 31 +++++++++++++++-- 5 files changed, 81 insertions(+), 6 deletions(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs index dc4d045e983..b82377f260e 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs @@ -17,12 +17,26 @@ public AIChatWebExecutionTests(TemplateExecutionTestFixture fixture, ITestOutput public static TemplateExecutionTestConfiguration Configuration { get; } = new() { TemplatePackageName = "Microsoft.Extensions.AI.Templates", - TestOutputFolderPrefix = "BuildWebTemplate" + TestOutputFolderPrefix = "AIChatWeb" }; [Theory] - [InlineData("BasicAppDefault")] - [InlineData("BasicAppWithAzureOpenAI", "--provider", "azureopenai")] + [InlineData("AzureApp", + "--provider", "azureopenai", + "--vector-store", "azureaisearch")] + [InlineData("AzureAppManagedIdentity", + "--provider", "azureopenai", + "--vector-store", "azureaisearch", + "--managed-identity")] + [InlineData("GitHubModelsWithLocalStore", + "--provider", "githubmodels", + "--vector-store", "local")] + [InlineData("OllamaWithLocalStore", + "--provider", "ollama", + "--vector-store", "local")] + [InlineData("OpenAIWithLocalStore", + "--provider", "openai", + "--vector-store", "local")] public async Task CreateRestoreAndBuild_BasicTemplate(string projectName, params string[] args) { var project = await Fixture.CreateProjectAsync( @@ -35,7 +49,18 @@ public async Task CreateRestoreAndBuild_BasicTemplate(string projectName, params } [Theory] - [InlineData("BasicAspireAppDefault")] + [InlineData("AzureApp", + "--provider", "azureopenai", + "--vector-store", "azureaisearch")] + [InlineData("GitHubModelsWithQdrant", + "--provider", "githubmodels", + "--vector-store", "qdrant")] + [InlineData("OllamaWithLocalStore", + "--provider", "ollama", + "--vector-store", "local")] + [InlineData("OpenAIWithLocalStore", + "--provider", "ollama", + "--vector-store", "local")] public async Task CreateRestoreAndBuild_AspireTemplate(string projectName, params string[] args) { var project = await Fixture.CreateProjectAsync( diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs index c919d2adc43..a8fe986cfc9 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs @@ -7,6 +7,10 @@ namespace Microsoft.Extensions.AI.Templates.Tests; +/// +/// Represents a test that executes a project template (create, restore, build, and run). +/// +/// A type defining global test execution settings. [Collection(TemplateExecutionTestCollection.Name)] public abstract class TemplateExecutionTestBase : IClassFixture.TemplateExecutionTestFixture>, IDisposable where TConfiguration : ITemplateExecutionTestConfigurationProvider @@ -46,6 +50,17 @@ public void Dispose() GC.SuppressFinalize(this); } + /// + /// An implementation of that utilizes + /// the configuration provided by TConfiguration. + /// + /// + /// The configuration has to be provided "statically" because the lifetime of the class fixture + /// is longer than the lifetime of each test class instance. In other words, it's not possible for + /// an instance of the test class to configure to the fixture directly, as the test class instance + /// gets created after the fixture has a chance to perform global setup. + /// + /// The The . public sealed class TemplateExecutionTestFixture(IMessageSink messageSink) : TemplateExecutionTestClassFixtureBase(TConfiguration.Configuration, messageSink); } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs index dfe3b181a22..b0385713575 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs @@ -9,6 +9,10 @@ namespace Microsoft.Extensions.AI.Templates.Tests; +/// +/// Provides functionality scoped to the duration of all the tests in a single test class +/// extending . +/// public abstract class TemplateExecutionTestClassFixtureBase : IAsyncLifetime { private readonly TemplateExecutionTestConfiguration _configuration; diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs index f8fd8126833..305a42820dd 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs @@ -5,6 +5,10 @@ namespace Microsoft.Extensions.AI.Templates.Tests; +/// +/// Provides functionality scoped to the lifetime of all tests defined in +/// test classes extending . +/// public sealed class TemplateExecutionTestCollectionFixture { public TemplateExecutionTestCollectionFixture() diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md index a9024a24011..47862657260 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md @@ -1,2 +1,29 @@ -This folder exists to serve as an isolated environment for templates to run in. -Generated projects will get placed here. +# Template test sandbox + +This folder exists to serve as an isolated environment for template execution tests. + +## Debugging template execution tests + +Before running template execution tests, make sure that packages defined in this solution have been packed by running the following commands from the repo root: +```sh +./build.cmd -build +./build.cmd -pack +``` + +Template tests can be debugged either in VS or by running `dotnet test`. + +However, it's sometimes helpful to debug failures by building, running, and modifying the generated projects directly instead of tinkering with test code. + +To help with this scenario: +* The `output/` folder containing the generated projects doesn't get cleared until the start of the next test run. +* An `activate.ps1` script can be used to simulate the environment that the template execution tests use. This script: + * Sets the active .NET installation to `/.dotnet`. + * Sets the `NUGET_PACKAGES` environment variable to the `output/packages` folder to use the isolated package cache. + * Sets a `LOCAL_SHIPPING_PATH` environment variable so that locally-built packages can get picked up during restore. + +As an example, here's how you can build a project generated by the tests: +```sh +. ./activate.ps1 +cd ./output/[test_collection]/[generated_template] +dotnet build +``` From ef5d2d732977414e1f5240676aefa9c35012a4bc Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 17:07:39 -0700 Subject: [PATCH 07/17] PR feedback --- .../Infrastructure/TemplateExecutionTestBase.cs | 1 - .../Microsoft.Extensions.AI.Templates.Tests.csproj | 2 ++ .../README.md | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/README.md diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs index a8fe986cfc9..b52e8cda3a6 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs @@ -64,4 +64,3 @@ public void Dispose() public sealed class TemplateExecutionTestFixture(IMessageSink messageSink) : TemplateExecutionTestClassFixtureBase(TConfiguration.Configuration, messageSink); } - diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj index 4e92f2145d7..d2fc26ea0ab 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj @@ -21,6 +21,8 @@ + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/README.md new file mode 100644 index 00000000000..ca589c22c2c --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/README.md @@ -0,0 +1,5 @@ +# Microsoft.Extensions.AI.Templates tests + +Contains snapshot and execution tests for `Microsoft.Extensions.AI.Templates`. + +For information on debugging template execution tests, see [this README](./TemplateSandbox/README.md). From 276f33332b070ad7660dc4a506fd6f34b26f9eb0 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 20:59:58 -0700 Subject: [PATCH 08/17] Add note to README --- .../TemplateSandbox/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md index 47862657260..7db29aaab19 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md @@ -10,6 +10,8 @@ Before running template execution tests, make sure that packages defined in this ./build.cmd -pack ``` +**Note:** These commands currently need to be run separately so that generated template content gets included in the template `.nupkg`. + Template tests can be debugged either in VS or by running `dotnet test`. However, it's sometimes helpful to debug failures by building, running, and modifying the generated projects directly instead of tinkering with test code. From b2e902c77d99ab5fd130624ec860a974037adde8 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 21:00:24 -0700 Subject: [PATCH 09/17] Update test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props Co-authored-by: Steve Sanderson --- .../TemplateSandbox/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props index 4508b7d4139..0853226a659 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props @@ -1,6 +1,6 @@ From fccdf15424be40df2e0dce367142022ce86d5d29 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Wed, 30 Apr 2025 21:00:33 -0700 Subject: [PATCH 10/17] Update test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets Co-authored-by: Steve Sanderson --- .../TemplateSandbox/Directory.Build.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets index f0643c8ea7b..ecef22f1080 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets @@ -1,6 +1,6 @@ From fa16de6b41866fe490f162771d008e70ec24fa98 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Thu, 1 May 2025 09:40:48 -0700 Subject: [PATCH 11/17] Fill out missing exception message --- .../Infrastructure/TemplateExecutionTestClassFixtureBase.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs index b0385713575..3592fd6474b 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs @@ -112,7 +112,10 @@ public void SetCurrentTestOutputHelper(ITestOutputHelper? outputHelper) { if (_currentTestOutputHelper is not null && outputHelper is not null) { - throw new InvalidOperationException(""); + throw new InvalidOperationException( + "Cannot set the template execution test output helper when one is already present. " + + "This might be a sign that template execution tests are running in parallel, " + + "which is not currently supported."); } _currentTestOutputHelper = outputHelper; From 6d2b0d78e6bab56a413a3229bfeb3fcee413812a Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Thu, 1 May 2025 10:19:58 -0700 Subject: [PATCH 12/17] Fix test command logging --- .../Infrastructure/TestCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs index b24d94d1323..697bf009f9c 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs @@ -83,7 +83,7 @@ DataReceivedEventHandler MakeOnDataReceivedHandler(StringBuilder outputBuilder) } }; - outputHelper.WriteLine($"Executing '{processStartInfo.FileName} {processStartInfo.Arguments}' in working directory '{processStartInfo.WorkingDirectory}'"); + outputHelper.WriteLine($"Executing '{processStartInfo.FileName} {string.Join(" ", Arguments)}' in working directory '{processStartInfo.WorkingDirectory}'"); using var timeoutCts = new CancellationTokenSource(); if (Timeout is { } timeout) From 0c54671317b568f7ed678c73f1e60684077eb460 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Thu, 1 May 2025 11:24:07 -0700 Subject: [PATCH 13/17] Treat warnings as errors + isolate .editorconfig --- .../TemplateSandbox/.editorconfig | 2 ++ .../TemplateSandbox/Directory.Build.props | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.editorconfig diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.editorconfig b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.editorconfig new file mode 100644 index 00000000000..b8f20c69836 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.editorconfig @@ -0,0 +1,2 @@ +# Don't apply the repo's editorconfig settings to generated templates. +root = true diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props index 0853226a659..e3b34086f94 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props @@ -3,4 +3,9 @@ This file exists to detach generated projects from other Directory.Build.props files in this repo. --> + + + + true + From e66a00bcc961ba7d99be092d364751bccfee547e Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Thu, 1 May 2025 13:24:16 -0700 Subject: [PATCH 14/17] Generate all possible valid template combinations --- .../AIChatWebExecutionTests.cs | 138 +++++++++++++----- 1 file changed, 105 insertions(+), 33 deletions(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs index b82377f260e..7e847466aac 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; +using System.Collections.Generic; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -20,28 +22,20 @@ public AIChatWebExecutionTests(TemplateExecutionTestFixture fixture, ITestOutput TestOutputFolderPrefix = "AIChatWeb" }; + public static IEnumerable GetBasicTemplateOptions() + => GetFilteredTemplateOptions("--aspire", "false"); + + public static IEnumerable GetAspireTemplateOptions() + => GetFilteredTemplateOptions("--aspire", "true"); + [Theory] - [InlineData("AzureApp", - "--provider", "azureopenai", - "--vector-store", "azureaisearch")] - [InlineData("AzureAppManagedIdentity", - "--provider", "azureopenai", - "--vector-store", "azureaisearch", - "--managed-identity")] - [InlineData("GitHubModelsWithLocalStore", - "--provider", "githubmodels", - "--vector-store", "local")] - [InlineData("OllamaWithLocalStore", - "--provider", "ollama", - "--vector-store", "local")] - [InlineData("OpenAIWithLocalStore", - "--provider", "openai", - "--vector-store", "local")] - public async Task CreateRestoreAndBuild_BasicTemplate(string projectName, params string[] args) + [MemberData(nameof(GetBasicTemplateOptions))] + public async Task CreateRestoreAndBuild_BasicTemplate(params string[] args) { + const string ProjectName = "BasicApp"; var project = await Fixture.CreateProjectAsync( templateName: "aichatweb", - projectName, + projectName: ProjectName, args); await Fixture.RestoreProjectAsync(project); @@ -49,28 +43,106 @@ public async Task CreateRestoreAndBuild_BasicTemplate(string projectName, params } [Theory] - [InlineData("AzureApp", - "--provider", "azureopenai", - "--vector-store", "azureaisearch")] - [InlineData("GitHubModelsWithQdrant", - "--provider", "githubmodels", - "--vector-store", "qdrant")] - [InlineData("OllamaWithLocalStore", - "--provider", "ollama", - "--vector-store", "local")] - [InlineData("OpenAIWithLocalStore", - "--provider", "ollama", - "--vector-store", "local")] - public async Task CreateRestoreAndBuild_AspireTemplate(string projectName, params string[] args) + [MemberData(nameof(GetAspireTemplateOptions))] + public async Task CreateRestoreAndBuild_AspireTemplate(params string[] args) { + const string ProjectName = "AspireApp"; var project = await Fixture.CreateProjectAsync( templateName: "aichatweb", - projectName, + ProjectName, args: ["--aspire", .. args]); - project.StartupProjectRelativePath = $"{projectName}.AppHost"; + project.StartupProjectRelativePath = $"{ProjectName}.AppHost"; await Fixture.RestoreProjectAsync(project); await Fixture.BuildProjectAsync(project); } + + private static readonly (string name, string[] values)[] _templateOptions = [ + ("--provider", ["azureopenai", "githubmodels", "ollama", "openai"]), + ("--vector-store", ["azureaisearch", "local", "qdrant"]), + ("--managed-identity", ["true", "false"]), + ("--aspire", ["true", "false"]), + ]; + + private static IEnumerable GetFilteredTemplateOptions(params string[] filter) + { + foreach (var options in GetAllPossibleOptions(_templateOptions)) + { + if (!MatchesFilter()) + { + continue; + } + + if (HasOption("--managed-identity", "true")) + { + if (HasOption("--aspire", "true")) + { + // The managed identity option is disabled for the Aspire template. + continue; + } + + if (!HasOption("--vector-store", "azureaisearch") && + !HasOption("--aspire", "false")) + { + // Can only use managed identity when using Azure in the non-Aspire template. + continue; + } + } + + if (HasOption("--vector-store", "qdrant") && + HasOption("--aspire", "false")) + { + // Can't use Qdrant without Aspire. + continue; + } + + yield return options; + + bool MatchesFilter() + { + for (var i = 0; i < filter.Length; i += 2) + { + if (!HasOption(filter[i], filter[i + 1])) + { + return false; + } + } + + return true; + } + + bool HasOption(string name, string value) + { + for (var i = 0; i < options.Length; i += 2) + { + if (string.Equals(name, options[i], StringComparison.Ordinal) && + string.Equals(value, options[i + 1], StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + } + } + + private static IEnumerable GetAllPossibleOptions(ReadOnlyMemory<(string name, string[] values)> options) + { + if (options.Length == 0) + { + yield return []; + yield break; + } + + var first = options.Span[0]; + foreach (var restSelection in GetAllPossibleOptions(options[1..])) + { + foreach (var value in first.values) + { + yield return [first.name, value, .. restSelection]; + } + } + } } From 25916b9c3bf5f351a6d9d3a311ebb7335c217e37 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Thu, 1 May 2025 13:36:06 -0700 Subject: [PATCH 15/17] Remove redundant Aspire args --- .../AIChatWebExecutionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs index 7e847466aac..71ff85b2488 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs @@ -50,7 +50,7 @@ public async Task CreateRestoreAndBuild_AspireTemplate(params string[] args) var project = await Fixture.CreateProjectAsync( templateName: "aichatweb", ProjectName, - args: ["--aspire", .. args]); + args); project.StartupProjectRelativePath = $"{ProjectName}.AppHost"; From d06918920536b1c8358e59661ae0590554bd48f9 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Thu, 1 May 2025 14:11:39 -0700 Subject: [PATCH 16/17] Explicitly exclude template content from build, just in case --- eng/build.proj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/build.proj b/eng/build.proj index 2f95df76500..e59868907db 100644 --- a/eng/build.proj +++ b/eng/build.proj @@ -1,6 +1,7 @@ <_SnapshotsToExclude Include="$(MSBuildThisFileDirectory)..\test\**\Snapshots\**\*.*proj" /> + <_GeneratedContentToExclude Include="$(MSBuildThisFileDirectory)..\test\**\TemplateSandbox\**\*.*proj" /> <_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\**\*.csproj" /> @@ -11,6 +12,6 @@ <_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\Packages\Microsoft.Internal.Extensions.DotNetApiDocs.Transport\Microsoft.Internal.Extensions.DotNetApiDocs.Transport.proj" /> - + - \ No newline at end of file + From 3a6dcb39cf4ef28e8a97dd79dbb77fe13e38acc4 Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Thu, 1 May 2025 17:01:09 -0700 Subject: [PATCH 17/17] Add comments about CG reporting --- .../AIChatWebExecutionTests.cs | 13 +++++++++++++ .../TemplateExecutionTestCollectionFixture.cs | 9 ++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs index 71ff85b2488..1b8c4177f40 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs @@ -9,6 +9,17 @@ namespace Microsoft.Extensions.AI.Templates.Tests; +/// +/// Contains execution tests for the "AI Chat Web" template. +/// +/// +/// In addition to validating that the templates build and restore correctly, +/// these tests are also responsible for template component governance reporting. +/// This is because the generated output is left on disk after tests complete, +/// most importantly the project.assets.json file that gets created during restore. +/// Therefore, it's *critical* that these tests remain in a working state, +/// as disabling them will also disable CG reporting. +/// public class AIChatWebExecutionTests : TemplateExecutionTestBase, ITemplateExecutionTestConfigurationProvider { public AIChatWebExecutionTests(TemplateExecutionTestFixture fixture, ITestOutputHelper outputHelper) @@ -28,6 +39,7 @@ public static IEnumerable GetBasicTemplateOptions() public static IEnumerable GetAspireTemplateOptions() => GetFilteredTemplateOptions("--aspire", "true"); + // Do not skip. See XML docs for this test class. [Theory] [MemberData(nameof(GetBasicTemplateOptions))] public async Task CreateRestoreAndBuild_BasicTemplate(params string[] args) @@ -42,6 +54,7 @@ public async Task CreateRestoreAndBuild_BasicTemplate(params string[] args) await Fixture.BuildProjectAsync(project); } + // Do not skip. See XML docs for this test class. [Theory] [MemberData(nameof(GetAspireTemplateOptions))] public async Task CreateRestoreAndBuild_AspireTemplate(params string[] args) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs index 305a42820dd..13140b0599e 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs @@ -13,7 +13,14 @@ public sealed class TemplateExecutionTestCollectionFixture { public TemplateExecutionTestCollectionFixture() { - // Clear output from previous test run, if it exists. + // Here, we clear execution test output from the previous test run, if it exists. + // + // It's critical that this clearing happens *before* the tests start, *not* after they complete. + // + // This is because: + // 1. This enables debugging the previous test run by building/running generated projects manually. + // 2. The existence of a project.assets.json file on disk is what allows template content to get discovered + // for component governance reporting. if (Directory.Exists(WellKnownPaths.TemplateSandboxOutputRoot)) { Directory.Delete(WellKnownPaths.TemplateSandboxOutputRoot, recursive: true);