Add tests for multiple in-process diagnosers with varying run modes

tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs

@@ -11,40 +11,107 @@
 
 namespace BenchmarkDotNet.IntegrationTests.Diagnosers;
 
-public sealed class MockInProcessDiagnoser : IInProcessDiagnoser
+public abstract class BaseMockInProcessDiagnoser : IInProcessDiagnoser
 {
     public Dictionary<BenchmarkCase, string> Results { get; } = [];
 
-    public IEnumerable<string> Ids => [nameof(MockInProcessDiagnoser)];
+    public abstract string DiagnoserName { get; }
+    public abstract RunMode DiagnoserRunMode { get; }
+    public abstract Type HandlerType { get; }
+    public abstract string ExpectedResult { get; }
+
+    public IEnumerable<string> Ids => [DiagnoserName];
 
     public IEnumerable<IExporter> Exporters => [];
 
     public IEnumerable<IAnalyser> Analysers => [];
 
-    public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoser)} results: [{string.Join(", ", Results.Values)}]");
+    public void DisplayResults(ILogger logger) => logger.WriteLine($"{DiagnoserName} results: [{string.Join(", ", Results.Values)}]");
 
-    public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
+    public RunMode GetRunMode(BenchmarkCase benchmarkCase) => DiagnoserRunMode;
 
     public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
 
     public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => [];
 
     public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
 
-    public (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
-        => (typeof(MockInProcessDiagnoserHandler), null);
+    public virtual (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
+        => (HandlerType, null);
 
-    public IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
-        => new MockInProcessDiagnoserHandler();
+    public virtual IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
+        => (IInProcessDiagnoserHandler)Activator.CreateInstance(HandlerType, ExpectedResult);
 
     public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results);
 }
 
-public sealed class MockInProcessDiagnoserHandler : IInProcessDiagnoserHandler
+public abstract class BaseMockInProcessDiagnoserHandler : IInProcessDiagnoserHandler
 {
+    private readonly string _result;
+
+    protected BaseMockInProcessDiagnoserHandler(string result) => _result = result;
+
     public void Initialize(string? serializedConfig) { }
 
     public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { }
 
-    public string SerializeResults() => "MockResult";
+    public string SerializeResults() => _result;
+}
+
+public sealed class MockInProcessDiagnoser : BaseMockInProcessDiagnoser
+{
+    public override string DiagnoserName => nameof(MockInProcessDiagnoser);
+    public override RunMode DiagnoserRunMode => RunMode.NoOverhead;
+    public override Type HandlerType => typeof(MockInProcessDiagnoserHandler);
+    public override string ExpectedResult => "MockResult";
+}
+
+public sealed class MockInProcessDiagnoserHandler : BaseMockInProcessDiagnoserHandler
+{
+    public MockInProcessDiagnoserHandler(string result) : base(result) { }
+}
+
+public sealed class MockInProcessDiagnoserNoOverhead : BaseMockInProcessDiagnoser
+{
+    public override string DiagnoserName => nameof(MockInProcessDiagnoserNoOverhead);
+    public override RunMode DiagnoserRunMode => RunMode.NoOverhead;
+    public override Type HandlerType => typeof(MockInProcessDiagnoserNoOverheadHandler);
+    public override string ExpectedResult => "NoOverheadResult";
+}
+
+public sealed class MockInProcessDiagnoserNoOverheadHandler : BaseMockInProcessDiagnoserHandler
+{
+    public MockInProcessDiagnoserNoOverheadHandler(string result) : base(result) { }
+}
+
+public sealed class MockInProcessDiagnoserExtraRun : BaseMockInProcessDiagnoser
+{
+    public override string DiagnoserName => nameof(MockInProcessDiagnoserExtraRun);
+    public override RunMode DiagnoserRunMode => RunMode.ExtraRun;
+    public override Type HandlerType => typeof(MockInProcessDiagnoserExtraRunHandler);
+    public override string ExpectedResult => "ExtraRunResult";
+}
+
+public sealed class MockInProcessDiagnoserExtraRunHandler : BaseMockInProcessDiagnoserHandler
+{
+    public MockInProcessDiagnoserExtraRunHandler(string result) : base(result) { }
+}
+
+public sealed class MockInProcessDiagnoserNone : BaseMockInProcessDiagnoser
+{
+    public override string DiagnoserName => nameof(MockInProcessDiagnoserNone);
+    public override RunMode DiagnoserRunMode => RunMode.None;
+    public override Type HandlerType => typeof(MockInProcessDiagnoserNoneHandler);
+    public override string ExpectedResult => "NoneResult";
+
+    public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
+        => default; // Returns default when RunMode is None
+
+    public override IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
+        => null; // Returns null when RunMode is None
+}
+
+public sealed class MockInProcessDiagnoserNoneHandler : BaseMockInProcessDiagnoserHandler
+{
+    public MockInProcessDiagnoserNoneHandler(string result) : base(result) { }
 }

tests/BenchmarkDotNet.IntegrationTests/MultipleInProcessDiagnosersTests.cs

@@ -0,0 +1,146 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.IntegrationTests.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Tests.Loggers;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace BenchmarkDotNet.IntegrationTests;
+
+public class MultipleInProcessDiagnosersTests : BenchmarkTestExecutor
+{
+    public MultipleInProcessDiagnosersTests(ITestOutputHelper output) : base(output) { }
+
+    public static IEnumerable<object[]> GetDiagnoserCombinations()
+    {
+        // Two diagnosers with NoOverhead
+        yield return new object[]
+        {
+            new BaseMockInProcessDiagnoser[] { new MockInProcessDiagnoserNoOverhead(), new MockInProcessDiagnoser() },
+            typeof(SimpleBenchmark),
+            new[] { true, true }
+        };
+
+        // Two diagnosers with ExtraRun and NoOverhead
+        yield return new object[]
+        {
+            new BaseMockInProcessDiagnoser[] { new MockInProcessDiagnoserExtraRun(), new MockInProcessDiagnoserNoOverhead() },
+            typeof(SimpleBenchmark),
+            new[] { true, true }
+        };
+
+        // Three diagnosers with varying run modes (None should not collect results)
+        yield return new object[]
+        {
+            new BaseMockInProcessDiagnoser[] { new MockInProcessDiagnoserNoOverhead(), new MockInProcessDiagnoserExtraRun(), new MockInProcessDiagnoserNone() },
+            typeof(SimpleBenchmark),
+            new[] { true, true, false }
+        };
+
+        // Three different types
+        yield return new object[]
+        {
+            new BaseMockInProcessDiagnoser[] { new MockInProcessDiagnoserNoOverhead(), new MockInProcessDiagnoser(), new MockInProcessDiagnoserExtraRun() },
+            typeof(SimpleBenchmark),
+            new[] { true, true, true }
+        };
+
+        // Multiple benchmarks
+        yield return new object[]
+        {
+            new BaseMockInProcessDiagnoser[] { new MockInProcessDiagnoserNoOverhead(), new MockInProcessDiagnoserExtraRun() },
+            typeof(MultipleBenchmarks),
+            new[] { true, true }
+        };
+    }
+
+    [Theory]
+    [MemberData(nameof(GetDiagnoserCombinations))]
+    public void MultipleInProcessDiagnosersWork(BaseMockInProcessDiagnoser[] diagnosers, Type benchmarkType, bool[] shouldHaveResults)
+    {
+        var logger = new OutputLogger(Output);
+        var config = CreateInProcessConfig(logger);
+
+        foreach (var diagnoser in diagnosers)
+        {
+            config = config.AddDiagnoser(diagnoser);
+        }
+
+        var summary = CanExecute(benchmarkType, config);
+
+        for (int i = 0; i < diagnosers.Length; i++)
+        {
+            var diagnoser = diagnosers[i];
+            var shouldHaveResult = shouldHaveResults[i];
+
+            if (shouldHaveResult)
+            {
+                Assert.NotEmpty(diagnoser.Results);
+                Assert.Equal(summary.BenchmarksCases.Length, diagnoser.Results.Count);
+                Assert.All(diagnoser.Results.Values, result => Assert.Equal(diagnoser.ExpectedResult, result));
+            }
+            else
+            {
+                Assert.Empty(diagnoser.Results);
+            }
+        }
+
+        // For multiple benchmarks, verify all benchmark methods are present
+        if (benchmarkType == typeof(MultipleBenchmarks))
+        {
+            var benchmarkMethods = summary.BenchmarksCases.Select(bc => bc.Descriptor.WorkloadMethod.Name).ToList();
+            Assert.Contains("Benchmark1", benchmarkMethods);
+            Assert.Contains("Benchmark2", benchmarkMethods);
+            Assert.Contains("Benchmark3", benchmarkMethods);
+        }
+    }
+
+    private IConfig CreateInProcessConfig(OutputLogger logger)
+    {
+        return new ManualConfig()
+            .AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.DontLogOutput))
+            .AddLogger(logger)
+            .AddColumnProvider(DefaultColumnProviders.Instance);
+    }
+
+    public class SimpleBenchmark
+    {
+        private int counter;
+
+        [Benchmark]
+        public void BenchmarkMethod()
+        {
+            Interlocked.Increment(ref counter);
+        }
+    }
+
+    public class MultipleBenchmarks
+    {
+        private int counter;
+
+        [Benchmark]
+        public void Benchmark1()
+        {
+            Interlocked.Increment(ref counter);
+        }
+
+        [Benchmark]
+        public void Benchmark2()
+        {
+            Interlocked.Increment(ref counter);
+        }
+
+        [Benchmark]
+        public void Benchmark3()
+        {
+            Interlocked.Increment(ref counter);
+        }
+    }
+}