diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs index 03a0fd09b6..c76ca3a861 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs @@ -46,7 +46,8 @@ public AsyncBenchmarksTests(ITestOutputHelper output) : base(output) { } [Fact] public void TaskReturningMethodsAreAwaited() { - var summary = CanExecute(); + var config = new SingleRunInProcessConfig(Output); + var summary = CanExecute(config); foreach (var report in summary.Reports) foreach (var measurement in report.AllMeasurements) diff --git a/tests/BenchmarkDotNet.IntegrationTests/AttributesTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AttributesTests.cs index 8130a7ce73..b73959d3c8 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AttributesTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AttributesTests.cs @@ -9,7 +9,9 @@ public AttributesTests(ITestOutputHelper output) : base(output) { } [Fact] public void AttributesAreNotSealed() { - CanExecute(); + var config = new SingleRunInProcessConfig(Output); + + CanExecute(config); } public class ConsumingCustomAttributes diff --git a/tests/BenchmarkDotNet.IntegrationTests/BaselineRatioColumnTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BaselineRatioColumnTest.cs index 83b3c1e98f..5d0a45e861 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BaselineRatioColumnTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BaselineRatioColumnTest.cs @@ -14,9 +14,11 @@ public BaselineRatioColumnTest(ITestOutputHelper output) : base(output) { } [Fact] public void ColumnsWithBaselineGetsRatio() { + var config = new SingleRunInProcessConfig(Output); + // This is the common way to run benchmarks, // BenchmarkTestExecutor.CanExecute(..) calls BenchmarkRunner.Run(..) under the hood - var summary = CanExecute(); + var summary = CanExecute(config); var table = summary.Table; var headerRow = table.FullHeader; @@ -59,7 +61,7 @@ public BaselineRatioResultExtenderNoBaselineTest(ITestOutputHelper output) : bas public void Test() { var testExporter = new TestExporter(); - var config = CreateSimpleConfig().AddExporter(testExporter); + var config = new SingleRunInProcessConfig(Output).AddExporter(testExporter); CanExecute(config); @@ -97,7 +99,9 @@ public BaselineRatioResultExtenderErrorTest(ITestOutputHelper output) : base(out [Fact] public void OnlyOneMethodCanBeBaseline() { - var summary = CanExecute(fullValidation: false); + var config = new SingleRunInProcessConfig(Output); + + var summary = CanExecute(config, fullValidation: false); // You can't have more than 1 method in a class with [Benchmark(Baseline = true)] Assert.True(summary.HasCriticalValidationErrors); @@ -122,7 +126,9 @@ public BaselineRatioColumnWithLongParamsTest(ITestOutputHelper output) : base(ou [Fact] public void ColumnsWithBaselineGetsRatio() { - var summary = CanExecute(fullValidation: false); + var config = new SingleRunInProcessConfig(Output); + + var summary = CanExecute(config, fullValidation: false); // Ensure that Params attribute values will not affect Baseline property Assert.False(summary.HasCriticalValidationErrors); diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index ab336fe92e..a5f9c2014d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -20,13 +20,19 @@ public class BenchmarkSwitcherTest public BenchmarkSwitcherTest(ITestOutputHelper output) => Output = output; + private IConfig GetRunInProcessConfig() + => new SingleRunInProcessConfig(Output); + + private IConfig GetRunOutOfProcessConfig() + => new SingleRunOutOfProcessConfig(Output); + [FactEnvSpecific( "When CommandLineParser wants to display help, it tries to get the Title of the Entry Assembly which is an xunit runner, which has no Title and fails..", EnvRequirement.DotNetCoreOnly)] public void WhenInvalidCommandLineArgumentIsPassedAnErrorMessageIsDisplayedAndNoBenchmarksAreExecuted() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); var summaries = BenchmarkSwitcher .FromTypes([]) @@ -39,8 +45,8 @@ public void WhenInvalidCommandLineArgumentIsPassedAnErrorMessageIsDisplayedAndNo [Fact] public void WhenUserAsksForInfoAnInfoIsDisplayedAndNoBenchmarksAreExecuted() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); var summaries = BenchmarkSwitcher .FromTypes([]) @@ -53,8 +59,8 @@ public void WhenUserAsksForInfoAnInfoIsDisplayedAndNoBenchmarksAreExecuted() [Fact] public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExecuted() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); var summaries = BenchmarkSwitcher .FromTypes([typeof(ClassC)]) @@ -67,8 +73,8 @@ public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAre [Fact] public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExecuted() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); var summaries = BenchmarkSwitcher .FromTypes([]) @@ -81,8 +87,8 @@ public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExe [Fact] public void WhenFilterReturnsNothingAnErrorMessageIsDisplayedAndNoBenchmarksAreExecuted() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); const string filter = "WRONG"; var summaries = BenchmarkSwitcher @@ -96,8 +102,8 @@ public void WhenFilterReturnsNothingAnErrorMessageIsDisplayedAndNoBenchmarksAreE [Fact] public void WhenUserAsksToPrintAListWePrintIt() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); var summaries = BenchmarkSwitcher .FromTypes([typeof(ClassA)]) @@ -111,8 +117,8 @@ public void WhenUserAsksToPrintAListWePrintIt() [Fact] public void WhenUserAsksToPrintAListAndProvidesAFilterWePrintFilteredList() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); var summaries = BenchmarkSwitcher .FromTypes([typeof(ClassA)]) @@ -127,8 +133,7 @@ public void WhenUserAsksToPrintAListAndProvidesAFilterWePrintFilteredList() [Fact] public void WhenDisableLogFileWeDontWriteToFile() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger).WithOptions(ConfigOptions.DisableLogFile).AddJob(Job.Dry); + var config = GetRunInProcessConfig().WithOptions(ConfigOptions.DisableLogFile); string? logFilePath = null; try @@ -153,8 +158,7 @@ public void WhenDisableLogFileWeDontWriteToFile() [Fact] public void EnsureLogFileIsWritten() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger).AddJob(Job.Dry); + var config = GetRunInProcessConfig(); string? logFilePath = null; try @@ -179,12 +183,11 @@ public void EnsureLogFileIsWritten() [Fact] public void WhenUserDoesNotProvideFilterOrCategoriesViaCommandLineWeAskToChooseBenchmark() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); var userInteractionMock = new UserInteractionMock(returnValue: []); var summaries = new BenchmarkSwitcher(userInteractionMock) - .With([typeof(WithDryAttributeAndCategory)]) + .With([typeof(WithCategory)]) .Run([], config); Assert.Empty(summaries); // summaries is empty because the returnValue configured for mock returns 0 types @@ -196,9 +199,8 @@ public void WhenUserDoesNotProvideFilterOrCategoriesViaCommandLineWeAskToChooseB [InlineData("--anyCategories")] public void WhenUserProvidesCategoriesWithoutFiltersWeDontAskToChooseBenchmarkJustRunGivenCategories(string categoriesConsoleLineArgument) { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); - var types = new[] { typeof(WithDryAttributeAndCategory) }; + var config = GetRunInProcessConfig(); + var types = new[] { typeof(WithCategory) }; var userInteractionMock = new UserInteractionMock(returnValue: types); var summaries = new BenchmarkSwitcher(userInteractionMock) @@ -214,9 +216,8 @@ public void WhenUserProvidesCategoriesWithoutFiltersWeDontAskToChooseBenchmarkJu [InlineData("--anyCategories")] public void WhenUserProvidesCategoriesWithFiltersWeDontAskToChooseBenchmarkJustUseCombinedFilterAndRunTheBenchmarks(string categoriesConsoleLineArgument) { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); - var types = new[] { typeof(WithDryAttributeAndCategory) }; + var config = GetRunInProcessConfig(); + var types = new[] { typeof(WithCategory) }; var userInteractionMock = new UserInteractionMock(returnValue: types); var summaries = new BenchmarkSwitcher(userInteractionMock) @@ -230,8 +231,7 @@ public void WhenUserProvidesCategoriesWithFiltersWeDontAskToChooseBenchmarkJustU [Fact] public void ValidCommandLineArgumentsAreProperlyHandled() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunOutOfProcessConfig(); // Don't cover every combination, just pick a complex scenario and check // it works end-to-end, i.e. "method=Method1" and "class=ClassB" @@ -253,7 +253,7 @@ public void WhenJobIsDefinedInTheConfigAndArgumentsDontContainJobArgumentOnlySin var types = new[] { typeof(ClassB) }; var switcher = new BenchmarkSwitcher(types); MockExporter mockExporter = new MockExporter(); - var configWithJobDefined = ManualConfig.CreateEmpty().AddExporter(mockExporter).AddJob(Job.Dry); + var configWithJobDefined = GetRunInProcessConfig().AddExporter(mockExporter); var results = switcher.Run(["--filter", "*Method3"], configWithJobDefined); @@ -262,7 +262,7 @@ public void WhenJobIsDefinedInTheConfigAndArgumentsDontContainJobArgumentOnlySin Assert.Single(results); Assert.Single(results.SelectMany(r => r.BenchmarksCases)); Assert.Single(results.SelectMany(r => r.BenchmarksCases.Select(bc => bc.Job))); - Assert.True(results.All(r => r.BenchmarksCases.All(bc => bc.Job == Job.Dry))); + Assert.True(results.All(r => r.BenchmarksCases.All(bc => bc.Job.Id == "Dry"))); } [Fact] @@ -271,7 +271,7 @@ public void WhenJobIsDefinedViaAttributeAndArgumentsDontContainJobArgumentOnlySi var types = new[] { typeof(WithDryAttributeAndCategory) }; var switcher = new BenchmarkSwitcher(types); MockExporter mockExporter = new MockExporter(); - var configWithoutJobDefined = ManualConfig.CreateEmpty().AddExporter(mockExporter); + var configWithoutJobDefined = GetRunOutOfProcessConfig().AddExporter(mockExporter); var results = switcher.Run(["--filter", "*WithDryAttribute*"], configWithoutJobDefined); @@ -289,7 +289,7 @@ public void JobNotDefinedButStillBenchmarkIsExecuted() var types = new[] { typeof(JustBenchmark) }; var switcher = new BenchmarkSwitcher(types); MockExporter mockExporter = new MockExporter(); - var configWithoutJobDefined = ManualConfig.CreateEmpty().AddExporter(mockExporter); + var configWithoutJobDefined = GetRunOutOfProcessConfig().AddExporter(mockExporter); var results = switcher.Run(["--filter", "*"], configWithoutJobDefined); @@ -304,8 +304,8 @@ public void JobNotDefinedButStillBenchmarkIsExecuted() [Fact] public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromTypes() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); var summariesForType = BenchmarkSwitcher .FromTypes([typeof(Static.BenchmarkClassWithStaticMethodsOnly)]) @@ -318,8 +318,8 @@ public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromTypes() [Fact] public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromAssembly() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunInProcessConfig(); + var logger = config.GetOutputLogger(); var summariesForAssembly = BenchmarkSwitcher .FromAssembly(typeof(Static.BenchmarkClassWithStaticMethodsOnly).Assembly) @@ -332,16 +332,15 @@ public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromAssembly() [FactEnvSpecific("For some reason this test is flaky on Full Framework", EnvRequirement.DotNetCoreOnly)] public void WhenUserAddTheResumeAttributeAndRunTheBenchmarks() { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); + var config = GetRunOutOfProcessConfig().AddJob(Job.Dry); - var types = new[] { typeof(WithDryAttributeAndCategory) }; + var types = new[] { typeof(WithCategory) }; var switcher = new BenchmarkSwitcher(types); // the first run should execute all benchmarks - Assert.Single(switcher.Run(["--filter", "*WithDryAttributeAndCategory*"], config)); + Assert.Single(switcher.Run(["--filter", "*WithCategory*"], config)); // resuming after succesfull run should run nothing - Assert.Empty(switcher.Run(["--resume", "--filter", "*WithDryAttributeAndCategory*"], config)); + Assert.Empty(switcher.Run(["--resume", "--filter", "*WithCategory*"], config)); } private class UserInteractionMock : IUserInteraction @@ -370,6 +369,12 @@ private string GetValidationErrorForType(Type type) return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; } } + + file static class ExtensionMethods + { + public static OutputLogger GetOutputLogger(this IConfig config) + => (OutputLogger)config.GetLoggers().Single(); + } } namespace BenchmarkDotNet.IntegrationTests @@ -402,6 +407,13 @@ public void Method2() { } public void Method3() { } } + [BenchmarkCategory(BenchmarkSwitcherTest.TestCategory)] + public class WithCategory + { + [Benchmark] + public void Method() { } + } + [DryJob] [BenchmarkCategory(BenchmarkSwitcherTest.TestCategory)] public class WithDryAttributeAndCategory diff --git a/tests/BenchmarkDotNet.IntegrationTests/Helpers/CaptureConsoleHelper.cs b/tests/BenchmarkDotNet.IntegrationTests/Helpers/CaptureConsoleHelper.cs new file mode 100644 index 0000000000..36b501167d --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/Helpers/CaptureConsoleHelper.cs @@ -0,0 +1,27 @@ +using System.Text; + +namespace BenchmarkDotNet.IntegrationTests; + +internal class CaptureConsoleHelper : IDisposable +{ + private readonly TextWriter originalOut; + private readonly StringWriter writer = new(new StringBuilder()); + + public CaptureConsoleHelper() + { + originalOut = Console.Out; + Console.SetOut(writer); + } + + public void Dispose() + { + Console.SetOut(originalOut); + writer.Dispose(); + } + + public string CapturedText + => writer.ToString(); + + public IReadOnlyList CapturedLines + => CapturedText.Split(["\r\n", "\n"], StringSplitOptions.None); +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs index 5db376879d..de54782eee 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs @@ -1,4 +1,6 @@ using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Toolchains; namespace BenchmarkDotNet.IntegrationTests { @@ -6,11 +8,36 @@ public class ParamsTests : BenchmarkTestExecutor { public ParamsTests(ITestOutputHelper output) : base(output) { } + private IConfig? GetConfig() + => new SingleRunInProcessConfig(Output); // Use `null` when running test with out-of-process toolchain. + + private IReadOnlyList Execute() + { + var config = GetConfig(); + + bool isInProcess = config == null + ? false + : config.GetJobs().Single().GetToolchain().IsInProcess; + + if (isInProcess) + { + // When using inprocess toolchain, Report don't contains StandardOutput. + using var captureConsoleHelper = new CaptureConsoleHelper(); + CanExecute(config); + return captureConsoleHelper.CapturedLines; // CapturedLines contains benchmark execution logs also. + } + else + { + // Use following code when running test with out-of-process. + var summary = CanExecute(); + return GetCombinedStandardOutput(summary); + } + } + [Fact] public void ParamsSupportPropertyWithPublicSetter() { - var summary = CanExecute(); - var standardOutput = GetCombinedStandardOutput(summary); + var standardOutput = Execute(); foreach (var param in new[] { 1, 2 }) Assert.Contains($"// ### New Parameter {param} ###", standardOutput); @@ -30,8 +57,7 @@ public class ParamsTestProperty [Fact] public void ParamsSupportPublicFields() { - var summary = CanExecute(); - var standardOutput = GetCombinedStandardOutput(summary); + var standardOutput = Execute(); foreach (var param in new[] { 1, 2 }) Assert.Contains($"// ### New Parameter {param} ###", standardOutput); @@ -54,7 +80,10 @@ public enum NestedOne } [Fact] - public void NestedEnumsAsParamsAreSupported() => CanExecute(); + public void NestedEnumsAsParamsAreSupported() + { + Execute(); + } public class NestedEnumsAsParams { @@ -66,7 +95,8 @@ public class NestedEnumsAsParams } [Fact] - public void CharactersAsParamsAreSupported() => CanExecute(); + public void CharactersAsParamsAreSupported() + => Execute(); public class CharactersAsParams { @@ -78,7 +108,8 @@ public class CharactersAsParams } [Fact] - public void NullableTypesAsParamsAreSupported() => CanExecute(); + public void NullableTypesAsParamsAreSupported() + => Execute(); public class NullableTypesAsParams { @@ -93,7 +124,8 @@ public void Benchmark() } [Fact] - public void InvalidFileNamesInParamsAreSupported() => CanExecute(); + public void InvalidFileNamesInParamsAreSupported() + => Execute(); public class InvalidFileNamesInParams { @@ -105,7 +137,8 @@ public class InvalidFileNamesInParams } [Fact] - public void SpecialCharactersInStringAreSupported() => CanExecute(); + public void SpecialCharactersInStringAreSupported() + => Execute(); public class CompileSpecialCharactersInString { @@ -140,7 +173,8 @@ public void Benchmark() } [Fact] - public void SpecialCharactersInCharAreSupported() => CanExecute(); + public void SpecialCharactersInCharAreSupported() + => Execute(); public class CompileSpecialCharactersInChar { @@ -156,7 +190,8 @@ public void Benchmark() { } } [Fact] - public void ParamsMustBeEscapedProperly() => CanExecute(); + public void ParamsMustBeEscapedProperly() + => Execute(); public class NeedEscaping { @@ -175,7 +210,8 @@ public void Benchmark(string argument) } [Fact] - public void ArrayCanBeUsedAsParameter() => CanExecute(); + public void ArrayCanBeUsedAsParameter() + => Execute(); public class WithArray { @@ -195,7 +231,8 @@ public void AcceptingArray() } [Fact] - public void StaticFieldsAndPropertiesCanBeParams() => CanExecute(); + public void StaticFieldsAndPropertiesCanBeParams() + => Execute(); public class WithStaticParams { @@ -219,8 +256,7 @@ public void Test() [Fact] public void ParamsSupportRequiredProperty() { - var summary = CanExecute(); - var standardOutput = GetCombinedStandardOutput(summary); + var standardOutput = Execute(); foreach (var param in new[] { "a", "b" }) { @@ -238,4 +274,4 @@ public class ParamsTestRequiredProperty } #endif } -} \ No newline at end of file +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/Shared/TestConfigs.cs b/tests/BenchmarkDotNet.IntegrationTests/Shared/TestConfigs.cs index 499b08e1a3..fcb3c7ee91 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Shared/TestConfigs.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/Shared/TestConfigs.cs @@ -1,6 +1,10 @@ +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Toolchains.InProcess.Emit; namespace BenchmarkDotNet.IntegrationTests { @@ -44,4 +48,29 @@ public DiagnoserConfig() AddJob(new Job { Run = { LaunchCount = 1, WarmupCount = 1, IterationCount = 50 } }); } } + + public class SingleRunInProcessConfig : ManualConfig + { + public SingleRunInProcessConfig(ITestOutputHelper? output = null, bool addDryJob = true) + { + AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()); + AddColumnProvider(DefaultColumnProviders.Instance); + AddLogger(output != null ? new OutputLogger(output) : ConsoleLogger.Default); + + var job = Job.Dry + .WithStrategy(RunStrategy.Monitoring) + .WithToolchain(InProcessEmitToolchain.Default); + AddJob(job); + } + } + + public class SingleRunOutOfProcessConfig : ManualConfig + { + public SingleRunOutOfProcessConfig(ITestOutputHelper? output = null) + { + AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()); + AddColumnProvider(DefaultColumnProviders.Instance); + AddLogger(output != null ? new OutputLogger(output) : ConsoleLogger.Default); + } + } } \ No newline at end of file