From 9aeb39c4246aee49749748e5f649f3c548ab8b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 28 Jun 2026 16:26:01 +0200 Subject: [PATCH 1/2] perf: add dotnet test server-mode scenario to performance runner Adds Scenario1_DotnetTest_PlainProcess which runs the 100x100 test asset via 'dotnet test --no-build' instead of direct executable invocation. This exercises the MTP JSON-RPC/named-pipe server-mode path that the existing PlainProcess scenarios do not cover. The new DotnetTestProcess step: - Invokes the repo-local SDK dotnet binary (consistent with DotnetMuxer) - Drains stdout/stderr async to avoid pipe deadlocks - Records wall-clock elapsed time and TotalProcessorTime (parent only) - Writes Result.json in the same format as PlainProcess The scenario name contains 'PlainProcess' so it is automatically captured by the existing nightly workflow filter (*PlainProcess*) without requiring any workflow file changes. Addresses the gap identified in #9480. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../MSTest.Performance.Runner/Program.cs | 11 ++ .../Steps/DotnetTestProcess.cs | 103 ++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs diff --git a/test/Performance/MSTest.Performance.Runner/Program.cs b/test/Performance/MSTest.Performance.Runner/Program.cs index 9343cfc9bf..127512aa01 100644 --- a/test/Performance/MSTest.Performance.Runner/Program.cs +++ b/test/Performance/MSTest.Performance.Runner/Program.cs @@ -101,6 +101,17 @@ private static int Pipelines(string pipelineNameFilter) .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results"))) .NextStep(() => new CleanupDisposable())); + // Server-mode variant: runs via `dotnet test --no-build` to exercise the MTP JSON-RPC + // (named-pipe) path rather than the plain-process standalone path. The "PlainProcess" + // suffix keeps this scenario captured by the existing nightly workflow filter (*PlainProcess*). + pipelineRunner.AddPipeline("Default", "Scenario1_DotnetTest_PlainProcess", [OSPlatform.Windows, OSPlatform.Linux, OSPlatform.OSX], parametersBag => + Pipeline + .FirstStep(() => new Scenario1(numberOfClass: 100, methodsPerClass: 100, tfm: "net9.0", executionScope: ExecutionScope.MethodLevel), parametersBag) + .NextStep(() => new DotnetMuxer(BuildConfiguration.Debug)) + .NextStep(() => new DotnetTestProcess("Scenario1_DotnetTest_PlainProcess.zip")) + .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results"))) + .NextStep(() => new CleanupDisposable())); + return pipelineRunner.Run(pipelineNameFilter); } } diff --git a/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs b/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs new file mode 100644 index 0000000000..bfed717beb --- /dev/null +++ b/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.IO.Compression; +using System.Text.Json; + +using Microsoft.Testing.TestInfrastructure; + +namespace MSTest.Performance.Runner.Steps; + +/// +/// Runs the test project via dotnet test --no-build to exercise the MTP +/// server-mode (JSON-RPC / named-pipe) path, and records wall-clock timing using +/// the same plain metrics as . +/// +/// +/// +/// When EnableMSTestRunner=true (MTP native mode), dotnet test invokes the +/// compiled test host in server mode, passing --server --protocol dotnet-test-protocol. +/// The host then communicates results back via a named pipe / TCP socket rather than running +/// standalone. This exercises the serialisation, JSON-RPC framing, and pipe I/O paths that +/// the plain-process scenario does not cover. +/// +/// +/// Measurement note: reflects only the +/// dotnet test parent process; the spawned test-host child's CPU time is not included. +/// minus (wall-clock) is the +/// primary metric and represents the end-to-end time a user observes when running +/// dotnet test. +/// +/// +internal class DotnetTestProcess : IStep +{ + private static readonly string s_root = RootFinder.Find(); + private readonly string _reportFileName; + private readonly int _numberOfRun; + private readonly CompressionLevel _compressionLevel; + + public string Description => "run dotnet test (MTP server mode)"; + + public DotnetTestProcess(string reportFileName, int numberOfRun = 3, CompressionLevel compressionLevel = CompressionLevel.Fastest) + { + _reportFileName = reportFileName; + _numberOfRun = numberOfRun; + _compressionLevel = compressionLevel; + } + + public async Task ExecuteAsync(BuildArtifact payload, IContext context) + { + string dotnet = Path.Combine(s_root, ".dotnet", $"dotnet{Constants.ExecutableExtension}"); + string projectDir = payload.TestAsset.TargetAssetPath; + + // Use the repo-local SDK consistently with the build step (DotnetMuxer). + ProcessStartInfo psi = new(dotnet, $"test \"{projectDir}\" --no-build") + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + + psi.EnvironmentVariables["DOTNET_CLI_TELEMETRY_OPTOUT"] = "1"; + psi.EnvironmentVariables["DOTNET_ROOT"] = Path.Combine(s_root, ".dotnet"); + psi.EnvironmentVariables["DOTNET_INSTALL_DIR"] = Path.Combine(s_root, ".dotnet"); + psi.EnvironmentVariables["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "1"; + psi.EnvironmentVariables["DOTNET_MULTILEVEL_LOOKUP"] = "0"; + + Console.WriteLine($"Process command: '{psi.FileName} {psi.Arguments.Trim()}' for {_numberOfRun} times"); + + List results = []; + for (int i = 0; i < _numberOfRun; i++) + { + using Process process = Process.Start(psi)!; + // Drain stdout/stderr asynchronously to prevent buffer deadlocks. + Task stdoutTask = process.StandardOutput.ReadToEndAsync(); + Task stderrTask = process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + await Task.WhenAll(stdoutTask, stderrTask); + + var result = new + { + ElapsedTime = process.ExitTime - process.StartTime, + process.TotalProcessorTime, + Environment.ProcessorCount, + GC.GetGCMemoryInfo().TotalAvailableMemoryBytes, + }; + + results.Add(result); + } + +#pragma warning disable CA1869 // Cache and reuse 'JsonSerializerOptions' instances + await File.AppendAllTextAsync( + Path.Combine(Path.GetDirectoryName(payload.TestHost.FullName)!, "Result.json"), + JsonSerializer.Serialize(results, new JsonSerializerOptions { WriteIndented = true })); +#pragma warning restore CA1869 + + string sample = Path.Combine(Path.GetTempPath(), _reportFileName); + File.Delete(sample); + Console.WriteLine($"Compressing to '{sample}'"); + ZipFile.CreateFromDirectory(payload.TestAsset.TargetAssetPath, sample, _compressionLevel, includeBaseDirectory: true); + + return new Files([sample]); + } +} From f43d0b3ad866572aea2e5ea1bfefe50a0ebfad4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Sun, 28 Jun 2026 21:54:44 +0200 Subject: [PATCH 2/2] Address review: fail fast on non-zero exit, pin WorkingDirectory and configuration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../MSTest.Performance.Runner/Program.cs | 2 +- .../Steps/DotnetTestProcess.cs | 24 ++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/test/Performance/MSTest.Performance.Runner/Program.cs b/test/Performance/MSTest.Performance.Runner/Program.cs index 127512aa01..84c7308305 100644 --- a/test/Performance/MSTest.Performance.Runner/Program.cs +++ b/test/Performance/MSTest.Performance.Runner/Program.cs @@ -108,7 +108,7 @@ private static int Pipelines(string pipelineNameFilter) Pipeline .FirstStep(() => new Scenario1(numberOfClass: 100, methodsPerClass: 100, tfm: "net9.0", executionScope: ExecutionScope.MethodLevel), parametersBag) .NextStep(() => new DotnetMuxer(BuildConfiguration.Debug)) - .NextStep(() => new DotnetTestProcess("Scenario1_DotnetTest_PlainProcess.zip")) + .NextStep(() => new DotnetTestProcess("Scenario1_DotnetTest_PlainProcess.zip", BuildConfiguration.Debug)) .NextStep(() => new MoveFiles("*.zip", Path.Combine(Directory.GetCurrentDirectory(), "Results"))) .NextStep(() => new CleanupDisposable())); diff --git a/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs b/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs index bfed717beb..de717570a7 100644 --- a/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs +++ b/test/Performance/MSTest.Performance.Runner/Steps/DotnetTestProcess.cs @@ -33,14 +33,16 @@ internal class DotnetTestProcess : IStep { private static readonly string s_root = RootFinder.Find(); private readonly string _reportFileName; + private readonly BuildConfiguration _buildConfiguration; private readonly int _numberOfRun; private readonly CompressionLevel _compressionLevel; public string Description => "run dotnet test (MTP server mode)"; - public DotnetTestProcess(string reportFileName, int numberOfRun = 3, CompressionLevel compressionLevel = CompressionLevel.Fastest) + public DotnetTestProcess(string reportFileName, BuildConfiguration buildConfiguration = BuildConfiguration.Debug, int numberOfRun = 3, CompressionLevel compressionLevel = CompressionLevel.Fastest) { _reportFileName = reportFileName; + _buildConfiguration = buildConfiguration; _numberOfRun = numberOfRun; _compressionLevel = compressionLevel; } @@ -50,12 +52,17 @@ public async Task ExecuteAsync(BuildArtifact payload, IContext context) string dotnet = Path.Combine(s_root, ".dotnet", $"dotnet{Constants.ExecutableExtension}"); string projectDir = payload.TestAsset.TargetAssetPath; - // Use the repo-local SDK consistently with the build step (DotnetMuxer). - ProcessStartInfo psi = new(dotnet, $"test \"{projectDir}\" --no-build") + // Use the repo-local SDK consistently with the build step (DotnetMuxer). The + // configuration must match the one used by DotnetMuxer so that --no-build finds the + // binaries that were actually produced. WorkingDirectory is pinned to the test asset so + // relative outputs (TestResults, logs, temp files) stay inside the generated asset rather + // than polluting the runner's current directory between scenarios. + ProcessStartInfo psi = new(dotnet, $"test \"{projectDir}\" --no-build --configuration {_buildConfiguration}") { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, + WorkingDirectory = projectDir, }; psi.EnvironmentVariables["DOTNET_CLI_TELEMETRY_OPTOUT"] = "1"; @@ -76,6 +83,17 @@ public async Task ExecuteAsync(BuildArtifact payload, IContext context) await process.WaitForExitAsync(); await Task.WhenAll(stdoutTask, stderrTask); + // Fail fast on a non-zero exit code: `dotnet test` has many infrastructure failure + // modes (restore issues, SDK mismatch, missing build artefacts) that would otherwise + // record timings for an invalid run and silently corrupt the perf baseline. + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"'dotnet test' exited with code {process.ExitCode}.{Environment.NewLine}" + + $"stdout:{Environment.NewLine}{await stdoutTask}{Environment.NewLine}" + + $"stderr:{Environment.NewLine}{await stderrTask}"); + } + var result = new { ElapsedTime = process.ExitTime - process.StartTime,