diff --git a/README.md b/README.md index 2725d727..8f09fc66 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,22 @@ Uses manual mod setup, including sub-mods and the EaW fallback game, and uses th --useDefaultBaseline ``` +#### Example 3: Layering a mod-specific baseline on top of the default baseline +A typical mod-dev workflow: filter the base game's known findings with the embedded default baseline, *and* filter your mod's own accepted findings with your own baseline. The two baselines stay independent — you can regenerate your mod baseline without touching the default. + +**Windows:** +```bat +.\ModVerify.exe verify --path "C:\My Games\FoC\Mods\MyMod" --useDefaultBaseline --baseline ./myModBaseline.json +``` + +**Linux:** +```bash +./ModVerify verify \ + --path "/home/user/games/FoC/Mods/MyMod" \ + --useDefaultBaseline \ + --baseline ./myModBaseline.json +``` + --- ## Available Checks @@ -156,3 +172,22 @@ ModVerify.exe createBaseline --outFile myBaseline.json --path "C:\My Games\FoC\M --outFile myBaseline.json \ --path "C:\My Games\FoC\Mods\MyMod" ``` + +### Creating a mod baseline on top of a base baseline + +If you maintain a mod and only want your baseline to contain findings your mod is responsible for, supply the base baselines you want to subtract out. Findings already covered by the base baselines are excluded from the new file. The base baselines themselves are not modified, so they can keep being maintained independently. + +**Windows** +```bat +.\ModVerify.exe createBaseline --outFile myModBaseline.json --path "C:\My Games\FoC\Mods\MyMod" --useDefaultBaseline +``` + +**Linux** +```bash +./ModVerify createBaseline \ + --outFile myModBaseline.json \ + --path "/home/user/games/FoC/Mods/MyMod" \ + --useDefaultBaseline +``` + +You can also chain a custom base baseline via `--baseline `, and combine it with `--useDefaultBaseline`. diff --git a/src/ModVerify.CliApp/App/CreateBaselineAction.cs b/src/ModVerify.CliApp/App/CreateBaselineAction.cs index b776e09f..ff33339a 100644 --- a/src/ModVerify.CliApp/App/CreateBaselineAction.cs +++ b/src/ModVerify.CliApp/App/CreateBaselineAction.cs @@ -1,8 +1,7 @@ -using AET.ModVerify.App.Reporting; +using AET.ModVerify.App.Reporting; using AET.ModVerify.App.Settings; using AET.ModVerify.App.Utilities; using AET.ModVerify.Reporting; -using AET.ModVerify.Reporting.Baseline; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; @@ -15,7 +14,7 @@ internal sealed class CreateBaselineAction(AppBaselineSettings settings, IServic : ModVerifyApplicationAction(settings, serviceProvider) { private readonly IFileSystem _fileSystem = serviceProvider.GetRequiredService(); - + protected override void PrintAction(VerificationTarget target) { Console.WriteLine(); @@ -29,11 +28,11 @@ protected override void PrintAction(VerificationTarget target) protected override async Task ProcessResult(VerificationResult result) { var baselineFactory = ServiceProvider.GetRequiredService(); - var baseline = baselineFactory.CreateBaseline(result.Target, Settings, result.Errors); + var baseline = baselineFactory.CreateBaseline(result.Target, Settings, result.Errors.NewErrors); var fullPath = _fileSystem.Path.GetFullPath(Settings.NewBaselinePath); - Logger?.LogInformation(ModVerifyConstants.ConsoleEventId, - "Writing Baseline to '{FullPath}' with {Number} findings", fullPath, result.Errors.Count); + Logger?.LogInformation(ModVerifyConstants.ConsoleEventId, + "Writing Baseline to '{FullPath}' with {Number} findings", fullPath, result.Errors.NewErrors.Count); await baselineFactory.WriteBaselineAsync(baseline, Settings.NewBaselinePath); @@ -46,9 +45,4 @@ protected override async Task ProcessResult(VerificationResult result) return ModVerifyConstants.Success; } - - protected override VerificationBaseline GetBaseline(VerificationTarget verificationTarget) - { - return VerificationBaseline.Empty; - } } \ No newline at end of file diff --git a/src/ModVerify.CliApp/App/ModVerifyApplicationAction.cs b/src/ModVerify.CliApp/App/ModVerifyApplicationAction.cs index 4fd10dc9..4ea4cc6c 100644 --- a/src/ModVerify.CliApp/App/ModVerifyApplicationAction.cs +++ b/src/ModVerify.CliApp/App/ModVerifyApplicationAction.cs @@ -6,6 +6,7 @@ using AET.ModVerify.App.Reporting; using AET.ModVerify.App.Settings; using AET.ModVerify.App.TargetSelectors; +using AET.ModVerify.App.Utilities; using AET.ModVerify.Reporting; using AET.ModVerify.Reporting.Baseline; using AET.ModVerify.Reporting.Suppressions; @@ -105,14 +106,30 @@ private int ReportVerificationFailure(Exception verificationException) } protected abstract Task ProcessResult(VerificationResult result); - - protected abstract VerificationBaseline GetBaseline(VerificationTarget verificationTarget); + + protected virtual BaselineCollection GetBaselines(VerificationTarget verificationTarget) + { + var baselineSelector = new BaselineSelector(Settings, ServiceProvider); + var baselines = baselineSelector.SelectBaselines(verificationTarget); + if (!baselines.IsEmpty) + { + Console.WriteLine(); + ModVerifyConsoleUtilities.WriteBaselineInfo(baselines); + foreach (var entry in baselines) + { + Logger?.LogDebug("Using baseline {Baseline} from source '{Identifier}'", + entry.Baseline.ToString(), entry.Identifier); + } + Console.WriteLine(); + } + return baselines; + } private async Task VerifyTargetAsync(VerificationTarget verificationTarget) { var progressReporter = new VerifyConsoleProgressReporter(verificationTarget.Name, Settings.ReportSettings); - var baseline = GetBaseline(verificationTarget); + var baselines = GetBaselines(verificationTarget); var suppressions = GetSuppressions(); try @@ -122,11 +139,11 @@ private async Task VerifyTargetAsync(VerificationTarget veri Logger?.LogInformation(ModVerifyConstants.ConsoleEventId, "Verifying '{Target}'...", verificationTarget.Name); var verificationResult = await verifierService.VerifyAsync( - verificationTarget, + verificationTarget, Settings.VerifierServiceSettings, - baseline, + baselines, suppressions, - progressReporter, + progressReporter, new EngineInitializeProgressReporter(verificationTarget.Engine)); progressReporter.Report(string.Empty, 1.0); diff --git a/src/ModVerify.CliApp/App/VerifyAction.cs b/src/ModVerify.CliApp/App/VerifyAction.cs index deeb053e..46ddf7bb 100644 --- a/src/ModVerify.CliApp/App/VerifyAction.cs +++ b/src/ModVerify.CliApp/App/VerifyAction.cs @@ -1,5 +1,4 @@ -using AET.ModVerify.App.Reporting; -using AET.ModVerify.App.Settings; +using AET.ModVerify.App.Settings; using AET.ModVerify.App.Utilities; using AET.ModVerify.Reporting; using Microsoft.Extensions.Logging; @@ -8,7 +7,6 @@ using System.Linq; using System.Threading.Tasks; using AET.ModVerify.Reporting.Reporters; -using AET.ModVerify.Reporting.Baseline; namespace AET.ModVerify.App; @@ -41,7 +39,7 @@ protected override async Task ProcessResult(VerificationResult result) await reportBroker.ReportAsync(result); if (Settings.AppFailsOnMinimumSeverity.HasValue && - result.Errors.Any(x => x.Severity >= Settings.AppFailsOnMinimumSeverity)) + result.Errors.NewErrors.Any(x => x.Severity >= Settings.AppFailsOnMinimumSeverity)) { Logger?.LogInformation(ModVerifyConstants.ConsoleEventId, "The verification of {Target} completed with findings of the specified failure severity {Severity}", @@ -53,32 +51,18 @@ protected override async Task ProcessResult(VerificationResult result) return ModVerifyConstants.Success; } - protected override VerificationBaseline GetBaseline(VerificationTarget verificationTarget) - { - var baselineSelector = new BaselineSelector(Settings, ServiceProvider); - var baseline = baselineSelector.SelectBaseline(verificationTarget, out var baselinePath); - if (!baseline.IsEmpty) - { - Console.WriteLine(); - ModVerifyConsoleUtilities.WriteBaselineInfo(baseline, baselinePath); - Logger?.LogDebug("Using baseline {Baseline} from location '{Path}'", - baseline.ToString(), baselinePath ?? "Embedded"); - Console.WriteLine(); - } - return baseline; - } - private IReadOnlyCollection CreateReporters() { - var reporters = new List(); - - reporters.Add(IVerificationReporter.CreateConsole(new ConsoleReporterSettings + var reporters = new List { - Verbose = Settings.ReportSettings.Verbose, - MinimumReportSeverity = Settings.VerifierServiceSettings.FailFastSettings.IsFailFast - ? VerificationSeverity.Information - : VerificationSeverity.Error - }, ServiceProvider)); + IVerificationReporter.CreateConsole(new ConsoleReporterSettings + { + Verbose = Settings.ReportSettings.Verbose, + MinimumReportSeverity = Settings.VerifierServiceSettings.FailFastSettings.IsFailFast + ? VerificationSeverity.Information + : VerificationSeverity.Error + }, ServiceProvider) + }; var outputDirectory = Settings.ReportDirectory; reporters.Add(IVerificationReporter.CreateJson(new JsonReporterSettings diff --git a/src/ModVerify.CliApp/Properties/launchSettings.json b/src/ModVerify.CliApp/Properties/launchSettings.json index 46fa0fc9..688267e0 100644 --- a/src/ModVerify.CliApp/Properties/launchSettings.json +++ b/src/ModVerify.CliApp/Properties/launchSettings.json @@ -6,7 +6,7 @@ }, "Verify (Interactive)": { "commandName": "Project", - "commandLineArgs": "verify -o verifyResults --offline --minFailSeverity Information --searchBaseline" + "commandLineArgs": "verify -o verifyResults --offline --minFailSeverity Information --useDefaultBaseline" }, "Verify (Automatic Target Selection)": { "commandName": "Project", diff --git a/src/ModVerify.CliApp/Reporting/BaselineSelector.cs b/src/ModVerify.CliApp/Reporting/BaselineSelector.cs index 4fff8129..544112aa 100644 --- a/src/ModVerify.CliApp/Reporting/BaselineSelector.cs +++ b/src/ModVerify.CliApp/Reporting/BaselineSelector.cs @@ -1,4 +1,4 @@ -using AET.ModVerify.App.Resources.Baselines; +using AET.ModVerify.App.Resources.Baselines; using AET.ModVerify.App.Settings; using AET.ModVerify.Reporting.Baseline; using AnakinRaW.ApplicationBase; @@ -6,65 +6,86 @@ using Microsoft.Extensions.Logging; using PG.StarWarsGame.Engine; using System; -using System.Diagnostics; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Text; namespace AET.ModVerify.App.Reporting; -internal sealed class BaselineSelector(AppVerifySettings settings, IServiceProvider services) +internal sealed class BaselineSelector(AppSettingsBase settings, IServiceProvider services) { private readonly ILogger? _logger = services.GetService()?.CreateLogger(typeof(ModVerifyApplication)); private readonly IBaselineFactory _baselineFactory = services.GetRequiredService(); - public VerificationBaseline SelectBaseline(VerificationTarget verificationTarget, out string? usedBaselinePath) + private bool IsCreatingBaseline => settings is AppBaselineSettings; + + public BaselineCollection SelectBaselines(VerificationTarget verificationTarget) { - var baselinePath = settings.ReportSettings.BaselinePath; - if (!string.IsNullOrEmpty(baselinePath)) + var report = settings.ReportSettings; + var collected = new List(); + var seenIdentifiers = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var path in report.BaselinePaths) + { + var entry = LoadExplicitBaseline(path); + if (seenIdentifiers.Add(entry.Identifier)) + collected.Add(entry); + } + + // In interactive mode, offer to discover a baseline near the target when none was supplied. + if (settings.IsInteractive && collected.Count == 0 && TryFindBaselineInteractive(verificationTarget, out var found)) + collected.Add(found); + + // Loading the engine's default baseline is meaningless when creating a baseline for the game itself + // (you'd be subtracting it from itself). Skip it in that case. + var defaultBaselineApplicable = !(IsCreatingBaseline && verificationTarget.IsGame); + + if (report.UseDefaultBaseline) { - try + if (!defaultBaselineApplicable) { - usedBaselinePath = baselinePath; - return _baselineFactory.ParseBaseline(baselinePath); + _logger?.LogWarning(ModVerifyConstants.ConsoleEventId, + "Ignoring --useDefaultBaseline: it does not apply when creating a baseline for the game itself."); } - catch (InvalidBaselineException e) + else if (TryLoadEmbeddedBaseline(verificationTarget.Engine, out var defaultBaseline, out var defaultId)) { - using (ConsoleUtilities.HorizontalLineSeparatedBlock('*')) - { - Console.WriteLine($"The baseline '{baselinePath}' is not a valid baseline file: {e.Message}" + - $"{Environment.NewLine}Please generate a new baseline file or download the latest version." + - $"{Environment.NewLine}"); - } - - // For now, we bubble up this exception because we except users - // to correctly specify their baselines through command line arguments. - throw; + collected.Add(new IdentifiedBaseline(defaultId, defaultBaseline, BaselineSource.EmbeddedDefault)); } } - - if (settings.ReportSettings is { SearchBaselineLocally: false, UseDefaultBaseline: false }) + else if (settings.IsInteractive && defaultBaselineApplicable) { - _logger?.LogDebug(ModVerifyConstants.ConsoleEventId, - "No baseline path specified and local search is not enabled. Using empty baseline."); - usedBaselinePath = null; - return VerificationBaseline.Empty; + // In interactive mode, offer the embedded default independently of any locally + // discovered or explicitly supplied baselines — they're typically stacked. + if (TryPromptForEmbeddedBaseline(verificationTarget.Engine, out var defaultBaseline, out var defaultId)) + collected.Add(new IdentifiedBaseline(defaultId, defaultBaseline, BaselineSource.EmbeddedDefault)); } - if (settings.IsInteractive) - return FindBaselineInteractive(verificationTarget, out usedBaselinePath); - - // If the application is not interactive, we only use a baseline file present in the directory of the verification target. - return FindBaselineNonInteractive(verificationTarget, out usedBaselinePath); + return new BaselineCollection(collected); + } + private IdentifiedBaseline LoadExplicitBaseline(string baselinePath) + { + try + { + return new IdentifiedBaseline(baselinePath, _baselineFactory.ParseBaseline(baselinePath), BaselineSource.File); + } + catch (InvalidBaselineException e) + { + using (ConsoleUtilities.HorizontalLineSeparatedBlock('*')) + { + Console.WriteLine($"The baseline '{baselinePath}' is not a valid baseline file: {e.Message}" + + $"{Environment.NewLine}Please generate a new baseline file or download the latest version." + + $"{Environment.NewLine}"); + } + throw; + } } - private VerificationBaseline FindBaselineInteractive(VerificationTarget verificationTarget, out string? baselinePath) + private bool TryFindBaselineInteractive(VerificationTarget verificationTarget, [NotNullWhen(true)] out IdentifiedBaseline? found) { - // The application is in interactive mode. We apply the following lookup: // 1. Use a baseline found in the directory of the verification target. - // 2. Use a baseline found in the directory ModVerify executable. - // 3. If the verification target is a mod, ask the user to apply the default game's baseline. - // In any case ask the use if they want to use the located baseline file, or they wish to continue using none/empty. + // 2. Use a baseline found in the directory of the ModVerify executable. + // Ask the user if they want to use the located baseline file. _logger?.LogInformation(ModVerifyConstants.ConsoleEventId, "Searching for local baseline files..."); @@ -72,47 +93,48 @@ private VerificationBaseline FindBaselineInteractive(VerificationTarget verifica verificationTarget.Location.TargetPath, b => IsBaselineCompatible(b, verificationTarget), out var baseline, - out baselinePath)) + out var baselinePath)) { if (!_baselineFactory.TryFindBaselineInDirectory( Environment.CurrentDirectory, - b => IsBaselineCompatible(b, verificationTarget), - out baseline, + b => IsBaselineCompatible(b, verificationTarget), + out baseline, out baselinePath)) { Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine("No baseline found locally."); Console.ResetColor(); - baselinePath = null; - TryGetDefaultBaseline(verificationTarget.Engine, out baseline); - return baseline ?? VerificationBaseline.Empty; + found = null; + return false; } } - Debug.Assert(baselinePath is not null && baseline is not null); - - return ShouldUseBaseline(baseline, baselinePath) - ? baseline - : VerificationBaseline.Empty; + if (ShouldUseBaseline(baseline, baselinePath)) + { + found = new IdentifiedBaseline(baselinePath, baseline, BaselineSource.File); + return true; + } + found = null; + return false; } - private static bool TryGetDefaultBaseline( - GameEngineType engineType, - [NotNullWhen(true)] out VerificationBaseline? baseline) + private bool TryLoadEmbeddedBaseline(GameEngineType engineType, + [NotNullWhen(true)] out VerificationBaseline? baseline, + [NotNullWhen(true)] out string? identifier) { baseline = null; - if (engineType == GameEngineType.Eaw) - { - // TODO: EAW currently not implemented - return false; - } + identifier = null; - if (!ConsoleUtilities.UserYesNoQuestion($"Do you want to load the default baseline for game engine '{engineType}'?")) + // TODO: EAW currently not implemented + if (engineType == GameEngineType.Eaw) return false; try { baseline = LoadEmbeddedBaseline(engineType); + identifier = MakeDefaultIdentifier(engineType); + _logger?.LogInformation(ModVerifyConstants.ConsoleEventId, + "Applying default embedded baseline for engine '{Engine}'.", engineType); return true; } catch (InvalidBaselineException) @@ -122,6 +144,27 @@ private static bool TryGetDefaultBaseline( } } + private bool TryPromptForEmbeddedBaseline(GameEngineType engineType, + [NotNullWhen(true)] out VerificationBaseline? baseline, + [NotNullWhen(true)] out string? identifier) + { + baseline = null; + identifier = null; + + // TODO: EAW currently not implemented + if (engineType == GameEngineType.Eaw) + return false; + + var question = IsCreatingBaseline + ? $"Apply the default baseline for engine '{engineType}' as a base? Findings already covered by it will be excluded from your new baseline." + : $"Do you want to load the default baseline for game engine '{engineType}'?"; + + if (!ConsoleUtilities.UserYesNoQuestion(question)) + return false; + + return TryLoadEmbeddedBaseline(engineType, out baseline, out identifier); + } + internal static VerificationBaseline LoadEmbeddedBaseline(GameEngineType engineType) { var baselineFileName = $"baseline-{engineType.ToString().ToLower()}.json"; @@ -131,46 +174,18 @@ internal static VerificationBaseline LoadEmbeddedBaseline(GameEngineType engineT return VerificationBaseline.FromJson(baselineStream); } - private VerificationBaseline FindBaselineNonInteractive(VerificationTarget target, out string? usedPath) - { - if (_baselineFactory.TryFindBaselineInDirectory( - target.Location.TargetPath, - b => IsBaselineCompatible(b, target), - out var baseline, - out usedPath)) - { - _logger?.LogInformation(ModVerifyConstants.ConsoleEventId, "Automatically applying local baseline file '{Path}'.", usedPath); - return baseline; - } - _logger?.LogTrace("No baseline file found in taget path '{TargetPath}'.", target.Location.TargetPath); - usedPath = null; - if (settings.ReportSettings.UseDefaultBaseline) - { - try - { - var defaultBaseline = LoadEmbeddedBaseline(target.Engine); - _logger?.LogInformation(ModVerifyConstants.ConsoleEventId, "Automatically applying default embedded baseline for engine '{Engine}'.", target.Engine); - return defaultBaseline; - } - catch (InvalidBaselineException) - { - throw new InvalidOperationException( - "Invalid baseline packed along ModVerify App. Please reach out to the creators. Thanks!"); - } - } - return VerificationBaseline.Empty; - } - + internal static string MakeDefaultIdentifier(GameEngineType engineType) + => $""; private static bool IsBaselineCompatible(VerificationBaseline baseline, VerificationTarget target) { return baseline.Target?.Engine == target.Engine; } - private static bool ShouldUseBaseline(VerificationBaseline baseline, string baselinePath) + private bool ShouldUseBaseline(VerificationBaseline baseline, string baselinePath) { var sb = new StringBuilder("Found baseline "); - if (baseline.Target is not null) + if (baseline.Target is not null) sb.Append($"for '{baseline.Target.Name}' "); sb.Append($"at '{baselinePath}'."); @@ -178,6 +193,10 @@ private static bool ShouldUseBaseline(VerificationBaseline baseline, string base Console.ForegroundColor = ConsoleColor.Cyan; Console.WriteLine(sb.ToString()); - return ConsoleUtilities.UserYesNoQuestion("Do you want to use it?"); + var question = IsCreatingBaseline + ? "Use it as a base? Findings already covered by it will be excluded from your new baseline." + : "Do you want to use it?"; + Console.ResetColor(); + return ConsoleUtilities.UserYesNoQuestion(question); } -} \ No newline at end of file +} diff --git a/src/ModVerify.CliApp/Reporting/VerifyConsoleProgressReporter.cs b/src/ModVerify.CliApp/Reporting/VerifyConsoleProgressReporter.cs index 66587ae4..4cff521a 100644 --- a/src/ModVerify.CliApp/Reporting/VerifyConsoleProgressReporter.cs +++ b/src/ModVerify.CliApp/Reporting/VerifyConsoleProgressReporter.cs @@ -61,10 +61,12 @@ protected override void DisposeResources() Console.WriteLine(); } + private const int MaxTicks = 10000; + private ProgressBar EnsureProgressBar() { return LazyInitializer.EnsureInitialized(ref _progressBar, - () => new ProgressBar(100, $"Verifying '{toVerifyName}'", ProgressBarOptions))!; + () => new ProgressBar(MaxTicks, $"Verifying '{toVerifyName}'", ProgressBarOptions))!; } private static int WriteQueuedMessage(ConsoleOutLine arg) diff --git a/src/ModVerify.CliApp/Settings/CommandLine/BaseModVerifyOptions.cs b/src/ModVerify.CliApp/Settings/CommandLine/BaseModVerifyOptions.cs index 521fb8c3..5212b1c2 100644 --- a/src/ModVerify.CliApp/Settings/CommandLine/BaseModVerifyOptions.cs +++ b/src/ModVerify.CliApp/Settings/CommandLine/BaseModVerifyOptions.cs @@ -1,5 +1,4 @@ -using System.Collections.Generic; -using AET.ModVerify.Reporting; +using AET.ModVerify.Reporting; using CommandLine; using PG.StarWarsGame.Engine; @@ -52,8 +51,12 @@ internal abstract class BaseModVerifyOptions "Multiple paths can be separated using the platform-specific path separator (';' on Windows, ':' on Linux).")] public string? AdditionalFallbackPath { get; init; } - [Option("parallel", Default = false, - HelpText = "When set, game verifiers will run in parallel. " + - "While this may reduce analysis time, console output might be harder to read.")] - public bool Parallel { get; init; } + [Option("baseline", Required = false, + HelpText = "Path(s) to one or more JSON baseline files. Multiple paths can be separated using the platform-specific path separator (';' on Windows, ':' on Linux). " + + "For 'verify' this is mutually exclusive with --searchBaseline. May be combined with --useDefaultBaseline.")] + public string? BaselinePaths { get; init; } + + [Option("useDefaultBaseline", Required = false, + HelpText = "When set, additionally applies the default embedded baseline for the detected game engine. May be combined with --baseline (or --searchBaseline for 'verify').")] + public bool UseDefaultBaseline { get; init; } } \ No newline at end of file diff --git a/src/ModVerify.CliApp/Settings/CommandLine/CreateBaselineVerbOption.cs b/src/ModVerify.CliApp/Settings/CommandLine/CreateBaselineVerbOption.cs index 6593245a..ca2f97d2 100644 --- a/src/ModVerify.CliApp/Settings/CommandLine/CreateBaselineVerbOption.cs +++ b/src/ModVerify.CliApp/Settings/CommandLine/CreateBaselineVerbOption.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; namespace AET.ModVerify.App.Settings.CommandLine; @@ -7,7 +7,7 @@ internal sealed class CreateBaselineVerbOption : BaseModVerifyOptions { [Option('o', "outFile", Required = true, HelpText = "The file path of the new baseline file.")] public required string OutputFile { get; init; } - + [Option("skipLocation", Required = false, HelpText = "Skips writing the target location to the baseline.")] public bool SkipLocation { get; init; } -} \ No newline at end of file +} diff --git a/src/ModVerify.CliApp/Settings/CommandLine/VerifyVerbOption.cs b/src/ModVerify.CliApp/Settings/CommandLine/VerifyVerbOption.cs index 5e01c5cb..47ae6207 100644 --- a/src/ModVerify.CliApp/Settings/CommandLine/VerifyVerbOption.cs +++ b/src/ModVerify.CliApp/Settings/CommandLine/VerifyVerbOption.cs @@ -1,4 +1,4 @@ -using AET.ModVerify.Reporting; +using AET.ModVerify.Reporting; using CommandLine; namespace AET.ModVerify.App.Settings.CommandLine; @@ -9,7 +9,6 @@ internal sealed class VerifyVerbOption : BaseModVerifyOptions internal static readonly VerifyVerbOption WithoutArguments = new() { IsRunningWithoutArguments = true, - SearchBaselineLocally = true, }; [Option('o', "outDir", Required = false, HelpText = "Directory where result files shall be stored to.")] @@ -28,18 +27,10 @@ internal sealed class VerifyVerbOption : BaseModVerifyOptions HelpText = "When this flag is present, the application will not report engine assertions.")] public bool IgnoreAsserts { get; init; } - - [Option("baseline", Required = false, - HelpText = "Path to a JSON baseline file. Cannot be used together with --searchBaseline or --useDefaultBaseline.")] - public string? Baseline { get; init; } - - [Option("searchBaseline", Required = false, - HelpText = "When set, the application will search for baseline files and use them for verification. Cannot be used together with --baseline or --useDefaultBaseline")] - public bool SearchBaselineLocally { get; init; } - - [Option("useDefaultBaseline", Required = false, - HelpText = "When set, the application will use the default embedded baseline for the detected game engine. Cannot be used together with --baseline or --searchBaseline.")] - public bool UseDefaultBaseline { get; init; } + [Option("parallel", Default = false, + HelpText = "When set, game verifiers will run in parallel. " + + "While this may reduce analysis time, console output might be harder to read.")] + public bool Parallel { get; init; } public bool IsRunningWithoutArguments { get; init; } -} \ No newline at end of file +} diff --git a/src/ModVerify.CliApp/Settings/ModVerifyAppSettings.cs b/src/ModVerify.CliApp/Settings/ModVerifyAppSettings.cs index 06107288..ba5272b8 100644 --- a/src/ModVerify.CliApp/Settings/ModVerifyAppSettings.cs +++ b/src/ModVerify.CliApp/Settings/ModVerifyAppSettings.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using AET.ModVerify.Reporting; using AET.ModVerify.Settings; @@ -7,16 +8,13 @@ namespace AET.ModVerify.App.Settings; public class AppReportSettings { public VerificationSeverity MinimumReportSeverity { get; init; } - + public string? SuppressionsPath { get; init; } public bool Verbose { get; init; } -} -public sealed class VerifyReportSettings : AppReportSettings -{ - public string? BaselinePath { get; init; } - public bool SearchBaselineLocally { get; init; } + public IReadOnlyList BaselinePaths { get; init; } = []; + public bool UseDefaultBaseline { get; init; } } @@ -39,13 +37,7 @@ public required VerifierServiceSettings VerifierServiceSettings public AppReportSettings ReportSettings { get; } = reportSettings ?? throw new ArgumentNullException(nameof(reportSettings)); } -internal abstract class AppSettingsBase(T reportSettings) : AppSettingsBase(reportSettings) - where T : AppReportSettings -{ - public new T ReportSettings { get; } = reportSettings ?? throw new ArgumentNullException(nameof(reportSettings)); -} - -internal sealed class AppVerifySettings(VerifyReportSettings reportSettings) : AppSettingsBase(reportSettings) +internal sealed class AppVerifySettings(AppReportSettings reportSettings) : AppSettingsBase(reportSettings) { public VerificationSeverity? AppFailsOnMinimumSeverity { get; init; } diff --git a/src/ModVerify.CliApp/Settings/SettingsBuilder.cs b/src/ModVerify.CliApp/Settings/SettingsBuilder.cs index fe315952..5a7fb4e9 100644 --- a/src/ModVerify.CliApp/Settings/SettingsBuilder.cs +++ b/src/ModVerify.CliApp/Settings/SettingsBuilder.cs @@ -51,27 +51,6 @@ private AppVerifySettings BuildFromVerifyVerb(VerifyVerbOption verifyOptions) void ValidateVerb() { - if (verifyOptions.SearchBaselineLocally && !string.IsNullOrEmpty(verifyOptions.Baseline)) - { - var searchOption = typeof(VerifyVerbOption).GetOptionName(nameof(VerifyVerbOption.SearchBaselineLocally)); - var baselineOption = typeof(VerifyVerbOption).GetOptionName(nameof(VerifyVerbOption.Baseline)); - throw new AppArgumentException($"Options {searchOption} and {baselineOption} cannot be used together."); - } - - if (verifyOptions.UseDefaultBaseline && !string.IsNullOrEmpty(verifyOptions.Baseline)) - { - var useDefaultOption = typeof(VerifyVerbOption).GetOptionName(nameof(VerifyVerbOption.UseDefaultBaseline)); - var baselineOption = typeof(VerifyVerbOption).GetOptionName(nameof(VerifyVerbOption.Baseline)); - throw new AppArgumentException($"Options {useDefaultOption} and {baselineOption} cannot be used together."); - } - - if (verifyOptions is { UseDefaultBaseline: true, SearchBaselineLocally: true }) - { - var useDefaultOption = typeof(VerifyVerbOption).GetOptionName(nameof(VerifyVerbOption.UseDefaultBaseline)); - var searchOption = typeof(VerifyVerbOption).GetOptionName(nameof(VerifyVerbOption.SearchBaselineLocally)); - throw new AppArgumentException($"Options {useDefaultOption} and {searchOption} cannot be used together."); - } - if (verifyOptions is { FailFast: true, MinimumFailureSeverity: null }) { var failFast = typeof(VerifyVerbOption).GetOptionName(nameof(VerifyVerbOption.FailFast)); @@ -94,13 +73,12 @@ string GetReportDirectory() verifyOptions.OutputDirectory ?? "ModVerifyResults")); } - VerifyReportSettings BuildReportSettings() + AppReportSettings BuildReportSettings() { - return new VerifyReportSettings + return new AppReportSettings { - BaselinePath = verifyOptions.Baseline, + BaselinePaths = SplitBaselinePaths(verifyOptions.BaselinePaths), MinimumReportSeverity = verifyOptions.MinimumSeverity, - SearchBaselineLocally = verifyOptions.SearchBaselineLocally, UseDefaultBaseline = verifyOptions.UseDefaultBaseline, SuppressionsPath = verifyOptions.Suppressions, Verbose = verifyOptions.Verbose @@ -114,7 +92,9 @@ private AppBaselineSettings BuildFromCreateBaselineVerb(CreateBaselineVerbOption { VerifierServiceSettings = new VerifierServiceSettings { - ParallelVerifiers = baselineVerb.Parallel ? 4 : 1, + // Always sequential: baseline creation must be deterministic — error ordering + // and any other parallelism-sensitive behavior would otherwise vary between runs. + ParallelVerifiers = 1, VerifiersProvider = new DefaultGameVerifiersProvider(), GameVerifySettings = GameVerifySettings.Default, FailFastSettings = FailFastSetting.NoFailFast, @@ -130,11 +110,24 @@ AppReportSettings BuildReportSettings() { MinimumReportSeverity = baselineVerb.MinimumSeverity, SuppressionsPath = baselineVerb.Suppressions, - Verbose = baselineVerb.Verbose + Verbose = baselineVerb.Verbose, + BaselinePaths = SplitBaselinePaths(baselineVerb.BaselinePaths), + UseDefaultBaseline = baselineVerb.UseDefaultBaseline }; } } + private IReadOnlyList SplitBaselinePaths(string? rawPaths) + { + if (string.IsNullOrEmpty(rawPaths)) + return []; + var separator = _fileSystem.Path.PathSeparator; + return [.. + rawPaths!.Split([separator], StringSplitOptions.RemoveEmptyEntries) + .Select(p => _fileSystem.Path.GetFullPath(p)) + ]; + } + private VerificationTargetSettings BuildTargetSettings(BaseModVerifyOptions options) { var separator = _fileSystem.Path.PathSeparator; diff --git a/src/ModVerify.CliApp/Utilities/ModVerifyConsoleUtilities.cs b/src/ModVerify.CliApp/Utilities/ModVerifyConsoleUtilities.cs index 9f9962b4..f6997647 100644 --- a/src/ModVerify.CliApp/Utilities/ModVerifyConsoleUtilities.cs +++ b/src/ModVerify.CliApp/Utilities/ModVerifyConsoleUtilities.cs @@ -2,6 +2,7 @@ using Figgle; using System; using System.Collections.Generic; +using System.Linq; using AET.ModVerify.Reporting.Baseline; namespace AET.ModVerify.App.Utilities; @@ -46,42 +47,58 @@ public static void WriteSelectedTarget(VerificationTarget target) Console.ResetColor(); } - public static void WriteBaselineInfo(VerificationBaseline baseline, string? filePath) + public static void WriteBaselineInfo(BaselineCollection baselines) { - if (baseline.IsEmpty) + var displayable = baselines.Where(b => !b.Baseline.IsEmpty).ToList(); + if (displayable.Count == 0) return; Console.ForegroundColor = ConsoleColor.Cyan; - Console.WriteLine("Using Baseline:"); + Console.WriteLine(displayable.Count == 1 ? "Using Baseline:" : "Using Baselines:"); + Console.ResetColor(); + + for (var i = 0; i < displayable.Count; i++) + { + Console.WriteLine(); + WriteSingleBaseline(displayable[i], displayable.Count > 1 ? i + 1 : null); + } + } + + private static void WriteSingleBaseline(IdentifiedBaseline entry, int? index) + { + var baseline = entry.Baseline; + var isDefault = entry.Source == BaselineSource.EmbeddedDefault; + + if (index is not null) + { + Console.ForegroundColor = ConsoleColor.DarkCyan; + Console.WriteLine($"[{index}]"); + } + Console.ForegroundColor = ConsoleColor.DarkGray; - - IList<(string, object)> baselineData = - [ + ConsoleUtilities.PrintAsTable([ + ("Source", isDefault ? "Default (embedded)" : entry.Identifier), ("Version", baseline.Version?.ToString(2) ?? "n/a"), - ("Is Default", filePath is null), ("Minimum Severity", baseline.MinimumSeverity.ToString()), - ("Entries", baseline.Count.ToString()) - ]; - if (!string.IsNullOrEmpty(filePath)) - baselineData.Add(("File Path", filePath)); - - ConsoleUtilities.PrintAsTable(baselineData, 120); + ("Entries", baseline.Count.ToString()), + ], 120); if (baseline.Target is not null) { Console.ForegroundColor = ConsoleColor.DarkMagenta; - Console.WriteLine("Baseline Target:"); + Console.WriteLine("Target:"); Console.ForegroundColor = ConsoleColor.DarkGray; + // Two-space prefix on each key indents the whole sub-table under "Target:". IList<(string, object)> targetData = [ - ("Name", baseline.Target.Name), - ("Type", baseline.Target.IsGame ? "Game" : "Mod"), - ("Engine", baseline.Target.Engine), - ("Version", baseline.Target.Version ?? "n/a"), + (" Name", baseline.Target.Name), + (" Type", baseline.Target.IsGame ? "Game" : "Mod"), + (" Engine", baseline.Target.Engine), + (" Version", baseline.Target.Version ?? "n/a"), ]; if (baseline.Target.Location is not null) - targetData.Add(("Location", baseline.Target.Location.TargetPath)); + targetData.Add((" Location", baseline.Target.Location.TargetPath)); ConsoleUtilities.PrintAsTable(targetData, 120); } diff --git a/src/ModVerify/GameVerifierService.cs b/src/ModVerify/GameVerifierService.cs index efdf3264..5592a7be 100644 --- a/src/ModVerify/GameVerifierService.cs +++ b/src/ModVerify/GameVerifierService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; using AET.ModVerify.Progress; @@ -13,11 +13,11 @@ namespace AET.ModVerify; internal sealed class GameVerifierService(IServiceProvider serviceProvider) : IGameVerifierService { public async Task VerifyAsync( - VerificationTarget verificationTarget, + VerificationTarget verificationTarget, VerifierServiceSettings settings, - VerificationBaseline baseline, + BaselineCollection baselines, SuppressionList suppressions, - IVerifyProgressReporter? progressReporter, + IVerifyProgressReporter? progressReporter, IGameEngineInitializationReporter? engineInitializationReporter, CancellationToken token = default) { @@ -25,12 +25,14 @@ public async Task VerifyAsync( throw new ArgumentNullException(nameof(verificationTarget)); if (settings == null) throw new ArgumentNullException(nameof(settings)); + if (baselines == null) + throw new ArgumentNullException(nameof(baselines)); using var pipeline = new GameVerifyPipeline( - verificationTarget, - settings, + verificationTarget, + settings, serviceProvider, - baseline, + baselines, suppressions, progressReporter, engineInitializationReporter); @@ -64,7 +66,7 @@ public async Task VerifyAsync( Errors = pipeline.Errors, Status = completionStatus, Target = verificationTarget, - UsedBaseline = baseline, + UsedBaselines = baselines, UsedSuppressions = suppressions, Verifiers = pipeline.Verifiers, Exception = exception diff --git a/src/ModVerify/GameVerifyPipeline.cs b/src/ModVerify/GameVerifyPipeline.cs index 4a7e0fb0..35e6e90f 100644 --- a/src/ModVerify/GameVerifyPipeline.cs +++ b/src/ModVerify/GameVerifyPipeline.cs @@ -24,18 +24,18 @@ namespace AET.ModVerify; internal sealed class GameVerifyPipeline : StepRunnerPipelineBase { private readonly List _verifiers = []; - private readonly List _errors = []; private readonly List _verificationSteps = []; - + private readonly GameEngineErrorCollection _engineErrorReporter = new(); private readonly VerificationTarget _verificationTarget; private readonly VerifierServiceSettings _serviceSettings; private readonly IVerifyProgressReporter? _progressReporter; private readonly IGameEngineInitializationReporter? _engineInitializationReporter; - private readonly VerificationBaseline _baseline; + private readonly BaselineCollection _baselines; private readonly SuppressionList _suppressions; + private VerificationErrors _errors = VerificationErrors.Empty; - internal IReadOnlyCollection Errors => [.._errors]; + internal VerificationErrors Errors => _errors; internal IReadOnlyCollection Verifiers => [.. _verifiers]; @@ -45,7 +45,7 @@ public GameVerifyPipeline( VerificationTarget verificationTarget, VerifierServiceSettings serviceSettings, IServiceProvider serviceProvider, - VerificationBaseline baseline, + BaselineCollection baselines, SuppressionList suppressions, IVerifyProgressReporter? progressReporter = null, IGameEngineInitializationReporter? engineInitializationReporter = null) @@ -53,7 +53,7 @@ public GameVerifyPipeline( { _verificationTarget = verificationTarget ?? throw new ArgumentNullException(nameof(verificationTarget)); _serviceSettings = serviceSettings ?? throw new ArgumentNullException(nameof(serviceSettings)); - _baseline = baseline ?? throw new ArgumentNullException(nameof(baseline)); + _baselines = baselines ?? throw new ArgumentNullException(nameof(baselines)); _suppressions = suppressions ?? throw new ArgumentNullException(nameof(suppressions)); _progressReporter = progressReporter; _engineInitializationReporter = engineInitializationReporter; @@ -76,7 +76,7 @@ protected override AsyncStepRunner CreateRunner() protected override async Task PrepareCoreAsync(CancellationToken token) { _verifiers.Clear(); - _errors.Clear(); + _errors = VerificationErrors.Empty; IStarWarsGameEngine gameEngine; @@ -117,8 +117,14 @@ protected override void OnExecuteStarted() protected override void OnExecuteCompleted() { Logger?.LogInformation("Game verifiers finished."); - _errors.AddRange(GetReportableErrors(_verifiers.SelectMany(s => s.VerifyErrors))); - _progressReporter?.Report(1.0, $"Finished Verifying {_verificationTarget.Name}", + // Suppressions are policy filters and run first so they never contaminate + // baseline attribution; the baselines then categorize what remains as + // new (unseen), existing (still present), or resolved (gone since baseline). + var afterSuppressions = _verifiers + .SelectMany(s => s.VerifyErrors) + .ApplySuppressions(_suppressions); + _errors = _baselines.Categorize(afterSuppressions); + _progressReporter?.Report(1.0, $"Finished Verifying {_verificationTarget.Name}", VerifyProgress.ProgressType, default); } @@ -129,7 +135,7 @@ protected override void OnRunnerExecutionError(object sender, StepRunnerErrorEve var minSeverity = _serviceSettings.FailFastSettings.MinumumSeverity; var ignoreError = verificationException.Errors .Where(error => error.Severity >= minSeverity) - .All(error => _baseline.Contains(error) || _suppressions.Suppresses(error)); + .All(error => _baselines.Contains(error) || _suppressions.Suppresses(error)); if (ignoreError) return; } @@ -157,14 +163,6 @@ private void AddStep(GameVerifier verifier) _verifiers.Add(verifier); } - private IEnumerable GetReportableErrors(IEnumerable errors) - { - Logger?.LogDebug("Applying baseline and suppressions."); - // NB: We don't filter for severity here, as the individual reporters handle that. - // This allows better control over what gets reported. - return errors.ApplyBaseline(_baseline).ApplySuppressions(_suppressions); - } - private IEnumerable CreateVerifiers(IStarWarsGameEngine engine) { return _serviceSettings.VerifiersProvider diff --git a/src/ModVerify/IGameVerifierService.cs b/src/ModVerify/IGameVerifierService.cs index c814779b..7095c44f 100644 --- a/src/ModVerify/IGameVerifierService.cs +++ b/src/ModVerify/IGameVerifierService.cs @@ -1,4 +1,4 @@ -using System.Threading; +using System.Threading; using System.Threading.Tasks; using AET.ModVerify.Progress; using AET.ModVerify.Reporting; @@ -14,9 +14,9 @@ public interface IGameVerifierService Task VerifyAsync( VerificationTarget verificationTarget, VerifierServiceSettings settings, - VerificationBaseline baseline, + BaselineCollection baselines, SuppressionList suppressions, IVerifyProgressReporter? progressReporter, IGameEngineInitializationReporter? engineInitializationReporter, CancellationToken token = default); -} \ No newline at end of file +} diff --git a/src/ModVerify/Reporting/Baseline/BaselineCollection.cs b/src/ModVerify/Reporting/Baseline/BaselineCollection.cs new file mode 100644 index 00000000..99099a5f --- /dev/null +++ b/src/ModVerify/Reporting/Baseline/BaselineCollection.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +namespace AET.ModVerify.Reporting.Baseline; + +public sealed class BaselineCollection : IReadOnlyCollection +{ + public static readonly BaselineCollection Empty = new([]); + + private readonly IReadOnlyList _baselines; + + public int Count => _baselines.Count; + + public bool IsEmpty => _baselines.Count == 0; + + public BaselineCollection(IEnumerable baselines) + { + if (baselines is null) + throw new ArgumentNullException(nameof(baselines)); + + var list = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var b in baselines) + { + if (b is null) + throw new ArgumentException("Baseline entries must not be null.", nameof(baselines)); + if (!seen.Add(b.Identifier)) + throw new ArgumentException($"Baseline identifier '{b.Identifier}' is not unique within the collection.", nameof(baselines)); + list.Add(b); + } + _baselines = list; + } + + public bool Contains(VerificationError error) + { + foreach (var entry in _baselines) + { + if (entry.Baseline.Contains(error)) + return true; + } + return false; + } + + public bool TryGetMatchingBaseline(VerificationError error, [NotNullWhen(true)] out string? identifier) + { + foreach (var entry in _baselines) + { + if (entry.Baseline.Contains(error)) + { + identifier = entry.Identifier; + return true; + } + } + identifier = null; + return false; + } + + public IEnumerable Apply(IEnumerable errors) + { + if (errors is null) + throw new ArgumentNullException(nameof(errors)); + if (_baselines.Count == 0) + return errors; + return errors.Where(e => !Contains(e)); + } + + public VerificationErrors Categorize(IEnumerable errors) + { + if (errors is null) + throw new ArgumentNullException(nameof(errors)); + + var newErrors = new List(); + var existingErrors = new Dictionary>(StringComparer.Ordinal); + foreach (var entry in _baselines) + existingErrors[entry.Identifier] = []; + + foreach (var error in errors) + { + if (TryGetMatchingBaseline(error, out var identifier)) + existingErrors[identifier].Add(error); + else + newErrors.Add(error); + } + + var resolvedErrors = new Dictionary>(StringComparer.Ordinal); + foreach (var entry in _baselines) + { + var seen = new HashSet(existingErrors[entry.Identifier]); + var solved = new List(); + foreach (var baselineError in entry.Baseline) + { + if (!seen.Contains(baselineError)) + solved.Add(baselineError); + } + resolvedErrors[entry.Identifier] = solved; + } + + var readOnlyExisting = new Dictionary>(StringComparer.Ordinal); + foreach (var kvp in existingErrors) + readOnlyExisting[kvp.Key] = kvp.Value; + + return new VerificationErrors(newErrors, readOnlyExisting, resolvedErrors); + } + + public IEnumerator GetEnumerator() => _baselines.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} \ No newline at end of file diff --git a/src/ModVerify/Reporting/Baseline/BaselineSource.cs b/src/ModVerify/Reporting/Baseline/BaselineSource.cs new file mode 100644 index 00000000..e018e418 --- /dev/null +++ b/src/ModVerify/Reporting/Baseline/BaselineSource.cs @@ -0,0 +1,7 @@ +namespace AET.ModVerify.Reporting.Baseline; + +public enum BaselineSource +{ + File, + EmbeddedDefault, +} \ No newline at end of file diff --git a/src/ModVerify/Reporting/Baseline/IdentifiedBaseline.cs b/src/ModVerify/Reporting/Baseline/IdentifiedBaseline.cs new file mode 100644 index 00000000..7726d84c --- /dev/null +++ b/src/ModVerify/Reporting/Baseline/IdentifiedBaseline.cs @@ -0,0 +1,21 @@ +using System; + +namespace AET.ModVerify.Reporting.Baseline; + +public sealed record IdentifiedBaseline +{ + public string Identifier { get; } + + public VerificationBaseline Baseline { get; } + + public BaselineSource Source { get; } + + public IdentifiedBaseline(string identifier, VerificationBaseline baseline, BaselineSource source) + { + if (string.IsNullOrEmpty(identifier)) + throw new ArgumentException("Identifier must be non-empty.", nameof(identifier)); + Identifier = identifier; + Baseline = baseline ?? throw new ArgumentNullException(nameof(baseline)); + Source = source; + } +} diff --git a/src/ModVerify/Reporting/Reporters/Console/ConsoleReporter.cs b/src/ModVerify/Reporting/Reporters/Console/ConsoleReporter.cs index 52665505..7b4433db 100644 --- a/src/ModVerify/Reporting/Reporters/Console/ConsoleReporter.cs +++ b/src/ModVerify/Reporting/Reporters/Console/ConsoleReporter.cs @@ -10,7 +10,7 @@ internal class ConsoleReporter(ConsoleReporterSettings settings, IServiceProvide { public override Task ReportAsync(VerificationResult verificationResult) { - var filteredErrors = FilteredErrors(verificationResult.Errors).OrderByDescending(x => x.Severity).ToList(); + var filteredErrors = FilteredErrors(verificationResult.Errors.NewErrors).OrderByDescending(x => x.Severity).ToList(); PrintErrorStats(verificationResult, filteredErrors); Console.WriteLine(); return Task.CompletedTask; @@ -24,7 +24,7 @@ private void PrintErrorStats(VerificationResult verificationResult, List x.Severity); + var groupedBySeverity = verificationResult.Errors.NewErrors.GroupBy(x => x.Severity); foreach (var group in groupedBySeverity) Console.WriteLine($" Severity {group.Key}: {group.Count()}"); Console.WriteLine(); if (filteredErrors.Count == 0) { - if (verificationResult.Errors.Count != 0) + if (verificationResult.Errors.NewErrors.Count != 0) Console.WriteLine("Some errors are not displayed to the console. Please check the created output files."); return; } diff --git a/src/ModVerify/Reporting/Reporters/JSON/JsonReporter.cs b/src/ModVerify/Reporting/Reporters/JSON/JsonReporter.cs index ac11bb6f..194cddbf 100644 --- a/src/ModVerify/Reporting/Reporters/JSON/JsonReporter.cs +++ b/src/ModVerify/Reporting/Reporters/JSON/JsonReporter.cs @@ -29,7 +29,7 @@ private JsonVerificationReport CreateJsonReport(VerificationResult result) IEnumerable errors; if (Settings.AggregateResults) { - errors = result.Errors + errors = result.Errors.NewErrors .OrderByDescending(x => x.Severity) .ThenBy(x => x.Id) .GroupBy(x => new GroupKey(x.Asset, x.Id, x.VerifierChain)) @@ -44,7 +44,7 @@ private JsonVerificationReport CreateJsonReport(VerificationResult result) } else { - errors = result.Errors + errors = result.Errors.NewErrors .OrderByDescending(x => x.Severity) .ThenBy(x => x.Id) .Select(x => new JsonVerificationError(x, Settings.Verbose)); diff --git a/src/ModVerify/Reporting/Reporters/Text/TextFileReporter.cs b/src/ModVerify/Reporting/Reporters/Text/TextFileReporter.cs index f3e81266..10a2f7f6 100644 --- a/src/ModVerify/Reporting/Reporters/Text/TextFileReporter.cs +++ b/src/ModVerify/Reporting/Reporters/Text/TextFileReporter.cs @@ -28,7 +28,7 @@ private async Task ReportWhole(VerificationResult result) await WriteHeader(result.Target, DateTime.Now, null, streamWriter); - foreach (var error in result.Errors.OrderBy(x => x.Id)) + foreach (var error in result.Errors.NewErrors.OrderBy(x => x.Id)) await WriteError(error, streamWriter); } @@ -36,7 +36,7 @@ private async Task ReportWhole(VerificationResult result) private async Task ReportByVerifier(VerificationResult result) { var time = DateTime.Now; - var grouped = result.Errors.GroupBy(x => x.VerifierChain.Last().Name); + var grouped = result.Errors.NewErrors.GroupBy(x => x.VerifierChain.Last().Name); foreach (var group in grouped) await ReportToSingleFile(group, result.Target, time); } diff --git a/src/ModVerify/Reporting/VerificationErrors.cs b/src/ModVerify/Reporting/VerificationErrors.cs new file mode 100644 index 00000000..7570b27d --- /dev/null +++ b/src/ModVerify/Reporting/VerificationErrors.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; + +namespace AET.ModVerify.Reporting; + +public sealed record VerificationErrors +{ + public static readonly VerificationErrors Empty = new( + [], + new Dictionary>(), + new Dictionary>()); + + public IReadOnlyList NewErrors { get; } + + public IReadOnlyDictionary> ExistingErrors { get; } + + public IReadOnlyDictionary> ResolvedErrors { get; } + + public VerificationErrors( + IReadOnlyList newErrors, + IReadOnlyDictionary> existingErrors, + IReadOnlyDictionary> resolvedErrors) + { + NewErrors = newErrors ?? throw new ArgumentNullException(nameof(newErrors)); + ExistingErrors = existingErrors ?? throw new ArgumentNullException(nameof(existingErrors)); + ResolvedErrors = resolvedErrors ?? throw new ArgumentNullException(nameof(resolvedErrors)); + } +} diff --git a/src/ModVerify/Reporting/VerificationResult.cs b/src/ModVerify/Reporting/VerificationResult.cs index 97c08e59..7333623d 100644 --- a/src/ModVerify/Reporting/VerificationResult.cs +++ b/src/ModVerify/Reporting/VerificationResult.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using AET.ModVerify.Reporting.Baseline; using AET.ModVerify.Reporting.Suppressions; @@ -10,13 +10,13 @@ public sealed record VerificationResult { public required VerificationCompletionStatus Status { get; init; } - public required IReadOnlyCollection Errors + public required VerificationErrors Errors { get; init => field = value ?? throw new ArgumentNullException(nameof(value)); } - public required VerificationBaseline UsedBaseline + public required BaselineCollection UsedBaselines { get; init => field = value ?? throw new ArgumentNullException(nameof(value)); @@ -28,8 +28,8 @@ public required SuppressionList UsedSuppressions init => field = value ?? throw new ArgumentNullException(nameof(value)); } - public required IReadOnlyCollection Verifiers - { + public required IReadOnlyCollection Verifiers + { get; init => field = value ?? throw new ArgumentNullException(nameof(value)); } @@ -43,4 +43,4 @@ public required VerificationTarget Target public required TimeSpan Duration { get; init; } public Exception? Exception { get; init; } -} \ No newline at end of file +} diff --git a/src/ModVerify/Utilities/VerificationErrorExtensions.cs b/src/ModVerify/Utilities/VerificationErrorExtensions.cs index f7eefbcc..53a93b5c 100644 --- a/src/ModVerify/Utilities/VerificationErrorExtensions.cs +++ b/src/ModVerify/Utilities/VerificationErrorExtensions.cs @@ -12,13 +12,22 @@ public static class VerificationErrorExtensions { public IEnumerable ApplyBaseline(VerificationBaseline baseline) { - if (errors == null) + if (errors == null) throw new ArgumentNullException(nameof(errors)); if (baseline == null) throw new ArgumentNullException(nameof(baseline)); return baseline.Apply(errors); } + public IEnumerable ApplyBaselines(BaselineCollection baselines) + { + if (errors == null) + throw new ArgumentNullException(nameof(errors)); + if (baselines == null) + throw new ArgumentNullException(nameof(baselines)); + return baselines.Apply(errors); + } + public IEnumerable ApplySuppressions(SuppressionList suppressions) { if (errors == null) diff --git a/src/ModVerify/Verifiers/Commons/SingleModelVerifier.cs b/src/ModVerify/Verifiers/Commons/SingleModelVerifier.cs index 0df4dd69..690d2505 100644 --- a/src/ModVerify/Verifiers/Commons/SingleModelVerifier.cs +++ b/src/ModVerify/Verifiers/Commons/SingleModelVerifier.cs @@ -439,7 +439,7 @@ private void VerifyShaderExists(IPetroglyphFileHolder model, string shader, IRea private bool CheckBinaryCorruptedFileIsActuallyRenderable(string fileName, out string actualFilePath) { var filePath = FileSystem.Path.Join(@"DATA\ART\MODELS", fileName); - var exists = GameEngine.GameRepository.FileExists(filePath, false, out _, out actualFilePath!); + var exists = GameEngine.GameRepository.ModelRepository.FileExists(filePath, false, out _, out actualFilePath!); Debug.Assert(exists); var extension = FileSystem.Path.GetExtension(actualFilePath); diff --git a/test/ModVerify.CliApp.Test/ModVerifyOptionsParserTest.cs b/test/ModVerify.CliApp.Test/ModVerifyOptionsParserTest.cs index ac98cfc1..0550b340 100644 --- a/test/ModVerify.CliApp.Test/ModVerifyOptionsParserTest.cs +++ b/test/ModVerify.CliApp.Test/ModVerifyOptionsParserTest.cs @@ -214,34 +214,18 @@ public void Parse_CreateBaseline_MissingRequired_Fails(string argString) } [Theory] - [InlineData("verify --mods myMod --baseline myBaseline.json", "myBaseline.json", false, false)] - [InlineData("verify --mods myMod --searchBaseline", null, true, false)] - [InlineData("verify --path myMod --useDefaultBaseline", null, false, true)] - public void Parse_Verify_BaselineOptions(string argString, string? expectedBaseline, bool expectedSearchBaseline, bool expectedUseDefaultBaseline) + [InlineData("verify --mods myMod --baseline myBaseline.json", "myBaseline.json", false)] + [InlineData("verify --path myMod --useDefaultBaseline", null, true)] + public void Parse_Verify_BaselineOptions(string argString, string? expectedBaseline, bool expectedUseDefaultBaseline) { var settings = Parser.Parse(argString.Split(' ', StringSplitOptions.RemoveEmptyEntries)); Assert.True(settings.HasOptions); var verify = Assert.IsType(settings.ModVerifyOptions); - Assert.Equal(expectedBaseline, verify.Baseline); - Assert.Equal(expectedSearchBaseline, verify.SearchBaselineLocally); + Assert.Equal(expectedBaseline, verify.BaselinePaths); Assert.Equal(expectedUseDefaultBaseline, verify.UseDefaultBaseline); } - [Fact] - public void Parse_Verify_Baseline_And_SearchBaseline_CanBeParsedTogether() - { - // Mutual exclusivity of --baseline and --searchBaseline is enforced later by SettingsBuilder, not by the parser. - const string argString = "verify --mods myMod --baseline myBaseline.json --searchBaseline"; - - var settings = Parser.Parse(argString.Split(' ', StringSplitOptions.RemoveEmptyEntries)); - - Assert.True(settings.HasOptions); - var verify = Assert.IsType(settings.ModVerifyOptions); - Assert.Equal("myBaseline.json", verify.Baseline); - Assert.True(verify.SearchBaselineLocally); - } - [Theory] [InlineData("verify --path myMod --outDir myOut", "myOut")] [InlineData("verify --path myMod -o myOut", "myOut")] diff --git a/test/ModVerify.CliApp.Test/SettingsBuilderTest.cs b/test/ModVerify.CliApp.Test/SettingsBuilderTest.cs index 429567b8..d79c7b08 100644 --- a/test/ModVerify.CliApp.Test/SettingsBuilderTest.cs +++ b/test/ModVerify.CliApp.Test/SettingsBuilderTest.cs @@ -70,41 +70,66 @@ public void BuildSettings_FallbackGamePath_RequiresGamePath() } [Fact] - public void BuildSettings_UseDefaultBaseline_And_Baseline_Throws() + public void BuildSettings_UseDefaultBaseline_And_Baseline() { var options = new VerifyVerbOption { UseDefaultBaseline = true, - Baseline = "myBaseline.json", + BaselinePaths = "myBaseline.json", TargetPath = "myPath", }; - Assert.Throws(() => _builder.BuildSettings(options)); + var settings = _builder.BuildSettings(options); + Assert.NotNull(settings); + var verifySettings = Assert.IsType(settings); + Assert.Equal([FileSystem.Path.GetFullPath("myBaseline.json")], verifySettings.ReportSettings.BaselinePaths); + Assert.True(verifySettings.ReportSettings.UseDefaultBaseline); } [Fact] - public void BuildSettings_UseDefaultBaseline_And_SearchBaseline_Throws() + public void BuildSettings_UseDefaultBaseline_Alone_DoesNotThrow() { var options = new VerifyVerbOption { UseDefaultBaseline = true, - SearchBaselineLocally = true, TargetPath = "myPath", }; - Assert.Throws(() => _builder.BuildSettings(options)); + var settings = _builder.BuildSettings(options); + Assert.NotNull(settings); } [Fact] - public void BuildSettings_UseDefaultBaseline_Alone_DoesNotThrow() + public void BuildSettings_CreateBaseline_Baseline_And_UseDefaultBaseline() { - var options = new VerifyVerbOption + var options = new CreateBaselineVerbOption { + OutputFile = "out.json", + BaselinePaths = "input.json", UseDefaultBaseline = true, TargetPath = "myPath", }; var settings = _builder.BuildSettings(options); - Assert.NotNull(settings); + var baselineSettings = Assert.IsType(settings); + Assert.Equal([FileSystem.Path.GetFullPath("input.json")], baselineSettings.ReportSettings.BaselinePaths); + Assert.True(baselineSettings.ReportSettings.UseDefaultBaseline); + } + + [Fact] + public void BuildSettings_Baselines_SplitsByPathSeparator() + { + var separator = FileSystem.Path.PathSeparator; + var options = new VerifyVerbOption + { + BaselinePaths = $"first.json{separator}second.json", + TargetPath = "myPath", + }; + + var settings = _builder.BuildSettings(options); + var verifySettings = Assert.IsType(settings); + Assert.Equal( + [FileSystem.Path.GetFullPath("first.json"), FileSystem.Path.GetFullPath("second.json")], + verifySettings.ReportSettings.BaselinePaths); } }