Skip to content

Commit 1093811

Browse files
exyiexyi
committed
Remove MedallionShell dependency, better error messages, Json: normalizePropertyOrder
1 parent 0b559ca commit 1093811

8 files changed

Lines changed: 260 additions & 19 deletions

File tree

‎exampleTest/SomeTest.cs‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public void MultilineCheck()
3434
public void JsonObjectCheck()
3535
{
3636
check.CheckJsonObject(
37-
new { number = 3, str = "jaja", list = new List<object> { 1, "23", new SomeTestObject { Prop = "hmm" } } }
37+
new { number = 3, str = "jaja", list = new List<object> { 1, "2313", new SomeTestObject { Prop = "hmm" } } }
3838
);
3939
}
4040

@@ -51,6 +51,48 @@ public void IncrementNumbers(string checkName, string testString)
5151
}, checkName);
5252
}
5353

54+
[Fact]
55+
public void JsonWithNormalizedOrder()
56+
{
57+
var dict = new Dictionary<string, object> {
58+
{ "a", 1 },
59+
{ "b", 2 },
60+
{ "c", 3 },
61+
{ "d", 4 },
62+
{ "e", 5 },
63+
{ "f", 6 },
64+
{ "g", 7 },
65+
{ "h", 8 },
66+
{ "o", 15 },
67+
{ "i", 9 },
68+
{ "j", 10 },
69+
{ "k", 11 },
70+
{ "l", 12 },
71+
{ "m", 13 },
72+
{ "n", 14 },
73+
{ "p", 16 },
74+
{ "q", 17 },
75+
{ "r", 18 },
76+
{ "s", 19 },
77+
{ "t", 20 },
78+
{ "u", 21 },
79+
{ "v", 22 },
80+
{ "w", 23 },
81+
{ "x", 24 },
82+
{ "y", 25 },
83+
{ "z", 26 },
84+
{ "lalala", new SomeTestObject() }
85+
};
86+
var random = new Random();
87+
// lol, Dictionary is bit too stable by default :D
88+
89+
var randomEl = dict.ElementAt(random.Next(0, dict.Count - 1));
90+
dict.Remove(randomEl.Key);
91+
dict.Add("lol", false);
92+
dict.Add(randomEl.Key, randomEl.Value);
93+
check.CheckJsonObject(dict, normalizePropertyOrder: true);
94+
}
95+
5496
class SomeTestObject
5597
{
5698
public string Prop { get; set; }

‎exampleTest/testoutputs/SomeTest.JsonObjectCheck.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"str": "jaja",
44
"list": [
55
1,
6-
"23",
6+
"2313",
77
{
88
"Prop": "hmm"
99
}

‎exampleTest/testoutputs/SomeTest.JsonWithNormalizedOrder.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"a": 1,
3+
"b": 2,
4+
"c": 3,
5+
"d": 4,
6+
"e": 5,
7+
"f": 6,
8+
"g": 7,
9+
"h": 8,
10+
"i": 9,
11+
"j": 10,
12+
"k": 11,
13+
"l": 12,
14+
"lalala": {
15+
"Prop": null
16+
},
17+
"lol": false,
18+
"m": 13,
19+
"n": 14,
20+
"o": 15,
21+
"p": 16,
22+
"q": 17,
23+
"r": 18,
24+
"s": 19,
25+
"t": 20,
26+
"u": 21,
27+
"v": 22,
28+
"w": 23,
29+
"x": 24,
30+
"y": 25,
31+
"z": 26
32+
}

‎img/vscode-stage-changes.png‎

60 KB
Loading

‎src/CheckTestOutput.csproj‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFrameworks>netstandard2.0;netcoreapp3.0;net5.0;net6.0</TargetFrameworks>
4+
<TargetFrameworks>netstandard2.1;net6.0</TargetFrameworks>
55

66
<Description>Simple helper which checks that output of a test matches a file. If not matching, just git staging the new file will accept the new version.</Description>
77
<PackageId>CheckTestOutput</PackageId>
88
<Version>$(PackageVersion)</Version>
99
<Authors>Standa Lukeš</Authors>
10-
<PackageLicenseFile>LICENSE</PackageLicenseFile>
10+
<PackageLicenseExpression>MIT</PackageLicenseExpression>
11+
<PackageRequireLicenseAcceptance>false</PackageRequireLicenseAcceptance>
1112
<RepositoryUrl>https://github.com/exyi/CheckTestOutput.git</RepositoryUrl>
1213
<RepositoryType>git</RepositoryType>
1314
<PackageProjectUrl>https://github.com/exyi/CheckTestOutput</PackageProjectUrl>
@@ -17,16 +18,20 @@
1718
<PublicSign>true</PublicSign>
1819

1920
<PackageReadmeFile>README.md</PackageReadmeFile>
21+
22+
<PublishRepositoryUrl>true</PublishRepositoryUrl>
23+
<IncludeSymbols>false</IncludeSymbols>
24+
<DebugType>embedded</DebugType>
25+
<EmbedAllSources>true</EmbedAllSources>
2026
</PropertyGroup>
2127

22-
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
28+
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.1'">
2329
<PackageReference Include="System.Text.Json" Version="5.0.2" />
2430
</ItemGroup>
2531

2632
<ItemGroup>
27-
<PackageReference Include="MedallionShell" Version="1.6.2" />
33+
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All"/>
2834

29-
<None Include="../LICENSE" Pack="true" PackagePath=""/>
3035
<None Include="../README.md" Pack="true" PackagePath=""/>
3136
</ItemGroup>
3237

‎src/JsonChecks.cs‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,27 @@ public static class JsonChecks
1212
public static void CheckJsonObject(
1313
this OutputChecker t,
1414
object output,
15+
JsonSerializerOptions jsonOptions = null,
16+
bool normalizePropertyOrder = false,
1517
string checkName = null,
1618
string fileExtension = "json",
1719
[System.Runtime.CompilerServices.CallerMemberName] string memberName = null,
1820
[System.Runtime.CompilerServices.CallerFilePath] string sourceFilePath = null)
1921
{
22+
jsonOptions ??= new JsonSerializerOptions() { WriteIndented = true };
2023
var strOutput =
21-
JsonSerializer.Serialize(output, new JsonSerializerOptions() { WriteIndented = true });
24+
JsonSerializer.Serialize(output, jsonOptions);
25+
26+
if (normalizePropertyOrder)
27+
{
28+
// this awesome, just give me back newtonsoft...
29+
var jsonDocument = JsonDocument.Parse(strOutput);
30+
var outputStream = new MemoryStream();
31+
var writer = new Utf8JsonWriter(outputStream, new JsonWriterOptions { Indented = true });
32+
NormalizePropertyOrder(jsonDocument.RootElement, writer);
33+
writer.Flush();
34+
strOutput = System.Text.Encoding.UTF8.GetString(outputStream.ToArray());
35+
}
2236

2337

2438
// indent using tabs for back compatibility
@@ -30,5 +44,38 @@ public static void CheckJsonObject(
3044
fileExtension
3145
);
3246
}
47+
48+
static void NormalizePropertyOrder(JsonElement e, Utf8JsonWriter output)
49+
{
50+
if (e.ValueKind == JsonValueKind.Object)
51+
{
52+
var properties = new List<(string name, JsonElement e)>();
53+
foreach (var prop in e.EnumerateObject())
54+
{
55+
properties.Add((prop.Name, prop.Value));
56+
}
57+
properties.Sort();
58+
output.WriteStartObject();
59+
foreach (var p in properties)
60+
{
61+
output.WritePropertyName(p.name);
62+
NormalizePropertyOrder(p.e, output);
63+
}
64+
output.WriteEndObject();
65+
}
66+
else if (e.ValueKind == JsonValueKind.Array)
67+
{
68+
output.WriteStartArray();
69+
foreach (var item in e.EnumerateArray())
70+
{
71+
NormalizePropertyOrder(item, output);
72+
}
73+
output.WriteEndArray();
74+
}
75+
else
76+
{
77+
e.WriteTo(output);
78+
}
79+
}
3380
}
3481
}

‎src/OutputChecker.cs‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
using System;
22
using System.Collections.Generic;
33
using System.ComponentModel;
4+
using System.Diagnostics;
45
using System.IO;
56
using System.Linq;
67
using System.Text.RegularExpressions;
7-
using Medallion.Shell;
88

99
namespace CheckTestOutput
1010
{
@@ -49,7 +49,7 @@ public OutputChecker(
4949
}
5050
catch (Win32Exception)
5151
{
52-
Console.WriteLine("CheckTestOutput warning: git command not found. Falling back to simple file-based checking");
52+
Console.WriteLine("CheckTestOutput warning: git command not found. Falling back to simple file-based checking. Make sure that git is installed and in the PATH.");
5353
return false;
5454
}
5555
catch (Exception e) when (e.Message.StartsWith("Git command failed: fatal: not a git repository"))
@@ -79,14 +79,39 @@ public OutputChecker(
7979

8080
private string[] RunGitCommand(params string[] args)
8181
{
82-
using(var cmd = Command.Run("git", args, o => { o.WorkingDirectory(CheckDirectory); o.Timeout(TimeSpan.FromSeconds(3)); }))
82+
// run `git ...args` in CheckDirectory working directory with 3 second timeout
83+
var procInfo = new ProcessStartInfo("git")
8384
{
84-
cmd.Wait();
85-
if (cmd.Task.Result.ExitCode != 0)
86-
throw new Exception($"Git command failed: {cmd.Task.Result.StandardError}");
87-
88-
return cmd.StandardOutput.GetLines().ToArray();
85+
WorkingDirectory = CheckDirectory,
86+
UseShellExecute = false,
87+
RedirectStandardOutput = true,
88+
RedirectStandardError = true,
89+
CreateNoWindow = true,
90+
StandardOutputEncoding = System.Text.Encoding.UTF8,
91+
StandardErrorEncoding = System.Text.Encoding.UTF8,
92+
};
93+
foreach (var a in args)
94+
procInfo.ArgumentList.Add(a);
95+
96+
var proc = Process.Start(procInfo);
97+
if (!proc.WaitForExit(3000))
98+
{
99+
proc.Kill();
100+
throw new Exception("Git command timed out");
89101
}
102+
103+
if (proc.ExitCode != 0)
104+
throw new Exception("Git command failed: " + proc.StandardError.ReadToEnd());
105+
106+
return proc.StandardOutput.ReadToEnd().Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
107+
}
108+
109+
static string[] ReadAllLines(StreamReader reader)
110+
{
111+
var lines = new List<string>();
112+
while (!reader.EndOfStream && reader.ReadLine() is {} line)
113+
lines.Add(line);
114+
return lines.ToArray();
90115
}
91116

92117
private string GetOldContent(string file)
@@ -167,13 +192,18 @@ internal void CheckOutputCore(string outputString, string checkName, string meth
167192
{
168193
var diff = RunGitCommand("diff", filename);
169194
if (diff.All(string.IsNullOrEmpty))
170-
throw new Exception($"Check {Path.GetFileName(filename)} - the file is probably untracked in git");
171-
throw new Exception($"Check {Path.GetFileName(filename)} - the expected output is different:\n{string.Join("\n", diff)}");
195+
throw new Exception($"{Path.GetFileName(filename)} is not explicitly accepted - the file is untracked in git. View the file and stage to let this test pass. Confused? See https://github.com/exyi/CheckTestOutput/blob/master/trouble.md#untracked-file\n");
196+
throw new Exception(
197+
$"{Path.GetFileName(filename)} has changed, the actual output differs from the previous accepted output:\n\n" +
198+
string.Join("\n", diff) + "\n\n" +
199+
"If this change OK? Stage the file in git to let the test pass. Confused? See https://github.com/exyi/CheckTestOutput/blob/master/trouble.md#changed-file\n"
200+
201+
);
172202
}
173203
}
174204
else
175205
{
176-
throw new Exception($"Check {Path.GetFileName(filename)} - the expected output is different:\n{outputString}");
206+
throw new Exception($"{Path.GetFileName(filename)}has changed, the previous accepted output differs from the actual output:\n\n{outputString}\n\nNote that CheckTestOutput could not use git on your system, so the \"UX\" is limited.");
177207
}
178208
}
179209
}

‎trouble.md‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Troubleshooting common issues
2+
3+
## Untracked File
4+
5+
Did you see error like?
6+
7+
```
8+
Error Message:
9+
System.Exception : SomeTests.MyTest.txt is not explicitly accepted - the file is untracked in git. Confused? See https://github.com/exyi/CheckTestOutput/blob/master/trouble.md
10+
Stack Trace:
11+
at CheckTestOutput.OutputChecker.CheckOutputCore(String outputString, String checkName, String method, String fileExtension)
12+
at CheckTestOutput.BasicChecks.CheckString(OutputChecker t, String output, String checkName, String fileExtension, String memberName, String sourceFilePath)
13+
at MyTest()
14+
```
15+
16+
When you run `CheckString(...)` for the first time, the test output file does not exist, so CheckTestOutput can't know if the output is correct or not.
17+
The only reasonable option is to initially fail your test and write the missing file to disk.
18+
You only need to view the file, manually verify that the test output is correct and stage it in git (using `git add $myfile`).
19+
20+
Using command line, run:
21+
22+
```bash
23+
$EDITOR <TheFile.txt>
24+
```
25+
26+
I you think this is a correct output, run
27+
28+
```bash
29+
git add <TheFile.txt>
30+
```
31+
32+
Now the test will pass. Alternatively, I'd recommend using VS Code or a similar editor with git integration.
33+
34+
## Changed file
35+
36+
Did you see error like?
37+
38+
```
39+
System.Exception : SomeTest.JsonObjectCheck.json has changed, the actual output differs from the previous accepted output:
40+
41+
diff --git a/exampleTest/testoutputs/SomeTest.JsonObjectCheck.json b/exampleTest/testoutputs/SomeTest.JsonObjectCheck.json
42+
index bc75540..2da815d 100644
43+
--- a/exampleTest/testoutputs/SomeTest.JsonObjectCheck.json
44+
+++ b/exampleTest/testoutputs/SomeTest.JsonObjectCheck.json
45+
@@ -3,7 +3,7 @@
46+
"str": "jaja",
47+
"list": [
48+
1,
49+
- "23",
50+
+ "2313",
51+
{
52+
"Prop": "hmm"
53+
}
54+
55+
If this change OK? Stage the file to let the test pass. Confused? See https://github.com/exyi/CheckTestOutput/blob/master/trouble.md#changed-file
56+
57+
Stack Trace:
58+
at CheckTestOutput.OutputChecker.CheckOutputCore(String outputString, String checkName, String method, String fileExtension)
59+
at CheckTestOutput.JsonChecks.CheckJsonObject(OutputChecker t, Object output, String checkName, String fileExtension, String memberName, String sourceFilePath)
60+
at CheckTestOutput.Example.SomeTest.JsonObjectCheck()
61+
```
62+
63+
This means that you or your coworker have previously accepted a different test output.
64+
Now it fails because the output is different, so either your test is non-deterministic or you changed something in logic.
65+
If you are happy with making the change, simply run `git add <TheFile>` or Stage the changes in your favorite editor / Git UI.
66+
67+
![Example: VS Code's "Stage Changes" button](img/vscode-stage-changes.png)
68+
69+
70+
## My test is not deterministic
71+
72+
This is a problem, you can't test a function which returns something different every time using CheckTestOutput.
73+
Fortunately, lot's of non-determinism can be dealt with:
74+
75+
* Preferably, you'd change your core logic to be deterministic ;)
76+
* If the test only fails with **low probability**, consider retrying it (simply put it in a loop with a try-catch, you don't need any exponential backoffs)
77+
* Often the problem comes from **different ordering** of output due to usage of **hash tables**.
78+
* In such case you just need to reorder the output alphabetically.
79+
* For example, [DotVVM reorders all HTML attributes when testing](https://github.com/riganti/dotvvm/blob/f4fd122218103b5b2ad1fda226ad9c8e131faaf3/src/Framework/Testing/ControlTestHelper.cs#L181)
80+
* For different order of json properties, use `CheckJsonObject(..., normalizePropertyOrder: true)`
81+
82+
* When the problem culprit is non-deterministic IDs (such as UUIDs/GUIDs or even sequential ids may depend on too many factors):
83+
* CheckTestOutput can sanitize these fragments into a set of sequential ids
84+
* See [README | Non-deterministic strings](https://github.com/exyi/CheckTestOutput#non-deterministic-strings)
85+
* TL;DR: use `OutputChecker(sanitizeGuids: true)` for UUIDs or `OutputChecker(nonDeterminismSanitizers: new [] { "Process Id: (\d+)" })` for matching anything else using a Regex.

0 commit comments

Comments
 (0)