Skip to content

Commit b19de96

Browse files
NucsNucs
committed
Revert "refactor(test): remove [SkipOnLowMemory] - CI passes without it"
This reverts commit ca05937.
1 parent ca05937 commit b19de96

5 files changed

Lines changed: 361 additions & 0 deletions

File tree

‎test/NumSharp.UnitTest/Backends/Unmanaged/AllocationTests.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using AwesomeAssertions;
33
using Microsoft.VisualStudio.TestTools.UnitTesting;
44
using NumSharp.Backends;
5+
using NumSharp.UnitTest.Utilities;
56

67
namespace NumSharp.UnitTest.Backends.Unmanaged
78
{
@@ -18,6 +19,7 @@ public class AllocationTests
1819

1920
[Test]
2021
[HighMemory]
22+
[SkipOnLowMemory(8)] // Actually allocates 4GB (Int32 * 1B elements)
2123
public void Allocate_1GB()
2224
{
2325
lock (_lock)
@@ -30,6 +32,7 @@ public void Allocate_1GB()
3032

3133
[Test]
3234
[HighMemory]
35+
[SkipOnLowMemory(12)] // Actually allocates 8GB (Int32 * 2B elements)
3336
public void Allocate_2GB()
3437
{
3538
lock (_lock)
@@ -42,6 +45,7 @@ public void Allocate_2GB()
4245

4346
[Test]
4447
[HighMemory]
48+
[SkipOnLowMemory(20)] // Actually allocates 16GB (Int32 * 4B elements)
4549
public void Allocate_4GB()
4650
{
4751
lock (_lock)
@@ -54,6 +58,7 @@ public void Allocate_4GB()
5458

5559
[Test]
5660
[HighMemory]
61+
[SkipOnLowMemory(50)] // Actually allocates 44GB+
5762
[OpenBugs]
5863
public void Allocate_44GB()
5964
{

‎test/NumSharp.UnitTest/LongIndexing/LongIndexingBroadcastTest.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Threading.Tasks;
33
using Microsoft.VisualStudio.TestTools.UnitTesting;
44
using NumSharp;
5+
using NumSharp.UnitTest.Utilities;
56
using TUnit.Core;
67

78
namespace NumSharp.UnitTest.LongIndexing;
@@ -20,8 +21,13 @@ namespace NumSharp.UnitTest.LongIndexing;
2021
/// - SliceDef is limited to int indices (cannot slice broadcast arrays with long indices)
2122
/// - Operations that produce output (add, multiply, etc.) allocate full-size output arrays
2223
/// even when input is broadcast, causing OutOfMemoryException
24+
///
25+
/// NOTE: Marked [HighMemory] because iterating over 2.36 billion elements causes
26+
/// excessive CPU/memory pressure when TUnit runs tests in parallel, leading to
27+
/// OOM kills on Ubuntu CI runners.
2328
/// </summary>
2429
[HighMemory]
30+
[SkipOnLowMemory(12)] // Skip on CI runners with limited memory
2531
public class LongIndexingBroadcastTest
2632
{
2733
/// <summary>

‎test/NumSharp.UnitTest/Utilities/MemoryMeasurementHook.cs‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
using System;
2+
using System.Collections.Concurrent;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Threading.Tasks;
6+
using TUnit.Core;
7+
8+
namespace NumSharp.UnitTest.Utilities;
9+
10+
/// <summary>
11+
/// Tracks test execution and memory usage.
12+
/// - Registers tests with TestMemoryTracker for active test monitoring
13+
/// - Logs high memory allocations during test run
14+
/// - Produces summary report at end of test run
15+
/// </summary>
16+
public class MemoryMeasurementHook
17+
{
18+
private static readonly ConcurrentDictionary<string, long> _beforeMemory = new();
19+
private static readonly ConcurrentDictionary<string, MemoryResult> _results = new();
20+
private static readonly string _logFile = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "memory-profile.log");
21+
private static bool _initialized = false;
22+
private static readonly object _initLock = new();
23+
24+
private static void Initialize()
25+
{
26+
lock (_initLock)
27+
{
28+
if (_initialized) return;
29+
_initialized = true;
30+
31+
try
32+
{
33+
File.WriteAllText(_logFile, $"Memory Profile Started: {DateTime.Now}\n");
34+
}
35+
catch { }
36+
37+
Console.Error.WriteLine("[MemoryHook] Memory tracking initialized");
38+
}
39+
}
40+
41+
private static void Log(string message)
42+
{
43+
try
44+
{
45+
lock (_logFile)
46+
{
47+
File.AppendAllText(_logFile, $"[{DateTime.Now:HH:mm:ss.fff}] {message}\n");
48+
}
49+
}
50+
catch { }
51+
}
52+
53+
[BeforeEvery(HookType.Test)]
54+
public static async Task BeforeEachTest(TestContext context)
55+
{
56+
Initialize();
57+
58+
var testName = GetTestName(context);
59+
60+
// Register with tracker for active test monitoring
61+
TestMemoryTracker.TestStarted(testName);
62+
63+
// Force GC to get accurate baseline
64+
GC.Collect();
65+
GC.WaitForPendingFinalizers();
66+
GC.Collect();
67+
68+
var memoryBefore = GC.GetTotalMemory(forceFullCollection: false);
69+
_beforeMemory[testName] = memoryBefore;
70+
71+
// Check memory and log if getting low (< 4GB available)
72+
TestMemoryTracker.LogIfMemoryLow(4.0);
73+
74+
await Task.CompletedTask;
75+
}
76+
77+
[AfterEvery(HookType.Test)]
78+
public static async Task AfterEachTest(TestContext context)
79+
{
80+
var testName = GetTestName(context);
81+
82+
// Unregister from tracker
83+
TestMemoryTracker.TestCompleted(testName);
84+
85+
if (!_beforeMemory.TryRemove(testName, out var memoryBefore))
86+
{
87+
memoryBefore = 0;
88+
}
89+
90+
// Measure after (don't force GC to see actual allocation)
91+
var memoryAfter = GC.GetTotalMemory(forceFullCollection: false);
92+
var allocated = memoryAfter - memoryBefore;
93+
94+
var result = new MemoryResult
95+
{
96+
TestName = testName,
97+
MemoryBefore = memoryBefore,
98+
MemoryAfter = memoryAfter,
99+
Allocated = allocated
100+
};
101+
102+
_results[testName] = result;
103+
104+
// Log if significant allocation (> 10 MB)
105+
var allocatedMB = allocated / (1024.0 * 1024.0);
106+
if (allocatedMB > 10.0)
107+
{
108+
var msg = $"{testName}: +{allocatedMB:F1} MB allocated";
109+
Log(msg);
110+
Console.Error.WriteLine($"[MemoryHook] {msg}");
111+
}
112+
113+
// Check memory after test completes
114+
TestMemoryTracker.LogIfMemoryLow(4.0);
115+
116+
await Task.CompletedTask;
117+
}
118+
119+
[After(HookType.Assembly)]
120+
public static async Task AfterAllTests(AssemblyHookContext context)
121+
{
122+
var sb = new System.Text.StringBuilder();
123+
sb.AppendLine("\n" + new string('=', 100));
124+
sb.AppendLine("MEMORY USAGE SUMMARY (sorted by allocation, descending)");
125+
sb.AppendLine(new string('=', 100));
126+
127+
var sorted = _results.Values
128+
.OrderByDescending(r => r.Allocated)
129+
.Take(100);
130+
131+
foreach (var result in sorted)
132+
{
133+
var allocatedMB = result.Allocated / (1024.0 * 1024.0);
134+
135+
if (allocatedMB > 1.0)
136+
{
137+
sb.AppendLine($" {allocatedMB,10:F2} MB | {result.TestName}");
138+
}
139+
}
140+
141+
sb.AppendLine(new string('=', 100));
142+
sb.AppendLine($"Total tests measured: {_results.Count}");
143+
144+
var totalAllocated = _results.Values.Sum(r => Math.Max(0, r.Allocated));
145+
sb.AppendLine($"Total allocated: {totalAllocated / (1024.0 * 1024.0 * 1024.0):F2} GB");
146+
147+
var testsOver100MB = _results.Values.Count(r => r.Allocated > 100 * 1024 * 1024);
148+
var testsOver1GB = _results.Values.Count(r => r.Allocated > 1024L * 1024 * 1024);
149+
sb.AppendLine($"Tests > 100 MB: {testsOver100MB}");
150+
sb.AppendLine($"Tests > 1 GB: {testsOver1GB}");
151+
sb.AppendLine(new string('=', 100));
152+
153+
var summary = sb.ToString();
154+
Log(summary);
155+
Console.WriteLine(summary);
156+
157+
await Task.CompletedTask;
158+
}
159+
160+
private static string GetTestName(TestContext context)
161+
{
162+
var className = context.Metadata.TestDetails?.Class?.ClassType?.Name ?? "Unknown";
163+
var testName = context.Metadata.TestName;
164+
return $"{className}.{testName}";
165+
}
166+
167+
private class MemoryResult
168+
{
169+
public string TestName { get; set; } = "";
170+
public long MemoryBefore { get; set; }
171+
public long MemoryAfter { get; set; }
172+
public long Allocated { get; set; }
173+
}
174+
}

‎test/NumSharp.UnitTest/Utilities/SkipOnLowMemoryAttribute.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
using System;
2+
using System.Threading.Tasks;
3+
using TUnit.Core;
4+
5+
namespace NumSharp.UnitTest.Utilities;
6+
7+
/// <summary>
8+
/// Skips the test when available system memory is below the required threshold.
9+
/// When skipping, logs all currently running tests to help diagnose memory pressure.
10+
///
11+
/// This is a runtime skip that works regardless of treenode-filter limitations.
12+
/// Use together with [HighMemory] for documentation and filtering attempts:
13+
/// <code>
14+
/// [HighMemory] // CategoryAttribute for documentation/filtering
15+
/// [SkipOnLowMemory(8)] // Runtime skip if less than 8GB available
16+
/// public void LargeAllocationTest() { }
17+
/// </code>
18+
/// </summary>
19+
public class SkipOnLowMemoryAttribute : SkipAttribute
20+
{
21+
private readonly long _requiredMemoryGB;
22+
23+
/// <summary>
24+
/// Creates a skip attribute that skips when available memory is below the threshold.
25+
/// </summary>
26+
/// <param name="requiredMemoryGB">Minimum required available memory in gigabytes.</param>
27+
public SkipOnLowMemoryAttribute(int requiredMemoryGB = 8)
28+
: base($"Requires {requiredMemoryGB}GB+ available memory")
29+
{
30+
_requiredMemoryGB = requiredMemoryGB;
31+
}
32+
33+
public override Task<bool> ShouldSkip(TestRegisteredContext context)
34+
{
35+
// DISABLED FOR TESTING - always run, never skip
36+
// This tests if CI can pass without memory-based skipping
37+
var availableMemoryGB = TestMemoryTracker.GetAvailableMemoryGB();
38+
Console.Error.WriteLine($"[SkipOnLowMemory] DISABLED - Running test. Available: {availableMemoryGB:F1}GB, Would need: {_requiredMemoryGB}GB");
39+
40+
return Task.FromResult(false); // Never skip
41+
42+
/* ORIGINAL CODE - re-enable after testing
43+
var shouldSkip = availableMemoryGB < _requiredMemoryGB;
44+
45+
if (shouldSkip)
46+
{
47+
// Get test name from TestRegisteredContext.TestDetails
48+
var className = context.TestDetails?.Class?.ClassType?.Name ?? "Unknown";
49+
var methodName = context.TestDetails?.TestName ?? context.TestName ?? "Unknown";
50+
var testName = $"{className}.{methodName}";
51+
52+
// Log skip reason with available memory
53+
Console.Error.WriteLine($"\n[SkipOnLowMemory] SKIPPING {testName}");
54+
Console.Error.WriteLine($"[SkipOnLowMemory] Available: {availableMemoryGB:F1}GB, Required: {_requiredMemoryGB}GB");
55+
56+
// Log what tests are currently running that might be consuming memory
57+
Console.Error.WriteLine(TestMemoryTracker.GetRunningTestsReport());
58+
Console.Error.WriteLine();
59+
}
60+
61+
return Task.FromResult(shouldSkip);
62+
*/
63+
}
64+
}

0 commit comments

Comments
 (0)