feat: Add Metrics to SharpHound to be collected - BED-7080 (#267)

* wip: stub MetricProcessor.cs

* wip: cleanup MetricProcessor and Writer

* chore: alter histogram value type

* fix: create Dictionary Directly

* wip: create platform for defining and observing metrics

* wip: adding some ldap metrics to LdapUtils and LdapConnectionPool

* feat: add FileMetricSink, MetricsFlushTimer, and MetricWriter. Update LabelValues to a struct

* feat: refine metric logic

* test: fix AdaptiveTimeout LatencyObservation, Add Tests

* test: fix AdaptiveTimeout LatencyObservation, Add MetricDefinitionTests.cs and DefaultLabelValuesCacheTests.cs

* test: add FileMetricSinkTests

* chore: add notes for IsExternalInit.cs

* tests: Add MetricAggregatorTests.cs, MetricRegistryTests.cs, and MetricRouterTests.cs

* tests: adjusting to relax ranges on AdaptiveTimeoutTests

* chore: coderabbit suggestions

* chore: more coderabbit suggestions

* chore: more coderabbit suggestions

* feat: update generics for aggregator to send values

* feat: add adaptive timeout metrics

* chore: coderabbit suggested changes

* chore: coderabbit suggested changes on FileMetricSink

* chore: coderabbit suggested changes

Author: mykeelium

src/CommonLib/AdaptiveTimeout.cs

@@ -5,6 +5,9 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using SharpHoundCommonLib.Exceptions;
+using SharpHoundCommonLib.Interfaces;
+using SharpHoundCommonLib.Models;
+using SharpHoundCommonLib.Static;
 using SharpHoundRPC.NetAPINative;
 
 namespace SharpHoundCommonLib;
@@ -26,6 +29,7 @@ public sealed class AdaptiveTimeout : IDisposable {
     private const int ExcessiveTimeoutsThreshold = 7;
     private const int StdDevMultiplier = 7; // 7 standard deviations should be a very conservative upper bound
     private const int CountOfLatestSuccessToKeep = 3;
+    private readonly IMetricRouter _metrics;
 
     public AdaptiveTimeout(TimeSpan maxTimeout, ILogger log, int sampleCount = 100, int logFrequency = 1000, int minSamplesForAdaptiveTimeout = 30, bool useAdaptiveTimeout = true, bool throwIfExcessiveTimeouts = false) {
         if (maxTimeout <= TimeSpan.Zero)
@@ -47,6 +51,7 @@ public AdaptiveTimeout(TimeSpan maxTimeout, ILogger log, int sampleCount = 100,
         _minSamplesForAdaptiveTimeout = minSamplesForAdaptiveTimeout;
         _useAdaptiveTimeout = useAdaptiveTimeout;
         _throwIfExcessiveTimeouts = throwIfExcessiveTimeouts;
+        _metrics = Metrics.Factory.CreateMetricRouter();
     }
 
     public AdaptiveTimeout(TimeSpan maxTimeout, TimeSpan minTimeout, ILogger log, int sampleCount = 100, int logFrequency = 1000, int minSamplesForAdaptiveTimeout = 30, bool useAdaptiveTimeout = true, bool throwIfExcessiveTimeouts = false)
@@ -75,15 +80,19 @@ public void ClearSamples() {
     /// <typeparam name="T"></typeparam>
     /// <param name="func"></param>
     /// <param name="parentToken"></param>
+    /// <param name="latencyObservation">A method that is used to observe the latency of the request.</param>
     /// <returns>Returns a Fail result if a task runs longer than its budgeted time.</returns>
-    public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, T> func, CancellationToken parentToken = default) {
+    public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, T> func, CancellationToken parentToken = default, Action<double> latencyObservation = null) {
         DateTime startTime = default;
         var result = await Timeout.ExecuteWithTimeout(GetAdaptiveTimeout(), (timeoutToken) =>
             _sampler.SampleExecutionTime(() => {
                 startTime = DateTime.Now; // for ordinal tracking; see use in TimeSpikeSafetyValve
                 return func(timeoutToken);
-            }), parentToken);
+            }, latencyObservation), parentToken);
         TimeSpikeSafetyValve(result.IsSuccess, startTime);
+        if (!result.IsSuccess) {
+            _metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
+        }
         return result;
     }
 
@@ -97,15 +106,19 @@ public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, T> fu
     /// </summary>
     /// <param name="func"></param>
     /// <param name="parentToken"></param>
+    /// <param name="latencyObservation">A method that is used to observe the latency of the request.</param>
     /// <returns>Returns a Fail result if a task runs longer than its budgeted time.</returns>
-    public async Task<Result> ExecuteWithTimeout(Action<CancellationToken> func, CancellationToken parentToken = default) {
+    public async Task<Result> ExecuteWithTimeout(Action<CancellationToken> func, CancellationToken parentToken = default, Action<double> latencyObservation = null) {
         DateTime startTime = default;
         var result = await Timeout.ExecuteWithTimeout(GetAdaptiveTimeout(), (timeoutToken) =>
             _sampler.SampleExecutionTime(() => {
                 startTime = DateTime.Now; // for ordinal tracking; see use in TimeSpikeSafetyValve
                 func(timeoutToken);
-            }), parentToken);
+            }, latencyObservation), parentToken);
         TimeSpikeSafetyValve(result.IsSuccess, startTime);
+        if (!result.IsSuccess) {
+            _metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
+        }
         return result;
     }
 
@@ -120,15 +133,19 @@ public async Task<Result> ExecuteWithTimeout(Action<CancellationToken> func, Can
     /// <typeparam name="T"></typeparam>
     /// <param name="func"></param>
     /// <param name="parentToken"></param>
+    /// <param name="latencyObservation">A method that is used to observe the latency of the request.</param>
     /// <returns>Returns a Fail result if a task runs longer than its budgeted time.</returns>
-    public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, Task<T>> func, CancellationToken parentToken = default) {
+    public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, Task<T>> func, CancellationToken parentToken = default, Action<double> latencyObservation = null) {
         DateTime startTime = default;
         var result = await Timeout.ExecuteWithTimeout(GetAdaptiveTimeout(), (timeoutToken) =>
             _sampler.SampleExecutionTime(() => {
                 startTime = DateTime.Now; // for ordinal tracking; see use in TimeSpikeSafetyValve
                 return func(timeoutToken);
-            }), parentToken);
+            }, latencyObservation), parentToken);
         TimeSpikeSafetyValve(result.IsSuccess, startTime);
+        if (!result.IsSuccess) {
+            _metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
+        }
         return result;
     }
 
@@ -142,15 +159,19 @@ public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, Task<
     /// </summary>
     /// <param name="func"></param>
     /// <param name="parentToken"></param>
+    /// <param name="latencyObservation">A method that is used to observe the latency of the request.</param>
     /// <returns>Returns a Fail result if a task runs longer than its budgeted time.</returns>
-    public async Task<Result> ExecuteWithTimeout(Func<CancellationToken, Task> func, CancellationToken parentToken = default) {
+    public async Task<Result> ExecuteWithTimeout(Func<CancellationToken, Task> func, CancellationToken parentToken = default, Action<double> latencyObservation = null) {
         DateTime startTime = default;
         var result = await Timeout.ExecuteWithTimeout(GetAdaptiveTimeout(), (timeoutToken) =>
             _sampler.SampleExecutionTime(() => {
                 startTime = DateTime.Now; // for ordinal tracking; see use in TimeSpikeSafetyValve
                 return func(timeoutToken);
-            }), parentToken);
+            }, latencyObservation), parentToken);
         TimeSpikeSafetyValve(result.IsSuccess, startTime);
+        if (!result.IsSuccess) {
+            _metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
+        }
         return result;
     }
 
@@ -174,6 +195,9 @@ public async Task<NetAPIResult<T>> ExecuteNetAPIWithTimeout<T>(Func<Cancellation
                 return func(timeoutToken);
             }), parentToken);
         TimeSpikeSafetyValve(result.IsSuccess, startTime);
+        if (!result.IsSuccess) {
+            _metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
+        }
         return result;
     }
 
@@ -197,6 +221,9 @@ public async Task<NetAPIResult<T>> ExecuteNetAPIWithTimeout<T>(Func<Cancellation
                 return func(timeoutToken);
             }), parentToken);
         TimeSpikeSafetyValve(result.IsSuccess, startTime);
+        if (!result.IsSuccess) {
+            _metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
+        }
         return result;
     }
 
@@ -220,6 +247,9 @@ public async Task<NetAPIResult<T>> ExecuteNetAPIWithTimeout<T>(Func<Cancellation
                 return func(timeoutToken);
             }), parentToken);
         TimeSpikeSafetyValve(result.IsSuccess, startTime);
+        if (!result.IsSuccess) {
+            _metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
+        }
         return result;
     }
 

src/CommonLib/ExecutionTimeSampler.cs

@@ -43,35 +43,39 @@ public double StandardDeviation() {
 
     public double Average() => _samples.Average();
 
-    public async Task<T> SampleExecutionTime<T>(Func<Task<T>> func) {
+    public async Task<T> SampleExecutionTime<T>(Func<Task<T>> func, Action<double> latencyObservation = null) {
         var stopwatch = Stopwatch.StartNew();
         var result = await func.Invoke();
         stopwatch.Stop();
+        latencyObservation?.Invoke(stopwatch.ElapsedMilliseconds);
         AddTimeSample(stopwatch.Elapsed);
 
         return result;
     }
 
-    public async Task SampleExecutionTime(Func<Task> func) {
+    public async Task SampleExecutionTime(Func<Task> func, Action<double> latencyObservation = null) {
         var stopwatch = Stopwatch.StartNew();
         await func.Invoke();
         stopwatch.Stop();
+        latencyObservation?.Invoke(stopwatch.ElapsedMilliseconds);
         AddTimeSample(stopwatch.Elapsed);
     }
 
-    public T SampleExecutionTime<T>(Func<T> func) {
+    public T SampleExecutionTime<T>(Func<T> func, Action<double> latencyObservation = null) {
         var stopwatch = Stopwatch.StartNew();
         var result = func.Invoke();
         stopwatch.Stop();
+        latencyObservation?.Invoke(stopwatch.ElapsedMilliseconds);
         AddTimeSample(stopwatch.Elapsed);
 
         return result;
     }
 
-    public void SampleExecutionTime(Action func) {
+    public void SampleExecutionTime(Action func, Action<double> latencyObservation = null) {
         var stopwatch = Stopwatch.StartNew();
         func.Invoke();
         stopwatch.Stop();
+        latencyObservation?.Invoke(stopwatch.ElapsedMilliseconds);
         AddTimeSample(stopwatch.Elapsed);
     }
 

src/CommonLib/Interfaces/ILabelValuesCache.cs

@@ -0,0 +1,5 @@
+namespace SharpHoundCommonLib.Interfaces;
+
+public interface ILabelValuesCache {
+    string[] Intern(string[] values);
+}

src/CommonLib/Interfaces/IMetricFactory.cs

@@ -0,0 +1,5 @@
+namespace SharpHoundCommonLib.Interfaces;
+
+public interface IMetricFactory {
+    IMetricRouter CreateMetricRouter();
+}

src/CommonLib/Interfaces/IMetricRegistry.cs

@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+using SharpHoundCommonLib.Models;
+
+namespace SharpHoundCommonLib.Interfaces;
+
+public interface IMetricRegistry {
+    bool TryRegister(MetricDefinition definition, out int definitionId);
+    IReadOnlyList<MetricDefinition> Definitions { get; }
+}

src/CommonLib/Interfaces/IMetricRouter.cs

@@ -0,0 +1,8 @@
+using SharpHoundCommonLib.Models;
+
+namespace SharpHoundCommonLib.Interfaces;
+
+public interface IMetricRouter {
+    void Observe(int definitionId, double value, LabelValues labelValues);
+    void Flush();
+}

src/CommonLib/Interfaces/IMetricSink.cs

@@ -0,0 +1,8 @@
+using SharpHoundCommonLib.Models;
+
+namespace SharpHoundCommonLib.Interfaces;
+
+public interface IMetricSink {
+    void Observe(in MetricObservation.DoubleMetricObservation observation);
+    void Flush();
+}

src/CommonLib/Interfaces/IMetricWriter.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Text;
+using SharpHoundCommonLib.Models;
+using SharpHoundCommonLib.Services;
+
+namespace SharpHoundCommonLib.Interfaces;
+
+public interface IMetricWriter {
+    void StringBuilderAppendMetric(
+        StringBuilder builder,
+        MetricDefinition definition,
+        LabelValues labelValues,
+        MetricAggregator aggregator,
+        DateTimeOffset timestamp,
+        string timestampOutputString = "yyyy-MM-dd HH:mm:ss.fff"
+    );
+}