From d1f90f1fb6aef9e62d55984554f056fb081216da Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 9 Jan 2026 14:39:26 -0800 Subject: [PATCH 01/38] fix: Retry after partial file reads. --- .../Internal/DataSources/FileDataSource.cs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs b/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs index 91cb989b..98ea34c8 100644 --- a/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs +++ b/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs @@ -28,6 +28,10 @@ internal sealed class FileDataSource : IDataSource private volatile int _lastVersion; private object _updateLock = new object(); + private const int MaxRetries = 5; + private readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(0.6); + private readonly Dictionary _retryCounts = new Dictionary(); + public FileDataSource(IDataSourceUpdates dataSourceUpdates, FileDataTypes.IFileReader fileReader, List paths, bool autoUpdate, Func alternateParser, bool skipMissingPaths, FileDataTypes.DuplicateKeysHandling duplicateKeysHandling, @@ -102,17 +106,51 @@ private void LoadAll() _logger.Debug("file data: {0}", content); var data = _parser.Parse(content, version); _dataMerger.AddToData(data, flags, segments); + // Remove any retry count associated with this path. + _retryCounts.Remove(path); } catch (FileNotFoundException) when (_skipMissingPaths) { _logger.Debug("{0}: {1}", path, "File not found"); } + catch (System.Text.Json.JsonException) + { + // We may have received the notification of a file change while the file was being written. + // So we may read an empty or partially written file. So, when we encounter a JSON parsing issue + // we will retry after a short delay. + // We will retry up to MaxRetries times before giving up. + if (!_retryCounts.ContainsKey(path)) + { + _retryCounts[path] = 0; + } + _retryCounts[path]++; + + if (_retryCounts[path] < MaxRetries) + { + _logger.Warn("{0}: {1}", path, "Failed to parse file, retrying in " + RetryDelay.TotalMilliseconds + " milliseconds"); + Task.Run(async () => + { + await Task.Delay(RetryDelay); + LoadAll(); + }); + } + else + { + _logger.Error("{0}: {1}", path, "Failed to parse file after " + MaxRetries + " retries"); + } + + return; + } catch (Exception e) { LogHelpers.LogException(_logger, "Failed to load " + path, e); return; } } + + // If any files failed to load, from anything other than not existing, then that + // update would fail. This behavior is retained with the addition of the retry. But it should be + // examined. var allData = new FullDataSet( ImmutableDictionary.Create>() From 28590643131bb7d43a7ddad8509ccb67bf5146d7 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 15:52:16 -0700 Subject: [PATCH 02/38] test: stop pinning segment version in flaky FileDataSource tests The two reload tests asserted an exact ExpectedDataSetForSegmentOnlyFile(2), which encoded an assumption of exactly two LoadAll attempts. With the file watcher firing on truncate-then-write plus the new JSON-parse retry, the successful attempt's version can be 3+ and the strict JSON comparison times out waiting for a version-2 event that never comes. Switch both tests to ExpectPredicate with a shared structural matcher and refactor the existing predicate in ModifiedFileIsReloadedIfAutoUpdateIsOn to use the same helper. Also tighten the retry path in FileDataSource: drop the trailing TODO, short-circuit LoadAll once Dispose() has been called, and skip the post- delay LoadAll if disposal raced the retry. --- .../Internal/DataSources/FileDataSource.cs | 31 +++--- .../DataSources/FileDataSourceTest.cs | 102 +++++++----------- 2 files changed, 56 insertions(+), 77 deletions(-) diff --git a/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs b/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs index 98ea34c8..1990622b 100644 --- a/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs +++ b/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs @@ -25,11 +25,13 @@ internal sealed class FileDataSource : IDataSource private readonly Logger _logger; private volatile bool _started; private volatile bool _loadedValidData; + private volatile bool _disposed; private volatile int _lastVersion; private object _updateLock = new object(); private const int MaxRetries = 5; - private readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(0.6); + private static readonly TimeSpan RetryDelay = TimeSpan.FromMilliseconds(600); + // Per-path JSON-parse retry counters. Only touched inside _updateLock. private readonly Dictionary _retryCounts = new Dictionary(); public FileDataSource(IDataSourceUpdates dataSourceUpdates, FileDataTypes.IFileReader fileReader, @@ -87,6 +89,7 @@ private void Dispose(bool disposing) { if (disposing) { + _disposed = true; _reloader?.Dispose(); } } @@ -95,6 +98,10 @@ private void LoadAll() { lock (_updateLock) { + if (_disposed) + { + return; + } var version = Interlocked.Increment(ref _lastVersion); var flags = new Dictionary(); var segments = new Dictionary(); @@ -106,7 +113,6 @@ private void LoadAll() _logger.Debug("file data: {0}", content); var data = _parser.Parse(content, version); _dataMerger.AddToData(data, flags, segments); - // Remove any retry count associated with this path. _retryCounts.Remove(path); } catch (FileNotFoundException) when (_skipMissingPaths) @@ -115,10 +121,9 @@ private void LoadAll() } catch (System.Text.Json.JsonException) { - // We may have received the notification of a file change while the file was being written. - // So we may read an empty or partially written file. So, when we encounter a JSON parsing issue - // we will retry after a short delay. - // We will retry up to MaxRetries times before giving up. + // A file-change notification can fire while the file is mid-write, so we may read an + // empty or partially written file. Retry up to MaxRetries times before giving up; the + // counter is cleared on the next successful load. if (!_retryCounts.ContainsKey(path)) { _retryCounts[path] = 0; @@ -127,16 +132,20 @@ private void LoadAll() if (_retryCounts[path] < MaxRetries) { - _logger.Warn("{0}: {1}", path, "Failed to parse file, retrying in " + RetryDelay.TotalMilliseconds + " milliseconds"); + _logger.Warn("{0}: Failed to parse file, retrying in {1} ms", path, RetryDelay.TotalMilliseconds); Task.Run(async () => { - await Task.Delay(RetryDelay); + await Task.Delay(RetryDelay).ConfigureAwait(false); + if (_disposed) + { + return; + } LoadAll(); }); } else { - _logger.Error("{0}: {1}", path, "Failed to parse file after " + MaxRetries + " retries"); + _logger.Error("{0}: Failed to parse file after {1} retries", path, MaxRetries); } return; @@ -147,10 +156,6 @@ private void LoadAll() return; } } - - // If any files failed to load, from anything other than not existing, then that - // update would fail. This behavior is retained with the addition of the retry. But it should be - // examined. var allData = new FullDataSet( ImmutableDictionary.Create>() diff --git a/pkgs/sdk/server/test/Internal/DataSources/FileDataSourceTest.cs b/pkgs/sdk/server/test/Internal/DataSources/FileDataSourceTest.cs index 5322a38c..d3f8a701 100644 --- a/pkgs/sdk/server/test/Internal/DataSources/FileDataSourceTest.cs +++ b/pkgs/sdk/server/test/Internal/DataSources/FileDataSourceTest.cs @@ -151,50 +151,7 @@ public void ModifiedFileIsReloadedIfAutoUpdateIsOn() file.SetContentFromPath(TestUtils.TestFilePath("segment-only.json")); - AssertHelpers.ExpectPredicate(_updateSink.Inits, actual => - { - var segments = actual.Data.First(item => item.Key == DataModel.Segments); - var features = actual.Data.First(item => item.Key == DataModel.Features); - if (!features.Value.Items.IsNullOrEmpty()) - { - return false; - } - - var segmentItems = segments.Value.Items.ToList(); - - if (segmentItems.Count != 1) - { - return false; - } - - var segmentDescriptor = segmentItems[0]; - if (segmentDescriptor.Key != "seg1") - { - return false; - } - - if (segmentDescriptor.Value.Version == 1) - { - return false; - } - - if (!(segmentDescriptor.Value.Item is Segment segment)) - { - return false; - } - - if (segment.Deleted) - { - return false; - } - - if (segment.Included.Count != 1) - { - return false; - } - - return segment.Included[0] == "user1"; - }, + AssertHelpers.ExpectPredicate(_updateSink.Inits, IsSegmentOnlyDataAfterReload, "Did not receive expected update from the file data source.", TimeSpan.FromSeconds(30)); } @@ -270,16 +227,9 @@ public void ModifiedFileIsReloadedEvenIfOneFileIsMissingIfSkipMissingPathsIsSet( file1.SetContentFromPath(TestUtils.TestFilePath("segment-only.json")); - // Use ExpectJsonValue to handle potential race conditions where the file watcher - // may trigger multiple reload events. This keeps checking events until one matches - // the expected JSON or the timeout expires. - var newData = AssertHelpers.ExpectJsonValue( - _updateSink.Inits, - DataSetAsJson(ExpectedDataSetForSegmentOnlyFile(2)), - DataSetAsJson, + AssertHelpers.ExpectPredicate(_updateSink.Inits, IsSegmentOnlyDataAfterReload, + "Did not receive expected update from the file data source.", TimeSpan.FromSeconds(30)); - - AssertJsonEqual(DataSetAsJson(ExpectedDataSetForSegmentOnlyFile(2)), DataSetAsJson(newData)); } } } @@ -298,18 +248,9 @@ public void IfFlagsAreBadAtStartTimeAutoUpdateCanStillLoadGoodDataLater() file.SetContentFromPath(TestUtils.TestFilePath("segment-only.json")); - // Use ExpectJsonValue to handle potential race conditions where the file watcher - // may trigger multiple reload events. This keeps checking events until one matches - // the expected JSON or the timeout expires. - // Note that the expected version is 2 because we increment the version on each - // *attempt* to load the files, not on each successful load. - var newData = AssertHelpers.ExpectJsonValue( - _updateSink.Inits, - DataSetAsJson(ExpectedDataSetForSegmentOnlyFile(2)), - DataSetAsJson, + AssertHelpers.ExpectPredicate(_updateSink.Inits, IsSegmentOnlyDataAfterReload, + "Did not receive expected update from the file data source.", TimeSpan.FromSeconds(30)); - - AssertJsonEqual(DataSetAsJson(ExpectedDataSetForSegmentOnlyFile(2)), DataSetAsJson(newData)); } } } @@ -365,5 +306,38 @@ private static FullDataSet ExpectedDataSetForSegmentOnlyFile(int new SegmentBuilder("seg1").Version(version).Included("user1").Build() ) .Build(); + + // Predicate that matches the structure of segment-only.json reloaded after the initial load. + // We deliberately don't pin the exact version: with the file watcher firing on truncate-then-write + // and the JsonException retry, the version of the successful load is non-deterministic, so we + // only require that it isn't the initial version 1. + private static bool IsSegmentOnlyDataAfterReload(FullDataSet actual) + { + var features = actual.Data.First(item => item.Key == DataModel.Features); + if (!features.Value.Items.IsNullOrEmpty()) + { + return false; + } + + var segments = actual.Data.First(item => item.Key == DataModel.Segments); + var segmentItems = segments.Value.Items.ToList(); + if (segmentItems.Count != 1) + { + return false; + } + + var segmentDescriptor = segmentItems[0]; + if (segmentDescriptor.Key != "seg1" || segmentDescriptor.Value.Version == 1) + { + return false; + } + + if (!(segmentDescriptor.Value.Item is Segment segment) || segment.Deleted) + { + return false; + } + + return segment.Included.Count == 1 && segment.Included[0] == "user1"; + } } } From f9eb45554d3bcd30ef9f54136031f4c087b99bca Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:04 -0700 Subject: [PATCH 03/38] ci: trigger flake check run 1/19 From d1add2aa20cfe5fc9cb371de34343517ce7c7982 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:36 -0700 Subject: [PATCH 04/38] ci: trigger flake check run 2/19 From 9924c10b72934d56ea486646514f04a826b4930b Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:38 -0700 Subject: [PATCH 05/38] ci: trigger flake check run 3/19 From a13e6da2acc631896c6c108e4f18c60afe7b4ab2 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:39 -0700 Subject: [PATCH 06/38] ci: trigger flake check run 4/19 From e35db5dee01ddec40f6087446a5a6efb3f03f82d Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:40 -0700 Subject: [PATCH 07/38] ci: trigger flake check run 5/19 From ce9dd1880a645831068087202ce669f8d7d01274 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:42 -0700 Subject: [PATCH 08/38] ci: trigger flake check run 6/19 From 8151abffc1fa6d96db5048c6b72a7b1c9477a66e Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:43 -0700 Subject: [PATCH 09/38] ci: trigger flake check run 7/19 From c04a01d4712910d5bcc25c6daf3f6146ea58e59b Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:45 -0700 Subject: [PATCH 10/38] ci: trigger flake check run 8/19 From 1baf7ecbdd824769b83f854434824f896d98df81 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:46 -0700 Subject: [PATCH 11/38] ci: trigger flake check run 9/19 From a458d0080f719291f0faf1c6a8e8aa15073b97e3 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:48 -0700 Subject: [PATCH 12/38] ci: trigger flake check run 10/19 From 81cf9ace93df3046ace7acf6023e3da8ac4e8797 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:49 -0700 Subject: [PATCH 13/38] ci: trigger flake check run 11/19 From d330eee2999d110a02910b93abcb5045aa2ca443 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:51 -0700 Subject: [PATCH 14/38] ci: trigger flake check run 12/19 From 98c80ececab2f7d0f0d7b9573970434d1cfc1ed8 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:52 -0700 Subject: [PATCH 15/38] ci: trigger flake check run 13/19 From 907e9bc3f4bfc79e0c06b976df19d8d231af1126 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:53 -0700 Subject: [PATCH 16/38] ci: trigger flake check run 14/19 From 50215b573c24cbcd1e2c39889d4f58a11ac33c6e Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:55 -0700 Subject: [PATCH 17/38] ci: trigger flake check run 15/19 From 75ab3bf34c1b23b6257c4047744293e477565ee6 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:56 -0700 Subject: [PATCH 18/38] ci: trigger flake check run 16/19 From 9ca9b0ebe697bca43b1438d46cb66ec459f2d054 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:58 -0700 Subject: [PATCH 19/38] ci: trigger flake check run 17/19 From 9d4913b965773506fbf9f6fc2eb082716b886179 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:00:59 -0700 Subject: [PATCH 20/38] ci: trigger flake check run 18/19 From 0066cd9d98abb6af3980fb216cfd8ed56e304f8f Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:01:01 -0700 Subject: [PATCH 21/38] ci: trigger flake check run 19/19 From 99a9cd0eba6b125b18c588535d4fd59c5d58a13c Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:02:15 -0700 Subject: [PATCH 22/38] ci: trigger flake check run 20 From b0b31e17ce7cd730c1db8527add0777b90bcb9e1 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:02:31 -0700 Subject: [PATCH 23/38] ci: trigger flake check run 21 From 764e7d3db360c4ef43858ec4567f74e076a5904c Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:02:47 -0700 Subject: [PATCH 24/38] ci: trigger flake check run 22 From dc58101f51755fbc51187c146d2c641b0af40e8a Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:03:04 -0700 Subject: [PATCH 25/38] ci: trigger flake check run 23 From 22c836c2b853393c42d37b759d74ca3bb7d7ccbd Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:03:20 -0700 Subject: [PATCH 26/38] ci: trigger flake check run 24 From 72cc40ab7c0c6c8644299a69600f8750a723d062 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:03:37 -0700 Subject: [PATCH 27/38] ci: trigger flake check run 25 From f90b545fb81b2fd1dba148f36ba6c1181aa356e9 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:03:53 -0700 Subject: [PATCH 28/38] ci: trigger flake check run 26 From ee1bf7ef5ed3eb559d044e2eea1b343b43965ad1 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:04:10 -0700 Subject: [PATCH 29/38] ci: trigger flake check run 27 From 824e929aaea5e22ca345be596d622f1ae2da458d Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:04:57 -0700 Subject: [PATCH 30/38] ci: trigger flake check run 28 From ba42d22021ff84100249eea02361f4299ede8950 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:05:14 -0700 Subject: [PATCH 31/38] ci: trigger flake check run 29 From fc329559c85adcf1f3ad3e8874919e0890c0e9d7 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:05:30 -0700 Subject: [PATCH 32/38] ci: trigger flake check run 30 From 5bc5f9b63325957ce568a4f3a10d7deab343c55d Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:05:47 -0700 Subject: [PATCH 33/38] ci: trigger flake check run 31 From 956bd47794c1e099cf24fb634b52367f80295293 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:06:04 -0700 Subject: [PATCH 34/38] ci: trigger flake check run 32 From 97bacd247444edcd3d165dbdb23255acd33247b7 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:06:20 -0700 Subject: [PATCH 35/38] ci: trigger flake check run 33 From 2de17efcc6c2bfb819922b9fcf6fc4c158c551cf Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:06:37 -0700 Subject: [PATCH 36/38] ci: trigger flake check run 34 From 0a142262e5d5dc8eb971bf5e9476707fa64886cb Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:06:53 -0700 Subject: [PATCH 37/38] ci: trigger flake check run 35 From c709ee4eb86ee57f5a00002a4a66c509ca17a2bf Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 26 May 2026 16:07:10 -0700 Subject: [PATCH 38/38] ci: trigger flake check run 36