diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs
index 609a8b35d..c23144eb1 100644
--- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs
+++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs
@@ -123,6 +123,11 @@ public class AzureStorageOrchestrationServiceSettings
///
public int MaxStorageOperationConcurrency { get; set; } = Environment.ProcessorCount * 25;
+ ///
+ /// Max wait time to wait for semaphore to process after which log should be added.
+ ///
+ public int MaxWaitForSemaphoreLoggingInSeconds { get; set; } = 20;
+
///
/// Gets the maximum number of orchestrator actions to checkpoint at a time.
///
diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj
index 7e281163d..49efcaad8 100644
--- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj
+++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj
@@ -24,7 +24,7 @@
1
$(MajorVersion).$(MinorVersion).$(PatchVersion)
- $(VersionPrefix).0
+ $(VersionPrefix)-semaphorelogs.4
$(VersionPrefix).$(FileVersionRevision)
diff --git a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs
index 004f09c71..92cc41b38 100644
--- a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs
+++ b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs
@@ -14,6 +14,7 @@
namespace DurableTask.AzureStorage.Storage
{
using System;
+ using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using DurableTask.AzureStorage.Monitoring;
@@ -127,9 +128,16 @@ public Task MakeTableStorageRequest(Func MakeStorageRequest(Func> storageRequest, string accountName, string operationName, string? clientRequestId = null, bool force = false)
{
+ Guid guid = Guid.NewGuid();
+
if (!force)
{
+ var cTkWait = new CancellationTokenSource();
+ _ = WaitAndLog(guid, "Wait", cTkWait.Token);
+
await requestThrottleSemaphore.WaitAsync();
+
+ cTkWait.Cancel();
}
try
@@ -144,7 +152,12 @@ private async Task MakeStorageRequest(Func
private static string GetAccountName(StorageCredentials credentials, AzureStorageOrchestrationServiceSettings settings, StorageUri serviceUri, string service) =>
credentials.AccountName ?? settings.StorageAccountDetails?.AccountName ?? serviceUri.GetAccountName(service) ?? "(unknown)";
+
+ private async Task WaitAndLog(Guid guid, string waitOrRelease, CancellationToken cTk)
+ {
+ if (this.Settings.MaxWaitForSemaphoreLoggingInSeconds >= 0)
+ {
+ await Task.Delay(this.Settings.MaxWaitForSemaphoreLoggingInSeconds * 1000, cTk);
+
+ if (!cTk.IsCancellationRequested)
+ {
+ StackTrace stackTrace = new StackTrace();
+
+ this.Settings?.Logger.PartitionManagerInfo(
+ this.Settings?.StorageAccountDetails?.AccountName,
+ this.Settings?.TaskHubName,
+ this.Settings?.WorkerId,
+ string.Empty,
+ $"Before {waitOrRelease} Id:{guid.ToString()}, processId: {Thread.CurrentThread.ManagedThreadId}" +
+ $"| Semaphore currentCount = {requestThrottleSemaphore.CurrentCount}" +
+ $"| Semaphore initialCount = {this.Settings?.MaxStorageOperationConcurrency}" +
+ $"| StackTrace : {stackTrace.ToString()}");
+ }
+ }
+ }
}
}