modify jitter handling

Signed-off-by: Mukil <mukils@nvidia.com>

Author: mukilsh

internal/exporter/writer/http.go

@@ -86,6 +86,32 @@ func (w *httpWriter) SetJWTRefreshFunc(refreshFunc JWTRefreshFunc) {
 
 // Send sends health data to the specified endpoint
 func (w *httpWriter) Send(ctx context.Context, data *collector.HealthData, metricsEndpoint string, logsEndpoint string, maxRetries int, authToken string, collectionInterval time.Duration) (string, error) {
+	// Add jitter before the initial push to spread agent requests and prevent
+	// thundering herd when many agents share the same collection tick.
+	// Use 5% of the collection interval, capped at maxInitialJitter
+	jitterCap := collectionInterval / 20 // 5%
+	if jitterCap > maxInitialJitter {
+		jitterCap = maxInitialJitter
+	}
+	if jitterCap <= 0 {
+		jitterCap = defaultRetryDelay / 2 // fallback for zero/negative intervals
+	}
+
+	// Apply a further limit to the initial jitter of half the remaining context deadline.
+	// This is to prevent the jitter from consuming the entire timeout budget.
+	if deadline, ok := ctx.Deadline(); ok {
+		if remaining := time.Until(deadline) / 2; remaining > 0 && jitterCap > remaining {
+			jitterCap = remaining
+		}
+	}
+	if jitter := calculateJitter(jitterCap); jitter > 0 {
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		case <-time.After(jitter):
+		}
+	}
+
 	// Convert to OTLP format
 	otlpData := w.otlpConverter.Convert(data)
 
@@ -98,7 +124,7 @@ func (w *httpWriter) Send(ctx context.Context, data *collector.HealthData, metri
 			return "", fmt.Errorf("failed to marshal OTLP metrics data: %w", err)
 		}
 
-		token, err := w.sendOTLPRequestWithRetry(ctx, metricsBytes, "metrics", data.CollectionID, data.MachineID, metricsEndpoint, maxRetries, authToken, collectionInterval)
+		token, err := w.sendOTLPRequestWithRetry(ctx, metricsBytes, "metrics", data.CollectionID, data.MachineID, metricsEndpoint, maxRetries, authToken)
 		if err != nil {
 			log.Logger.Errorw("Failed to send metrics data after all retries",
 				"collection_id", data.CollectionID,
@@ -117,7 +143,7 @@ func (w *httpWriter) Send(ctx context.Context, data *collector.HealthData, metri
 			return newToken, fmt.Errorf("failed to marshal OTLP logs data: %w", err)
 		}
 
-		token, err := w.sendOTLPRequestWithRetry(ctx, logsBytes, "logs", data.CollectionID, data.MachineID, logsEndpoint, maxRetries, authToken, collectionInterval)
+		token, err := w.sendOTLPRequestWithRetry(ctx, logsBytes, "logs", data.CollectionID, data.MachineID, logsEndpoint, maxRetries, authToken)
 		if err != nil {
 			return newToken, fmt.Errorf("failed to send critical logs data (includes events): %w", err)
 		} else if token != "" {
@@ -130,7 +156,7 @@ func (w *httpWriter) Send(ctx context.Context, data *collector.HealthData, metri
 }
 
 // sendOTLPRequestWithRetry sends the OTLP data with retry logic
-func (w *httpWriter) sendOTLPRequestWithRetry(ctx context.Context, reqData []byte, dataType, collectionID, machineID, endpoint string, maxRetries int, authToken string, collectionInterval time.Duration) (string, error) {
+func (w *httpWriter) sendOTLPRequestWithRetry(ctx context.Context, reqData []byte, dataType, collectionID, machineID, endpoint string, maxRetries int, authToken string) (string, error) {
 	if maxRetries <= 0 {
 		maxRetries = 1 // At least one attempt
 	}
@@ -139,25 +165,6 @@ func (w *httpWriter) sendOTLPRequestWithRetry(ctx context.Context, reqData []byt
 	var lastErr error
 	jwtRefreshAttempted := false
 
-	// Add jitter before the initial push to spread agent requests and prevent
-	// thundering herd when many agents share the same collection tick.
-	// Use 5% of the collection interval, capped at maxInitialJitter.
-	initialMaxJitter := collectionInterval / 20 // 5%
-	if initialMaxJitter > maxInitialJitter {
-		initialMaxJitter = maxInitialJitter
-	}
-	if initialMaxJitter <= 0 {
-		initialMaxJitter = defaultRetryDelay / 2 // fallback for zero/negative intervals
-	}
-	initialJitter := calculateJitter(initialMaxJitter)
-	if initialJitter > 0 {
-		select {
-		case <-ctx.Done():
-			return "", ctx.Err()
-		case <-time.After(initialJitter):
-		}
-	}
-
 	for attempt := 1; attempt <= maxRetries; attempt++ {
 		token, err := w.sendOTLPRequest(ctx, reqData, dataType, collectionID, machineID, endpoint, currentAuthToken)
 		if err == nil {