-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathretry.go
More file actions
226 lines (189 loc) · 6 KB
/
Copy pathretry.go
File metadata and controls
226 lines (189 loc) · 6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
package llmprovider
import (
"context"
"fmt"
"math"
"math/rand"
"net/http"
"time"
)
// RetryConfig defines retry behavior for LLM API calls
type RetryConfig struct {
// MaxRetries is the maximum number of retry attempts (0 = no retries)
MaxRetries int
// InitialDelay is the initial delay before first retry
InitialDelay time.Duration
// MaxDelay is the maximum delay between retries
MaxDelay time.Duration
// Multiplier is the factor by which delay increases after each retry
Multiplier float64
// JitterFactor adds randomness to delays (0.0-1.0)
JitterFactor float64
}
// DefaultRetryConfig returns sensible defaults for LLM API retry behavior
func DefaultRetryConfig() RetryConfig {
return RetryConfig{
MaxRetries: 3,
InitialDelay: 1 * time.Second,
MaxDelay: 30 * time.Second,
Multiplier: 2.0,
JitterFactor: 0.1,
}
}
// RetryableFunc is a function that can be retried
type RetryableFunc func() (*http.Response, error)
// RetryResult contains the result of a retry operation
type RetryResult struct {
Response *http.Response
Attempts int
LastError error
TotalDelay time.Duration
}
// IsRetryableStatusCode determines if an HTTP status code warrants a retry
func IsRetryableStatusCode(statusCode int) bool {
switch statusCode {
case http.StatusTooManyRequests, // 429 - Rate limited
http.StatusInternalServerError, // 500
http.StatusBadGateway, // 502
http.StatusServiceUnavailable, // 503
http.StatusGatewayTimeout: // 504
return true
default:
return false
}
}
// IsRetryableError determines if an error warrants a retry
func IsRetryableError(err error) bool {
if err == nil {
return false
}
// Context cancelled or deadline exceeded - don't retry
if err == context.Canceled || err == context.DeadlineExceeded {
return false
}
// Network errors are generally retryable
// This includes connection refused, timeout, DNS errors, etc.
return true
}
// ExecuteWithRetry executes a function with retry logic and exponential backoff
func ExecuteWithRetry(ctx context.Context, config RetryConfig, fn RetryableFunc) (*RetryResult, error) {
result := &RetryResult{
Attempts: 0,
}
delay := config.InitialDelay
for attempt := 0; attempt <= config.MaxRetries; attempt++ {
result.Attempts = attempt + 1
// Check context before making request
select {
case <-ctx.Done():
result.LastError = ctx.Err()
return result, fmt.Errorf("context cancelled before attempt %d: %w", attempt+1, ctx.Err())
default:
}
// Execute the function
resp, err := fn()
// Success - return immediately
if err == nil && resp != nil && !IsRetryableStatusCode(resp.StatusCode) {
result.Response = resp
return result, nil
}
// If we got a response with retryable status, close it before retrying
if resp != nil && IsRetryableStatusCode(resp.StatusCode) {
result.LastError = fmt.Errorf("HTTP %d: retryable server error", resp.StatusCode)
_ = resp.Body.Close()
} else if err != nil {
result.LastError = err
}
// Check if we should retry
shouldRetry := false
if err != nil && IsRetryableError(err) {
shouldRetry = true
} else if resp != nil && IsRetryableStatusCode(resp.StatusCode) {
shouldRetry = true
}
// Last attempt or non-retryable error - return
if !shouldRetry || attempt >= config.MaxRetries {
if result.LastError != nil {
return result, fmt.Errorf("all %d attempts failed: %w", result.Attempts, result.LastError)
}
result.Response = resp
return result, nil
}
// Calculate delay with jitter
jitteredDelay := addJitter(delay, config.JitterFactor)
// Wait before retry
select {
case <-ctx.Done():
result.LastError = ctx.Err()
return result, fmt.Errorf("context cancelled during backoff: %w", ctx.Err())
case <-time.After(jitteredDelay):
result.TotalDelay += jitteredDelay
}
// Increase delay for next retry (exponential backoff)
delay = time.Duration(float64(delay) * config.Multiplier)
if delay > config.MaxDelay {
delay = config.MaxDelay
}
}
return result, fmt.Errorf("max retries exceeded: %w", result.LastError)
}
// addJitter adds randomness to a duration
// Note: Using math/rand for jitter is acceptable - it doesn't require cryptographic randomness
func addJitter(d time.Duration, factor float64) time.Duration {
if factor <= 0 {
return d
}
// Calculate jitter range
jitterRange := float64(d) * factor
// Add random jitter (can be positive or negative)
jitter := (rand.Float64() - 0.5) * 2 * jitterRange // #nosec G404 - jitter doesn't require cryptographic randomness
result := time.Duration(float64(d) + jitter)
if result < 0 {
result = 0
}
return result
}
// CalculateBackoff calculates the backoff duration for a given attempt
func CalculateBackoff(attempt int, config RetryConfig) time.Duration {
if attempt <= 0 {
return config.InitialDelay
}
delay := float64(config.InitialDelay) * math.Pow(config.Multiplier, float64(attempt-1))
if delay > float64(config.MaxDelay) {
delay = float64(config.MaxDelay)
}
return addJitter(time.Duration(delay), config.JitterFactor)
}
// RetryableHTTPClient wraps an http.Client with retry logic
type RetryableHTTPClient struct {
client *http.Client
config RetryConfig
}
// NewRetryableHTTPClient creates a new RetryableHTTPClient
func NewRetryableHTTPClient(client *http.Client, config RetryConfig) *RetryableHTTPClient {
if client == nil {
client = &http.Client{
Timeout: 60 * time.Second,
}
}
return &RetryableHTTPClient{
client: client,
config: config,
}
}
// Do executes an HTTP request with retry logic
func (c *RetryableHTTPClient) Do(ctx context.Context, req *http.Request) (*http.Response, error) {
result, err := ExecuteWithRetry(ctx, c.config, func() (*http.Response, error) {
// Clone the request for each attempt (body needs to be re-readable)
clonedReq := req.Clone(ctx)
return c.client.Do(clonedReq)
})
if err != nil {
return nil, err
}
return result.Response, nil
}
// GetAttempts returns the number of attempts from the last request
func (c *RetryableHTTPClient) GetConfig() RetryConfig {
return c.config
}