Skip to content

Commit eab8d90

Browse files
zajcazajca
authored
Merge pull request #6541 from keboola/devin/DMD-987-1769163786-deadlock-handling
feat(DMD-987): handle MySQL deadlocks during configuration row creation
2 parents 35db1aa + dc80705 commit eab8d90

2 files changed

Lines changed: 322 additions & 0 deletions

File tree

‎src/Keboola/StorageApi/HandlerStack.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ private static function createDefaultDecider(int $maxRetries, bool $retryOnMaint
6666

6767
if ($retries >= $maxRetries) {
6868
return false;
69+
} elseif ($response && $response->getStatusCode() === 409) {
70+
// Retry on 409 Conflict if it's a version conflict (deadlock) error
71+
return self::isVersionConflictResponse($response);
6972
} elseif ($response && $response->getStatusCode() > 499) {
7073
return true;
7174
} elseif ($error) {
@@ -76,6 +79,21 @@ private static function createDefaultDecider(int $maxRetries, bool $retryOnMaint
7679
};
7780
}
7881

82+
private static function isVersionConflictResponse(ResponseInterface $response): bool
83+
{
84+
$body = (string) $response->getBody();
85+
if ($response->getBody()->isSeekable()) {
86+
$response->getBody()->rewind();
87+
}
88+
89+
$data = json_decode($body, true);
90+
if (!is_array($data)) {
91+
return false;
92+
}
93+
94+
return isset($data['code']) && $data['code'] === 'storage.components.configurations.versionConflict';
95+
}
96+
7997
private static function createExponentialDelay(): callable
8098
{
8199
return function ($retries) {

‎tests/Client/HandlerStackTest.php‎

Lines changed: 304 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Keboola\UnitTest\Client;
6+
7+
use GuzzleHttp\Client as GuzzleClient;
8+
use GuzzleHttp\Handler\MockHandler;
9+
use GuzzleHttp\Psr7\Response;
10+
use Keboola\StorageApi\HandlerStack;
11+
use PHPUnit\Framework\TestCase;
12+
13+
class HandlerStackTest extends TestCase
14+
{
15+
public function testCreateWithCustomHandler(): void
16+
{
17+
$mockHandler = new MockHandler([
18+
new Response(200, [], 'ok'),
19+
]);
20+
21+
$handlerStack = HandlerStack::create(['handler' => $mockHandler]);
22+
$client = new GuzzleClient(['handler' => $handlerStack]);
23+
24+
$response = $client->request('GET', 'http://example.com');
25+
self::assertSame(200, $response->getStatusCode());
26+
}
27+
28+
public function testNoRetryWhen501NotImplemented(): void
29+
{
30+
$mockHandler = new MockHandler([
31+
new Response(501, [], 'Not Implemented'),
32+
new Response(200, [], 'ok'),
33+
]);
34+
35+
$handlerStack = HandlerStack::create([
36+
'handler' => $mockHandler,
37+
'backoffMaxTries' => 3,
38+
]);
39+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
40+
41+
$response = $client->request('GET', 'http://example.com');
42+
self::assertSame(501, $response->getStatusCode());
43+
self::assertSame(1, $mockHandler->count(), 'Second response should not be consumed');
44+
}
45+
46+
public function testNoRetryWhen503AndRetryOnMaintenanceDisabled(): void
47+
{
48+
$mockHandler = new MockHandler([
49+
new Response(503, [], 'Service Unavailable'),
50+
new Response(200, [], 'ok'),
51+
]);
52+
53+
$handlerStack = HandlerStack::create([
54+
'handler' => $mockHandler,
55+
'backoffMaxTries' => 3,
56+
'retryOnMaintenance' => false,
57+
]);
58+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
59+
60+
$response = $client->request('GET', 'http://example.com');
61+
self::assertSame(503, $response->getStatusCode());
62+
self::assertSame(1, $mockHandler->count(), 'Second response should not be consumed');
63+
}
64+
65+
public function testRetryWhen503AndRetryOnMaintenanceEnabled(): void
66+
{
67+
$mockHandler = new MockHandler([
68+
new Response(503, [], 'Service Unavailable'),
69+
new Response(200, [], 'ok'),
70+
]);
71+
72+
$handlerStack = HandlerStack::create([
73+
'handler' => $mockHandler,
74+
'backoffMaxTries' => 3,
75+
'retryOnMaintenance' => true,
76+
]);
77+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
78+
79+
$response = $client->request('GET', 'http://example.com');
80+
self::assertSame(200, $response->getStatusCode());
81+
self::assertSame(0, $mockHandler->count(), 'Both responses should be consumed');
82+
}
83+
84+
public function testRetryOn5xxErrors(): void
85+
{
86+
$mockHandler = new MockHandler([
87+
new Response(500, [], 'Internal Server Error'),
88+
new Response(200, [], 'ok'),
89+
]);
90+
91+
$handlerStack = HandlerStack::create([
92+
'handler' => $mockHandler,
93+
'backoffMaxTries' => 3,
94+
]);
95+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
96+
97+
$response = $client->request('GET', 'http://example.com');
98+
self::assertSame(200, $response->getStatusCode());
99+
self::assertSame(0, $mockHandler->count(), 'Both responses should be consumed');
100+
}
101+
102+
public function testNoRetryWhenMaxRetriesExceeded(): void
103+
{
104+
$mockHandler = new MockHandler([
105+
new Response(500, [], 'Internal Server Error'),
106+
new Response(500, [], 'Internal Server Error'),
107+
new Response(500, [], 'Internal Server Error'),
108+
new Response(200, [], 'ok'),
109+
]);
110+
111+
$handlerStack = HandlerStack::create([
112+
'handler' => $mockHandler,
113+
'backoffMaxTries' => 2,
114+
]);
115+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
116+
117+
$response = $client->request('GET', 'http://example.com');
118+
self::assertSame(500, $response->getStatusCode());
119+
self::assertSame(1, $mockHandler->count(), 'Should stop after max retries');
120+
}
121+
122+
public function testRetryOn409VersionConflict(): void
123+
{
124+
$versionConflictBody = (string) json_encode([
125+
'code' => 'storage.components.configurations.versionConflict',
126+
'message' => 'Configuration row creation conflict. Please retry the operation.',
127+
]);
128+
129+
$mockHandler = new MockHandler([
130+
new Response(409, [], $versionConflictBody),
131+
new Response(200, [], 'ok'),
132+
]);
133+
134+
$handlerStack = HandlerStack::create([
135+
'handler' => $mockHandler,
136+
'backoffMaxTries' => 3,
137+
]);
138+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
139+
140+
$response = $client->request('GET', 'http://example.com');
141+
self::assertSame(200, $response->getStatusCode());
142+
self::assertSame(0, $mockHandler->count(), 'Both responses should be consumed');
143+
}
144+
145+
public function testNoRetryOn409WithDifferentErrorCode(): void
146+
{
147+
$otherConflictBody = (string) json_encode([
148+
'code' => 'storage.buckets.alreadyExists',
149+
'message' => 'Bucket already exists.',
150+
]);
151+
152+
$mockHandler = new MockHandler([
153+
new Response(409, [], $otherConflictBody),
154+
new Response(200, [], 'ok'),
155+
]);
156+
157+
$handlerStack = HandlerStack::create([
158+
'handler' => $mockHandler,
159+
'backoffMaxTries' => 3,
160+
]);
161+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
162+
163+
$response = $client->request('GET', 'http://example.com');
164+
self::assertSame(409, $response->getStatusCode());
165+
self::assertSame(1, $mockHandler->count(), 'Second response should not be consumed');
166+
}
167+
168+
public function testNoRetryOn409WithInvalidJson(): void
169+
{
170+
$mockHandler = new MockHandler([
171+
new Response(409, [], 'not valid json'),
172+
new Response(200, [], 'ok'),
173+
]);
174+
175+
$handlerStack = HandlerStack::create([
176+
'handler' => $mockHandler,
177+
'backoffMaxTries' => 3,
178+
]);
179+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
180+
181+
$response = $client->request('GET', 'http://example.com');
182+
self::assertSame(409, $response->getStatusCode());
183+
self::assertSame(1, $mockHandler->count(), 'Second response should not be consumed');
184+
}
185+
186+
public function testNoRetryOn409WithEmptyBody(): void
187+
{
188+
$mockHandler = new MockHandler([
189+
new Response(409, [], ''),
190+
new Response(200, [], 'ok'),
191+
]);
192+
193+
$handlerStack = HandlerStack::create([
194+
'handler' => $mockHandler,
195+
'backoffMaxTries' => 3,
196+
]);
197+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
198+
199+
$response = $client->request('GET', 'http://example.com');
200+
self::assertSame(409, $response->getStatusCode());
201+
self::assertSame(1, $mockHandler->count(), 'Second response should not be consumed');
202+
}
203+
204+
public function testNoRetryOn409WithMissingCodeField(): void
205+
{
206+
$bodyWithoutCode = (string) json_encode([
207+
'message' => 'Some conflict error.',
208+
]);
209+
210+
$mockHandler = new MockHandler([
211+
new Response(409, [], $bodyWithoutCode),
212+
new Response(200, [], 'ok'),
213+
]);
214+
215+
$handlerStack = HandlerStack::create([
216+
'handler' => $mockHandler,
217+
'backoffMaxTries' => 3,
218+
]);
219+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
220+
221+
$response = $client->request('GET', 'http://example.com');
222+
self::assertSame(409, $response->getStatusCode());
223+
self::assertSame(1, $mockHandler->count(), 'Second response should not be consumed');
224+
}
225+
226+
public function testNoRetryOnSuccessfulResponse(): void
227+
{
228+
$mockHandler = new MockHandler([
229+
new Response(200, [], 'ok'),
230+
new Response(200, [], 'second'),
231+
]);
232+
233+
$handlerStack = HandlerStack::create([
234+
'handler' => $mockHandler,
235+
'backoffMaxTries' => 3,
236+
]);
237+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
238+
239+
$response = $client->request('GET', 'http://example.com');
240+
self::assertSame(200, $response->getStatusCode());
241+
self::assertSame('ok', (string) $response->getBody());
242+
self::assertSame(1, $mockHandler->count(), 'Second response should not be consumed');
243+
}
244+
245+
public function testNoRetryOn4xxErrors(): void
246+
{
247+
$mockHandler = new MockHandler([
248+
new Response(400, [], 'Bad Request'),
249+
new Response(200, [], 'ok'),
250+
]);
251+
252+
$handlerStack = HandlerStack::create([
253+
'handler' => $mockHandler,
254+
'backoffMaxTries' => 3,
255+
]);
256+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
257+
258+
$response = $client->request('GET', 'http://example.com');
259+
self::assertSame(400, $response->getStatusCode());
260+
self::assertSame(1, $mockHandler->count(), 'Second response should not be consumed');
261+
}
262+
263+
public function testDefaultBackoffMaxTriesIsZero(): void
264+
{
265+
$mockHandler = new MockHandler([
266+
new Response(500, [], 'Internal Server Error'),
267+
new Response(200, [], 'ok'),
268+
]);
269+
270+
$handlerStack = HandlerStack::create([
271+
'handler' => $mockHandler,
272+
]);
273+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
274+
275+
$response = $client->request('GET', 'http://example.com');
276+
self::assertSame(500, $response->getStatusCode());
277+
self::assertSame(1, $mockHandler->count(), 'Should not retry when backoffMaxTries is 0');
278+
}
279+
280+
public function testRetryOn409VersionConflictRespectsMaxRetries(): void
281+
{
282+
$versionConflictBody = (string) json_encode([
283+
'code' => 'storage.components.configurations.versionConflict',
284+
'message' => 'Configuration row creation conflict. Please retry the operation.',
285+
]);
286+
287+
$mockHandler = new MockHandler([
288+
new Response(409, [], $versionConflictBody),
289+
new Response(409, [], $versionConflictBody),
290+
new Response(409, [], $versionConflictBody),
291+
new Response(200, [], 'ok'),
292+
]);
293+
294+
$handlerStack = HandlerStack::create([
295+
'handler' => $mockHandler,
296+
'backoffMaxTries' => 2,
297+
]);
298+
$client = new GuzzleClient(['handler' => $handlerStack, 'http_errors' => false]);
299+
300+
$response = $client->request('GET', 'http://example.com');
301+
self::assertSame(409, $response->getStatusCode());
302+
self::assertSame(1, $mockHandler->count(), 'Should stop after max retries');
303+
}
304+
}

0 commit comments

Comments
 (0)