Skip to content

Commit bf471e6

Browse files
eccleseccles
authored
Fix confirmation timeouts (#260)
* Fix confirmation timeouts Reduced max timeout from 1200 to 300s Set maximum retry interval to 30s handles merkleLog confirmation correctly AB#9103
1 parent 4cf032c commit bf471e6

6 files changed

Lines changed: 85 additions & 19 deletions

File tree

‎archivist/confirmer.py‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from .errors import ArchivistUnconfirmedError
2121
from .utils import backoff_handler
2222

23-
MAX_TIME = 1200
23+
MAX_TIME = 300
2424
LOGGER = getLogger(__name__)
2525

2626
# pylint: disable=protected-access
@@ -78,6 +78,7 @@ def _wait_for_confirmation(
7878

7979
@backoff.on_predicate(
8080
backoff.expo,
81+
max_value=30.0,
8182
logger=None, # pyright: ignore
8283
max_time=__lookup_max_time,
8384
on_backoff=backoff_handler,
@@ -94,19 +95,17 @@ def _wait_for_confirmation(self: Managers, identity: str) -> ReturnTypes:
9495
f"cannot confirm {identity} as confirmation_status is not present"
9596
)
9697

97-
if entity[CONFIRMATION_STATUS] == ConfirmationStatus.FAILED.name:
98+
status = entity[CONFIRMATION_STATUS]
99+
if status == ConfirmationStatus.FAILED.name:
98100
raise ArchivistUnconfirmedError(
99101
f"confirmation for {identity} FAILED - this is unusable"
100102
)
101103

102-
# Simple hash
103-
if entity[CONFIRMATION_STATUS] == ConfirmationStatus.CONFIRMED.name:
104-
return entity
105-
106-
# merkle_log
107-
if (
108-
ConfirmationStatus[entity[CONFIRMATION_STATUS]].value
109-
>= ConfirmationStatus.COMMITTED.value
104+
# Simple hash and merkleLog
105+
if status in (
106+
ConfirmationStatus.CONFIRMED.name,
107+
ConfirmationStatus.COMMITTED.name,
108+
ConfirmationStatus.UNEQUIVOCAL.name,
110109
):
111110
return entity
112111

@@ -124,6 +123,7 @@ def __on_giveup_confirmed(details: "Details"):
124123

125124
@backoff.on_predicate(
126125
backoff.expo,
126+
max_value=30.0,
127127
logger=None, # pyright: ignore
128128
max_time=__lookup_max_time,
129129
on_backoff=backoff_handler,
@@ -137,12 +137,19 @@ def _wait_for_confirmed(
137137
) -> bool:
138138
"""Return False until all entities are confirmed"""
139139

140-
# look for unconfirmed entities
140+
# look for pending entities
141141
newprops = deepcopy(props) if props else {}
142142
newprops[CONFIRMATION_STATUS] = ConfirmationStatus.PENDING.name
143+
LOGGER.debug("Count pending entities %s", newprops)
144+
pending_count = self.count(props=newprops, **kwargs)
145+
146+
# look for stored entities
147+
newprops = deepcopy(props) if props else {}
148+
newprops[CONFIRMATION_STATUS] = ConfirmationStatus.STORED.name
149+
LOGGER.debug("Count stored entities %s", newprops)
150+
stored_count = self.count(props=newprops, **kwargs)
143151

144-
LOGGER.debug("Count unconfirmed entities %s", newprops)
145-
count = self.count(props=newprops, **kwargs)
152+
count = pending_count + stored_count
146153

147154
if count == 0:
148155
# did any fail

‎archivist/subjects_confirmer.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
# pylint:disable=cyclic-import # but pylint doesn't understand this feature
1919
from .utils import backoff_handler
2020

21-
MAX_TIME = 1200
21+
MAX_TIME = 300
2222

2323
LOGGER = getLogger(__name__)
2424

@@ -37,6 +37,7 @@ def __on_giveup_confirmation(details):
3737

3838
@backoff.on_predicate(
3939
backoff.expo,
40+
max_value=30.0,
4041
logger=None, # pyright: ignore
4142
max_time=__lookup_max_time,
4243
on_backoff=backoff_handler,

‎requirements.txt‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
# for the published wheel - the file docs/requirements.txt
33
# must be kept in sync with this file.
44
#
5-
# upgrading to 2.2 of backoff raises all kinds of type hint errors so
6-
# this will require more work - stick to 1.11 for now.
75
backoff~=2.2.1
86
certifi
97
flatten-dict~=0.4

‎unittests/testassetsconstants.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,12 @@
537537
"attributes": ATTRS,
538538
"confirmation_status": "PENDING",
539539
}
540+
RESPONSE_STORED = {
541+
"identity": IDENTITY,
542+
"behaviours": ASSET_BEHAVIOURS,
543+
"attributes": ATTRS,
544+
"confirmation_status": "STORED",
545+
}
540546
RESPONSE_FAILED = {
541547
"identity": IDENTITY,
542548
"behaviours": ASSET_BEHAVIOURS,
@@ -553,7 +559,7 @@ class TestAssetsBase(TestCase):
553559
maxDiff = None
554560

555561
def setUp(self):
556-
self.arch = Archivist("url", "authauthauth", max_time=1)
562+
self.arch = Archivist("url", "authauthauth", max_time=0.5)
557563

558564
def tearDown(self):
559565
self.arch.close()

‎unittests/testassetswait.py‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from .testassetsconstants import (
1919
RESPONSE_FAILED,
2020
RESPONSE_PENDING,
21+
RESPONSE_STORED,
2122
SUBPATH,
2223
TestAssetsBase,
2324
)
@@ -45,9 +46,11 @@ def test_assets_wait_for_confirmed(self):
4546
status = (
4647
{"page_size": 1},
4748
{"page_size": 1, "confirmation_status": "PENDING"},
49+
{"page_size": 1, "confirmation_status": "STORED"},
4850
{"page_size": 1, "confirmation_status": "FAILED"},
4951
)
5052
with mock.patch.object(self.arch.session, "get") as mock_get:
53+
# there are 2 gets for each retry - one for PENDING and one for STORED
5154
mock_get.side_effect = [
5255
MockResponse(
5356
200,
@@ -66,6 +69,16 @@ def test_assets_wait_for_confirmed(self):
6669
headers={HEADERS_TOTAL_COUNT: 0},
6770
assets=[],
6871
),
72+
MockResponse(
73+
200,
74+
headers={HEADERS_TOTAL_COUNT: 0},
75+
assets=[],
76+
),
77+
MockResponse(
78+
200,
79+
headers={HEADERS_TOTAL_COUNT: 0},
80+
assets=[],
81+
),
6982
]
7083

7184
self.arch.assets.wait_for_confirmed()
@@ -110,6 +123,8 @@ def test_assets_wait_for_confirmed_timeout(self):
110123
"""
111124
## last call to get looks for FAILED assets
112125
with mock.patch.object(self.arch.session, "get") as mock_get:
126+
# there are 2 gets for each retry - one for PENDING and one for STORED
127+
# enough entries to be supplied so that timeout occurs
113128
mock_get.side_effect = [
114129
MockResponse(
115130
200,
@@ -122,7 +137,7 @@ def test_assets_wait_for_confirmed_timeout(self):
122137
200,
123138
headers={HEADERS_TOTAL_COUNT: 2},
124139
assets=[
125-
RESPONSE_PENDING,
140+
RESPONSE_STORED,
126141
],
127142
),
128143
MockResponse(
@@ -132,27 +147,55 @@ def test_assets_wait_for_confirmed_timeout(self):
132147
RESPONSE_PENDING,
133148
],
134149
),
150+
MockResponse(
151+
200,
152+
headers={HEADERS_TOTAL_COUNT: 2},
153+
assets=[
154+
RESPONSE_STORED,
155+
],
156+
),
135157
MockResponse(
136158
200,
137159
headers={HEADERS_TOTAL_COUNT: 2},
138160
assets=[
139161
RESPONSE_PENDING,
140162
],
141163
),
164+
MockResponse(
165+
200,
166+
headers={HEADERS_TOTAL_COUNT: 2},
167+
assets=[
168+
RESPONSE_STORED,
169+
],
170+
),
142171
MockResponse(
143172
200,
144173
headers={HEADERS_TOTAL_COUNT: 2},
145174
assets=[
146175
RESPONSE_PENDING,
147176
],
148177
),
178+
MockResponse(
179+
200,
180+
headers={HEADERS_TOTAL_COUNT: 2},
181+
assets=[
182+
RESPONSE_STORED,
183+
],
184+
),
149185
MockResponse(
150186
200,
151187
headers={HEADERS_TOTAL_COUNT: 2},
152188
assets=[
153189
RESPONSE_PENDING,
154190
],
155191
),
192+
MockResponse(
193+
200,
194+
headers={HEADERS_TOTAL_COUNT: 2},
195+
assets=[
196+
RESPONSE_STORED,
197+
],
198+
),
156199
]
157200

158201
with self.assertRaises(ArchivistUnconfirmedError):
@@ -177,6 +220,11 @@ def test_assets_wait_for_confirmed_failed(self):
177220
headers={HEADERS_TOTAL_COUNT: 0},
178221
assets=[],
179222
),
223+
MockResponse(
224+
200,
225+
headers={HEADERS_TOTAL_COUNT: 0},
226+
assets=[],
227+
),
180228
MockResponse(
181229
200,
182230
headers={HEADERS_TOTAL_COUNT: 1},

‎unittests/testeventswait.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,13 @@ class TestEventsWait(TestEventsBase):
3535

3636
def test_events_wait_for_confirmed(self):
3737
"""
38-
Test event counting
38+
Test event confirmation
3939
"""
4040
## last call to get looks for FAILED assets
4141
status = (
4242
{"page_size": 1},
4343
{"page_size": 1, "confirmation_status": "PENDING"},
44+
{"page_size": 1, "confirmation_status": "STORED"},
4445
{"page_size": 1, "confirmation_status": "FAILED"},
4546
)
4647
with mock.patch.object(self.arch.session, "get") as mock_get:
@@ -62,6 +63,11 @@ def test_events_wait_for_confirmed(self):
6263
headers={HEADERS_TOTAL_COUNT: 0},
6364
assets=[],
6465
),
66+
MockResponse(
67+
200,
68+
headers={HEADERS_TOTAL_COUNT: 0},
69+
assets=[],
70+
),
6571
]
6672

6773
self.arch.events.wait_for_confirmed()

0 commit comments

Comments
 (0)