|
12 | 12 | from __future__ import annotations |
13 | 13 |
|
14 | 14 | import json |
15 | | -import os |
16 | 15 | import sys |
| 16 | +import time |
| 17 | +from pathlib import Path |
17 | 18 |
|
18 | 19 | import impit |
19 | 20 |
|
| 21 | +REPO_ROOT = Path(__file__).resolve().parent.parent |
| 22 | + |
20 | 23 | # The published, bundled specification. It is built and deployed from the `apify/apify-docs` repository. |
21 | 24 | SPEC_URL = 'https://docs.apify.com/api/openapi.json' |
22 | 25 |
|
23 | | -# Path of the committed snapshot, relative to the repository root. |
24 | | -SPEC_PATH = os.path.join('spec', 'openapi.json') |
| 26 | +SPEC_PATH = REPO_ROOT / 'spec' / 'openapi.json' |
25 | 27 |
|
26 | 28 | # A truncated response or an error page must never overwrite the snapshot. The real specification is roughly |
27 | 29 | # 1 MB, so anything remotely this small is broken. |
28 | 30 | MIN_SPEC_SIZE_BYTES = 100_000 |
29 | 31 |
|
30 | | -# Top-level members every specification we can generate models from has to contain. |
31 | | -REQUIRED_SPEC_KEYS = ('openapi', 'paths', 'components') |
| 32 | +# Top-level members every specification we can generate models from has to contain. `info` is included because |
| 33 | +# the version stamp reported below is read from it. |
| 34 | +REQUIRED_SPEC_KEYS = ('openapi', 'info', 'paths', 'components') |
32 | 35 |
|
33 | 36 | REQUEST_TIMEOUT_SECS = 60 |
34 | 37 |
|
| 38 | +# The nightly workflow alerts the team when this fails, so a single network blip shouldn't be worth a ping. |
| 39 | +DOWNLOAD_ATTEMPTS = 3 |
| 40 | +RETRY_DELAY_SECS = 5 |
| 41 | + |
| 42 | + |
| 43 | +def download_spec() -> bytes: |
| 44 | + """Download the published specification, retrying transient failures.""" |
| 45 | + last_error = '' |
35 | 46 |
|
36 | | -def main() -> None: |
37 | 47 | with impit.Client(follow_redirects=True) as client: |
38 | | - response = client.request('GET', SPEC_URL, timeout=REQUEST_TIMEOUT_SECS) |
| 48 | + for attempt in range(1, DOWNLOAD_ATTEMPTS + 1): |
| 49 | + try: |
| 50 | + response = client.request('GET', SPEC_URL, timeout=REQUEST_TIMEOUT_SECS) |
| 51 | + except Exception as exc: |
| 52 | + last_error = f'{type(exc).__name__}: {exc}' |
| 53 | + else: |
| 54 | + if response.status_code == 200: |
| 55 | + return response.content |
| 56 | + last_error = f'HTTP {response.status_code}' |
39 | 57 |
|
40 | | - if response.status_code != 200: |
41 | | - print(f'Failed to download {SPEC_URL}: HTTP {response.status_code}.') |
42 | | - sys.exit(1) |
| 58 | + print(f'Attempt {attempt}/{DOWNLOAD_ATTEMPTS} to download {SPEC_URL} failed ({last_error}).') |
| 59 | + if attempt < DOWNLOAD_ATTEMPTS: |
| 60 | + time.sleep(RETRY_DELAY_SECS) |
| 61 | + |
| 62 | + print(f'Failed to download {SPEC_URL} after {DOWNLOAD_ATTEMPTS} attempts: {last_error}.', file=sys.stderr) |
| 63 | + sys.exit(1) |
| 64 | + |
| 65 | + |
| 66 | +def main() -> None: |
| 67 | + payload = download_spec() |
43 | 68 |
|
44 | | - payload = response.content |
45 | 69 | if len(payload) < MIN_SPEC_SIZE_BYTES: |
46 | | - print(f'Downloaded specification is only {len(payload)} bytes, which cannot be the real one - aborting.') |
| 70 | + print( |
| 71 | + f'Downloaded specification is only {len(payload)} bytes, which cannot be the real one - aborting.', |
| 72 | + file=sys.stderr, |
| 73 | + ) |
47 | 74 | sys.exit(1) |
48 | 75 |
|
49 | 76 | try: |
50 | 77 | spec = json.loads(payload) |
51 | 78 | except json.JSONDecodeError as exc: |
52 | | - print(f'Downloaded specification is not valid JSON: {exc}.') |
| 79 | + print(f'Downloaded specification is not valid JSON: {exc}.', file=sys.stderr) |
53 | 80 | sys.exit(1) |
54 | 81 |
|
55 | 82 | missing_keys = [key for key in REQUIRED_SPEC_KEYS if key not in spec] |
56 | 83 | if missing_keys: |
57 | | - print(f'Downloaded specification is missing top-level {", ".join(missing_keys)} - aborting.') |
| 84 | + print(f'Downloaded specification is missing top-level {", ".join(missing_keys)} - aborting.', file=sys.stderr) |
| 85 | + sys.exit(1) |
| 86 | + |
| 87 | + # Read the version stamp before writing anything, so a malformed `info` aborts with the snapshot intact. |
| 88 | + version = spec['info'].get('version') |
| 89 | + if not version: |
| 90 | + print('Downloaded specification has no `info.version` - aborting.', file=sys.stderr) |
58 | 91 | sys.exit(1) |
59 | 92 |
|
60 | 93 | # Normalize the formatting so the committed diffs stay readable no matter how the published bundle is |
61 | 94 | # formatted. Key order is deliberately preserved: `keep_model_order` makes the generated model order follow |
62 | 95 | # the specification, so sorting keys here would reshuffle `_models.py` on the next run. |
63 | 96 | normalized = json.dumps(spec, indent=2, ensure_ascii=False) + '\n' |
64 | 97 |
|
65 | | - os.makedirs(os.path.dirname(SPEC_PATH), exist_ok=True) |
66 | | - with open(SPEC_PATH, 'w', encoding='utf-8') as spec_file: |
67 | | - spec_file.write(normalized) |
| 98 | + SPEC_PATH.parent.mkdir(parents=True, exist_ok=True) |
| 99 | + SPEC_PATH.write_text(normalized, encoding='utf-8', newline='\n') |
68 | 100 |
|
69 | | - print(f'Wrote {SPEC_PATH} (version {spec["info"]["version"]}, {len(normalized.encode())} bytes).') |
| 101 | + print(f'Wrote {SPEC_PATH.relative_to(REPO_ROOT)} (version {version}, {len(normalized.encode())} bytes).') |
70 | 102 |
|
71 | 103 |
|
72 | 104 | if __name__ == '__main__': |
|
0 commit comments