From 08c2c0d1302a26af754e544e94a615e883e9f139 Mon Sep 17 00:00:00 2001 From: Niall Date: Wed, 15 Apr 2026 15:29:14 +0100 Subject: [PATCH 1/3] chore(cli): raise WS heartbeat + Node heap headroom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two configuration tweaks that independently reduce the chance of a bulletin upload failing on the polkadot-api WS heartbeat or hitting the default Node heap ceiling. No behavioural change to upload code. - packages/cli/src/bulletin/store.ts: Pass heartbeatTimeout=300_000 and timeout=10_000 to getWsProvider. The 40_000 ms ws-provider default is shorter than a single Bulletin chunk's worst-case best-block inclusion latency we measured (>50 s outliers under contention), so the transport was tearing down healthy WS connections while the chain was still acknowledging an in-flight extrinsic. - .github/actions/bulletin/action.yml: Set NODE_OPTIONS=--max-old-space-size=4096 in both the authorize and upload steps. ubuntu-latest runners have ≥16 GiB RAM; default ~2 GiB Node heap leaves no headroom if a transient leak window opens. Cheap safety net. --- .github/actions/bulletin/action.yml | 7 +++++++ packages/cli/src/bulletin/store.ts | 18 +++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/actions/bulletin/action.yml b/.github/actions/bulletin/action.yml index 0f4d7a6..35349f4 100644 --- a/.github/actions/bulletin/action.yml +++ b/.github/actions/bulletin/action.yml @@ -72,6 +72,11 @@ runs: CI: "true" FORCE_COLOR: "0" NODE_NO_WARNINGS: "1" + # ubuntu-latest runners have ≥16 GiB RAM; default ~2 GiB Node heap + # leaves no headroom if a transient leak window opens during a WS + # halt teardown. Raising it is a cheap safety net independent of + # the in-CLI cancellation fixes. + NODE_OPTIONS: "--max-old-space-size=4096" RETRY_DELAY: ${{ inputs.retry-delay }} - name: Upload to Bulletin @@ -124,5 +129,7 @@ runs: CI: "true" FORCE_COLOR: "0" NODE_NO_WARNINGS: "1" + # See note in the authorize step above. + NODE_OPTIONS: "--max-old-space-size=4096" RETRY_DELAY: ${{ inputs.retry-delay }} CACHE: ${{ inputs.cache }} diff --git a/packages/cli/src/bulletin/store.ts b/packages/cli/src/bulletin/store.ts index a09f652..5d7932d 100644 --- a/packages/cli/src/bulletin/store.ts +++ b/packages/cli/src/bulletin/store.ts @@ -43,6 +43,15 @@ const WAVE_TIMEOUT_MS = 60_000; const STORE_CALL_TIMEOUT_MS = 60_000; export const FINAL_STORE_CALL_TIMEOUT_MS = 180_000; const FETCH_NONCE_TIMEOUT_MS = 15_000; +// polkadot-api ws-provider defaults to 40_000 ms, which is shorter than a +// single Bulletin chunk's worst-case best-block inclusion latency under +// contention (we observed >50 s outliers). Raise it well above WAVE_TIMEOUT_MS +// so the transport layer never tears down a healthy WS while the chain is +// still acknowledging an in-flight extrinsic. +const WS_HEARTBEAT_TIMEOUT_MS = 300_000; +// polkadot-api ws-provider default is 3_500 ms; give the handshake more +// headroom on slow links (CI runners → Scaleway can spike past that). +const WS_CONNECT_TIMEOUT_MS = 10_000; let rxUnhandledErrorGuardInstalled = false; export type AdaptiveWindowUpdateInput = { @@ -468,7 +477,14 @@ export function formatTransactionWatchFailure( export function createBulletinClient(rpc: string): PolkadotClient { installRxUnhandledErrorGuard(); - return createPolkadotClient(withPolkadotSdkCompat(getWsProvider(rpc))); + return createPolkadotClient( + withPolkadotSdkCompat( + getWsProvider(rpc, { + heartbeatTimeout: WS_HEARTBEAT_TIMEOUT_MS, + timeout: WS_CONNECT_TIMEOUT_MS, + }), + ), + ); } async function storeContentOnBulletin( From 4e9b6a81795f992ed10c8c7a2fc44a113beeffe5 Mon Sep 17 00:00:00 2001 From: Niall Date: Wed, 15 Apr 2026 15:30:36 +0100 Subject: [PATCH 2/3] Release chunk buffer regardless of submit outcome Move chunk.bytes = null into a finally so a failed submission also releases its 2 MiB. Today the buffer is only freed on success; on failure (stall, retry, abort) it stays alive via the closure, pinning memory for the lifetime of the upload. The payload is re-streamed from disk on the next attempt, so dropping the in-memory copy is always safe. --- packages/cli/src/bulletin/store.ts | 48 +++++++++++++++++------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/packages/cli/src/bulletin/store.ts b/packages/cli/src/bulletin/store.ts index 5d7932d..6d463ab 100644 --- a/packages/cli/src/bulletin/store.ts +++ b/packages/cli/src/bulletin/store.ts @@ -788,26 +788,34 @@ export async function storeChunkedFileToBulletin( submitChunk: async (chunk) => { const nonce = waveNonces.get(chunk.index)!; - await storeContentOnBulletin({ - rpc: parameters.rpc, - signer: parameters.signer, - contentBytes: chunk.bytes, - contentCid: chunk.cid, - codecValue: CODEC.RAW, - hashCodeValue: HASH.SHA2_256, - nonce, - client: activeClient, - waitForFinalization, - }); - - manifestState.completedBlocks.set(chunk.index, { - index: chunk.index, - cid: chunk.cid, - length: chunk.length, - }); - - completedChunks += 1; - parameters.onProgress?.(chunk.index + 1, totalChunks, "stored"); + try { + await storeContentOnBulletin({ + rpc: parameters.rpc, + signer: parameters.signer, + contentBytes: chunk.bytes, + contentCid: chunk.cid, + codecValue: CODEC.RAW, + hashCodeValue: HASH.SHA2_256, + nonce, + client: activeClient, + waitForFinalization, + }); + + manifestState.completedBlocks.set(chunk.index, { + index: chunk.index, + cid: chunk.cid, + length: chunk.length, + }); + + completedChunks += 1; + parameters.onProgress?.(chunk.index + 1, totalChunks, "stored"); + } finally { + // Release the chunk's 2 MiB Buffer regardless of outcome. On + // success it is already persisted; on failure it must not pin + // memory while the caller decides whether to retry — the + // payload is re-streamed from disk on the next attempt. + chunk.bytes = null as unknown as Uint8Array; + } }, }); From d60a631c40b92195256d8abef0556c15ee1f28c8 Mon Sep 17 00:00:00 2001 From: Niall Date: Wed, 15 Apr 2026 16:08:46 +0100 Subject: [PATCH 3/3] Revert "Release chunk buffer regardless of submit outcome" This reverts commit 4e9b6a81795f992ed10c8c7a2fc44a113beeffe5. --- packages/cli/src/bulletin/store.ts | 48 +++++++++++++----------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/packages/cli/src/bulletin/store.ts b/packages/cli/src/bulletin/store.ts index 6d463ab..5d7932d 100644 --- a/packages/cli/src/bulletin/store.ts +++ b/packages/cli/src/bulletin/store.ts @@ -788,34 +788,26 @@ export async function storeChunkedFileToBulletin( submitChunk: async (chunk) => { const nonce = waveNonces.get(chunk.index)!; - try { - await storeContentOnBulletin({ - rpc: parameters.rpc, - signer: parameters.signer, - contentBytes: chunk.bytes, - contentCid: chunk.cid, - codecValue: CODEC.RAW, - hashCodeValue: HASH.SHA2_256, - nonce, - client: activeClient, - waitForFinalization, - }); - - manifestState.completedBlocks.set(chunk.index, { - index: chunk.index, - cid: chunk.cid, - length: chunk.length, - }); - - completedChunks += 1; - parameters.onProgress?.(chunk.index + 1, totalChunks, "stored"); - } finally { - // Release the chunk's 2 MiB Buffer regardless of outcome. On - // success it is already persisted; on failure it must not pin - // memory while the caller decides whether to retry — the - // payload is re-streamed from disk on the next attempt. - chunk.bytes = null as unknown as Uint8Array; - } + await storeContentOnBulletin({ + rpc: parameters.rpc, + signer: parameters.signer, + contentBytes: chunk.bytes, + contentCid: chunk.cid, + codecValue: CODEC.RAW, + hashCodeValue: HASH.SHA2_256, + nonce, + client: activeClient, + waitForFinalization, + }); + + manifestState.completedBlocks.set(chunk.index, { + index: chunk.index, + cid: chunk.cid, + length: chunk.length, + }); + + completedChunks += 1; + parameters.onProgress?.(chunk.index + 1, totalChunks, "stored"); }, });