From c8ee030488938b1185a4fd983618cea675f41cf9 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Mon, 27 Jul 2026 09:58:55 +0200 Subject: [PATCH 01/14] temp --- core/consensus/src/client_table.rs | 693 ++++++++++++++++--------- core/consensus/src/metadata_helpers.rs | 227 +++++--- core/metadata/src/impls/metadata.rs | 161 +++--- 3 files changed, 700 insertions(+), 381 deletions(-) diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 6b0a8526fc..09a87292e9 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -17,7 +17,7 @@ use iggy_binary_protocol::ReplyHeader; use server_common::{MESSAGE_ALIGN, Message, iobuf::Frozen}; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::mem::size_of; use tracing::trace; @@ -71,19 +71,46 @@ impl CachedReply { /// Real requests start at 1 (header validation enforces `request > 0`). pub const REGISTER_REQUEST_ID: u64 = 0; -/// Per-client entry (VR paper ยง4, Fig. 2): session + latest committed reply. +/// Displaced replies retained per entry for below-watermark duplicate hits. /// -/// `session` is assigned at registration and fixed for the entry's lifetime. +/// The SDK enforces one request in flight per session, so the only reply a +/// live client can be waiting for is its latest (`request == watermark`). +/// The ring answers old retransmits and post-rebind stragglers with the +/// original bytes instead of a bare "already applied"; losing an entry +/// degrades the answer, never correctness. In-memory only: ring contents are +/// refcount bumps and are never persisted or transferred. +const REPLY_RING_CAPACITY: usize = 4; + +/// Per-session entry: fence epoch + committed-request watermark + replies. +/// +/// The key (`client_id` today, the stable `session_id` once SDK identity +/// stability lands) is client-supplied; `epoch` is the server-minted fence +/// that orders rebinds of that key. #[derive(Debug)] pub struct ClientEntry { - /// Session number = commit op of the register. Monotonic across - /// registrations; new register always gets a higher session. - pub session: u64, - /// Acting user id captured at register. Fixed for the entry's lifetime; - /// lets every replica resolve session -> user without a metadata lookup. - pub user_id: u32, - /// Cached reply for client's latest committed request. - pub reply: CachedReply, + /// Fence epoch: 1 at first register, +1 per committed re-register. + /// Minted here, in apply order, so every replica derives the same value. + /// Requests stamped with an older epoch are zombies and get fenced; + /// a newer epoch than minted is a protocol violation. + epoch: u64, + /// Acting user id captured at register (re-register refreshes it: the + /// rebind re-authenticated). Lets every replica resolve session -> user + /// without a metadata lookup. + user_id: u32, + /// Highest committed request number. `REGISTER_REQUEST_ID` (0) until the + /// first app op commits. Survives re-register: a resumed session keeps + /// its dedup history. + watermark: u64, + /// `request_checksum` of the watermark request; catches a client reusing + /// a request id for a different operation. Zero when unstamped (integrity + /// fields are zeroed on the wire today), which disables the comparison. + watermark_checksum: u128, + /// Latest committed reply (register or app op). + reply: CachedReply, + /// Displaced app replies, oldest at front, bounded by + /// [`REPLY_RING_CAPACITY`]. Register replies never enter (their + /// `request == REGISTER_REQUEST_ID` can never match a lookup). + ring: VecDeque, } /// Result of checking a request against the client table. @@ -93,49 +120,71 @@ pub struct ClientEntry { /// committed state. #[derive(Debug)] pub enum RequestStatus { - /// Not seen; proceed with consensus. + /// Above the watermark; proceed with consensus. Jumps are allowed: the + /// watermark records the highest committed request, not a contiguous + /// sequence, so `watermark + k` for any `k >= 1` is new. New, - /// Exact request already committed; re-send cached reply. + /// At or below the watermark with the original reply still cached; + /// re-send it. Duplicate(CachedReply), - /// Older than client's latest committed request; drop silently. - Stale, - /// No session for this client; must register first. + /// At or below the watermark, original reply no longer cached. Applied + /// once already; must not re-execute, nothing to replay. + AlreadyApplied { request: u64, watermark: u64 }, + /// Request number matches the watermark but its `request_checksum` + /// differs: the client reused a request id for a different operation. + /// Returning the cached reply would answer the wrong request. + ChecksumMismatch { request: u64 }, + /// No entry for this client; must register first. NoSession, - /// Session number doesn't match the entry. - SessionMismatch { expected: u64, received: u64 }, - /// Request != `committed + 1`. Skipped numbers would be lost permanently. - RequestGap { expected: u64, received: u64 }, - /// Client already has a session. From `check_register`. + /// Stamped epoch is older than the entry's: a zombie holdover from + /// before a re-register. Terminal for that holder. + Fenced { current: u64, received: u64 }, + /// Stamped epoch is newer than any this table minted: client bug + /// (epochs are only handed out by register replies). + EpochAhead { current: u64, received: u64 }, + /// Client already has an entry. From `check_register`. AlreadyRegistered { - session: u64, + epoch: u64, cached_reply: CachedReply, }, } -/// VSR client-table: durable per-client session state. +/// VSR client table: per-session fence epoch + request-watermark dedup. /// /// Fixed-size slot array (source of truth) + `HashMap` index (O(1) lookup). /// -/// ## Plane: metadata-only +/// ## Semantics (v2) +/// +/// - **Epoch, not commit.** Session identity is the client-supplied key; +/// the entry's `epoch` is a plain counter minted at `commit_register` +/// (1, then +1 per rebind). No field derives from a commit op number, so +/// the same table logic serves any consensus group. +/// - **Watermark, not contiguity.** A request above the watermark executes +/// (gaps allowed); at or below is a duplicate. There is no `RequestGap`: +/// a client that jumps its counter loses nothing but the skipped ids. +/// - **Replies are volatile.** Latest reply plus a small ring of displaced +/// ones, all in-memory refcounts. A duplicate whose reply aged out is +/// still refused execution ([`RequestStatus::AlreadyApplied`]). /// -/// Backs Register session, request contiguity, metadata-retry dedup, and -/// `NoSession`/`SessionTooLow` eviction. Partition plane is at-least-once; -/// `SendMessages` retries can re-commit at a new offset and consumers -/// dedup via message ID (`server_common::MessageDeduplicator`). +/// ## Plane /// -/// Do not add per-partition `ClientTable` or `(client_id, request)` dedup -/// on the partition side, that flips iggy's contract toward at-most-once. -/// See project memory `project_vsr_clients_table_integration`. +/// Metadata-plane today. The design spans planes (one logical table, +/// group-resident slices); partition-plane integration arrives once +/// partition prepares carry real `(session_id, request)` instead of the +/// transport id (data-plane request numbering, IGGY-137). Until then the +/// partition plane stays at-least-once with no dedup. /// /// ## Tracking /// -/// Committed state only, latest reply per client. In-flight state -/// (acks, subscribers, in-progress dedup) lives on [`crate::PipelineEntry`]. -/// Updated by `commit_reply` / `commit_register`. +/// Committed state only. In-flight state (acks, subscribers, in-progress +/// dedup) lives on [`crate::PipelineEntry`]. Updated by `commit_reply` / +/// `commit_register` in the apply path, so every replica of the group +/// derives an identical table from the committed log. /// /// ## Known gaps /// -/// - **Checkpoint serialization**: slot layout deterministic, encode/decode TODO. +/// - **Serialization**: encode/decode for rejoin slice-fetch and state +/// transfer TODO (IGGY-137). #[derive(Debug)] pub struct ClientTable { /// `None` = free slot. Deterministic iteration for eviction + serialization. @@ -156,52 +205,70 @@ impl ClientTable { } } - /// Check request against table. Session first, then request progression. - /// For Register, use [`check_register`]. + /// Check a request against the table. Epoch fence first, then the + /// watermark. For Register, use [`Self::check_register`]. + /// + /// `request_checksum` is the request's integrity stamp; zero (unstamped) + /// disables the reuse check. /// /// # Panics /// If index points to empty slot (invariant violation). #[must_use] - pub fn check_request(&self, client_id: u128, session: u64, request: u64) -> RequestStatus { + pub fn check_request( + &self, + client_id: u128, + epoch: u64, + request: u64, + request_checksum: u128, + ) -> RequestStatus { assert!(client_id != 0, "client_id 0 is reserved for internal use"); // Header validation guarantees both > 0 at wire layer. - debug_assert!(session > 0, "check_request: session must be > 0"); + debug_assert!(epoch > 0, "check_request: epoch must be > 0"); debug_assert!(request > 0, "check_request: request must be > 0"); - // Session check before request: wrong-session must be rejected even if - // (client_id, request) matches a correct-session pending entry. + // Epoch check before request: a fenced zombie must be rejected even + // if its request number would read as a clean duplicate. let Some(&slot_idx) = self.index.get(&client_id) else { return RequestStatus::NoSession; }; let entry = self.slots[slot_idx].as_ref().expect("index/slot mismatch"); - if session != entry.session { - return RequestStatus::SessionMismatch { - expected: entry.session, - received: session, + if epoch < entry.epoch { + return RequestStatus::Fenced { + current: entry.epoch, + received: epoch, }; } - - let committed_request = entry.reply.header().request; - - if request < committed_request { - return RequestStatus::Stale; + if epoch > entry.epoch { + return RequestStatus::EpochAhead { + current: entry.epoch, + received: epoch, + }; } - if request == committed_request { - return RequestStatus::Duplicate(entry.reply.clone()); + + if request > entry.watermark { + return RequestStatus::New; } - if request != committed_request + 1 { - return RequestStatus::RequestGap { - expected: committed_request + 1, - received: request, - }; + + if request == entry.watermark + && entry.watermark_checksum != 0 + && request_checksum != 0 + && entry.watermark_checksum != request_checksum + { + return RequestStatus::ChecksumMismatch { request }; } - RequestStatus::New + match entry.find_cached(request) { + Some(cached) => RequestStatus::Duplicate(cached.clone()), + None => RequestStatus::AlreadyApplied { + request, + watermark: entry.watermark, + }, + } } - /// Check register. Valid without existing session; returns - /// `AlreadyRegistered { session, cached_reply }`. + /// Check register. Valid without existing entry; returns + /// `AlreadyRegistered { epoch, cached_reply }` otherwise. /// /// Caller does in-flight dedup via `pipeline.has_message_from_client`. /// @@ -216,33 +283,25 @@ impl ClientTable { }; let entry = self.slots[slot_idx].as_ref().expect("index/slot mismatch"); RequestStatus::AlreadyRegistered { - session: entry.session, + epoch: entry.epoch, cached_reply: entry.reply.clone(), } } - /// Record committed register; create or update session. + /// Record a committed register: create the entry at epoch 1, or bump the + /// existing entry's epoch (rebind). /// - /// Session = `reply.header().commit`. Monotonic, deterministic. - /// Idempotent on same-session WAL replay. - /// - /// # Session mismatch (no panic, log + skip) - /// - /// - `existing.session > new`: stale WAL replay; newer slot is authoritative. - /// - `existing.session < new`: duplicate Register at different ops, - /// protocol violation; keep existing (other replicas may have agreed on it). - /// - /// Was `assert_eq!` pre-fix. `commit_journal` runs without the - /// `is_caught_up_primary` gate (it's what opens the gate), so a - /// malformed WAL or capacity-evict-then-reregister race could reach - /// here and panic the shard pump. + /// The epoch is minted HERE, in apply order, so it is deterministic + /// across replicas without reading any commit number. A rebind refreshes + /// `user_id` (the bind re-authenticated), replaces the latest reply with + /// the register reply (the displaced app reply moves into the ring), and + /// preserves the watermark - session resume keeps dedup history. /// /// Full table evicts oldest commit; `in_flight` protects pipeline /// holders, see [`Self::evict_oldest`]. /// /// # Panics - /// If `client_id == 0`, `session == 0`, or `client_id != reply.header().client`. - /// Session mismatch does NOT panic. + /// If `client_id == 0` or `client_id != reply.header().client`. pub fn commit_register( &mut self, client_id: u128, @@ -260,55 +319,37 @@ impl ClientTable { reply.header().client ); - let session = reply.header().commit; - assert!(session > 0, "commit_register: session must be > 0"); - - let existing = self.index.get(&client_id).copied(); - - // Mismatch on re-register: log + skip, not panic. See doc above. - if let Some(slot_idx) = existing { - let slot = self.slots[slot_idx].as_ref().expect("index/slot mismatch"); - if slot.session != session { - tracing::warn!( - client_id, - existing_session = slot.session, - replay_session = session, - "commit_register: session mismatch (stale WAL replay or \ - duplicate Register at different ops); skipping update" - ); - return; - } - } - // Freeze once; later dedup-hit clones Arc-bump. let cached: CachedReply = CachedReply::from_message(reply); - // Update in place on re-register, else new slot. Reply-delivery - // channel lives on popped `PipelineEntry`, fired by commit caller - // after this returns, slot-first ordering, see `commit_reply`. - if let Some(slot_idx) = existing { - self.slots[slot_idx] - .as_mut() - .expect("index/slot mismatch") - .reply = cached; + if let Some(&slot_idx) = self.index.get(&client_id) { + let entry = self.slots[slot_idx].as_mut().expect("index/slot mismatch"); + entry.epoch += 1; + entry.user_id = user_id; + let displaced = std::mem::replace(&mut entry.reply, cached); + entry.push_ring(displaced); } else { if self.index.len() >= self.slots.len() { self.evict_oldest(&in_flight); } let slot_idx = self.first_free_slot().expect("eviction must free a slot"); self.slots[slot_idx] = Some(ClientEntry { - session, + epoch: 1, user_id, + watermark: REGISTER_REQUEST_ID, + watermark_checksum: 0, reply: cached, + ring: VecDeque::with_capacity(REPLY_RING_CAPACITY), }); self.index.insert(client_id, slot_idx); } } - /// Record committed reply, update in place. Client must be registered. + /// Record a committed reply: advance the watermark, cache the reply, + /// move the displaced one into the ring. /// - /// `session` is asserted against stored session to guard WAL replay - /// from clobbering a newer entry. + /// `epoch` is asserted against the entry to guard a mis-attributed apply + /// from clobbering a rebound session's state. /// /// Reply delivery is caller's job, `Sender` lives on the popped /// `PipelineEntry` ([`crate::PipelineEntry::take_reply_sender`]), @@ -319,18 +360,23 @@ impl ClientTable { /// ships; cache skipped; client gets `NoSession` next request. /// /// # Panics - /// On session mismatch or commit/request regression. Missing client + /// On epoch mismatch or commit/watermark regression. Missing client /// does NOT panic. - pub fn commit_reply(&mut self, client_id: u128, session: u64, reply: Message) { + pub fn commit_reply(&mut self, client_id: u128, epoch: u64, reply: Message) { assert!(client_id != 0, "client_id 0 is reserved for internal use"); let new_header = reply.header(); let new_client = new_header.client; let new_request = new_header.request; let new_commit = new_header.commit; + let new_checksum = new_header.request_checksum; assert_eq!( client_id, new_client, "commit_reply: client_id mismatch (arg={client_id}, header={new_client})", ); + debug_assert!( + new_request > REGISTER_REQUEST_ID, + "commit_reply: register replies go through commit_register" + ); let Some(&slot_idx) = self.index.get(&client_id) else { // Evicted between prepare and commit (WAL replay or @@ -345,33 +391,40 @@ impl ClientTable { return; }; - let slot = self.slots[slot_idx].as_ref().expect("index/slot mismatch"); - let slot_header = slot.reply.header(); - let slot_commit = slot_header.commit; - let slot_request = slot_header.request; + let entry = self.slots[slot_idx].as_mut().expect("index/slot mismatch"); assert_eq!( - slot.session, session, - "commit_reply: session mismatch for client {client_id}: \ - entry={}, prepare={session}", - slot.session + entry.epoch, epoch, + "commit_reply: epoch mismatch for client {client_id}: \ + entry={}, prepare={epoch}", + entry.epoch ); + let latest_commit = entry.reply.header().commit; assert!( - new_commit >= slot_commit, - "commit_reply: commit regression for client {client_id}: {slot_commit} -> {new_commit}", + new_commit >= latest_commit, + "commit_reply: commit regression for client {client_id}: {latest_commit} -> {new_commit}", ); assert!( - new_request >= slot_request, - "commit_reply: request regression for client {client_id}: {slot_request} -> {new_request}", + new_request >= entry.watermark, + "commit_reply: watermark regression for client {client_id}: {} -> {new_request}", + entry.watermark ); // Freeze once; later dedup-hit clones Arc-bump. - self.slots[slot_idx] - .as_mut() - .expect("index/slot mismatch") - .reply = CachedReply::from_message(reply); + let cached = CachedReply::from_message(reply); + if new_request == entry.watermark { + // Same request re-committed (WAL replay shape): replace in + // place, never push the stale twin into the ring - two cached + // replies for one request number would make lookups ambiguous. + entry.reply = cached; + } else { + let displaced = std::mem::replace(&mut entry.reply, cached); + entry.push_ring(displaced); + entry.watermark = new_request; + } + entry.watermark_checksum = new_checksum; } - /// Remove a client session and cached reply. + /// Remove a client session and cached replies. /// /// **LOCAL ONLY -- does NOT replicate.** Two correct call sites: /// @@ -412,9 +465,9 @@ impl ClientTable { /// state -> identical choice. `commit_journal` catch-up has empty pipeline, /// so `in_flight` returns `false` everywhere, matches pre-fix policy. /// - /// **Metadata caveat**: pre-checkpoint, eviction breaks at-most-once - /// for the evicted client, next metadata retry treated as `New`. - /// Partition plane unaffected (at-least-once, doesn't use this table). + /// **Caveat**: eviction erases the evicted session's watermark, so its + /// next retry is treated as `New` (re-executes). Bounded by table + /// capacity; the op-TTL + slice persistence work (IGGY-137) shrinks it. fn evict_oldest(&mut self, in_flight: &F) where F: Fn(u128) -> bool, @@ -453,7 +506,7 @@ impl ClientTable { self.slots.iter().position(Option::is_none) } - /// Cached reply for a client (duplicate re-sends). + /// Latest cached reply for a client. /// /// Borrow avoids Arc bump for header-only inspection. Wire-senders /// `.clone()` (Arc bump) then `.into_wire_bytes()`. @@ -463,11 +516,21 @@ impl ClientTable { self.slots[slot_idx].as_ref().map(|entry| &entry.reply) } - /// Session number for a registered client. + /// Fence epoch for a registered client. This is the u64 the register + /// reply hands the client and the wire `session` field carries back. #[must_use] - pub fn get_session(&self, client_id: u128) -> Option { + pub fn get_epoch(&self, client_id: u128) -> Option { let &slot_idx = self.index.get(&client_id)?; - self.slots[slot_idx].as_ref().map(|entry| entry.session) + self.slots[slot_idx].as_ref().map(|entry| entry.epoch) + } + + /// Committed-request watermark for a registered client. A (re)bind reply + /// surfaces this so a restarted client resumes numbering at + /// `watermark + 1` instead of silently colliding below it. + #[must_use] + pub fn get_watermark(&self, client_id: u128) -> Option { + let &slot_idx = self.index.get(&client_id)?; + self.slots[slot_idx].as_ref().map(|entry| entry.watermark) } /// Acting user id captured when the client registered. @@ -484,6 +547,33 @@ impl ClientTable { } } +impl ClientEntry { + /// Cached reply whose `request` matches, latest first then the ring + /// (newest displaced entries sit at the back; scan order is irrelevant + /// because request numbers in the ring are unique). + fn find_cached(&self, request: u64) -> Option<&CachedReply> { + if self.reply.header().request == request { + return Some(&self.reply); + } + self.ring + .iter() + .find(|cached| cached.header().request == request) + } + + /// Retain a displaced reply for below-watermark duplicates. Register + /// replies never enter: `request == REGISTER_REQUEST_ID` can never match + /// a `check_request` lookup (wire validation enforces `request > 0`). + fn push_ring(&mut self, displaced: CachedReply) { + if displaced.header().request == REGISTER_REQUEST_ID { + return; + } + if self.ring.len() == REPLY_RING_CAPACITY { + self.ring.pop_front(); + } + self.ring.push_back(displaced); + } +} + #[cfg(test)] mod tests { use super::*; @@ -512,6 +602,15 @@ mod tests { } fn make_reply_for(client: u128, request: u64, commit: u64) -> Message { + make_reply_with_checksum(client, request, commit, 0) + } + + fn make_reply_with_checksum( + client: u128, + request: u64, + commit: u64, + request_checksum: u128, + ) -> Message { let header_size = std::mem::size_of::(); let mut msg = Message::::new(header_size); let header = bytemuck::checked::try_from_bytes_mut::( @@ -522,6 +621,7 @@ mod tests { client, request, commit, + request_checksum, command: Command2::Reply, operation: Operation::SendMessages, ..ReplyHeader::default() @@ -534,30 +634,59 @@ mod tests { |_| false } - /// Register client 1 at commit 10. Returns (table, session=10). + /// Register client 1 (register commit stamped at op 10). Returns + /// (table, epoch=1). fn table_with_client() -> (ClientTable, u64) { let mut table = ClientTable::new(10); - let session = 10; - table.commit_register( - 1, - TEST_USER_ID, - make_register_reply(1, session), - no_in_flight(), - ); - (table, session) + table.commit_register(1, TEST_USER_ID, make_register_reply(1, 10), no_in_flight()); + (table, 1) } // Registration tests #[test] - fn register_creates_session() { + fn register_mints_epoch_one() { let mut table = ClientTable::new(10); table.commit_register(1, TEST_USER_ID, make_register_reply(1, 42), no_in_flight()); - assert_eq!(table.get_session(1), Some(42)); + assert_eq!(table.get_epoch(1), Some(1)); + assert_eq!(table.get_watermark(1), Some(0)); assert_eq!(table.get_user_id(1), Some(TEST_USER_ID)); assert_eq!(table.count(), 1); } + // Re-register = rebind: epoch bumps, watermark (dedup history) survives. + #[test] + fn reregister_bumps_epoch_and_preserves_watermark() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 5, 15)); + assert_eq!(table.get_watermark(1), Some(5)); + + table.commit_register(1, TEST_USER_ID, make_register_reply(1, 20), no_in_flight()); + assert_eq!(table.get_epoch(1), Some(2), "rebind mints the next epoch"); + assert_eq!( + table.get_watermark(1), + Some(5), + "session resume keeps dedup history" + ); + assert_eq!(table.count(), 1); + + // The displaced app reply moved into the ring: the watermark request + // still answers with its original bytes under the new epoch. + match table.check_request(1, 2, 5, 0) { + RequestStatus::Duplicate(cached) => assert_eq!(cached.header().request, 5), + other => panic!("expected Duplicate from ring, got {other:?}"), + } + } + + // A rebind re-authenticates; the fresh register's user wins. + #[test] + fn reregister_refreshes_user_id() { + let mut table = ClientTable::new(10); + table.commit_register(1, 11, make_register_reply(1, 10), no_in_flight()); + table.commit_register(1, 22, make_register_reply(1, 20), no_in_flight()); + assert_eq!(table.get_user_id(1), Some(22)); + } + // Each entry keeps the user id it registered with; lookups are per-client. #[test] fn register_stores_user_id() { @@ -581,16 +710,15 @@ mod tests { #[test] fn check_register_already_registered() { - let (table, session) = table_with_client(); + let (table, epoch) = table_with_client(); match table.check_register(1) { RequestStatus::AlreadyRegistered { - session: s, + epoch: e, cached_reply, } => { - assert_eq!(s, session); + assert_eq!(e, epoch); // Cached reply IS the register reply, preflight replays it. assert_eq!(cached_reply.header().request, REGISTER_REQUEST_ID); - assert_eq!(cached_reply.header().commit, session); } other => panic!("expected AlreadyRegistered, got {other:?}"), } @@ -598,17 +726,17 @@ mod tests { #[test] fn check_register_already_registered_after_progress() { - let (mut table, session) = table_with_client(); + let (mut table, epoch) = table_with_client(); // Client progresses past registration. - table.commit_reply(1, 10, make_reply_for(1, 1, 11)); - table.commit_reply(1, 10, make_reply_for(1, 2, 12)); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); + table.commit_reply(1, epoch, make_reply_for(1, 2, 12)); // Cached reply is now latest app reply; preflight must silent-drop. match table.check_register(1) { RequestStatus::AlreadyRegistered { - session: s, + epoch: e, cached_reply, } => { - assert_eq!(s, session); + assert_eq!(e, epoch); assert_eq!( cached_reply.header().request, 2, @@ -619,50 +747,124 @@ mod tests { } } - // Session validation tests + // Epoch fence tests #[test] fn check_request_no_session() { let table = ClientTable::new(10); - // Not registered: valid session/request but no entry. + // Not registered: valid epoch/request but no entry. assert!(matches!( - table.check_request(1, 99, 1), + table.check_request(1, 99, 1, 0), RequestStatus::NoSession )); } + // Zombie fencing: requests stamped with a pre-rebind epoch are terminal. #[test] - fn check_request_session_mismatch() { - let (table, session) = table_with_client(); - match table.check_request(1, session + 1, 1) { - RequestStatus::SessionMismatch { expected, received } => { - assert_eq!(expected, session); - assert_eq!(received, session + 1); + fn check_request_stale_epoch_is_fenced() { + let (mut table, _) = table_with_client(); + table.commit_register(1, TEST_USER_ID, make_register_reply(1, 20), no_in_flight()); + assert_eq!(table.get_epoch(1), Some(2)); + match table.check_request(1, 1, 1, 0) { + RequestStatus::Fenced { current, received } => { + assert_eq!(current, 2); + assert_eq!(received, 1); } - other => panic!("expected SessionMismatch, got {other:?}"), + other => panic!("expected Fenced, got {other:?}"), } } + // Epochs are only handed out by register replies; a newer-than-minted + // epoch is a client bug, distinct from the zombie case. #[test] - fn check_request_correct_session_new() { - let (mut table, session) = table_with_client(); - table.commit_reply(1, 10, make_reply_for(1, 1, 11)); + fn check_request_future_epoch_is_client_bug() { + let (table, epoch) = table_with_client(); + match table.check_request(1, epoch + 1, 1, 0) { + RequestStatus::EpochAhead { current, received } => { + assert_eq!(current, epoch); + assert_eq!(received, epoch + 1); + } + other => panic!("expected EpochAhead, got {other:?}"), + } + } + + // Watermark tests + + #[test] + fn check_request_above_watermark_is_new() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); + assert!(matches!( + table.check_request(1, epoch, 2, 0), + RequestStatus::New + )); + } + + // No contiguity requirement: a jump past the watermark executes. The + // watermark records the highest committed request, not a sequence. + #[test] + fn check_request_jump_above_watermark_is_new() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); assert!(matches!( - table.check_request(1, session, 2), + table.check_request(1, epoch, 9, 0), RequestStatus::New )); + // And committing the jump moves the watermark to it. + table.commit_reply(1, epoch, make_reply_for(1, 9, 12)); + assert_eq!(table.get_watermark(1), Some(9)); } #[test] - fn check_request_duplicate_after_commit() { - let (mut table, session) = table_with_client(); - table.commit_reply(1, 10, make_reply_for(1, 1, 11)); - match table.check_request(1, session, 1) { + fn check_request_duplicate_at_watermark() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); + match table.check_request(1, epoch, 1, 0) { RequestStatus::Duplicate(cached) => assert_eq!(cached.header().request, 1), other => panic!("expected Duplicate, got {other:?}"), } } + // Below-watermark duplicate with the original still in the ring answers + // with the original bytes. + #[test] + fn check_request_below_watermark_hits_ring() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); + table.commit_reply(1, epoch, make_reply_for(1, 2, 12)); + match table.check_request(1, epoch, 1, 0) { + RequestStatus::Duplicate(cached) => { + assert_eq!(cached.header().request, 1, "original reply, not latest"); + assert_eq!(cached.header().commit, 11, "original commit op"); + } + other => panic!("expected Duplicate from ring, got {other:?}"), + } + } + + // Below-watermark duplicate whose reply aged out of the ring is refused + // execution with nothing to replay. + #[test] + fn check_request_below_watermark_past_ring_is_already_applied() { + let (mut table, epoch) = table_with_client(); + // Requests 1..=6: request 1's reply is displaced beyond the ring + // (capacity 4 holds 2,3,4,5 once 6 is latest). + for request in 1..=6u64 { + table.commit_reply(1, epoch, make_reply_for(1, request, 10 + request)); + } + match table.check_request(1, epoch, 1, 0) { + RequestStatus::AlreadyApplied { request, watermark } => { + assert_eq!(request, 1); + assert_eq!(watermark, 6); + } + other => panic!("expected AlreadyApplied, got {other:?}"), + } + // The oldest retained entry still answers. + match table.check_request(1, epoch, 2, 0) { + RequestStatus::Duplicate(cached) => assert_eq!(cached.header().request, 2), + other => panic!("expected Duplicate, got {other:?}"), + } + } + // Dedup across view change. Backup inherits client_table via // commit_journal; on failover, retry must return ORIGINAL cached reply // (same request, same commit op), no re-execution. Pipeline state is @@ -670,10 +872,10 @@ mod tests { // Simulator test covers end-to-end; this is the unit invariant. #[test] fn duplicate_survives_view_change_reset() { - let (mut table, session) = table_with_client(); - table.commit_reply(1, session, make_reply_for(1, 1, 11)); + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); - match table.check_request(1, session, 1) { + match table.check_request(1, epoch, 1, 0) { RequestStatus::Duplicate(cached) => { assert_eq!(cached.header().client, 1, "original client_id"); assert_eq!(cached.header().request, 1, "ORIGINAL request, not re-issue"); @@ -687,50 +889,78 @@ mod tests { } } + // Checksum tests + + // Same request id, different request bytes: returning the cached reply + // would answer the wrong request. Refused loudly. #[test] - fn check_request_stale() { - let (mut table, session) = table_with_client(); - table.commit_reply(1, 10, make_reply_for(1, 5, 15)); + fn check_request_checksum_mismatch_at_watermark() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_with_checksum(1, 1, 11, 0xAA)); + match table.check_request(1, epoch, 1, 0xBB) { + RequestStatus::ChecksumMismatch { request } => assert_eq!(request, 1), + other => panic!("expected ChecksumMismatch, got {other:?}"), + } + // Matching stamp replays. assert!(matches!( - table.check_request(1, session, 3), - RequestStatus::Stale + table.check_request(1, epoch, 1, 0xAA), + RequestStatus::Duplicate(_) )); } + // Integrity fields are zeroed on the wire today; a zero on either side + // must not trip the mismatch (rollout compatibility). #[test] - fn check_request_gap_rejected() { - let (mut table, session) = table_with_client(); - table.commit_reply(1, 10, make_reply_for(1, 1, 11)); - // Skip from 1 to 3, reject. - match table.check_request(1, session, 3) { - RequestStatus::RequestGap { expected, received } => { - assert_eq!(expected, 2); - assert_eq!(received, 3); - } - other => panic!("expected RequestGap, got {other:?}"), - } + fn check_request_zero_checksum_disables_comparison() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_with_checksum(1, 1, 11, 0xAA)); + assert!(matches!( + table.check_request(1, epoch, 1, 0), + RequestStatus::Duplicate(_) + )); + + table.commit_reply(1, epoch, make_reply_for(1, 2, 12)); // stored zero + assert!(matches!( + table.check_request(1, epoch, 2, 0xBB), + RequestStatus::Duplicate(_) + )); } // Commit tests #[test] fn commit_caches_reply() { - let (mut table, _) = table_with_client(); - table.commit_reply(1, 10, make_reply_for(1, 1, 11)); + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); let cached = table.get_reply(1).expect("should have cached reply"); assert_eq!(cached.header().request, 1); } #[test] - fn commit_updates_preserves_session() { - let (mut table, session) = table_with_client(); - table.commit_reply(1, 10, make_reply_for(1, 1, 11)); - table.commit_reply(1, 10, make_reply_for(1, 2, 12)); + fn commit_updates_preserves_epoch() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); + table.commit_reply(1, epoch, make_reply_for(1, 2, 12)); assert_eq!(table.get_reply(1).unwrap().header().request, 2); - assert_eq!(table.get_session(1), Some(session)); + assert_eq!(table.get_epoch(1), Some(epoch)); assert_eq!(table.count(), 1); } + // Same request re-committed (WAL replay shape): replace in place, no + // ring push - two cached replies for one request number would make + // duplicate lookups ambiguous. + #[test] + fn commit_reply_same_request_replaces_in_place() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); + table.commit_reply(1, epoch, make_reply_for(1, 1, 11)); + assert_eq!(table.get_watermark(1), Some(1)); + match table.check_request(1, epoch, 1, 0) { + RequestStatus::Duplicate(cached) => assert_eq!(cached.header().request, 1), + other => panic!("expected Duplicate, got {other:?}"), + } + } + // Eviction tests #[test] @@ -872,62 +1102,53 @@ mod tests { // Edge cases - #[test] - fn commit_register_idempotent_on_replay() { - let mut table = ClientTable::new(10); - table.commit_register(1, TEST_USER_ID, make_register_reply(1, 10), no_in_flight()); - // Same client_id + session = idempotent (WAL replay). - table.commit_register(1, TEST_USER_ID, make_register_reply(1, 10), no_in_flight()); - assert_eq!(table.get_session(1), Some(10)); - assert_eq!(table.count(), 1); - } - - // Re-register with mismatched session must not panic shard pump. - // Stale WAL replay or duplicate Register at different ops; either way - // log + skip, existing slot stays authoritative. - #[test] - fn commit_register_different_session_logs_and_skips() { - let mut table = ClientTable::new(10); - table.commit_register(1, TEST_USER_ID, make_register_reply(1, 10), no_in_flight()); - // existing=10, replay=20. - table.commit_register(1, TEST_USER_ID, make_register_reply(1, 20), no_in_flight()); - assert_eq!(table.get_session(1), Some(10), "first session stays"); - // Smaller replay session: same skip. - table.commit_register(1, TEST_USER_ID, make_register_reply(1, 5), no_in_flight()); - assert_eq!(table.get_session(1), Some(10)); - } - // commit_reply for unregistered/evicted client must not panic; // wire reply still ships, cache silently skipped. #[test] fn commit_reply_for_unregistered_client_is_noop() { let mut table = ClientTable::new(10); // No register: index has no entry. - table.commit_reply(1, 10, make_reply_for(1, 1, 10)); + table.commit_reply(1, 1, make_reply_for(1, 1, 10)); assert!(table.get_reply(1).is_none(), "no entry must be created"); assert_eq!(table.count(), 0); } #[test] - #[should_panic(expected = "session mismatch")] - fn commit_reply_wrong_session_panics() { - let (mut table, _session) = table_with_client(); - // Registered session=10, commit session=99. + #[should_panic(expected = "epoch mismatch")] + fn commit_reply_wrong_epoch_panics() { + let (mut table, _epoch) = table_with_client(); + // Entry epoch=1, commit claims epoch=99. table.commit_reply(1, 99, make_reply_for(1, 1, 11)); } #[test] - fn different_clients_independent_sessions() { + #[should_panic(expected = "watermark regression")] + fn commit_reply_watermark_regression_panics() { + let (mut table, epoch) = table_with_client(); + table.commit_reply(1, epoch, make_reply_for(1, 5, 15)); + table.commit_reply(1, epoch, make_reply_for(1, 3, 16)); + } + + #[test] + fn different_clients_independent_epochs() { let mut table = ClientTable::new(10); table.commit_register(1, TEST_USER_ID, make_register_reply(1, 10), no_in_flight()); table.commit_register(2, TEST_USER_ID, make_register_reply(2, 20), no_in_flight()); - assert_eq!(table.get_session(1), Some(10)); - assert_eq!(table.get_session(2), Some(20)); - assert!(matches!(table.check_request(1, 10, 1), RequestStatus::New)); - assert!(matches!(table.check_request(2, 20, 1), RequestStatus::New)); + // Rebind client 2 only. + table.commit_register(2, TEST_USER_ID, make_register_reply(2, 30), no_in_flight()); + assert_eq!(table.get_epoch(1), Some(1)); + assert_eq!(table.get_epoch(2), Some(2)); + assert!(matches!( + table.check_request(1, 1, 1, 0), + RequestStatus::New + )); + assert!(matches!( + table.check_request(2, 2, 1, 0), + RequestStatus::New + )); assert!(matches!( - table.check_request(1, 20, 1), - RequestStatus::SessionMismatch { .. } + table.check_request(2, 1, 1, 0), + RequestStatus::Fenced { .. } )); } } diff --git a/core/consensus/src/metadata_helpers.rs b/core/consensus/src/metadata_helpers.rs index 6191f08967..fdc3878bbb 100644 --- a/core/consensus/src/metadata_helpers.rs +++ b/core/consensus/src/metadata_helpers.rs @@ -56,17 +56,23 @@ pub enum PreflightOutcome { Drop, } -/// Request preflight (metadata only): session validation, dedup, in-flight check. +/// Request preflight (metadata only): epoch fence, watermark dedup, +/// in-flight check. /// /// Pure decision -- emits no frames (see [`PreflightOutcome`]). Callers turn /// the outcome into a reply: the home-shard path resends by transport id, the /// message-plane paths fall back to [`apply_preflight_consensus_plane`]. +/// +/// `session` is the wire `session` field, which carries the entry's fence +/// epoch; `request_checksum` is the request's integrity stamp (zero = +/// unstamped, disables the reuse check). pub fn request_preflight( consensus: &VsrConsensus, client_table: &RefCell, client_id: u128, session: u64, request: u64, + request_checksum: u128, ) -> PreflightOutcome where B: MessageBus, @@ -107,7 +113,7 @@ where let status = client_table .borrow() - .check_request(client_id, session, request); + .check_request(client_id, session, request, request_checksum); match status { // Frozen-backed cache -> refcount handoff to the home shard, no copy. RequestStatus::Duplicate(cached_reply) => { @@ -116,29 +122,45 @@ where // Session evicted under capacity pressure. SAFETY: catch-up gate makes // this replica authoritative for session truth. RequestStatus::NoSession => PreflightOutcome::Evict(EvictionReason::NoSession), - RequestStatus::SessionMismatch { expected, received } => { - // expected > received: stale session (rotated post-eviction) -> terminal eviction. - // expected < received: client bug; silent drop, log. - // SAFETY: catch-up gate makes this replica authoritative. - if expected > received { - PreflightOutcome::Evict(EvictionReason::SessionTooLow) - } else { - // Catch-up gate rules out network race; newer-than-issued - // session = client bug. Error log, no eviction (transient bug - // must not kill session), no rate limit (per-event). - tracing::error!( - client_id, - expected, - received, - "request_preflight: ignoring newer session (client bug)" - ); - PreflightOutcome::Drop - } + // Zombie holdover from before a re-register: terminal for that + // holder. SAFETY: catch-up gate makes this replica authoritative. + RequestStatus::Fenced { current, received } => { + tracing::debug!( + client_id, + current, + received, + "request_preflight: fencing stale-epoch request" + ); + PreflightOutcome::Evict(EvictionReason::SessionTooLow) + } + // Catch-up gate rules out network race; an epoch newer than any this + // table minted = client bug. Error log, no eviction (transient bug + // must not kill the session), no rate limit (per-event). + RequestStatus::EpochAhead { current, received } => { + tracing::error!( + client_id, + current, + received, + "request_preflight: ignoring future epoch (client bug)" + ); + PreflightOutcome::Drop + } + // Same request id, different request bytes: replaying the cached + // reply would answer the wrong request, and re-executing would + // double-apply. Loud drop; the client must fix its numbering. + RequestStatus::ChecksumMismatch { request } => { + tracing::error!( + client_id, + request, + "request_preflight: request id reused for a different operation (client bug)" + ); + PreflightOutcome::Drop + } + // Applied once, original reply aged out of the ring: refuse + // re-execution, nothing to replay. Silent drop. + RequestStatus::AlreadyApplied { .. } | RequestStatus::AlreadyRegistered { .. } => { + PreflightOutcome::Drop } - // Client bug; recovered by client retry. Silent drop. - RequestStatus::Stale - | RequestStatus::RequestGap { .. } - | RequestStatus::AlreadyRegistered { .. } => PreflightOutcome::Drop, RequestStatus::New => PreflightOutcome::Dispatch, } } @@ -213,8 +235,9 @@ where // Catch-up gate: new primary may have inherited Register(client, op=N) // committed in WAL but not yet applied. Without gate, check_register - // returns New -> fresh register -> two register entries -> commit_register's - // session-equality assert panics on replay. SDK retry recovers post-catch-up. + // returns New -> a second register commits -> the epoch bumps past the + // one the first register's reply handed the client, fencing a live + // client for no reason. SDK retry recovers post-catch-up. if !is_caught_up_primary(consensus) { tracing::debug!( client_id, @@ -231,7 +254,7 @@ where let status = client_table.borrow().check_register(client_id); match status { RequestStatus::AlreadyRegistered { - session, + epoch, cached_reply, } => { // cached.request == REGISTER_REQUEST_ID: replay cached bytes @@ -247,13 +270,13 @@ where .await; tracing::debug!( client_id, - session, + epoch, "register_preflight: replayed cached register reply" ); } else { tracing::debug!( client_id, - session, + epoch, cached_request = cached_reply.header().request, "register_preflight: retry past register, drop" ); @@ -368,8 +391,9 @@ fn build_eviction_from_header(header: EvictionHeader) -> Message /// `NoSession`/`SessionTooLow` against stale table erases live clients. /// - **Dispatch**: primary with `commit_min < commit_max` may hold an /// inherited `Register(client, op=N)` in WAL but not yet applied. -/// Fresh `Register(client, op=M>N)` panics `commit_register`'s -/// session-equality assert on replay. +/// Admitting a fresh Register commits a second register, bumping the +/// epoch past the one the inherited register's reply handed the client +/// and fencing a live client for no reason. /// /// `false` -> caller silent-drops; client retry lands on peer or here /// post-catch-up. @@ -479,10 +503,10 @@ mod tests { let client_table = fresh_client_table(); let client_id: u128 = 0xBEEF; - let session: u64 = 17; + let register_commit: u64 = 17; // Cached reply IS the register reply (request == REGISTER_REQUEST_ID). - let initial_reply = synthesize_register_reply(&consensus, client_id, session); + let initial_reply = synthesize_register_reply(&consensus, client_id, register_commit); let original_checksum = initial_reply.header().checksum; client_table .borrow_mut() @@ -505,7 +529,7 @@ mod tests { "must be original cached bytes, not fresh synthesis" ); assert_eq!(header.request, REGISTER_REQUEST_ID); - assert_eq!(header.commit, session); + assert_eq!(header.commit, register_commit); } // Past-register retry: silent drop, no replay, no eviction. Read-timeout @@ -517,18 +541,18 @@ mod tests { let client_table = fresh_client_table(); let client_id: u128 = 0xBEEF; - let session: u64 = 17; + let epoch: u64 = 1; - let initial_reply = synthesize_register_reply(&consensus, client_id, session); + let initial_reply = synthesize_register_reply(&consensus, client_id, 17); client_table .borrow_mut() .commit_register(client_id, ACTING_USER_ID, initial_reply, |_| false); // SendMessages commits -> cached is no longer the register reply. - let app_reply = synthesize_send_messages_reply(&consensus, client_id, session, 1, 18); + let app_reply = synthesize_send_messages_reply(&consensus, client_id, 1, 18); client_table .borrow_mut() - .commit_reply(client_id, session, app_reply); + .commit_reply(client_id, epoch, app_reply); let result = futures::executor::block_on(register_preflight(&consensus, &client_table, client_id)); @@ -556,8 +580,9 @@ mod tests { &consensus, &client_table, client_id, - 10, // session + 10, // epoch (wire session field) 1, // request + 0, // request_checksum (unstamped) ), client_id, )); @@ -576,29 +601,33 @@ mod tests { assert_eq!(header.client, client_id); } - // Stale session (post capacity-evict + re-register): terminal SessionTooLow. + // Zombie fencing: a request stamped with a pre-rebind epoch gets a + // terminal SessionTooLow eviction. #[test] - fn request_preflight_session_too_low_evicts_client() { + fn request_preflight_stale_epoch_evicts_client() { let consensus = VsrConsensus::new(1, 0, 3, 0, ClientSpyBus::new(), LocalPipeline::new()); consensus.init(); let client_table = fresh_client_table(); let client_id: u128 = 0xBEEF; - let real_session: u64 = 99; - // Slot at session 99. - let initial_reply = synthesize_register_reply(&consensus, client_id, real_session); + // Register, then rebind: entry epoch is now 2. + let initial_reply = synthesize_register_reply(&consensus, client_id, 17); client_table .borrow_mut() .commit_register(client_id, ACTING_USER_ID, initial_reply, |_| false); + let rebind_reply = synthesize_register_reply(&consensus, client_id, 25); + client_table + .borrow_mut() + .commit_register(client_id, ACTING_USER_ID, rebind_reply, |_| false); - // Older retry (17 < 99): stale-session case. + // Zombie still stamping epoch 1: fenced. let result = futures::executor::block_on(apply_preflight_consensus_plane( &consensus, - request_preflight(&consensus, &client_table, client_id, 17, 1), + request_preflight(&consensus, &client_table, client_id, 1, 1, 0), client_id, )); - assert!(!result, "SessionMismatch short-circuits"); + assert!(!result, "Fenced short-circuits"); let sends = consensus.message_bus().client_sends.borrow(); assert_eq!(sends.len(), 1, "one Eviction"); @@ -611,36 +640,33 @@ mod tests { assert_eq!(header.client, client_id); } - // Newer-than-cluster session: sessions monotonic; healthy SDK can't reach - // this. Client bug -> silent drop, no eviction. + // Newer-than-minted epoch: epochs are only handed out by register + // replies; healthy SDK can't reach this. Client bug -> silent drop, no + // eviction. #[test] - fn request_preflight_session_too_high_is_silent_drop() { + fn request_preflight_future_epoch_is_silent_drop() { let consensus = VsrConsensus::new(1, 0, 3, 0, ClientSpyBus::new(), LocalPipeline::new()); consensus.init(); let client_table = fresh_client_table(); let client_id: u128 = 0xBEEF; - let real_session: u64 = 17; - // Slot at session 17. - let initial_reply = synthesize_register_reply(&consensus, client_id, real_session); + // Entry at epoch 1. + let initial_reply = synthesize_register_reply(&consensus, client_id, 17); client_table .borrow_mut() .commit_register(client_id, ACTING_USER_ID, initial_reply, |_| false); - // Client claims newer session (99 > 17), client bug. + // Client claims epoch 99 (> 1), client bug. let result = futures::executor::block_on(apply_preflight_consensus_plane( &consensus, - request_preflight(&consensus, &client_table, client_id, 99, 1), + request_preflight(&consensus, &client_table, client_id, 99, 1, 0), client_id, )); - assert!(!result, "SessionMismatch short-circuits"); + assert!(!result, "EpochAhead short-circuits"); let sends = consensus.message_bus().client_sends.borrow(); - assert!( - sends.is_empty(), - "newer-session mismatch must be silent drop" - ); + assert!(sends.is_empty(), "future epoch must be silent drop"); } // Backups never send NoSession: their ClientTable lags. Without gate, @@ -661,8 +687,9 @@ mod tests { &consensus, &client_table, client_id, - 10, // session + 10, // epoch (wire session field) 1, // request + 0, // request_checksum (unstamped) ), client_id, )); @@ -675,41 +702,75 @@ mod tests { ); } - // Stale + RequestGap: silent drop, no eviction. + // Below-watermark retry whose reply is still cached: replayed, not + // re-executed and not dropped. #[test] - fn request_preflight_stale_is_silent_drop() { + fn request_preflight_below_watermark_replays_ring_hit() { let consensus = VsrConsensus::new(1, 0, 3, 0, ClientSpyBus::new(), LocalPipeline::new()); consensus.init(); let client_table = fresh_client_table(); let client_id: u128 = 0xABCD; - let session: u64 = 5; + let epoch: u64 = 1; - let initial_reply = synthesize_register_reply(&consensus, client_id, session); + let initial_reply = synthesize_register_reply(&consensus, client_id, 5); client_table .borrow_mut() .commit_register(client_id, ACTING_USER_ID, initial_reply, |_| false); - // Cache at request 5 -> request 3 is stale. - let advanced = synthesize_send_messages_reply(&consensus, client_id, session, 5, 100); - client_table - .borrow_mut() - .commit_reply(client_id, session, advanced); + for (request, commit) in [(3u64, 98u64), (5, 100)] { + let reply = synthesize_send_messages_reply(&consensus, client_id, request, commit); + client_table + .borrow_mut() + .commit_reply(client_id, epoch, reply); + } let result = futures::executor::block_on(apply_preflight_consensus_plane( &consensus, - request_preflight(&consensus, &client_table, client_id, session, 3), // stale + request_preflight(&consensus, &client_table, client_id, epoch, 3, 0), client_id, )); - assert!(!result); + assert!(!result, "duplicate short-circuits"); let sends = consensus.message_bus().client_sends.borrow(); - assert!(sends.is_empty(), "stale = silent drop"); + assert_eq!(sends.len(), 1, "ring hit replays the original reply"); + let header = bytemuck::checked::try_from_bytes::( + &sends[0].1.as_slice()[..HEADER_SIZE], + ) + .expect("valid ReplyHeader"); + assert_eq!(header.request, 3, "original reply for the retried request"); + } + + // Watermark jump: request numbers above the watermark dispatch even + // when non-contiguous (there is no RequestGap). + #[test] + fn request_preflight_jump_above_watermark_dispatches() { + let consensus = VsrConsensus::new(1, 0, 3, 0, ClientSpyBus::new(), LocalPipeline::new()); + consensus.init(); + let client_table = fresh_client_table(); + + let client_id: u128 = 0xABCD; + let epoch: u64 = 1; + + let initial_reply = synthesize_register_reply(&consensus, client_id, 5); + client_table + .borrow_mut() + .commit_register(client_id, ACTING_USER_ID, initial_reply, |_| false); + let advanced = synthesize_send_messages_reply(&consensus, client_id, 2, 99); + client_table + .borrow_mut() + .commit_reply(client_id, epoch, advanced); + + let outcome = request_preflight(&consensus, &client_table, client_id, epoch, 9, 0); + assert!( + matches!(outcome, PreflightOutcome::Dispatch), + "watermark jump must dispatch" + ); } - // Catch-up gate prevents WAL-replay race in commit_register: primary - // with commit_min < commit_max may hold inherited Register(client) in - // WAL not yet applied. Fresh Register would panic session-equality - // assert on replay. + // Catch-up gate prevents the WAL-replay race in commit_register: primary + // with commit_min < commit_max may hold an inherited Register(client) in + // WAL not yet applied. A fresh Register would commit a second register + // and bump the epoch past the inherited reply's, fencing a live client. #[test] fn register_preflight_silently_drops_when_behind_on_commits() { let consensus = VsrConsensus::new(1, 0, 3, 0, ClientSpyBus::new(), LocalPipeline::new()); @@ -775,11 +836,13 @@ mod tests { // Fixture: register reply mirroring `commit_register` storage. Test-only // production replays cached via `AlreadyRegistered { cached_reply }`. + // `register_commit` stamps the reply's op/commit (recency for eviction + // ordering); the entry's epoch is minted by the table, not read from it. #[allow(clippy::cast_possible_truncation)] fn synthesize_register_reply( consensus: &VsrConsensus, client_id: u128, - session: u64, + register_commit: u64, ) -> Message where B: MessageBus, @@ -798,8 +861,8 @@ mod tests { command: Command2::Reply, replica: consensus.replica(), client: client_id, - op: session, - commit: session, + op: register_commit, + commit: register_commit, request: REGISTER_REQUEST_ID, operation: Operation::Register, ..ReplyHeader::default() @@ -807,12 +870,11 @@ mod tests { msg } - // SendMessages reply fixture: advances cached request number. + // SendMessages reply fixture: advances the cached watermark. #[allow(clippy::cast_possible_truncation)] fn synthesize_send_messages_reply( consensus: &VsrConsensus, client_id: u128, - session: u64, request: u64, commit: u64, ) -> Message @@ -826,7 +888,6 @@ mod tests { &mut msg.as_mut_slice()[..header_size], ) .expect("zeroed bytes are valid"); - let _ = session; *header = ReplyHeader { cluster: consensus.cluster(), size: header_size as u32, diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 7ba69922ba..e6caae4509 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -396,7 +396,8 @@ pub enum MetadataSubmitError { NotPrimary, /// Primary but `commit_min < commit_max` (committed prefix not yet /// drained). Dispatching now would race ops inherited from a prior view; - /// for `Register` that trips `commit_register`'s session-eq assert. + /// for `Register` that double-commits a register and bumps the epoch + /// past the first reply's, fencing a live client. NotCaughtUp, /// Prepare queue full. PipelineFull, @@ -615,6 +616,7 @@ where let client_id = message.header().client; let session = message.header().session; let request = message.header().request; + let request_checksum = message.header().request_checksum; let operation = message.header().operation; // Preflight first: dedup, eviction sends, cached-reply replay all @@ -624,8 +626,14 @@ where let dispatch = if operation == Operation::Register { register_preflight(consensus, &self.client_table, client_id).await } else { - let outcome = - request_preflight(consensus, &self.client_table, client_id, session, request); + let outcome = request_preflight( + consensus, + &self.client_table, + client_id, + session, + request, + request_checksum, + ); apply_preflight_consensus_plane(consensus, outcome, client_id).await }; if !dispatch { @@ -995,9 +1003,12 @@ where .as_ref() .expect("submit_register_in_process: consensus only exists on shard 0"); - // Idempotent fast path: existing session skips pipeline + wire-reply. - if let Some(session) = self.client_table.borrow().get_session(client_id) { - return Ok(session); + // Idempotent fast path: existing entry skips pipeline + wire-reply. + // Returns the current epoch without bumping it; rebind-bumps-epoch + // (zombie fencing per reconnect) arrives with the stable-session-id + // auth flow, which commits a Register per bind. + if let Some(epoch) = self.client_table.borrow().get_epoch(client_id) { + return Ok(epoch); } // Wrong node: waiting or queueing cannot fix that, the client must @@ -1006,10 +1017,10 @@ where return Err(MetadataSubmitError::NotPrimary); } - // Mirror wire-path register_preflight: a racing second prepare fails - // check_register on commit. Surface pre-synthesis. Scans both the - // prepare queue and the request queue, so a register absorbed below - // dedups its own replays. + // Mirror wire-path register_preflight: a racing second prepare would + // commit a second register and bump the epoch past the first reply's. + // Surface pre-synthesis. Scans both the prepare queue and the request + // queue, so a register absorbed below dedups its own replays. if consensus .pipeline() .borrow() @@ -1030,7 +1041,7 @@ where ); // Catch-up gate (Register only: admitting one while a committed op - // is still unapplied races `commit_register`'s session-eq assert) or + // is still unapplied risks a double-register epoch bump) or // prepare queue full: absorb into the request queue instead of // bouncing with a transient error. The queued // entry carries this caller's reply subscriber; the commit path @@ -1049,14 +1060,20 @@ where return Err(MetadataSubmitError::PipelineFull); } return match receiver.await { - Ok(reply) => Ok(reply.header().commit), + // The commit's `commit_register` minted the epoch; read it + // from the table (the reply header does not carry it). + Ok(_reply) => self + .client_table + .borrow() + .get_epoch(client_id) + .ok_or(MetadataSubmitError::Canceled), // Entry dropped before commit: view-change reset or a // promotion-time preflight rejection. Same re-check as the // direct path's cancel arm below. Err(Canceled) => self .client_table .borrow() - .get_session(client_id) + .get_epoch(client_id) .ok_or(MetadataSubmitError::Canceled), }; } @@ -1068,18 +1085,24 @@ where .expect("Operation::Register is client-allowed; prepare projection cannot fail"); match self.dispatch_prepare_and_await(consensus, prepare).await { - Ok(reply) => Ok(reply.header().commit), + // The commit's `commit_register` minted the epoch; read it from + // the table (the reply header does not carry it). + Ok(_reply) => self + .client_table + .borrow() + .get_epoch(client_id) + .ok_or(MetadataSubmitError::Canceled), Err(Canceled) => { // View-change cancel. Re-check is correct-by-VSR: any // inherited Register applied via local commit_journal between - // cancel and read produces a cluster-authoritative session - // (`session = commit-op`, deterministic). Own surviving - // Register would have routed through `AlreadyRegistered` - // against the same entry, so no "this primary vs inherited - // primary" split. + // cancel and read mints the same epoch on every replica + // (`commit_register` counts in apply order, deterministic). + // Own surviving Register would have routed through + // `AlreadyRegistered` against the same entry, so no "this + // primary vs inherited primary" split. self.client_table .borrow() - .get_session(client_id) + .get_epoch(client_id) .ok_or(MetadataSubmitError::Canceled) } } @@ -1113,12 +1136,12 @@ where .as_ref() .expect("submit_logout_in_process: consensus only exists on shard 0"); - // Session guard: only propose a Logout when the slot still holds the - // exact session this logout targets. A late disconnect-logout for a - // reused client id (slot since rebound to a newer session) carries the - // stale session and is dropped here, so it can never wipe the fresh + // Epoch guard: only propose a Logout when the slot still holds the + // exact epoch this logout targets. A late disconnect-logout for a + // reused client id (slot since rebound to a newer epoch) carries the + // stale epoch and is dropped here, so it can never wipe the fresh // registration. A missing slot also fails the match and short-circuits. - if self.client_table.borrow().get_session(client_id) != Some(session) { + if self.client_table.borrow().get_epoch(client_id) != Some(session) { return Ok(consensus.commit_min()); } @@ -1165,7 +1188,7 @@ where return match receiver.await { Ok(reply) => Ok(reply.header().commit), Err(Canceled) => { - if self.client_table.borrow().get_session(client_id).is_none() { + if self.client_table.borrow().get_epoch(client_id).is_none() { Ok(consensus.commit_min()) } else { Err(MetadataSubmitError::Canceled) @@ -1180,7 +1203,7 @@ where match self.dispatch_prepare_and_await(consensus, prepare).await { Ok(reply) => Ok(reply.header().commit), Err(Canceled) => { - if self.client_table.borrow().get_session(client_id).is_none() { + if self.client_table.borrow().get_epoch(client_id).is_none() { Ok(consensus.commit_min()) } else { Err(MetadataSubmitError::Canceled) @@ -1295,7 +1318,7 @@ where /// /// No client session exists, so this skips `request_preflight` (like /// the logout precedent) and uses the reserved internal `client` id - /// `0`: never registered, so the commit path's `get_session(0)` is + /// `0`: never registered, so the commit path's `get_epoch(0)` is /// `None` and skips `commit_reply` (and its `assert!(client_id != 0)`), /// while the preflight and register asserts never run. Delete is /// idempotent, so the dropped dedup is harmless and a re-proposal on the @@ -1400,6 +1423,7 @@ where let client_id = request_header.client; let session = request_header.session; let request = request_header.request; + let request_checksum = request_header.request_checksum; let consensus = self .consensus @@ -1429,13 +1453,21 @@ where .into_generic()); } - // Dedup / session / eviction. shard 0 cannot route by the VSR + // Dedup / epoch fence / eviction. shard 0 cannot route by the VSR // consensus `client_id` (its top bits are random, not home-shard // routing), so a Replay/Evict/NotReady is returned to the home shard as // the reply -- `handle_client_request` writes it to the originating // socket by transport id, exactly like a fresh commit. Drop (client-bug - // stale/gap) surfaces as Canceled so the home shard stays silent. - match request_preflight(consensus, &self.client_table, client_id, session, request) { + // already-applied / future-epoch) surfaces as Canceled so the home + // shard stays silent. + match request_preflight( + consensus, + &self.client_table, + client_id, + session, + request, + request_checksum, + ) { PreflightOutcome::Dispatch => {} PreflightOutcome::Replay(reply) => { return server_common::Message::::try_from( @@ -1545,7 +1577,7 @@ where consensus.verify_pipeline(); let receiver = consensus.pipeline_message_with_subscriber(PlaneKind::Metadata, &prepare); // Register is the one op whose admission requires the catch-up gate - // (session-eq assert at commit); its submit path checks the gate and + // (double-register epoch bump); its submit path checks the gate and // the check-to-dispatch section is synchronous. Non-register ops // dispatch mid-window by design (they pipeline behind the in-flight // batch, like the wire path always has). @@ -1815,14 +1847,11 @@ where // Cache only if session exists. Client evicted between // prepare and commit: skip cache (`commit_reply` no-ops), // wire reply still ships. - let session = self - .client_table - .borrow() - .get_session(prepare_header.client); - if let Some(session) = session { + let epoch = self.client_table.borrow().get_epoch(prepare_header.client); + if let Some(epoch) = epoch { self.client_table.borrow_mut().commit_reply( prepare_header.client, - session, + epoch, reply.clone(), ); } else { @@ -1931,8 +1960,9 @@ where /// /// # Safety /// Re-preflight per iteration: `commit_journal` may have advanced the - /// client's request between push and drain (Stale / Duplicate / - /// `AlreadyRegistered`). Skipping produces a duplicate prepare and panics. + /// client's watermark between push and drain (Duplicate / AlreadyApplied + /// / `AlreadyRegistered`). Skipping produces a duplicate prepare and + /// panics. #[allow(clippy::future_not_send)] async fn drain_request_queue_into_prepares(&self) { let consensus = self.consensus.as_ref().unwrap(); @@ -1952,6 +1982,7 @@ where let client_id = req.message.header().client; let session = req.message.header().session; let request = req.message.header().request; + let request_checksum = req.message.header().request_checksum; let operation = req.message.header().operation; // If preflight or projection rejects below, dropping `req` (and // the sender taken from it) wakes an in-process awaiter with @@ -1960,8 +1991,14 @@ where let dispatch = if operation == Operation::Register { register_preflight(consensus, &self.client_table, client_id).await } else { - let outcome = - request_preflight(consensus, &self.client_table, client_id, session, request); + let outcome = request_preflight( + consensus, + &self.client_table, + client_id, + session, + request, + request_checksum, + ); apply_preflight_consensus_plane(consensus, outcome, client_id).await }; if !dispatch { @@ -2260,8 +2297,8 @@ where /// between. [`crate::metadata_helpers::is_caught_up_primary`] reads /// `commit_min == commit_max` as proof the table is caught up; an await /// here lets another task observe transient equality with stale table, - /// dispatch a fresh Register on an already-registered client, and panic - /// `commit_register`'s session-eq assert. + /// dispatch a fresh Register on an already-registered client, and bump + /// the epoch past the reply the live client holds. /// /// Inner block sync today. Future async state-machine must either: /// 1. Apply SM + bump `commit_min` in one `RefCell` borrow, or @@ -2331,11 +2368,11 @@ where }); // Cache only if session still exists. WAL replay may carry a // reply for a later-evicted client; `commit_reply` no-ops. - let session = self.client_table.borrow().get_session(header.client); - if let Some(session) = session { + let epoch = self.client_table.borrow().get_epoch(header.client); + if let Some(epoch) = epoch { self.client_table .borrow_mut() - .commit_reply(header.client, session, reply); + .commit_reply(header.client, epoch, reply); } else { tracing::trace!( client = header.client, @@ -3507,20 +3544,20 @@ mod tests { "resumed driver commits nothing new" ); assert_eq!( - md.client_table.borrow().get_session(CLIENT_A), + md.client_table.borrow().get_epoch(CLIENT_A), None, "session removed by the committed logout" ); } /// Register is the one op that still honors the catch-up gate (its - /// admission races `commit_register`'s session-eq assert against - /// committed-but-unapplied ops). New contract: a register arriving in - /// the mid-commit window is ABSORBED into the pipeline's request queue - /// with its reply subscriber attached, promoted by - /// the commit path once the batch drains, and the caller's await - /// resolves with the committed session โ€” instead of the historical - /// `NotCaughtUp` bounce that one-shot CLI clients surfaced as + /// admission races a committed-but-unapplied register; a double commit + /// bumps the epoch past the first reply's and fences a live client). + /// New contract: a register arriving in the mid-commit window is + /// ABSORBED into the pipeline's request queue with its reply subscriber + /// attached, promoted by the commit path once the batch drains, and the + /// caller's await resolves with the committed epoch โ€” instead of the + /// historical `NotCaughtUp` bounce that one-shot CLI clients surfaced as /// "Disconnected" login failures. #[compio::test] async fn register_in_mid_commit_window_is_queued_then_committed() { @@ -3634,13 +3671,13 @@ mod tests { } assert_eq!( outcome.expect("absorbed register must resolve"), - Ok(2), - "queued register commits with the next batch; session = commit op" + Ok(1), + "queued register commits with the next batch; first bind mints epoch 1" ); assert_eq!( - md.client_table.borrow().get_session(CLIENT_C), - Some(2), - "session created by the promoted register" + md.client_table.borrow().get_epoch(CLIENT_C), + Some(1), + "entry created by the promoted register" ); } @@ -3765,8 +3802,8 @@ mod tests { } compio::time::sleep(std::time::Duration::from_millis(1)).await; } - assert_eq!(outcome.expect("promoted register must resolve"), Ok(2)); - assert_eq!(md.client_table.borrow().get_session(CLIENT_C), Some(2)); + assert_eq!(outcome.expect("promoted register must resolve"), Ok(1)); + assert_eq!(md.client_table.borrow().get_epoch(CLIENT_C), Some(1)); assert!(is_caught_up_primary(consensus)); } } From d2461e340821784b7503cd740c177c99c75e4e9f Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Mon, 27 Jul 2026 10:49:02 +0200 Subject: [PATCH 02/14] feat(consensus): rework client table for session resume across restarts --- .../src/consensus/operation.rs | 3 +- core/consensus/src/client_table.rs | 8 +- core/consensus/src/metadata_helpers.rs | 7 +- .../tests/cluster/client_table_restart.rs | 61 +++--- core/metadata/src/impls/metadata.rs | 14 +- core/metadata/src/impls/recovery.rs | 178 +++++++++++++++++- core/metadata/src/stm/user.rs | 2 +- core/server-ng/src/bootstrap.rs | 17 +- core/server-ng/src/dispatch.rs | 178 +++++++++++++++++- core/server-ng/src/users.rs | 10 +- core/shard/src/lib.rs | 9 + 11 files changed, 422 insertions(+), 65 deletions(-) diff --git a/core/binary_protocol/src/consensus/operation.rs b/core/binary_protocol/src/consensus/operation.rs index 9546231322..11d15a1064 100644 --- a/core/binary_protocol/src/consensus/operation.rs +++ b/core/binary_protocol/src/consensus/operation.rs @@ -30,7 +30,8 @@ pub enum Operation { /// Register a client session with the cluster. Goes through the same /// consensus pipeline (prepare/replicate/commit) as normal operations /// but skips state machine dispatch at commit time, the metadata - /// plane calls `commit_register` directly. Session number = commit op. + /// plane calls `commit_register` directly, which mints the session's + /// fence epoch (1 at first register, +1 per rebind). Register = 1, /// Non-replicated client request carried in VSR framing. The concrete diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 09a87292e9..2a82fc14ff 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -258,13 +258,13 @@ impl ClientTable { return RequestStatus::ChecksumMismatch { request }; } - match entry.find_cached(request) { - Some(cached) => RequestStatus::Duplicate(cached.clone()), - None => RequestStatus::AlreadyApplied { + entry.find_cached(request).map_or( + RequestStatus::AlreadyApplied { request, watermark: entry.watermark, }, - } + |cached| RequestStatus::Duplicate(cached.clone()), + ) } /// Check register. Valid without existing entry; returns diff --git a/core/consensus/src/metadata_helpers.rs b/core/consensus/src/metadata_helpers.rs index fdc3878bbb..1ffbe86912 100644 --- a/core/consensus/src/metadata_helpers.rs +++ b/core/consensus/src/metadata_helpers.rs @@ -733,10 +733,9 @@ mod tests { let sends = consensus.message_bus().client_sends.borrow(); assert_eq!(sends.len(), 1, "ring hit replays the original reply"); - let header = bytemuck::checked::try_from_bytes::( - &sends[0].1.as_slice()[..HEADER_SIZE], - ) - .expect("valid ReplyHeader"); + let header = + bytemuck::checked::try_from_bytes::(&sends[0].1.as_slice()[..HEADER_SIZE]) + .expect("valid ReplyHeader"); assert_eq!(header.request, 3, "original reply for the retried request"); } diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs index 3eb6195ce3..2e57893352 100644 --- a/core/integration/tests/cluster/client_table_restart.rs +++ b/core/integration/tests/cluster/client_table_restart.rs @@ -21,41 +21,40 @@ //! node crash must be able to continue: a retry of an already-committed //! request id must be answered from the dedup cache (never re-applied, //! never silently dropped), and the next request id must be admitted. -//! Today the table lives only in memory, so a rebooted node has no record -//! of the session or its request watermark and both scenarios fail. //! -//! The Rust SDK cannot drive this: it resets its `ConsensusSession` on every -//! disconnect and re-registers under a fresh identity. The frames are -//! therefore hand-crafted on a raw TCP socket, same technique as the -//! protocol-version gate tests. +//! Server-side this rests on three landed pieces: //! -//! What has to land for these tests to go green, in order: +//! 1. WAL-replay table recovery: `metadata::impls::recovery::recover` +//! replays registers (minting the same epochs) and re-caches committed +//! replies byte-identically, so a rebooted node remembers where each +//! client left off. Sessions whose register fell below the snapshot +//! floor are not recovered yet (no checkpoint artifact - IGGY-137 +//! remainder). +//! 2. Implicit rebind: a replicated request on an unbound transport whose +//! `(client, session)` matches a live table entry rebinds the transport +//! (`try_resume_session`); the same session arriving on two connections +//! evicts the older binding (`SessionManager::bind_session`). +//! 3. Sessions survive transport disconnect: `submit_disconnect_logout` +//! tears down only consumer-group members (the group must rebalance off +//! a dead consumer); everything else keeps its slot for resume until an +//! explicit `Logout` or capacity eviction. //! -//! 1. Persist the clients table (IGGY-137, standalone): include the -//! (client id, last request id, cached reply) entries in the checkpoint -//! and recover them on boot from WAL replay, so a rebooted node -//! remembers where each client left off. -//! 2. The client stops forgetting itself on disconnect: keep client id, -//! session id and request counter across reconnects and present the old -//! identity instead of a fresh Register. -//! 3. The server accepts a resumed identity: look the session up in the -//! replicated table, rebind the new transport to it, and answer with -//! the last committed request id so the client knows whether its -//! in-doubt request went through. Define the conflict rule for the same -//! session arriving on two connections (evict the older). -//! 4. SDK retry rule change: a replicated write may only be retried under -//! the same (client id, request id); the path that re-issues an -//! in-doubt write under a fresh session after failover goes away. +//! The Rust SDK cannot drive this yet: it resets its `ConsensusSession` on +//! every disconnect and re-registers under a fresh identity (the +//! `sdk/vsr.rs` retry TODO). The frames are therefore hand-crafted on a raw +//! TCP socket, same technique as the protocol-version gate tests. SDK-side +//! identity stability (keep client id + request counter across reconnects, +//! retry replicated writes only under the same identity) is the remaining +//! client half. //! -//! Steps 2+3 must ship together; 1 is standalone. An alternative to 2-4 is -//! a per-request idempotency key that is independent of the session, the -//! way TigerBeetle does it: no session resume at all, retries from a fresh -//! client session stay safe because dedup keys off the request, not the -//! (client, session) pair. +//! Single-node topology on purpose: the raw client pins one address, and a +//! follower cannot commit replicated TCP writes (no follower forwarding for +//! TCP yet), so post-failover resume against a 3-node cluster is future +//! work alongside that forwarding. //! //! These tests pin the implicit-rebind contract: a resumed client simply //! keeps sending under its old `(client, session)` on a fresh connection and -//! the server rebinds the transport from the persisted table. If the +//! the server rebinds the transport from the recovered table. If the //! session-resume work settles on an explicit resume handshake instead, //! adjust `resume_request` to speak it. @@ -102,8 +101,7 @@ const REPLY_WAIT: Duration = Duration::from_secs(5); const RETRY_PAUSE: Duration = Duration::from_millis(100); -#[iggy_harness] -#[ignore = "red until clients-table persistence + session resume land"] +#[iggy_harness(cluster_nodes = 1)] async fn given_committed_request_when_node_restarts_should_dedup_same_id_retry( harness: &mut TestHarness, ) { @@ -122,8 +120,7 @@ async fn given_committed_request_when_node_restarts_should_dedup_same_id_retry( resume_request(addr, session, 1, &create_stream).await; } -#[iggy_harness] -#[ignore = "red until clients-table persistence + session resume land"] +#[iggy_harness(cluster_nodes = 1)] async fn given_bound_session_when_node_restarts_should_accept_next_request_id( harness: &mut TestHarness, ) { diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index e6caae4509..92ea95c5ed 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -547,6 +547,14 @@ impl IggyMetadata { *self.commit_notifier.borrow_mut() = notifier; } + /// Install the client table rebuilt by WAL-replay recovery + /// ([`crate::impls::recovery::recover`]). Boot-time only, on the owning + /// shard, before it serves traffic - replacing a live table would drop + /// committed session state. + pub fn install_client_table(&self, client_table: ClientTable) { + *self.client_table.borrow_mut() = client_table; + } + /// Install the resolved byte value used for `MaxTopicSize::ServerDefault`. /// Server-ng bootstrap calls this with `system.topic.max_size` on every /// shard (responses read it too); only shard 0's copy feeds admission. @@ -1960,9 +1968,9 @@ where /// /// # Safety /// Re-preflight per iteration: `commit_journal` may have advanced the - /// client's watermark between push and drain (Duplicate / AlreadyApplied - /// / `AlreadyRegistered`). Skipping produces a duplicate prepare and - /// panics. + /// client's watermark between push and drain (`Duplicate` / + /// `AlreadyApplied` / `AlreadyRegistered`). Skipping produces a duplicate + /// prepare and panics. #[allow(clippy::future_not_send)] async fn drain_request_queue_into_prepares(&self) { let consensus = self.consensus.as_ref().unwrap(); diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs index 88413b14d6..51e3cf56f5 100644 --- a/core/metadata/src/impls/recovery.rs +++ b/core/metadata/src/impls/recovery.rs @@ -19,7 +19,8 @@ use crate::impls::metadata::IggySnapshot; use crate::stm::StateMachine; use crate::stm::authz::GatedApply; use crate::stm::snapshot::{MetadataSnapshot, RestoreSnapshot, Snapshot, SnapshotError}; -use iggy_binary_protocol::consensus::PrepareHeader; +use consensus::{CLIENTS_TABLE_MAX, ClientTable, build_reply_message, build_reply_message_with}; +use iggy_binary_protocol::consensus::{Operation, PrepareHeader}; use iggy_common::IggyError; use journal::prepare_journal::{JournalError, PrepareJournal}; use server_common::Message; @@ -86,6 +87,13 @@ pub struct RecoveredMetadata { pub journal: PrepareJournal, pub snapshot: Option, pub mux_stm: M, + /// Client table rebuilt from the replayed committed prefix: registers + /// re-mint epochs in apply order, replies re-cache byte-identically + /// (`build_reply_message*` reads only the prepare header + deterministic + /// apply output). Sessions whose register fell below the snapshot floor + /// are NOT recovered - the table has no checkpoint artifact yet + /// (IGGY-137); those clients re-register and their epoch restarts at 1. + pub client_table: ClientTable, /// `None` means no snapshot existed and no journal entries were replayed. /// `Some(op)` is the highest op applied, either from the snapshot or journal replay. /// @@ -105,7 +113,9 @@ pub struct RecoveredMetadata { /// 1. Load snapshot from `{data_dir}/metadata/snapshot.bin` if present /// 2. Restore state machine from snapshot, or initialize empty state /// 3. Open WAL at `{data_dir}/metadata/journal.wal`, scan and rebuild index -/// 4. Replay journal entries from the first post-snapshot op through the state machine +/// 4. Replay journal entries from the first post-snapshot op through the +/// state machine, rebuilding the client table alongside (registers mint +/// epochs, replies re-cache) exactly as the commit paths did live /// 5. Return the assembled `RecoveredMetadata` /// /// Only the owning shard (shard 0) should call this. Peer shards receive @@ -188,6 +198,7 @@ where .fold(snapshot_floor, u64::max) }; + let mut client_table = ClientTable::new(CLIENTS_TABLE_MAX); let mut last_applied_op: Option = None; let mut last_journaled_op: Option = None; for header in &headers_to_replay { @@ -200,6 +211,26 @@ where continue; } + // Register/Logout mutate the client table and skip the state + // machine, mirroring the commit paths (`on_ack` / `commit_journal`). + // Replaying them from a fresh table in apply order re-mints the same + // epochs every replica derived live. + if header.operation == Operation::Register { + let reply = build_reply_message(header, &bytes::Bytes::new()); + client_table.commit_register(header.client, header.user_id, reply, |_| false); + last_applied_op = Some(header.op); + continue; + } + if header.operation == Operation::Logout { + client_table.remove_client(header.client); + // TODO: the commit paths also run `remove_consumer_group_member` + // here; recovery has no `StreamsFrontend` bound, so replayed + // logouts leave stale group members (pre-existing, harmless for + // dead connections but a divergence from the live apply). + last_applied_op = Some(header.op); + continue; + } + let entry = journal.entry_at(header).await?.ok_or_else(|| { RecoveryError::Io(std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -209,6 +240,16 @@ where // WAL replay must recompute authorization denials identically to the // primary/backup commit paths, so it goes through the same gate. let reply = mux_stm.gated_update(entry)?; + // Re-cache the reply exactly like the commit paths: same prepare + // header + deterministic apply output = the original bytes. Skipped + // when the session is absent (server-originated ops, or the client + // was evicted / registered below the snapshot floor). + if let Some(epoch) = client_table.get_epoch(header.client) { + let cached = build_reply_message_with(header, reply.reply_body_len(), |dst| { + reply.write_reply_body(dst); + }); + client_table.commit_reply(header.client, epoch, cached); + } tracing::debug!( target: "iggy.metadata.diag", op = header.op, @@ -224,6 +265,7 @@ where journal, snapshot, mux_stm, + client_table, last_applied_op, last_journaled_op, }) @@ -264,6 +306,31 @@ mod tests { Message::try_from(buffer).unwrap() } + /// A client-attributed prepare (Register / app op / Logout) as the + /// admission path stamps it. + fn make_client_prepare( + op: u64, + operation: Operation, + client: u128, + user_id: u32, + request: u64, + ) -> Message { + let total_size = HEADER_SIZE; + let mut buffer = Owned::<4096>::zeroed(total_size); + let header = bytemuck::checked::from_bytes_mut::( + &mut buffer.as_mut_slice()[..HEADER_SIZE], + ); + header.size = total_size as u32; + header.command = Command2::Prepare; + header.op = op; + header.commit = op.saturating_sub(1); + header.operation = operation; + header.client = client; + header.user_id = user_id; + header.request = request; + Message::try_from(buffer).unwrap() + } + #[compio::test] async fn recover_empty_state() { let dir = tempdir().unwrap(); @@ -422,6 +489,113 @@ mod tests { ); } + // The IGGY-137 restart contract: a rebooted node must remember where + // each client left off. Replay re-mints the epoch, restores the + // watermark, and re-caches the reply so a retry of the last committed + // request id dedups instead of re-executing or silently dropping. + #[compio::test] + async fn recover_rebuilds_client_table_from_wal() { + use consensus::client_table::RequestStatus; + + const CLIENT: u128 = 0x1337; + const USER: u32 = 7; + + let dir = tempdir().unwrap(); + let metadata_dir = dir.path().join("metadata"); + std::fs::create_dir_all(&metadata_dir).unwrap(); + + { + let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0) + .await + .unwrap(); + journal + .append(make_client_prepare(1, Operation::Register, CLIENT, USER, 0)) + .await + .unwrap(); + journal + .append(make_client_prepare( + 2, + Operation::CreateStream, + CLIENT, + USER, + 1, + )) + .await + .unwrap(); + journal.storage_ref().fsync().await.unwrap(); + } + + // Solo: every journaled op is committed. + let recovered = recover::( + dir.path(), + true, + journal::prepare_journal::DEFAULT_SLOT_COUNT, + |_| {}, + ) + .await + .unwrap(); + + let table = &recovered.client_table; + assert_eq!(table.get_epoch(CLIENT), Some(1), "register minted epoch 1"); + assert_eq!(table.get_user_id(CLIENT), Some(USER)); + assert_eq!( + table.get_watermark(CLIENT), + Some(1), + "committed request 1 restored the watermark" + ); + match table.check_request(CLIENT, 1, 1, 0) { + RequestStatus::Duplicate(cached) => { + assert_eq!(cached.header().request, 1, "retry replays the cached reply"); + } + other => panic!("expected Duplicate, got {other:?}"), + } + assert!( + matches!(table.check_request(CLIENT, 1, 2, 0), RequestStatus::New), + "the next request id is admitted" + ); + } + + // A replayed Logout removes the entry, mirroring the commit paths. + #[compio::test] + async fn recover_replays_logout_as_session_removal() { + const CLIENT: u128 = 0x1337; + const USER: u32 = 7; + + let dir = tempdir().unwrap(); + let metadata_dir = dir.path().join("metadata"); + std::fs::create_dir_all(&metadata_dir).unwrap(); + + { + let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0) + .await + .unwrap(); + journal + .append(make_client_prepare(1, Operation::Register, CLIENT, USER, 0)) + .await + .unwrap(); + journal + .append(make_client_prepare(2, Operation::Logout, CLIENT, USER, 1)) + .await + .unwrap(); + journal.storage_ref().fsync().await.unwrap(); + } + + let recovered = recover::( + dir.path(), + true, + journal::prepare_journal::DEFAULT_SLOT_COUNT, + |_| {}, + ) + .await + .unwrap(); + assert_eq!( + recovered.client_table.get_epoch(CLIENT), + None, + "logged-out session must not be resurrected" + ); + assert_eq!(recovered.last_applied_op, Some(2)); + } + #[test] fn snapshot_persist_load_roundtrip() { let dir = tempdir().unwrap(); diff --git a/core/metadata/src/stm/user.rs b/core/metadata/src/stm/user.rs index e7784a81e1..c8ea95f9db 100644 --- a/core/metadata/src/stm/user.rs +++ b/core/metadata/src/stm/user.rs @@ -563,7 +563,7 @@ impl StateHandler for ChangePasswordRequest { // `verify_and_rewrite_change_password`): the accept path always // replicates a non-empty Argon2 hash, so this is unambiguous. Rejecting // here (rather than denying pre-consensus) commits the op as a no-op, - // keeping the client's request sequence contiguous in the ClientTable. + // recording the request id in the ClientTable so a retry of it dedups. if self.new_password.is_empty() { return ApplyReply::err(ChangePasswordResult::InvalidCredentials); } diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs index b7d578a3ff..592fbe8de0 100644 --- a/core/server-ng/src/bootstrap.rs +++ b/core/server-ng/src/bootstrap.rs @@ -925,6 +925,7 @@ async fn shard_main( recovered.snapshot, recovered.last_applied_op, recovered.last_journaled_op, + recovered.client_table, )), ) } @@ -943,8 +944,10 @@ async fn shard_main( // Metadata consensus + journal + snapshot live only on shard 0. // `IggyShard::tick_metadata` short-circuits when `consensus.is_none()`, // so peer shards have no caller that reads `journal` or `snapshot`. - let (metadata_consensus, journal_for_metadata, snapshot_for_metadata) = - if let Some((journal, snapshot, last_applied_op, last_journaled_op)) = owner_state { + let (metadata_consensus, journal_for_metadata, snapshot_for_metadata, recovered_client_table) = + if let Some((journal, snapshot, last_applied_op, last_journaled_op, client_table)) = + owner_state + { let snapshot_floor = snapshot.as_ref().map_or(0, IggySnapshot::sequence_number); let commit_watermark = last_applied_op.unwrap_or(snapshot_floor); let restored_op = last_journaled_op.unwrap_or(snapshot_floor); @@ -959,9 +962,9 @@ async fn shard_main( config.metadata.prepare_queue_depth, cluster_heartbeat_ticks(config), ); - (Some(consensus), Some(journal), snapshot) + (Some(consensus), Some(journal), snapshot, Some(client_table)) } else { - (None, None, None) + (None, None, None, None) }; let metadata = ServerNgMetadata::new( metadata_consensus, @@ -970,6 +973,12 @@ async fn shard_main( mux_stm, Some(PathBuf::from(&config.system.path)), ); + // Reinstall the sessions the WAL replay rebuilt, so a rebooted node + // dedups retries and admits continuations from clients that kept their + // identity across the restart (IGGY-137). + if let Some(client_table) = recovered_client_table { + metadata.install_client_table(client_table); + } // Shard 0's copy resolves the `ServerDefault` sentinels (max topic size and // message expiry) at admission; every shard's copy backs the same resolution in responses. metadata.set_default_max_topic_size(config.system.topic.max_size.as_bytes_u64()); diff --git a/core/server-ng/src/dispatch.rs b/core/server-ng/src/dispatch.rs index 89160a63a1..d02a3d1e90 100644 --- a/core/server-ng/src/dispatch.rs +++ b/core/server-ng/src/dispatch.rs @@ -107,7 +107,7 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet, VecDeque}; use std::rc::Rc; use std::sync::Arc; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; pub(crate) type ClientRequestQueues = Rc>>>>; pub(crate) type ActiveClientRequests = Rc>>; @@ -578,11 +578,142 @@ where .ok(); let _ = reply.try_send(commit); } + shard::MetadataSubmit::ResumeLookup { + vsr_client_id, + reply, + } => { + let entry = lookup_resumable_session(&shard, vsr_client_id); + let _ = reply.try_send(entry); + } } }); }) } +/// Shard 0's read side of a session-resume attempt: `(epoch, user_id)` for a +/// registered client, `None` otherwise. Table reads are only authoritative on +/// a caught-up primary (same rule as `request_preflight`'s gate); a stale +/// answer here would rebind a transport to a session the cluster has already +/// rotated, so fail closed and let the client retry. +fn lookup_resumable_session( + shard: &Rc>, + vsr_client_id: u128, +) -> Option<(u64, u32)> +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal, Header = PrepareHeader>, + S: 'static, +{ + let metadata = shard.plane.metadata(); + let consensus = metadata.consensus.as_ref()?; + if !consensus::is_caught_up_primary(consensus) { + debug!( + vsr_client_id, + is_primary = consensus.is_primary(), + commit_min = consensus.commit_min(), + commit_max = consensus.commit_max(), + "resume lookup refused: not a caught-up primary" + ); + return None; + } + let table = metadata.client_table.borrow(); + let entry = table + .get_epoch(vsr_client_id) + .zip(table.get_user_id(vsr_client_id)); + if entry.is_none() { + debug!(vsr_client_id, "resume lookup: no table entry"); + } + entry +} + +/// Rebind a reconnecting transport that presents its pre-restart identity +/// (IGGY-137 session resume, the implicit-rebind contract): a replicated +/// request on an unbound transport whose `(client, session)` matches a live +/// entry in the replicated client table binds this connection to that +/// session and proceeds as if bound all along. +/// +/// The `(client_id, epoch)` pair acts as a bearer token here: the client id +/// is a client-generated random u128, so presenting it proves the caller is +/// (or eavesdropped on) the original registrant. `bind_session` evicts any +/// previous connection bound to the same client id, which is the conflict +/// rule for one session arriving on two connections. +/// +/// Returns the `(client_id, session)` binding on success, `None` when the +/// identity does not check out (caller falls back to the unbound-transport +/// reply and the client re-registers). +#[allow(clippy::future_not_send)] +async fn try_resume_session( + shard: &Rc>, + sessions: &Rc>, + transport_client_id: u128, + header: &RequestHeader, +) -> Option<(u128, u64)> +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal, Header = PrepareHeader>, + S: 'static, +{ + // A resumable frame carries the full old identity; anything less is a + // plain unbound request (e.g. a fresh SDK that has not registered yet). + if header.client == 0 || header.session == 0 || header.request == 0 { + return None; + } + + let entry = if shard.id == 0 { + lookup_resumable_session(shard, header.client) + } else { + let (reply, rx) = shard::channel::>(1); + shard.forward_metadata_submit(shard::MetadataSubmit::ResumeLookup { + vsr_client_id: header.client, + reply, + }); + rx.recv().await.ok().flatten() + }; + let (epoch, user_id) = entry?; + if epoch != header.session { + // Stale epoch = zombie holdover (the preflight would fence it); + // future epoch = client bug. Neither may rebind. + warn!( + transport_client_id, + client = header.client, + presented = header.session, + current = epoch, + "refusing session resume with mismatched epoch" + ); + return None; + } + + { + let mut sessions = sessions.borrow_mut(); + if let Err(error) = sessions.login(transport_client_id, user_id) { + warn!( + transport_client_id, + error = %error, + "session resume: login transition failed" + ); + return None; + } + if let Err(error) = sessions.bind_session(transport_client_id, header.client, epoch) { + warn!( + transport_client_id, + error = %error, + "session resume: bind failed" + ); + return None; + } + } + info!( + transport_client_id, + client = header.client, + epoch, + user_id, + "rebound transport to resumed session" + ); + Some((header.client, epoch)) +} + fn enqueue_client_request( shard: Rc>, sessions: Rc>, @@ -748,6 +879,13 @@ async fn handle_client_request( } let bound = sessions.borrow().get_session(transport_client_id); + // Unbound transport presenting a full old identity: session resume + // (IGGY-137). The table survived the restart via WAL-replay recovery; + // a matching `(client, epoch)` rebinds this connection in place. + let bound = match bound { + Some(bound) => Some(bound), + None => try_resume_session(shard, sessions, transport_client_id, &header).await, + }; if bound.is_none() { // Replicated request on an unbound transport. Without this short- // circuit, the rewrite below overwrites `header.client` with @@ -2168,11 +2306,10 @@ async fn handle_delete_segments_request( // offset on the owning shard, then replicate a `TruncatePartition(offset)` // AS the client's own request through the standard owner path: the commit // records (client, session, request) in the `ClientTable` on every replica, - // keeping the sequence contiguous. Skipping the commit (or attributing it - // to an internal id) leaves a hole that fails the next metadata op's - // `request == committed + 1` preflight -> RequestGap -> silent drop -> the - // SDK blocks until timeout. A no-op delete still commits `up_to_offset = 0` - // (monotonic apply) for the same reason. + // advancing the watermark. Skipping the commit (or attributing it to an + // internal id) leaves this request id unrecorded, so the SDK's own retry + // of it would re-execute instead of deduping. A no-op delete still + // commits `up_to_offset = 0` (monotonic apply) for the same reason. let truncate = match resolve_delete_segments_truncate( shard, &header, @@ -2370,9 +2507,17 @@ where } /// Disconnect cleanup: the local `SessionManager` connection is already -/// dropped by the caller; this submits a session-matched `Logout` so the -/// committed apply releases the `ClientTable` slot on every replica (shard 0 -/// included, since shard 0 is itself a replica). +/// dropped by the caller; for a consumer-group member this submits a +/// session-matched `Logout` so the committed apply releases the +/// `ClientTable` slot on every replica (shard 0 included, since shard 0 is +/// itself a replica) and the group rebalances off the dead consumer. +/// +/// A connection with NO group membership keeps its session: the client may +/// reconnect and resume under its old `(client, session)` identity +/// (IGGY-137, `try_resume_session`), which is exactly the crash-retry window +/// dedup exists for. Its slot is reclaimed by an explicit client `Logout` or +/// the table's capacity eviction. Same membership rule as the heartbeat +/// verifier, which deliberately reaps only group members. /// /// Deliberately does NOT drop the local `ClientTable` slot first: /// `submit_logout_*` short-circuits when the slot is already gone, so a @@ -2397,6 +2542,21 @@ fn submit_disconnect_logout( // The logout apply keys on (client, session) only, so any non-zero id // is valid here. const DISCONNECT_LOGOUT_REQUEST_ID: u64 = u64::MAX; + + let is_group_member = !shard + .plane + .metadata() + .mux_stm + .streams() + .consumer_group_memberships(vsr_client_id) + .is_empty(); + if !is_group_member { + debug!( + vsr_client_id, + "transport disconnected; keeping session for resume" + ); + return; + } let bus = shard.bus.clone(); bus.spawn(async move { if let Err(error) = diff --git a/core/server-ng/src/users.rs b/core/server-ng/src/users.rs index e5e08f2ca4..2fb03af403 100644 --- a/core/server-ng/src/users.rs +++ b/core/server-ng/src/users.rs @@ -97,11 +97,11 @@ where /// /// When the target resolves (`stored_hash` is `Some`) the supplied /// `current_password` is verified against it first. A mismatch does NOT deny -/// pre-consensus: that would consume the client's request id without advancing -/// the replicated `ClientTable`, gapping the sequence so the next replicated op -/// is dropped (`RequestGap`) and the caller stalls. Instead the new password is -/// emptied, which signals the committed apply to reject with -/// `InvalidCredentials` (a committed no-op that keeps the sequence contiguous). +/// pre-consensus: that would consume the client's request id without recording +/// it in the replicated `ClientTable`, so a retry of that id would re-execute +/// instead of deduping. Instead the new password is emptied, which signals the +/// committed apply to reject with `InvalidCredentials` (a committed no-op that +/// advances the watermark). /// /// Either way the current password is stripped and the accepted new one hashed, /// so no plaintext credential ever enters consensus. An unresolved target keeps diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 60803f5b57..27b0b2a10f 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -199,6 +199,15 @@ pub enum MetadataSubmit { partition_id: u32, reply: Sender>, }, + /// A home shard asks shard 0 whether `vsr_client_id` has a live entry in + /// the replicated client table, to rebind a reconnecting transport that + /// presents its old identity (session resume, IGGY-137). Read-only. + /// `reply` carries `(epoch, user_id)` for a registered client, `None` + /// otherwise. + ResumeLookup { + vsr_client_id: u128, + reply: Sender>, + }, } /// Handler shard 0 runs for an inbound [`MetadataSubmit`]. From 9244af7ae70a981ab153df23f0abc4e644387852 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Mon, 27 Jul 2026 11:16:51 +0200 Subject: [PATCH 03/14] drop distinct cached reply, run dedup tests under 3 node cluster --- core/consensus/src/client_table.rs | 117 +++++++++++------- .../tests/cluster/client_table_restart.rs | 67 ++++++---- core/sdk/src/vsr.rs | 9 +- core/simulator/src/workload/mod.rs | 4 +- 4 files changed, 121 insertions(+), 76 deletions(-) diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 2a82fc14ff..daa5a66460 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -71,15 +71,17 @@ impl CachedReply { /// Real requests start at 1 (header validation enforces `request > 0`). pub const REGISTER_REQUEST_ID: u64 = 0; -/// Displaced replies retained per entry for below-watermark duplicate hits. +/// Committed replies retained per entry, newest at the back. /// +/// The back is the latest committed reply and is structurally safe: +/// eviction pops the front, and only pushing a newer reply triggers it. /// The SDK enforces one request in flight per session, so the only reply a /// live client can be waiting for is its latest (`request == watermark`). -/// The ring answers old retransmits and post-rebind stragglers with the -/// original bytes instead of a bare "already applied"; losing an entry +/// Older entries answer old retransmits and post-rebind stragglers with the +/// original bytes instead of a bare "already applied"; losing one /// degrades the answer, never correctness. In-memory only: ring contents are /// refcount bumps and are never persisted or transferred. -const REPLY_RING_CAPACITY: usize = 4; +const REPLY_RING_CAPACITY: usize = 5; /// Per-session entry: fence epoch + committed-request watermark + replies. /// @@ -105,11 +107,11 @@ pub struct ClientEntry { /// a request id for a different operation. Zero when unstamped (integrity /// fields are zeroed on the wire today), which disables the comparison. watermark_checksum: u128, - /// Latest committed reply (register or app op). - reply: CachedReply, - /// Displaced app replies, oldest at front, bounded by - /// [`REPLY_RING_CAPACITY`]. Register replies never enter (their - /// `request == REGISTER_REQUEST_ID` can never match a lookup). + /// Committed replies, oldest at front, latest at back; never empty + /// (registration seeds the register reply). Bounded by + /// [`REPLY_RING_CAPACITY`]. Request numbers are unique: same-request + /// recommits replace in place, and a rebind drops the previous + /// register reply before pushing the new one. ring: VecDeque, } @@ -162,9 +164,9 @@ pub enum RequestStatus { /// - **Watermark, not contiguity.** A request above the watermark executes /// (gaps allowed); at or below is a duplicate. There is no `RequestGap`: /// a client that jumps its counter loses nothing but the skipped ids. -/// - **Replies are volatile.** Latest reply plus a small ring of displaced -/// ones, all in-memory refcounts. A duplicate whose reply aged out is -/// still refused execution ([`RequestStatus::AlreadyApplied`]). +/// - **Replies are volatile.** A small per-entry ring of recent replies +/// (latest at the back), all in-memory refcounts. A duplicate whose reply +/// aged out is still refused execution ([`RequestStatus::AlreadyApplied`]). /// /// ## Plane /// @@ -284,7 +286,7 @@ impl ClientTable { let entry = self.slots[slot_idx].as_ref().expect("index/slot mismatch"); RequestStatus::AlreadyRegistered { epoch: entry.epoch, - cached_reply: entry.reply.clone(), + cached_reply: entry.latest().clone(), } } @@ -293,9 +295,10 @@ impl ClientTable { /// /// The epoch is minted HERE, in apply order, so it is deterministic /// across replicas without reading any commit number. A rebind refreshes - /// `user_id` (the bind re-authenticated), replaces the latest reply with - /// the register reply (the displaced app reply moves into the ring), and - /// preserves the watermark - session resume keeps dedup history. + /// `user_id` (the bind re-authenticated), pushes the register reply as + /// the latest (cached app replies stay put; the previous register reply + /// is dropped), and preserves the watermark - session resume keeps dedup + /// history. /// /// Full table evicts oldest commit; `in_flight` protects pipeline /// holders, see [`Self::evict_oldest`]. @@ -326,27 +329,34 @@ impl ClientTable { let entry = self.slots[slot_idx].as_mut().expect("index/slot mismatch"); entry.epoch += 1; entry.user_id = user_id; - let displaced = std::mem::replace(&mut entry.reply, cached); - entry.push_ring(displaced); + // Drop the previous register reply (if still retained) before + // pushing the new one: only the newest rebind's reply is + // replayable, and two request-0 entries would break the ring's + // unique-request invariant. + entry + .ring + .retain(|stored| stored.header().request != REGISTER_REQUEST_ID); + entry.push_latest(cached); } else { if self.index.len() >= self.slots.len() { self.evict_oldest(&in_flight); } let slot_idx = self.first_free_slot().expect("eviction must free a slot"); + let mut ring = VecDeque::with_capacity(REPLY_RING_CAPACITY); + ring.push_back(cached); self.slots[slot_idx] = Some(ClientEntry { epoch: 1, user_id, watermark: REGISTER_REQUEST_ID, watermark_checksum: 0, - reply: cached, - ring: VecDeque::with_capacity(REPLY_RING_CAPACITY), + ring, }); self.index.insert(client_id, slot_idx); } } - /// Record a committed reply: advance the watermark, cache the reply, - /// move the displaced one into the ring. + /// Record a committed reply: advance the watermark, push the reply into + /// the ring (evicting the oldest when full). /// /// `epoch` is asserted against the entry to guard a mis-attributed apply /// from clobbering a rebound session's state. @@ -398,7 +408,7 @@ impl ClientTable { entry={}, prepare={epoch}", entry.epoch ); - let latest_commit = entry.reply.header().commit; + let latest_commit = entry.latest().header().commit; assert!( new_commit >= latest_commit, "commit_reply: commit regression for client {client_id}: {latest_commit} -> {new_commit}", @@ -413,12 +423,19 @@ impl ClientTable { let cached = CachedReply::from_message(reply); if new_request == entry.watermark { // Same request re-committed (WAL replay shape): replace in - // place, never push the stale twin into the ring - two cached - // replies for one request number would make lookups ambiguous. - entry.reply = cached; + // place, never push a stale twin - two cached replies for one + // request number would make lookups ambiguous. + if let Some(stored) = entry + .ring + .iter_mut() + .find(|stored| stored.header().request == new_request) + { + *stored = cached; + } else { + entry.push_latest(cached); + } } else { - let displaced = std::mem::replace(&mut entry.reply, cached); - entry.push_ring(displaced); + entry.push_latest(cached); entry.watermark = new_request; } entry.watermark_checksum = new_checksum; @@ -477,8 +494,9 @@ impl ClientTable { for (idx, slot) in self.slots.iter().enumerate() { let Some(entry) = slot else { continue }; - let commit = entry.reply.header().commit; - let client_id = entry.reply.header().client; + let latest = entry.latest().header(); + let commit = latest.commit; + let client_id = latest.client; let target = if in_flight(client_id) { &mut fallback } else { @@ -496,7 +514,7 @@ impl ClientTable { let pick = evictee.or(fallback); if let Some((slot_idx, _)) = pick { let entry = self.slots[slot_idx].take().expect("evictee must exist"); - let client_id = entry.reply.header().client; + let client_id = entry.latest().header().client; self.index.remove(&client_id); trace!(client_id, "evict_oldest: removed client from session table"); } @@ -513,7 +531,7 @@ impl ClientTable { #[must_use] pub fn get_reply(&self, client_id: u128) -> Option<&CachedReply> { let &slot_idx = self.index.get(&client_id)?; - self.slots[slot_idx].as_ref().map(|entry| &entry.reply) + self.slots[slot_idx].as_ref().map(ClientEntry::latest) } /// Fence epoch for a registered client. This is the u64 the register @@ -548,29 +566,31 @@ impl ClientTable { } impl ClientEntry { - /// Cached reply whose `request` matches, latest first then the ring - /// (newest displaced entries sit at the back; scan order is irrelevant + /// Latest committed reply (register or app op). + /// + /// # Panics + /// Unreachable: registration seeds the ring and pops happen only when + /// displaced by a newer push. + fn latest(&self) -> &CachedReply { + self.ring + .back() + .expect("ring is never empty after registration") + } + + /// Cached reply whose `request` matches (scan order is irrelevant /// because request numbers in the ring are unique). fn find_cached(&self, request: u64) -> Option<&CachedReply> { - if self.reply.header().request == request { - return Some(&self.reply); - } self.ring .iter() .find(|cached| cached.header().request == request) } - /// Retain a displaced reply for below-watermark duplicates. Register - /// replies never enter: `request == REGISTER_REQUEST_ID` can never match - /// a `check_request` lookup (wire validation enforces `request > 0`). - fn push_ring(&mut self, displaced: CachedReply) { - if displaced.header().request == REGISTER_REQUEST_ID { - return; - } + /// Push the newest committed reply, evicting the oldest when full. + fn push_latest(&mut self, cached: CachedReply) { if self.ring.len() == REPLY_RING_CAPACITY { self.ring.pop_front(); } - self.ring.push_back(displaced); + self.ring.push_back(cached); } } @@ -670,8 +690,8 @@ mod tests { ); assert_eq!(table.count(), 1); - // The displaced app reply moved into the ring: the watermark request - // still answers with its original bytes under the new epoch. + // The app reply stays cached across the rebind: the watermark + // request still answers with its original bytes under the new epoch. match table.check_request(1, 2, 5, 0) { RequestStatus::Duplicate(cached) => assert_eq!(cached.header().request, 5), other => panic!("expected Duplicate from ring, got {other:?}"), @@ -847,7 +867,8 @@ mod tests { fn check_request_below_watermark_past_ring_is_already_applied() { let (mut table, epoch) = table_with_client(); // Requests 1..=6: request 1's reply is displaced beyond the ring - // (capacity 4 holds 2,3,4,5 once 6 is latest). + // (capacity 5 holds 2..=6 once 6 commits; the register reply and + // request 1 aged out first). for request in 1..=6u64 { table.commit_reply(1, epoch, make_reply_for(1, request, 10 + request)); } diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs index 2e57893352..b5cac4c5fb 100644 --- a/core/integration/tests/cluster/client_table_restart.rs +++ b/core/integration/tests/cluster/client_table_restart.rs @@ -47,10 +47,16 @@ //! retry replicated writes only under the same identity) is the remaining //! client half. //! -//! Single-node topology on purpose: the raw client pins one address, and a -//! follower cannot commit replicated TCP writes (no follower forwarding for -//! TCP yet), so post-failover resume against a 3-node cluster is future -//! work alongside that forwarding. +//! The topology matrix covers two distinct recovery paths: +//! +//! - **1 node**: the restarted node rebuilds the table from its own WAL and +//! the client resumes against it (restart recovery). +//! - **3 nodes**: `restart_server` reboots node 0 only; the survivors elect +//! a new primary, whose table was maintained by its own `commit_journal` +//! applies all along. The client resumes against whichever node answers +//! as primary โ€” a follower cannot commit replicated TCP writes (no +//! follower forwarding for TCP yet), so the continuation loop probes +//! every node the way a leader-aware SDK re-routes (failover resume). //! //! These tests pin the implicit-rebind contract: a resumed client simply //! keeps sending under its old `(client, session)` on a fresh connection and @@ -101,7 +107,7 @@ const REPLY_WAIT: Duration = Duration::from_secs(5); const RETRY_PAUSE: Duration = Duration::from_millis(100); -#[iggy_harness(cluster_nodes = 1)] +#[iggy_harness(cluster_nodes = [1, 3])] async fn given_committed_request_when_node_restarts_should_dedup_same_id_retry( harness: &mut TestHarness, ) { @@ -116,11 +122,11 @@ async fn given_committed_request_when_node_restarts_should_dedup_same_id_retry( // The reply for request 1 was already delivered, but the client cannot // know that in the crash window; retrying the same id must converge on // the cached reply, never on a second apply or a silent drop. - let addr = tcp_addr(harness); - resume_request(addr, session, 1, &create_stream).await; + let addrs = tcp_addrs(harness); + resume_request(&addrs, session, 1, &create_stream).await; } -#[iggy_harness(cluster_nodes = 1)] +#[iggy_harness(cluster_nodes = [1, 3])] async fn given_bound_session_when_node_restarts_should_accept_next_request_id( harness: &mut TestHarness, ) { @@ -140,8 +146,8 @@ async fn given_bound_session_when_node_restarts_should_accept_next_request_id( // Continuation, not retry: the session advances to the next id. A node // that forgot the watermark sees request 2 on an unknown session and // either drops it as a gap or bounces the session entirely. - let addr = tcp_addr(harness); - resume_request(addr, session, 2, &create_stream_payload("iggy137-second")).await; + let addrs = tcp_addrs(harness); + resume_request(&addrs, session, 2, &create_stream_payload("iggy137-second")).await; } fn tcp_addr(harness: &TestHarness) -> SocketAddr { @@ -151,6 +157,21 @@ fn tcp_addr(harness: &TestHarness) -> SocketAddr { .expect("server must expose a TCP address") } +/// Every node's TCP address. The continuation loop probes all of them the +/// way a leader-aware SDK re-routes: after a node restart in a cluster the +/// primary may be any survivor, and only the primary commits (or answers +/// dedup for) replicated requests. +fn tcp_addrs(harness: &TestHarness) -> Vec { + (0..harness.cluster_size()) + .map(|node| { + harness + .node(node) + .tcp_addr() + .expect("every node must expose a TCP address") + }) + .collect() +} + fn create_stream_payload(name: &str) -> Bytes { CreateStreamRequest { name: WireName::new(name).unwrap(), @@ -232,16 +253,20 @@ async fn commit_request(stream: &mut TcpStream, session: u64, request: u64, body } } -/// Post-restart continuation: keep presenting the old identity until the -/// server commits (or serves the cached reply for) the request. Every -/// attempt uses a fresh connection, both because the old one died with the -/// node and so an unanswered frame cannot desync the next attempt. Panics -/// with the last observed failure mode when the budget runs out. -async fn resume_request(addr: SocketAddr, session: u64, request: u64, body: &Bytes) { +/// Post-restart continuation: keep presenting the old identity, round-robin +/// across every node, until some node commits (or serves the cached reply +/// for) the request. Every attempt uses a fresh connection, both because the +/// old one died with the node and so an unanswered frame cannot desync the +/// next attempt. Panics with the last observed failure mode when the budget +/// runs out. +async fn resume_request(addrs: &[SocketAddr], session: u64, request: u64, body: &Bytes) { let header = request_header(Operation::CreateStream, session, request, body.len()); let deadline = Instant::now() + RESUME_BUDGET; let mut last_failure = "the listener never came back".to_string(); + let mut attempt = 0usize; while Instant::now() < deadline { + let addr = addrs[attempt % addrs.len()]; + attempt += 1; let Ok(mut stream) = TcpStream::connect(addr).await else { sleep(RETRY_PAUSE).await; continue; @@ -259,21 +284,21 @@ async fn resume_request(addr: SocketAddr, session: u64, request: u64, body: &Byt } Verdict::NoResultSection => { last_failure = format!( - "request {request} got the unbound-transport empty Reply: the \ - restarted node does not recognize session {session}" + "request {request} got the unbound-transport empty Reply on \ + {addr}: that node does not recognize session {session}" ); } Verdict::Ignored => { last_failure = format!( "request {request} on session {session} was silently ignored for \ - {REPLY_WAIT:?} (RequestGap-style drop: the restarted node lost \ - the request watermark)" + {REPLY_WAIT:?} (RequestGap-style drop: the node lost the \ + request watermark)" ); } Verdict::Evicted(reason) => { last_failure = format!( "session {session} was evicted with reason {reason} instead of \ - being rebound from the persisted table" + being rebound from the recovered table" ); } } diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index d1e9ab859c..671fbd5e9b 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs @@ -47,11 +47,10 @@ const NON_REPLICATED_CODE_RANGE: std::ops::Range = 0..4; // the retry from the current ConsensusSession. If disconnect created a fresh VSR // client/session, the retried request gets a new (client_id, request_id) tuple, so // server-side deduplication cannot match a mutation that may already have committed -// before the transport failure. TigerBeetle avoids session resume and relies on -// idempotency for requests retried from a new client session. For Iggy, either stop -// transparent retries for replicated mutations after VSR session reset, add explicit -// session resume/rebind semantics, or add a protocol-level idempotency key that is -// independent of (client_id, request_id). +// before the transport failure. The server now rebinds a transport that presents +// its old (client, session) identity (`try_resume_session`), so the fix here is +// to keep the ConsensusSession across reconnects and retry replicated writes +// under the same (client_id, request_id) instead of re-registering fresh. pub(crate) fn encode_contiguous_request( session: &mut ConsensusSession, code: u32, diff --git a/core/simulator/src/workload/mod.rs b/core/simulator/src/workload/mod.rs index 985890f1ba..045e946fa3 100644 --- a/core/simulator/src/workload/mod.rs +++ b/core/simulator/src/workload/mod.rs @@ -203,7 +203,7 @@ impl Workload { OnReply::Unknown => return Vec::new(), }; - // Decode the committed result code. Metadata replies carry a TB-style + // Decode the committed result code. Metadata replies carry a // result section (see `ApplyReply::to_reply_body`); partition-plane // replies do not, hence the `is_metadata` gate. let committed_code = if header.operation.is_metadata() { @@ -233,7 +233,7 @@ impl Workload { }; // The state machine only commits codes its own result enum declares, // so an unrecognized one is a server bug (a race still yields a - // declared code). TB's "classify never guesses". + // declared code). Classify never guesses. assert!( result_code_recognized(header.operation, code), "metadata op {:?} returned unrecognized result code {code} \ From 767b0c6e05d26aae8679b01ed023dc4e7787b649 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Mon, 27 Jul 2026 12:32:45 +0200 Subject: [PATCH 04/14] temp --- Cargo.lock | 2 + core/binary_protocol/src/consensus/command.rs | 19 +- core/binary_protocol/src/consensus/header.rs | 282 ++++++ core/binary_protocol/src/consensus/mod.rs | 5 +- core/binary_protocol/src/lib.rs | 6 +- core/consensus/Cargo.toml | 1 + core/consensus/src/client_table.rs | 290 +++++- core/consensus/src/impls.rs | 181 +++- core/consensus/src/lib.rs | 2 +- core/consensus/src/metadata_helpers.rs | 27 +- core/consensus/src/observability.rs | 4 +- core/consensus/src/plane_helpers.rs | 11 +- .../integration/src/harness/config/resolve.rs | 7 +- core/integration/src/harness/handle/server.rs | 10 +- .../tests/cluster/client_table_restart.rs | 17 +- .../tests/cluster/metadata_state_transfer.rs | 104 +++ core/integration/tests/cluster/mod.rs | 1 + core/journal/src/lib.rs | 11 + core/journal/src/prepare_journal.rs | 13 + core/metadata/src/impls/metadata.rs | 255 +++++- core/metadata/src/lib.rs | 2 +- core/metadata/src/stm/mod.rs | 10 + core/metadata/src/stm/mux.rs | 22 +- core/metadata/src/stm/snapshot.rs | 47 + core/metadata/src/stm/stream.rs | 14 +- core/metadata/src/stm/user.rs | 17 +- core/partitions/src/iggy_partition.rs | 6 +- core/partitions/src/iggy_partitions.rs | 2 +- core/server-ng/src/bootstrap.rs | 6 + core/server-ng/src/dispatch.rs | 2 +- core/server_common/src/consensus_message.rs | 34 +- core/shard/Cargo.toml | 1 + core/shard/src/lib.rs | 841 +++++++++++++++++- core/shard/src/router.rs | 26 +- 34 files changed, 2147 insertions(+), 131 deletions(-) create mode 100644 core/integration/tests/cluster/metadata_state_transfer.rs diff --git a/Cargo.lock b/Cargo.lock index 7591cdbf30..f336698105 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3227,6 +3227,7 @@ dependencies = [ "rand_xoshiro", "server_common", "tracing", + "twox-hash", ] [[package]] @@ -12018,6 +12019,7 @@ dependencies = [ "server_common", "thiserror 2.0.19", "tracing", + "twox-hash", ] [[package]] diff --git a/core/binary_protocol/src/consensus/command.rs b/core/binary_protocol/src/consensus/command.rs index d2d0ada7cc..6dcf1298ae 100644 --- a/core/binary_protocol/src/consensus/command.rs +++ b/core/binary_protocol/src/consensus/command.rs @@ -59,6 +59,17 @@ pub enum Command2 { RepairPrepare = 19, RepairDone = 20, RangeEvicted = 21, + + // State transfer (metadata plane): a restarted replica replaces its + // snapshot-shaped state (metadata snapshot + client table) from the + // current primary, then journal repair covers the tail. Pull-based: + // the requester asks for the target descriptor, then fetches each + // artifact in bounded chunks (per-peer bus queues drop overruns + // silently, so push cannot work). + RequestStateTransfer = 22, + StateTransferTarget = 23, + RequestStateChunk = 24, + StateChunk = 25, } // SAFETY: Command2 is #[repr(u8)] with no padding bytes. @@ -69,7 +80,7 @@ unsafe impl CheckedBitPattern for Command2 { type Bits = u8; fn is_valid_bit_pattern(bits: &u8) -> bool { - *bits <= Self::RangeEvicted as u8 + *bits <= Self::StateChunk as u8 } } @@ -90,8 +101,8 @@ mod tests { #[test] fn replica_auth_commands_are_valid_bit_patterns() { - // Locks the is_valid_bit_pattern bump: 14..=21 parse, 22 still rejects. - for command in 14u8..=21 { + // Locks the is_valid_bit_pattern bump: 14..=25 parse, 26 still rejects. + for command in 14u8..=25 { let mut buf: AVec> = AVec::new(16); buf.resize(256, 0); buf[60] = command; @@ -99,7 +110,7 @@ mod tests { } let mut buf: AVec> = AVec::new(16); buf.resize(256, 0); - buf[60] = 22; + buf[60] = 26; assert!(bytemuck::checked::try_from_bytes::(&buf).is_err()); } } diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index 7c852e150e..39bb4acc22 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -1250,6 +1250,288 @@ impl ConsensusHeader for RepairRangeReplyHeader { } } +// State transfer (metadata plane): descriptor + chunk pull frames. + +/// Artifact id for the metadata snapshot bytes (`snapshot.bin` content). +pub const STATE_TRANSFER_ARTIFACT_SNAPSHOT: u8 = 0; +/// Artifact id for the encoded client table. +pub const STATE_TRANSFER_ARTIFACT_TABLE: u8 = 1; + +// RequestStateTransferHeader - restarted replica -> current primary + +/// Ask the current primary for a state-transfer target descriptor. +/// +/// Answered with `StateTransferTarget`. Sent by a restarted replica after it +/// adopts a view from a live primary; `nonce` correlates the whole transfer +/// session. +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[repr(C)] +pub struct RequestStateTransferHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub nonce: u128, + pub namespace: u64, + pub reserved: [u8; 104], +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!( + offset_of!(RequestStateTransferHeader, nonce) + == offset_of!(RequestStateTransferHeader, reserved_frame) + size_of::<[u8; 66]>() + ); + assert!( + offset_of!(RequestStateTransferHeader, reserved) + size_of::<[u8; 104]>() == HEADER_SIZE + ); +}; + +impl ConsensusHeader for RequestStateTransferHeader { + const COMMAND: Command2 = Command2::RequestStateTransfer; + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::RequestStateTransfer { + return Err(ConsensusError::InvalidCommand { + expected: Command2::RequestStateTransfer, + found: self.command, + }); + } + Ok(()) + } +} + +// StateTransferTargetHeader - serving primary -> requester + +/// The transfer target descriptor. +/// +/// Carries artifact lengths + checksums and the frontiers the receiver +/// installs at. `available == 0` means the serving peer cannot serve right +/// now (not a caught-up primary, or it has never checkpointed, so the +/// requester's journal repair can cover the whole gap); every other field is +/// meaningful only when `available == 1`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[repr(C)] +pub struct StateTransferTargetHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub nonce: u128, + /// Serving primary's applied frontier (`commit_min`) when the descriptor + /// was built. The receiver's tail repair targets past this. + pub commit_op: u64, + /// `sequence_number` of the offered snapshot; the receiver's applied + /// frontier after install. + pub snapshot_seq: u64, + pub snapshot_len: u64, + /// `XxHash3_64` over the snapshot bytes. + pub snapshot_checksum: u64, + /// Client-table mutations at or below this op are already reflected in + /// the offered table; the receiver's commit walk skips them. + pub table_frontier: u64, + pub table_len: u64, + /// `XxHash3_64` over the encoded table bytes. + pub table_checksum: u64, + pub namespace: u64, + pub available: u8, + pub reserved: [u8; 47], +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!( + offset_of!(StateTransferTargetHeader, nonce) + == offset_of!(StateTransferTargetHeader, reserved_frame) + size_of::<[u8; 66]>() + ); + assert!(offset_of!(StateTransferTargetHeader, reserved) + size_of::<[u8; 47]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for StateTransferTargetHeader { + const COMMAND: Command2 = Command2::StateTransferTarget; + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::StateTransferTarget { + return Err(ConsensusError::InvalidCommand { + expected: Command2::StateTransferTarget, + found: self.command, + }); + } + if self.available > 1 { + return Err(ConsensusError::InvalidField( + "available must be 0 or 1".to_string(), + )); + } + Ok(()) + } +} + +// RequestStateChunkHeader - requester -> serving primary + +/// Pull one bounded chunk of an artifact. +/// +/// Lockstep per artifact: the requester keeps at most one chunk in flight, +/// so the bounded per-peer bus queue can never drop a burst tail. +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[repr(C)] +pub struct RequestStateChunkHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub nonce: u128, + pub offset: u64, + pub namespace: u64, + pub len: u32, + /// [`STATE_TRANSFER_ARTIFACT_SNAPSHOT`] or [`STATE_TRANSFER_ARTIFACT_TABLE`]. + pub artifact: u8, + pub reserved: [u8; 91], +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!( + offset_of!(RequestStateChunkHeader, nonce) + == offset_of!(RequestStateChunkHeader, reserved_frame) + size_of::<[u8; 66]>() + ); + assert!(offset_of!(RequestStateChunkHeader, reserved) + size_of::<[u8; 91]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for RequestStateChunkHeader { + const COMMAND: Command2 = Command2::RequestStateChunk; + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::RequestStateChunk { + return Err(ConsensusError::InvalidCommand { + expected: Command2::RequestStateChunk, + found: self.command, + }); + } + if self.len == 0 { + return Err(ConsensusError::InvalidField( + "chunk len must be non-zero".to_string(), + )); + } + if self.artifact > STATE_TRANSFER_ARTIFACT_TABLE { + return Err(ConsensusError::InvalidField( + "unknown state transfer artifact".to_string(), + )); + } + Ok(()) + } +} + +// StateChunkHeader - serving primary -> requester (carries payload) + +/// One chunk of artifact bytes at `offset`. +/// +/// The payload rides the body (`size` spans header + payload). Transit +/// integrity is checked at the artifact level (descriptor checksums), not +/// per chunk. +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[repr(C)] +pub struct StateChunkHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub nonce: u128, + pub offset: u64, + pub namespace: u64, + /// [`STATE_TRANSFER_ARTIFACT_SNAPSHOT`] or [`STATE_TRANSFER_ARTIFACT_TABLE`]. + pub artifact: u8, + pub reserved: [u8; 95], +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!( + offset_of!(StateChunkHeader, nonce) + == offset_of!(StateChunkHeader, reserved_frame) + size_of::<[u8; 66]>() + ); + assert!(offset_of!(StateChunkHeader, reserved) + size_of::<[u8; 95]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for StateChunkHeader { + const COMMAND: Command2 = Command2::StateChunk; + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::StateChunk { + return Err(ConsensusError::InvalidCommand { + expected: Command2::StateChunk, + found: self.command, + }); + } + if (self.size as usize) < HEADER_SIZE { + return Err(ConsensusError::InvalidField( + "state chunk size below header size".to_string(), + )); + } + if self.artifact > STATE_TRANSFER_ARTIFACT_TABLE { + return Err(ConsensusError::InvalidField( + "unknown state transfer artifact".to_string(), + )); + } + Ok(()) + } +} + // Tests #[cfg(test)] diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs index 594d1ed989..e188511313 100644 --- a/core/binary_protocol/src/consensus/mod.rs +++ b/core/binary_protocol/src/consensus/mod.rs @@ -48,8 +48,9 @@ pub use header::{ CommitHeader, ConsensusHeader, DoViewChangeHeader, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, - RequestStartViewHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, - read_size_field, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, SIZE_FIELD_OFFSET, + STATE_TRANSFER_ARTIFACT_SNAPSHOT, STATE_TRANSFER_ARTIFACT_TABLE, StartViewChangeHeader, + StartViewHeader, StateChunkHeader, StateTransferTargetHeader, read_size_field, }; pub use operation::Operation; pub use reply_result::{RESULT_COUNT_LEN, RESULT_ENTRY_LEN, result_code, result_section_len}; diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index 1d2010f3f6..da739ea68c 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -74,8 +74,10 @@ pub use consensus::{ Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, - StartViewHeader, read_size_field, result_code, result_section_len, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, SIZE_FIELD_OFFSET, STATE_TRANSFER_ARTIFACT_SNAPSHOT, + STATE_TRANSFER_ARTIFACT_TABLE, StartViewChangeHeader, StartViewHeader, StateChunkHeader, + StateTransferTargetHeader, read_size_field, result_code, result_section_len, }; pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_command}; pub use error::WireError; diff --git a/core/consensus/Cargo.toml b/core/consensus/Cargo.toml index a612dca77f..5aec679f58 100644 --- a/core/consensus/Cargo.toml +++ b/core/consensus/Cargo.toml @@ -41,6 +41,7 @@ rand = { workspace = true } rand_xoshiro = { workspace = true } server_common = { workspace = true } tracing = { workspace = true } +twox-hash = { workspace = true } [dev-dependencies] futures = { workspace = true } diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index daa5a66460..0d22d5e594 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -15,11 +15,16 @@ // specific language governing permissions and limitations // under the License. -use iggy_binary_protocol::ReplyHeader; -use server_common::{MESSAGE_ALIGN, Message, iobuf::Frozen}; +use iggy_binary_protocol::{GenericHeader, ReplyHeader}; +use server_common::{ + MESSAGE_ALIGN, Message, + iobuf::{Frozen, Owned}, +}; use std::collections::{HashMap, VecDeque}; +use std::hash::Hasher; use std::mem::size_of; use tracing::trace; +use twox_hash::XxHash3_64; /// Refcounted wrapper around a committed reply. /// @@ -183,10 +188,12 @@ pub enum RequestStatus { /// `commit_register` in the apply path, so every replica of the group /// derives an identical table from the committed log. /// -/// ## Known gaps +/// ## Serialization /// -/// - **Serialization**: encode/decode for rejoin slice-fetch and state -/// transfer TODO (IGGY-137). +/// [`Self::encode`] / [`Self::decode`] carry the table across state transfer +/// (a cluster-restart rejoin replaces its table with the primary's copy). +/// Deterministic: slot-order walk over apply-derived state, so every +/// caught-up replica encodes identical bytes. #[derive(Debug)] pub struct ClientTable { /// `None` = free slot. Deterministic iteration for eviction + serialization. @@ -565,6 +572,191 @@ impl ClientTable { } } +/// Failure decoding an encoded client table (state transfer install path). +#[derive(Debug)] +pub enum ClientTableCodecError { + /// Byte stream ended mid-field. + Truncated, + /// Leading magic is not [`CLIENT_TABLE_MAGIC`]. + BadMagic, + /// Trailing hash does not match the content. + ChecksumMismatch { expected: u64, actual: u64 }, + /// Encoded entry count exceeds this table's capacity. + TooManyEntries { count: u32, max: usize }, + /// A cached reply's bytes do not parse as a valid reply message. + InvalidReply, + /// An entry carries an empty reply ring (violates the never-empty + /// invariant registration establishes). + EmptyRing, +} + +impl std::fmt::Display for ClientTableCodecError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Truncated => write!(f, "encoded client table truncated"), + Self::BadMagic => write!(f, "encoded client table has wrong magic"), + Self::ChecksumMismatch { expected, actual } => write!( + f, + "client table checksum mismatch: expected {expected:#018x}, actual {actual:#018x}" + ), + Self::TooManyEntries { count, max } => { + write!(f, "encoded client table holds {count} entries, max {max}") + } + Self::InvalidReply => write!(f, "encoded client table holds an invalid cached reply"), + Self::EmptyRing => write!(f, "encoded client table entry has an empty reply ring"), + } + } +} + +impl std::error::Error for ClientTableCodecError {} + +/// Format tag for [`ClientTable::encode`]; bump on layout change. +pub const CLIENT_TABLE_MAGIC: [u8; 4] = *b"ICT1"; + +impl ClientTable { + /// Encode the table for state transfer. + /// + /// Layout (all little-endian): `magic(4) count(u32)` then per entry in + /// slot order `client(u128) epoch(u64) user_id(u32) watermark(u64) + /// watermark_checksum(u128) ring_len(u8) [reply_len(u32) reply_bytes]*`, + /// terminated by an `XxHash3_64(8)` over everything before it. + /// + /// Slot order makes the bytes deterministic across caught-up replicas: + /// slots are apply-derived state. + #[must_use] + #[allow(clippy::cast_possible_truncation)] + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(64 * self.index.len() + 16); + out.extend_from_slice(&CLIENT_TABLE_MAGIC); + out.extend_from_slice(&(self.index.len() as u32).to_le_bytes()); + for (slot_idx, slot) in self.slots.iter().enumerate() { + let Some(entry) = slot else { continue }; + let client = entry.latest().header().client; + debug_assert_eq!(self.index.get(&client), Some(&slot_idx)); + out.extend_from_slice(&client.to_le_bytes()); + out.extend_from_slice(&entry.epoch.to_le_bytes()); + out.extend_from_slice(&entry.user_id.to_le_bytes()); + out.extend_from_slice(&entry.watermark.to_le_bytes()); + out.extend_from_slice(&entry.watermark_checksum.to_le_bytes()); + out.push(entry.ring.len() as u8); + for reply in &entry.ring { + let bytes = reply.bytes.as_slice(); + out.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + out.extend_from_slice(bytes); + } + } + let mut hasher = XxHash3_64::new(); + hasher.write(&out); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + out + } + + /// Decode a table encoded by [`Self::encode`] into a fresh table of + /// `max_clients` capacity. + /// + /// # Errors + /// [`ClientTableCodecError`] on truncation, magic/checksum mismatch, + /// capacity overflow, or an undecodable cached reply. + /// + /// # Panics + /// Unreachable: slice-to-array conversions are length-checked first. + pub fn decode(bytes: &[u8], max_clients: usize) -> Result { + const TRAILER: usize = size_of::(); + let content_len = bytes + .len() + .checked_sub(TRAILER) + .ok_or(ClientTableCodecError::Truncated)?; + let (content, trailer) = bytes.split_at(content_len); + let expected = u64::from_le_bytes(trailer.try_into().expect("trailer is 8 bytes")); + let mut hasher = XxHash3_64::new(); + hasher.write(content); + let actual = hasher.finish(); + if expected != actual { + return Err(ClientTableCodecError::ChecksumMismatch { expected, actual }); + } + + let mut reader = CodecReader { bytes: content }; + if reader.take(CLIENT_TABLE_MAGIC.len())? != CLIENT_TABLE_MAGIC { + return Err(ClientTableCodecError::BadMagic); + } + let count = reader.u32()?; + if count as usize > max_clients { + return Err(ClientTableCodecError::TooManyEntries { + count, + max: max_clients, + }); + } + + let mut table = Self::new(max_clients); + for slot_idx in 0..count as usize { + let client = reader.u128()?; + let epoch = reader.u64()?; + let user_id = reader.u32()?; + let watermark = reader.u64()?; + let watermark_checksum = reader.u128()?; + let ring_len = reader.u8()?; + if ring_len == 0 { + return Err(ClientTableCodecError::EmptyRing); + } + let mut ring = VecDeque::with_capacity(REPLY_RING_CAPACITY); + for _ in 0..ring_len { + let reply_len = reader.u32()? as usize; + let reply_bytes = reader.take(reply_len)?; + let owned = Owned::::copy_from_slice(reply_bytes); + let message = Message::::try_from(owned) + .map_err(|_| ClientTableCodecError::InvalidReply)? + .try_into_typed::() + .map_err(|_| ClientTableCodecError::InvalidReply)?; + ring.push_back(CachedReply::from_message(message)); + } + table.slots[slot_idx] = Some(ClientEntry { + epoch, + user_id, + watermark, + watermark_checksum, + ring, + }); + table.index.insert(client, slot_idx); + } + if !reader.bytes.is_empty() { + return Err(ClientTableCodecError::Truncated); + } + Ok(table) + } +} + +/// Little-endian cursor over the encoded table content. +struct CodecReader<'a> { + bytes: &'a [u8], +} + +impl<'a> CodecReader<'a> { + const fn take(&mut self, len: usize) -> Result<&'a [u8], ClientTableCodecError> { + if self.bytes.len() < len { + return Err(ClientTableCodecError::Truncated); + } + let (head, tail) = self.bytes.split_at(len); + self.bytes = tail; + Ok(head) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().expect("4B"))) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.take(8)?.try_into().expect("8B"))) + } + + fn u128(&mut self) -> Result { + Ok(u128::from_le_bytes(self.take(16)?.try_into().expect("16B"))) + } +} + impl ClientEntry { /// Latest committed reply (register or app op). /// @@ -603,6 +795,7 @@ mod tests { /// assert on it (see `register_stores_user_id` for the accessor check). const TEST_USER_ID: u32 = 7; + #[allow(clippy::cast_possible_truncation)] fn make_register_reply(client: u128, commit: u64) -> Message { let header_size = std::mem::size_of::(); let mut msg = Message::::new(header_size); @@ -614,6 +807,8 @@ mod tests { client, request: REGISTER_REQUEST_ID, commit, + // Real size so codec-roundtripped replies re-parse. + size: header_size as u32, command: Command2::Reply, operation: Operation::Register, ..ReplyHeader::default() @@ -625,6 +820,7 @@ mod tests { make_reply_with_checksum(client, request, commit, 0) } + #[allow(clippy::cast_possible_truncation)] fn make_reply_with_checksum( client: u128, request: u64, @@ -642,6 +838,8 @@ mod tests { request, commit, request_checksum, + // Real size so codec-roundtripped replies re-parse. + size: header_size as u32, command: Command2::Reply, operation: Operation::SendMessages, ..ReplyHeader::default() @@ -1150,6 +1348,88 @@ mod tests { table.commit_reply(1, epoch, make_reply_for(1, 3, 16)); } + // Codec tests + + // State transfer ships the table as bytes; the decoded table must answer + // every dedup question identically to the original. + #[test] + fn encode_decode_roundtrip_preserves_dedup_state() { + let mut table = ClientTable::new(10); + table.commit_register(1, 11, make_register_reply(1, 10), no_in_flight()); + table.commit_reply(1, 1, make_reply_with_checksum(1, 1, 11, 0xAA)); + table.commit_reply(1, 1, make_reply_for(1, 2, 12)); + // Rebind: epoch 2, watermark preserved. + table.commit_register(1, 22, make_register_reply(1, 20), no_in_flight()); + table.commit_register(2, 33, make_register_reply(2, 30), no_in_flight()); + + let encoded = table.encode(); + let decoded = ClientTable::decode(&encoded, 10).expect("roundtrip decodes"); + + assert_eq!(decoded.count(), 2); + assert_eq!(decoded.get_epoch(1), Some(2)); + assert_eq!(decoded.get_user_id(1), Some(22)); + assert_eq!(decoded.get_watermark(1), Some(2)); + assert_eq!(decoded.get_epoch(2), Some(1)); + match decoded.check_request(1, 2, 2, 0) { + RequestStatus::Duplicate(cached) => { + assert_eq!(cached.header().request, 2, "latest reply survives"); + assert_eq!(cached.header().commit, 12, "original bytes survive"); + } + other => panic!("expected Duplicate, got {other:?}"), + } + match decoded.check_request(1, 2, 1, 0xBB) { + RequestStatus::ChecksumMismatch { request } => { + assert_eq!(request, 1, "watermark checksum survives"); + } + // Request 1 is below the watermark, so the stored checksum only + // fences at the watermark itself; a ring duplicate is also fine. + RequestStatus::Duplicate(cached) => assert_eq!(cached.header().request, 1), + other => panic!("expected duplicate-or-mismatch, got {other:?}"), + } + assert!(matches!( + decoded.check_request(1, 1, 1, 0), + RequestStatus::Fenced { .. } + )); + + // Deterministic bytes: encoding the decoded table reproduces them. + assert_eq!(decoded.encode(), encoded); + } + + #[test] + fn decode_rejects_corruption() { + let mut table = ClientTable::new(4); + table.commit_register(1, TEST_USER_ID, make_register_reply(1, 10), no_in_flight()); + let encoded = table.encode(); + + // Flipped content byte -> checksum mismatch. + let mut flipped = encoded.clone(); + flipped[8] ^= 0xFF; + assert!(matches!( + ClientTable::decode(&flipped, 4), + Err(ClientTableCodecError::ChecksumMismatch { .. }) + )); + + // Truncation. + assert!(matches!( + ClientTable::decode(&encoded[..encoded.len() - 1], 4), + Err(ClientTableCodecError::ChecksumMismatch { .. } | ClientTableCodecError::Truncated) + )); + + // Capacity overflow. + assert!(matches!( + ClientTable::decode(&encoded, 0), + Err(ClientTableCodecError::TooManyEntries { .. }) + )); + + let empty = ClientTable::new(4).encode(); + assert_eq!( + ClientTable::decode(&empty, 4) + .expect("empty table decodes") + .count(), + 0 + ); + } + #[test] fn different_clients_independent_epochs() { let mut table = ClientTable::new(10); diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index 8869f870fb..f103d09e93 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -654,6 +654,47 @@ pub enum Status { Recovering, } +/// State-transfer progress, orthogonal to [`Status`]. +/// +/// A transferring replica stays a normal protocol participant (it adopts +/// views, votes, answers probes) but withholds `PrepareOk` and ignores live +/// prepares (`replicate_preflight` / `send_prepare_ok` gate on +/// [`Consensus::is_transferring`]) -- it must not vouch for state it is +/// about to replace. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StateTransferStage { + /// No transfer in progress. + Idle, + /// Restarted into a cluster: waiting for the view probe to find a live + /// primary to fetch from. The probe-exhausted election fallback proves + /// full-cluster bootstrap instead and abandons the transfer (local + /// recovery is then authoritative). + AwaitingTarget, + /// Target descriptor accepted; pulling artifact chunks. + Fetching, + /// Artifacts verified; installing. + Installing, +} + +impl StateTransferStage { + /// Legal stage transitions; [`VsrConsensus::set_state_transfer_stage`] + /// asserts against this so a mis-sequenced handler fails loudly instead + /// of corrupting the install. + #[must_use] + pub const fn valid_transition(from: Self, to: Self) -> bool { + matches!( + (from, to), + (Self::Idle, Self::AwaitingTarget) + | (Self::AwaitingTarget, Self::Fetching | Self::Idle) + | ( + Self::Fetching, + Self::Installing | Self::AwaitingTarget | Self::Idle + ) + | (Self::Installing, Self::Idle) + ) + } +} + /// What a received `Commit` heartbeat did, so the caller knows whether to /// drain the journal or correct a stale peer. #[derive(Debug, Clone, Copy)] @@ -792,6 +833,11 @@ where /// legitimately (`StartView` adoption, DVC completion). ceded_primaryship: Cell, status: Cell, + /// See [`StateTransferStage`]. Set to `AwaitingTarget` by a cluster + /// restart boot (`begin_state_transfer_await`), driven through + /// `Fetching`/`Installing` by the shard's transfer session, cleared by + /// the probe-exhausted election fallback (full-cluster bootstrap). + state_transfer_stage: Cell, /// Highest op number that has been locally executed (state machine applied, /// client table updated). Advances one-by-one in `commit_journal` (backup) @@ -906,6 +952,7 @@ impl> VsrConsensus { recovery_barrier: Cell::new(0), ceded_primaryship: Cell::new(false), status: Cell::new(Status::Recovering), + state_transfer_stage: Cell::new(StateTransferStage::Idle), sequencer: LocalSequencer::new(0), commit_min: Cell::new(0), commit_max: Cell::new(0), @@ -1437,6 +1484,18 @@ impl> VsrConsensus { let attempts = self.probe_attempts.get() + 1; self.probe_attempts.set(attempts); if attempts >= PROBE_ATTEMPTS_MAX { + // Nobody answered: full-cluster bootstrap, so there + // is no live primary to fetch state from. Local + // recovery is authoritative; abandon the transfer + // and elect on recovered logs. + if self.state_transfer_stage.get() != StateTransferStage::Idle { + tracing::info!( + replica = self.replica, + namespace_raw = self.namespace, + "view probe exhausted; abandoning state transfer (cluster bootstrap)" + ); + self.set_state_transfer_stage(StateTransferStage::Idle); + } self.finish_view_probe(); actions.extend( self.start_election(plane, ViewChangeReason::ViewProbeUnanswered), @@ -2077,6 +2136,40 @@ impl> VsrConsensus { timeouts.start(TimeoutKind::RequestStartViewMessage); } + /// Arm state transfer for a cluster restart: the replica will replace + /// its snapshot-shaped state from the live primary the view probe finds + /// (the shard's transfer session drives the stage from there). If the + /// probe exhausts instead -- full-cluster bootstrap, nobody to fetch + /// from -- the election fallback clears the stage and local recovery + /// stands. + pub fn begin_state_transfer_await(&self) { + self.set_state_transfer_stage(StateTransferStage::AwaitingTarget); + } + + #[must_use] + pub const fn state_transfer_stage(&self) -> StateTransferStage { + self.state_transfer_stage.get() + } + + /// # Panics + /// On an illegal stage transition (see + /// [`StateTransferStage::valid_transition`]). + pub fn set_state_transfer_stage(&self, to: StateTransferStage) { + let from = self.state_transfer_stage.get(); + assert!( + StateTransferStage::valid_transition(from, to), + "state transfer stage transition {from:?} -> {to:?} is illegal" + ); + tracing::info!( + replica = self.replica, + namespace_raw = self.namespace, + ?from, + ?to, + "state transfer stage" + ); + self.state_transfer_stage.set(to); + } + /// Peer side of the probe (sent by a Recovering replica at boot, or by /// a `ViewChange` backup whose copy of the concluding `StartView` was /// lost). Only the current view's PRIMARY answers, with a `StartView`; @@ -2487,9 +2580,9 @@ impl> VsrConsensus { } // Ignore if syncing - if self.is_syncing() { + if self.is_transferring() { return PrepareOkOutcome::Ignored { - reason: IgnoreReason::Syncing, + reason: IgnoreReason::StateTransfer, }; } @@ -2709,9 +2802,8 @@ where self.status() == Status::Normal } - fn is_syncing(&self) -> bool { - // TODO: for now return false. we have to add syncing related setup to VsrConsensus to make this work. - false + fn is_transferring(&self) -> bool { + self.state_transfer_stage.get() != StateTransferStage::Idle } } @@ -3051,3 +3143,82 @@ mod timestamp_clamp_tests { ); } } + +#[cfg(test)] +mod state_transfer_stage_tests { + use super::*; + + #[test] + fn stage_transitions_follow_the_machine() { + use StateTransferStage::{AwaitingTarget, Fetching, Idle, Installing}; + let legal = [ + (Idle, AwaitingTarget), + (AwaitingTarget, Fetching), + (AwaitingTarget, Idle), + (Fetching, Installing), + (Fetching, AwaitingTarget), + (Fetching, Idle), + (Installing, Idle), + ]; + for (from, to) in legal { + assert!( + StateTransferStage::valid_transition(from, to), + "{from:?} -> {to:?} must be legal" + ); + } + let illegal = [ + (Idle, Fetching), + (Idle, Installing), + (AwaitingTarget, Installing), + (Installing, Fetching), + (Installing, AwaitingTarget), + (Idle, Idle), + ]; + for (from, to) in illegal { + assert!( + !StateTransferStage::valid_transition(from, to), + "{from:?} -> {to:?} must be illegal" + ); + } + } + + /// Bus stub: stage plumbing never touches the wire. + struct StageNoopBus; + + impl MessageBus for StageNoopBus { + async fn send_to_client( + &self, + _client_id: u128, + _data: server_common::iobuf::Frozen<{ server_common::MESSAGE_ALIGN }>, + ) -> Result<(), message_bus::SendError> { + Ok(()) + } + + async fn send_to_replica( + &self, + _replica: u8, + _data: server_common::iobuf::Frozen<{ server_common::MESSAGE_ALIGN }>, + ) -> Result<(), message_bus::SendError> { + Ok(()) + } + + fn track_background(&self, _handle: message_bus::JoinHandle<()>) {} + fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {} + fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {} + fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {} + } + + #[test] + fn is_transferring_tracks_stage() { + let consensus = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + assert!(!consensus.is_transferring()); + consensus.begin_state_transfer_await(); + assert!(consensus.is_transferring()); + consensus.set_state_transfer_stage(StateTransferStage::Fetching); + consensus.set_state_transfer_stage(StateTransferStage::Installing); + assert!(consensus.is_transferring()); + consensus.set_state_transfer_stage(StateTransferStage::Idle); + assert!(!consensus.is_transferring()); + } +} diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 9c68f17ac9..03420203c7 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -109,7 +109,7 @@ pub trait Consensus: Sized { fn is_follower(&self) -> bool; fn is_normal(&self) -> bool; - fn is_syncing(&self) -> bool; + fn is_transferring(&self) -> bool; } /// Shared consensus lifecycle interface for control/data planes. diff --git a/core/consensus/src/metadata_helpers.rs b/core/consensus/src/metadata_helpers.rs index 1ffbe86912..6df6f42f80 100644 --- a/core/consensus/src/metadata_helpers.rs +++ b/core/consensus/src/metadata_helpers.rs @@ -103,7 +103,7 @@ where request, is_primary = consensus.is_primary(), is_normal = consensus.is_normal(), - is_syncing = consensus.is_syncing(), + is_transferring = consensus.is_transferring(), commit_min = consensus.commit_min(), commit_max = consensus.commit_max(), "request_preflight: not caught up, not ready" @@ -243,7 +243,7 @@ where client_id, is_primary = consensus.is_primary(), is_normal = consensus.is_normal(), - is_syncing = consensus.is_syncing(), + is_transferring = consensus.is_transferring(), commit_min = consensus.commit_min(), commit_max = consensus.commit_max(), "register_preflight: not caught up, drop" @@ -398,9 +398,10 @@ fn build_eviction_from_header(header: EvictionHeader) -> Message /// `false` -> caller silent-drops; client retry lands on peer or here /// post-catch-up. /// -/// `is_syncing` is currently hardcoded `false` (inert clause); safety rests -/// on `commit_min == commit_max`, correlated (a syncing replica also has -/// `commit_min < commit_max`). Do not weaken commit-equality. +/// `is_transferring` is real (state transfer replaces snapshot-shaped state +/// on a cluster-restart rejoin), but safety still rests on +/// `commit_min == commit_max` -- a transferring replica also fails that. Do +/// not weaken commit-equality. pub fn is_caught_up_primary(consensus: &VsrConsensus) -> bool where B: MessageBus, @@ -409,7 +410,7 @@ where consensus.is_primary() && !consensus.has_ceded_primaryship() && consensus.is_normal() - && !consensus.is_syncing() + && !consensus.is_transferring() && consensus.commit_min() == consensus.commit_max() // Recovery re-pipelines the WAL's prepared-but-uncommitted suffix; // those ops were acked to clients before the restart, so admitting @@ -791,9 +792,8 @@ mod tests { assert!(sends.is_empty(), "silent drop until catch-up"); } - // Direct gate test. is_syncing is hardcoded false today; clause is - // inert. Test exercises primary, normal, commit_min == commit_max. - // Extends naturally when is_syncing becomes real. + // Direct gate test: primary, normal, commit_min == commit_max, and not + // mid-state-transfer. #[test] fn is_caught_up_primary_gate_states() { // Primary, normal, equal commits -> true. @@ -801,10 +801,17 @@ mod tests { primary.init(); assert!(primary.is_primary()); assert!(primary.is_normal()); - assert!(!primary.is_syncing(), "is_syncing inert; pin"); + assert!(!primary.is_transferring()); assert_eq!(primary.commit_min(), primary.commit_max()); assert!(is_caught_up_primary(&primary)); + // Mid-transfer -> false, even with equal commits. + primary.begin_state_transfer_await(); + assert!(primary.is_transferring()); + assert!(!is_caught_up_primary(&primary)); + primary.set_state_transfer_stage(crate::StateTransferStage::Idle); + assert!(is_caught_up_primary(&primary)); + // commit_min < commit_max -> false. primary.advance_commit_max(5); assert_ne!(primary.commit_min(), primary.commit_max()); diff --git a/core/consensus/src/observability.rs b/core/consensus/src/observability.rs index 03a9b1b088..4cd041bc4f 100644 --- a/core/consensus/src/observability.rs +++ b/core/consensus/src/observability.rs @@ -88,7 +88,7 @@ impl ViewChangeReason { pub enum IgnoreReason { NotPrimary, NotNormal, - Syncing, + StateTransfer, NewerView, OlderView, OldPrepare, @@ -106,7 +106,7 @@ impl IgnoreReason { match self { Self::NotPrimary => "not_primary", Self::NotNormal => "not_normal", - Self::Syncing => "syncing", + Self::StateTransfer => "state_transfer", Self::NewerView => "newer_view", Self::OlderView => "older_view", Self::OldPrepare => "old_prepare", diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index e385566a40..c0694bcc4e 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -40,7 +40,10 @@ pub async fn pipeline_prepare_common( { assert!(!consensus.is_follower(), "on_request: primary only"); assert!(consensus.is_normal(), "on_request: status must be normal"); - assert!(!consensus.is_syncing(), "on_request: must not be syncing"); + assert!( + !consensus.is_transferring(), + "on_request: must not be syncing" + ); consensus.verify_pipeline(); consensus.pipeline_message(plane, &prepare); @@ -131,8 +134,8 @@ where { assert_eq!(header.command, Command2::Prepare); - if consensus.is_syncing() { - return Err(IgnoreReason::Syncing); + if consensus.is_transferring() { + return Err(IgnoreReason::StateTransfer); } let current_op = consensus.sequencer().current_sequence(); @@ -540,7 +543,7 @@ pub async fn send_prepare_ok( return; } - if consensus.is_syncing() { + if consensus.is_transferring() { return; } diff --git a/core/integration/src/harness/config/resolve.rs b/core/integration/src/harness/config/resolve.rs index 6a82c29655..99d89f4d30 100644 --- a/core/integration/src/harness/config/resolve.rs +++ b/core/integration/src/harness/config/resolve.rs @@ -52,7 +52,12 @@ pub fn resolve_config_paths( }; let mapping = ServerConfig::find_by_config_path(resolved_path) - .or_else(|| ServerConfig::find_by_config_path(&format!("system.{}", resolved_path))); + .or_else(|| ServerConfig::find_by_config_path(&format!("system.{}", resolved_path))) + // Fields that exist only in the next-gen server's config (e.g. + // `metadata.*`, `cluster.*`, `message_bus.*`). Env names share + // the `IGGY_` prefix, so the resolved variable reaches whichever + // binary the harness spawns; the legacy server ignores unknowns. + .or_else(|| configs::server_ng::ServerNgConfig::find_by_config_path(resolved_path)); match mapping { Some(m) => { diff --git a/core/integration/src/harness/handle/server.rs b/core/integration/src/harness/handle/server.rs index 62913f95a7..0c28f57bbb 100644 --- a/core/integration/src/harness/handle/server.rs +++ b/core/integration/src/harness/handle/server.rs @@ -184,10 +184,18 @@ impl ServerHandle { /// a late joiner that misses early ops. #[must_use] pub fn replica_mesh_complete(&self) -> bool { + self.stdout_contains("replica mesh complete") + } + + /// True once this node's stdout log contains `marker`. Spec tests use + /// this to pin server-side behavior that has no client-visible effect on + /// this node (e.g. a follower completing a state transfer). + #[must_use] + pub fn stdout_contains(&self, marker: &str) -> bool { self.stdout_path .as_ref() .and_then(|path| fs::read_to_string(path).ok()) - .is_some_and(|log| log.contains("replica mesh complete")) + .is_some_and(|log| log.contains(marker)) } /// Returns a `ClientBuilder` using the test transport. diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs index b5cac4c5fb..098f511a85 100644 --- a/core/integration/tests/cluster/client_table_restart.rs +++ b/core/integration/tests/cluster/client_table_restart.rs @@ -150,7 +150,7 @@ async fn given_bound_session_when_node_restarts_should_accept_next_request_id( resume_request(&addrs, session, 2, &create_stream_payload("iggy137-second")).await; } -fn tcp_addr(harness: &TestHarness) -> SocketAddr { +pub(super) fn tcp_addr(harness: &TestHarness) -> SocketAddr { harness .server() .tcp_addr() @@ -161,7 +161,7 @@ fn tcp_addr(harness: &TestHarness) -> SocketAddr { /// way a leader-aware SDK re-routes: after a node restart in a cluster the /// primary may be any survivor, and only the primary commits (or answers /// dedup for) replicated requests. -fn tcp_addrs(harness: &TestHarness) -> Vec { +pub(super) fn tcp_addrs(harness: &TestHarness) -> Vec { (0..harness.cluster_size()) .map(|node| { harness @@ -172,7 +172,7 @@ fn tcp_addrs(harness: &TestHarness) -> Vec { .collect() } -fn create_stream_payload(name: &str) -> Bytes { +pub(super) fn create_stream_payload(name: &str) -> Bytes { CreateStreamRequest { name: WireName::new(name).unwrap(), } @@ -205,7 +205,7 @@ fn request_header( /// so the pre-restart request must reuse the returned stream. Replays on /// transient rejections: right after boot the single node may not have /// elected itself yet. -async fn register(addr: SocketAddr) -> (TcpStream, u64) { +pub(super) async fn register(addr: SocketAddr) -> (TcpStream, u64) { let body = LoginRegisterRequest { version_info: ClientVersionInfo { protocol_version: IGGY_PROTOCOL_VERSION, @@ -239,7 +239,12 @@ async fn register(addr: SocketAddr) -> (TcpStream, u64) { /// Send one replicated metadata request on the registered connection and /// require a committed success within `COMMIT_BUDGET`. -async fn commit_request(stream: &mut TcpStream, session: u64, request: u64, body: &Bytes) { +pub(super) async fn commit_request( + stream: &mut TcpStream, + session: u64, + request: u64, + body: &Bytes, +) { let header = request_header(Operation::CreateStream, session, request, body.len()); let deadline = Instant::now() + COMMIT_BUDGET; loop { @@ -259,7 +264,7 @@ async fn commit_request(stream: &mut TcpStream, session: u64, request: u64, body /// old one died with the node and so an unanswered frame cannot desync the /// next attempt. Panics with the last observed failure mode when the budget /// runs out. -async fn resume_request(addrs: &[SocketAddr], session: u64, request: u64, body: &Bytes) { +pub(super) async fn resume_request(addrs: &[SocketAddr], session: u64, request: u64, body: &Bytes) { let header = request_header(Operation::CreateStream, session, request, body.len()); let deadline = Instant::now() + RESUME_BUDGET; let mut last_failure = "the listener never came back".to_string(); diff --git a/core/integration/tests/cluster/metadata_state_transfer.rs b/core/integration/tests/cluster/metadata_state_transfer.rs new file mode 100644 index 0000000000..67d967eb3d --- /dev/null +++ b/core/integration/tests/cluster/metadata_state_transfer.rs @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Spec test for metadata state transfer: a node that restarts into a live +//! cluster replaces its snapshot-shaped state (metadata snapshot + client +//! table) from the current primary instead of relying on its own WAL. +//! +//! The scenario forces the interesting case: enough committed metadata ops +//! to trip a checkpoint on every node (`journal_slots` shrunk below), which +//! drains the early ops -- including the client's register -- out of every +//! WAL. A restarted node's local recovery can then neither replay those ops +//! nor journal-repair them (the serving peers evicted them too); only the +//! transferred snapshot + table carry that history. +//! +//! The transferred node is a follower afterwards, so its installed state has +//! no client-visible surface to assert against (followers neither commit nor +//! serve resume lookups). The install is pinned via its log marker; the +//! functional assert (post-restart continuation commits cluster-wide) rides +//! on top. + +#![cfg(feature = "vsr")] + +use super::client_table_restart::{ + commit_request, create_stream_payload, register, resume_request, tcp_addr, tcp_addrs, +}; +use integration::iggy_harness; +use std::time::Duration; +use tokio::time::{Instant, sleep}; + +/// Committed ops before the restart. `journal_slots = 256` with the built-in +/// checkpoint margin (64) forces a checkpoint at ~192 committed ops, so this +/// guarantees at least one checkpoint+drain on every node, pushing the +/// register (op 1) below every snapshot floor. +const OPS_BEFORE_RESTART: u64 = 220; + +/// How long the restarted follower gets to probe, fetch, and install. +const TRANSFER_BUDGET: Duration = Duration::from_secs(30); + +const MARKER_POLL: Duration = Duration::from_millis(200); + +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_checkpointed_cluster_when_node_restarts_should_state_transfer_metadata( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + for request in 1..=OPS_BEFORE_RESTART { + commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-transfer-{request}")), + ) + .await; + } + drop(stream); + + harness.restart_server().await.unwrap(); + + // Functional: the session (registered below every snapshot floor by now) + // still continues cluster-wide. + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + OPS_BEFORE_RESTART + 1, + &create_stream_payload("iggy-transfer-continuation"), + ) + .await; + + // The restarted node itself: it must have entered state transfer, + // fetched the primary's snapshot + client table, and installed them. + // Its own WAL no longer holds the pre-checkpoint history, so nothing + // short of the transfer can restore that state on it. + let deadline = Instant::now() + TRANSFER_BUDGET; + loop { + if harness + .node(0) + .stdout_contains("metadata state transfer installed") + { + break; + } + assert!( + Instant::now() < deadline, + "restarted node never completed the metadata state transfer \ + within {TRANSFER_BUDGET:?}" + ); + sleep(MARKER_POLL).await; + } +} diff --git a/core/integration/tests/cluster/mod.rs b/core/integration/tests/cluster/mod.rs index 93c631bd00..ffcf3507c3 100644 --- a/core/integration/tests/cluster/mod.rs +++ b/core/integration/tests/cluster/mod.rs @@ -16,3 +16,4 @@ // under the License. mod client_table_restart; +mod metadata_state_transfer; diff --git a/core/journal/src/lib.rs b/core/journal/src/lib.rs index 629a2e4ec7..691b3e30eb 100644 --- a/core/journal/src/lib.rs +++ b/core/journal/src/lib.rs @@ -58,6 +58,17 @@ where ) -> impl Future>> { async { Ok(Vec::new()) } } + + /// Snapshot watermark: entries at or below it are evictable. `0` for + /// journals without snapshot bookkeeping. + fn snapshot_op(&self) -> u64 { + 0 + } + + /// Advance the snapshot watermark (see [`Self::snapshot_op`]). No-op for + /// journals without snapshot bookkeeping. State transfer uses this to + /// mark pre-transfer residents superseded by the installed snapshot. + fn set_snapshot_op(&self, _op: u64) {} } // TODO: Move to other crate. diff --git a/core/journal/src/prepare_journal.rs b/core/journal/src/prepare_journal.rs index 802b36d819..674b94e209 100644 --- a/core/journal/src/prepare_journal.rs +++ b/core/journal/src/prepare_journal.rs @@ -413,6 +413,11 @@ impl PrepareJournal { self.last_op.get() } + /// Current snapshot watermark: entries at or below it are evictable. + pub const fn snapshot_op(&self) -> u64 { + self.snapshot_op.get() + } + /// Advance the snapshot watermark. The caller must ensure `op` is /// monotonically increasing and corresponds to a durable snapshot. /// @@ -524,6 +529,14 @@ impl Journal for PrepareJournal { type Entry = Message; type HeaderRef<'a> = Ref<'a, PrepareHeader>; + fn snapshot_op(&self) -> u64 { + Self::snapshot_op(self) + } + + fn set_snapshot_op(&self, op: u64) { + Self::set_snapshot_op(self, op); + } + fn header(&self, idx: usize) -> Option> { let headers = self.headers.borrow(); Ref::filter_map(headers, |h| { diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 92ea95c5ed..7fa76a4d1c 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -18,7 +18,9 @@ use crate::MuxStateMachine; use crate::stm::authz::gated_apply; use crate::stm::consumer_group::CompleteConsumerGroupRevocationRequest; -use crate::stm::snapshot::{FillSnapshot, MetadataSnapshot, Snapshot, SnapshotError}; +use crate::stm::snapshot::{ + FillSnapshot, MetadataSnapshot, RestoreSnapshotInPlace, Snapshot, SnapshotError, +}; use crate::stm::stream::{Streams, TruncatePartitionRequest}; use crate::stm::user::{DeletePersonalAccessTokenRequest, Users}; use crate::stm::{ConsensusGroupAllocator, StateMachine}; @@ -243,6 +245,16 @@ impl SnapshotCoordinator { /// /// # Errors /// Returns `SnapshotError` if snapshotting, persistence, or drain fails. + /// On-disk location of the persisted snapshot; also the artifact state + /// transfer serves and installs. + #[must_use] + pub fn snapshot_path(&self) -> std::path::PathBuf { + self.data_dir.join(super::METADATA_DIR).join("snapshot.bin") + } + + /// # Errors + /// Returns `SnapshotError` if snapshotting, persistence, or the journal + /// drain fails. #[allow(clippy::future_not_send)] pub async fn checkpoint( &self, @@ -255,7 +267,7 @@ impl SnapshotCoordinator { J: JournalHandle, { let snapshot = (self.create_snapshot)(stm, last_op, created_at)?; - let path = self.data_dir.join(super::METADATA_DIR).join("snapshot.bin"); + let path = self.snapshot_path(); snapshot.persist(&path)?; let _ = journal @@ -502,6 +514,11 @@ pub struct IggyMetadata { /// set from server config at bootstrap ([`Self::set_default_message_expiry`]). /// Defaults to never-expire, matching the shipped server config. default_message_expiry: Cell, + /// Client-table mutations at or below this op are already reflected in a + /// state-transferred table, so the tail-repair commit walk must skip + /// them (re-running `commit_register` would double-bump epochs). `0` + /// outside state transfer (no op is skipped). Monotone per install. + client_table_frontier: Cell, } impl IggyMetadata @@ -535,6 +552,7 @@ where commit_notifier: RefCell::new(None), default_max_topic_size: Cell::new(u64::MAX), default_message_expiry: Cell::new(u64::MAX), + client_table_frontier: Cell::new(0), } } } @@ -550,11 +568,21 @@ impl IggyMetadata { /// Install the client table rebuilt by WAL-replay recovery /// ([`crate::impls::recovery::recover`]). Boot-time only, on the owning /// shard, before it serves traffic - replacing a live table would drop - /// committed session state. + /// committed session state. (State transfer replaces a LIVE table via + /// [`IggyMetadata::install_state_transfer`], which also stamps the + /// frontier.) pub fn install_client_table(&self, client_table: ClientTable) { *self.client_table.borrow_mut() = client_table; } + /// Client-table mutations at or below the frontier are already in the + /// state-transferred table; the commit walk skips them (re-running + /// `commit_register` would double-bump epochs). STM effects still apply + /// -- the frontier fences the TABLE only. + const fn client_table_mutation_allowed(&self, op: u64) -> bool { + op > self.client_table_frontier.get() + } + /// Install the resolved byte value used for `MaxTopicSize::ServerDefault`. /// Server-ng bootstrap calls this with `system.topic.max_size` on every /// shard (responses read it too); only shard 0's copy feeds admission. @@ -970,6 +998,26 @@ where } } +/// One state-transfer serving payload. +/// +/// The on-disk snapshot bytes plus the live client table, both +/// frontier-stamped. Built by [`IggyMetadata::state_transfer_offer`] on the +/// serving primary; the shard caches it per requester and serves chunks +/// from it. +pub struct StateTransferOffer { + /// Serving primary's applied frontier when the offer was built; the + /// receiver's tail repair targets past this. + pub commit_op: u64, + /// `sequence_number` of the offered snapshot bytes. + pub snapshot_seq: u64, + pub snapshot: Vec, + /// [`ClientTable::encode`] output. + pub table: Vec, + /// Client-table mutations at or below this op are in `table` already; + /// the receiver's commit walk skips them. + pub table_frontier: u64, +} + impl IggyMetadata, J, S, M> where B: MessageBus, @@ -982,6 +1030,123 @@ where Error = iggy_common::IggyError, >, { + /// Build a state-transfer offer for a restarted peer, or `None` when + /// this replica cannot serve one right now: not a caught-up primary + /// (table reads would not be authoritative), or no snapshot has ever + /// been persisted (the WAL then still holds the full history and the + /// requester's journal repair covers the whole gap). + /// + /// The snapshot is served as the on-disk bytes verbatim -- possibly + /// stale, which costs nothing: the receiver journal-repairs + /// `(snapshot_seq, commit_max]` afterwards through the existing repair + /// machinery. The table is encoded live at this instant; both frontier + /// stamps read `commit_min` inside one synchronous region, so they are + /// mutually consistent. + #[must_use] + pub fn state_transfer_offer(&self) -> Option { + let consensus = self.consensus.as_ref()?; + if !is_caught_up_primary(consensus) { + return None; + } + let coordinator = self.coordinator.as_ref()?; + let snapshot = std::fs::read(coordinator.snapshot_path()).ok()?; + let snapshot_seq = IggySnapshot::decode(&snapshot).ok()?.sequence_number(); + let commit_op = consensus.commit_min(); + let table = self.client_table.borrow().encode(); + Some(StateTransferOffer { + commit_op, + snapshot_seq, + snapshot, + table, + table_frontier: commit_op, + }) + } + + /// Install a fetched state transfer: persist + restore the snapshot, + /// replace the client table, and jump the commit state to the snapshot + /// floor so the tail repair takes over from there. + /// + /// Ordering: persist FIRST (a crash mid-install must reboot from the + /// transferred state, not the pre-transfer one), then the in-place STM + /// restore (readers observe it on their next read), then the table + + /// frontier, then journal/commit bookkeeping. + /// + /// The partition plane is deliberately untouched: partitions load + /// whatever their disks hold at boot and repair through their own + /// consensus groups. Topology changes the snapshot carries below the + /// receiver's old frontier (topics created/deleted while it was down) + /// fire no commit notifier -- convergence rests on the partition + /// reconciler's periodic full diff against the committed STM, which + /// reads the restored state on its next tick. + /// + /// Returns the installed snapshot sequence (the receiver's new applied + /// frontier). + /// + /// # Errors + /// [`SnapshotError`] when the snapshot bytes do not decode, the persist + /// fails, or the in-place restore is rejected. + /// + /// # Panics + /// If called on a shard without consensus (state transfer is a shard-0 + /// concern). + pub fn install_state_transfer( + &self, + snapshot_bytes: &[u8], + client_table: ClientTable, + table_frontier: u64, + commit_op: u64, + ) -> Result + where + M: RestoreSnapshotInPlace, + { + let consensus = self + .consensus + .as_ref() + .expect("install_state_transfer: consensus only exists on shard 0"); + + let snapshot = IggySnapshot::decode(snapshot_bytes)?; + let snapshot_seq = snapshot.sequence_number(); + + if let Some(coordinator) = &self.coordinator { + snapshot.persist(&coordinator.snapshot_path())?; + } else { + tracing::warn!( + snapshot_seq, + "installing state transfer without a snapshot coordinator; \ + the transferred state will not survive a further restart" + ); + } + + self.mux_stm + .restore_snapshot_in_place(snapshot.snapshot())?; + + *self.client_table.borrow_mut() = client_table; + self.client_table_frontier.set(table_frontier); + + // Entries at or below the installed floor are superseded by the + // snapshot; without this the journal's wrap-eviction assert trips + // on pre-transfer residents the next time slots recycle. + if let Some(journal) = &self.journal { + let handle = journal.handle(); + if snapshot_seq > handle.snapshot_op() { + handle.set_snapshot_op(snapshot_seq); + } + } + + // The snapshot IS ops `..=snapshot_seq` applied: jump the applied + // frontier (this is the op-jump the tail repair resumes from) and + // let the announced commit point pull the walk target forward. + if snapshot_seq > consensus.commit_min() { + consensus.set_commit_floor(snapshot_seq); + } + consensus.advance_commit_max(commit_op); + if snapshot_seq > consensus.sequencer().current_sequence() { + consensus.sequencer().set_sequence(snapshot_seq); + } + + Ok(snapshot_seq) + } + /// Submit `Register` from in-process, await commit. Wire reply still fires /// via `message_bus.send_to_client`; subscriber is additive. /// @@ -1021,7 +1186,7 @@ where // Wrong node: waiting or queueing cannot fix that, the client must // re-route to the primary. - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { return Err(MetadataSubmitError::NotPrimary); } @@ -1159,7 +1324,7 @@ where // logout-vs-logout race. It simply pipelines behind the in-flight // batch and commits with it, so a one-shot client's session // teardown is latency, never an error. - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { return Err(MetadataSubmitError::NotPrimary); } @@ -1273,7 +1438,7 @@ where // client submits compete for. if !is_caught_up_primary(consensus) { return Err( - if consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing() { + if consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring() { MetadataSubmitError::NotCaughtUp } else { MetadataSubmitError::NotPrimary @@ -1355,7 +1520,7 @@ where // deletion on its next sweep. if !is_caught_up_primary(consensus) { return Err( - if consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing() { + if consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring() { MetadataSubmitError::NotCaughtUp } else { MetadataSubmitError::NotPrimary @@ -1452,7 +1617,7 @@ where // simply pipelines behind the in-flight batch (the wire path has // always done this); the register-specific invariant is guarded in // `submit_register_in_process` / `register_preflight`. - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { return Ok(build_result_rejection_reply( &request_header, consensus.commit_max(), @@ -1644,7 +1809,7 @@ where let Some(consensus) = self.consensus.as_ref() else { return; }; - if !consensus.is_primary() || !consensus.is_normal() || consensus.is_syncing() { + if !consensus.is_primary() || !consensus.is_normal() || consensus.is_transferring() { return; } let Some(journal) = self.journal.as_ref() else { @@ -1812,23 +1977,31 @@ where let reply = if prepare_header.operation == Operation::Register { // Register: commit_register creates session, no SM. let reply = build_reply_message(&prepare_header, &bytes::Bytes::new()); - let in_flight = |c: u128| consensus.pipeline().borrow().has_message_from_client(c); - self.client_table.borrow_mut().commit_register( - prepare_header.client, - prepare_header.user_id, - reply.clone(), - in_flight, - ); + if self.client_table_mutation_allowed(prepare_header.op) { + let in_flight = + |c: u128| consensus.pipeline().borrow().has_message_from_client(c); + self.client_table.borrow_mut().commit_register( + prepare_header.client, + prepare_header.user_id, + reply.clone(), + in_flight, + ); + } reply } else if prepare_header.operation == Operation::Logout { // Logout unregisters the VSR client session on every replica. let reply = build_reply_message(&prepare_header, &bytes::Bytes::new()); - self.client_table - .borrow_mut() - .remove_client(prepare_header.client); + if self.client_table_mutation_allowed(prepare_header.op) { + self.client_table + .borrow_mut() + .remove_client(prepare_header.client); + } // Drop the disconnected client from every consumer group it // joined and rebalance. Deterministic side-effect of the - // Logout commit, applied identically on every replica. + // Logout commit, applied identically on every replica. Runs + // regardless of the table frontier: the STM was restored at + // the snapshot floor, so ops above it still owe their STM + // effects even where the table already reflects them. self.mux_stm.streams().remove_consumer_group_member( prepare_header.client, iggy_common::IggyTimestamp::from(prepare_header.timestamp), @@ -1856,7 +2029,9 @@ where // prepare and commit: skip cache (`commit_reply` no-ops), // wire reply still ships. let epoch = self.client_table.borrow().get_epoch(prepare_header.client); - if let Some(epoch) = epoch { + if !self.client_table_mutation_allowed(prepare_header.op) { + // Already reflected in the state-transferred table. + } else if let Some(epoch) = epoch { self.client_table.borrow_mut().commit_reply( prepare_header.client, epoch, @@ -1949,7 +2124,7 @@ where let Some(consensus) = self.consensus.as_ref() else { return; }; - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { return; } let stranded_commits = consensus.commit_min() < consensus.commit_max(); @@ -2038,7 +2213,10 @@ where // at enqueue time resolves on this prepare's commit. assert!(!consensus.is_follower(), "promotion: primary only"); assert!(consensus.is_normal(), "promotion: status must be normal"); - assert!(!consensus.is_syncing(), "promotion: must not be syncing"); + assert!( + !consensus.is_transferring(), + "promotion: must not be syncing" + ); consensus.verify_pipeline(); match reply_sender { Some(sender) => { @@ -2343,18 +2521,25 @@ where // mutation and counter bump. if header.operation == Operation::Register { // Register: commit_register creates session, no SM. - let reply = build_reply_message(&header, &bytes::Bytes::new()); - let in_flight = |c: u128| consensus.pipeline().borrow().has_message_from_client(c); - self.client_table.borrow_mut().commit_register( - header.client, - header.user_id, - reply, - in_flight, - ); + if self.client_table_mutation_allowed(header.op) { + let reply = build_reply_message(&header, &bytes::Bytes::new()); + let in_flight = + |c: u128| consensus.pipeline().borrow().has_message_from_client(c); + self.client_table.borrow_mut().commit_register( + header.client, + header.user_id, + reply, + in_flight, + ); + } } else if header.operation == Operation::Logout { - self.client_table.borrow_mut().remove_client(header.client); + if self.client_table_mutation_allowed(header.op) { + self.client_table.borrow_mut().remove_client(header.client); + } // Mirror the on_ack path: drop the disconnected client from - // every consumer group it joined and rebalance. + // every consumer group it joined and rebalance. Not fenced by + // the table frontier: the STM was restored at the snapshot + // floor and still owes this effect. self.mux_stm.streams().remove_consumer_group_member( header.client, iggy_common::IggyTimestamp::from(header.timestamp), @@ -2377,7 +2562,9 @@ where // Cache only if session still exists. WAL replay may carry a // reply for a later-evicted client; `commit_reply` no-ops. let epoch = self.client_table.borrow().get_epoch(header.client); - if let Some(epoch) = epoch { + if !self.client_table_mutation_allowed(header.op) { + // Already reflected in the state-transferred table. + } else if let Some(epoch) = epoch { self.client_table .borrow_mut() .commit_reply(header.client, epoch, reply); diff --git a/core/metadata/src/lib.rs b/core/metadata/src/lib.rs index 5aee2f77aa..6c26e0e5e5 100644 --- a/core/metadata/src/lib.rs +++ b/core/metadata/src/lib.rs @@ -22,7 +22,7 @@ pub mod permissioner; pub mod stm; // Re-export IggyMetadata for use in other modules -pub use impls::metadata::{CommitNotifier, IggyMetadata, MetadataSubmitError}; +pub use impls::metadata::{CommitNotifier, IggyMetadata, MetadataSubmitError, StateTransferOffer}; // Re-export MuxStateMachine for use in other modules pub use stm::mux::MuxStateMachine; diff --git a/core/metadata/src/stm/mod.rs b/core/metadata/src/stm/mod.rs index 5fc0c10d1d..0763f44017 100644 --- a/core/metadata/src/stm/mod.rs +++ b/core/metadata/src/stm/mod.rs @@ -396,6 +396,12 @@ macro_rules! collect_handlers { $( $operation([<$operation Request>], ::iggy_common::IggyTimestamp), )* + /// Replace the whole state from a snapshot section, in place. + /// Never parsed off the wire (state transfer installs it via + /// `RestoreSnapshotInPlace`); absorbed on both left-right + /// buffers like any op, which is what makes an in-place + /// restore sound under the double-apply contract. + RestoreSnapshot([<$state Snapshot>]), } impl $crate::stm::Command for [<$state Inner>] { @@ -434,6 +440,10 @@ macro_rules! collect_handlers { $crate::stm::StateHandler::apply(payload, self, *ts) }, )* + [<$state Command>]::RestoreSnapshot(snapshot) => { + *self = Self::inner_from_snapshot(snapshot.clone()); + $crate::stm::result::ApplyReply::default() + }, }); } } diff --git a/core/metadata/src/stm/mux.rs b/core/metadata/src/stm/mux.rs index 864d40e2fa..027b4bcaf7 100644 --- a/core/metadata/src/stm/mux.rs +++ b/core/metadata/src/stm/mux.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::stm::snapshot::{FillSnapshot, RestoreSnapshot, SnapshotError}; +use crate::stm::snapshot::{FillSnapshot, RestoreSnapshot, RestoreSnapshotInPlace, SnapshotError}; use iggy_binary_protocol::PrepareHeader; use iggy_common::Either; use iggy_common::variadic; @@ -188,6 +188,26 @@ where } } +impl RestoreSnapshotInPlace for variadic!(Head, ...Tail) +where + Head: RestoreSnapshotInPlace, + Tail: RestoreSnapshotInPlace, +{ + fn restore_snapshot_in_place(&self, snapshot: &SnapshotData) -> Result<(), SnapshotError> { + self.0.restore_snapshot_in_place(snapshot)?; + self.1.restore_snapshot_in_place(snapshot) + } +} + +impl RestoreSnapshotInPlace for MuxStateMachine +where + T: StateMachine + RestoreSnapshotInPlace, +{ + fn restore_snapshot_in_place(&self, snapshot: &SnapshotData) -> Result<(), SnapshotError> { + self.inner.restore_snapshot_in_place(snapshot) + } +} + impl FillSnapshot for MuxStateMachine where T: StateMachine + FillSnapshot, diff --git a/core/metadata/src/stm/snapshot.rs b/core/metadata/src/stm/snapshot.rs index 8d61cb23e4..72c1086e85 100644 --- a/core/metadata/src/stm/snapshot.rs +++ b/core/metadata/src/stm/snapshot.rs @@ -243,6 +243,27 @@ pub trait RestoreSnapshot: Sized { fn restore_snapshot(snapshot: &S) -> Result; } +/// Restore a LIVE state machine from a snapshot without reconstructing it. +/// +/// [`RestoreSnapshot`] builds a fresh instance -- boot-time only, because a +/// running system shares read handles (left-right factories) across shards +/// that a swap would orphan. This variant replaces the state THROUGH the +/// existing write handle (an absorbed `RestoreSnapshot` command), so every +/// reader observes the restored state on its next read. State transfer +/// installs with this. +#[allow(clippy::missing_errors_doc)] +pub trait RestoreSnapshotInPlace { + /// Replace this state machine's contents from the snapshot. + fn restore_snapshot_in_place(&self, snapshot: &S) -> Result<(), SnapshotError>; +} + +/// Base case for the recursive tuple pattern - unit type terminates the recursion. +impl RestoreSnapshotInPlace for () { + fn restore_snapshot_in_place(&self, _snapshot: &S) -> Result<(), SnapshotError> { + Ok(()) + } +} + /// Base case for the recursive tuple pattern - unit type terminates the recursion. impl FillSnapshot for () { fn fill_snapshot(&self, _snapshot: &mut S) -> Result<(), SnapshotError> { @@ -298,6 +319,32 @@ macro_rules! impl_fill_restore { Self::from_snapshot(snap) } } + + impl $crate::stm::snapshot::RestoreSnapshotInPlace<$crate::stm::snapshot::MetadataSnapshot> + for $wrapper + { + fn restore_snapshot_in_place( + &self, + snapshot: &$crate::stm::snapshot::MetadataSnapshot, + ) -> Result<(), $crate::stm::snapshot::SnapshotError> { + use serde::de::Error as _; + use $crate::stm::snapshot::SnapshotError; + paste::paste! { + let snap = snapshot.$field.clone().ok_or_else(|| { + SnapshotError::Deserialize(rmp_serde::decode::Error::custom( + format_args!("Snapshot Restore Error: {}", stringify!($field)), + )) + })?; + self.inner + .try_apply([<$wrapper Command>]::RestoreSnapshot(snap)) + .map_err(|err| { + SnapshotError::Io(std::io::Error::other(format!( + "in-place restore on a reader-only state machine: {err}" + ))) + }) + } + } + } }; } diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs index 1b5f6b9e63..fcc37b2359 100644 --- a/core/metadata/src/stm/stream.rs +++ b/core/metadata/src/stm/stream.rs @@ -1807,6 +1807,16 @@ impl Snapshotable for Streams { fn from_snapshot( snapshot: Self::Snapshot, ) -> Result { + Ok(StreamsInner::inner_from_snapshot(snapshot).into()) + } +} + +impl StreamsInner { + /// Build a complete `StreamsInner` from a snapshot section. Shared by + /// wrapper construction ([`Snapshotable::from_snapshot`]) and the + /// in-place restore command (state transfer), which absorbs it on both + /// left-right buffers. + pub(crate) fn inner_from_snapshot(snapshot: StreamsSnapshot) -> Self { let mut index: AHashMap, usize> = AHashMap::new(); let mut stream_entries: Vec<(usize, Stream)> = Vec::new(); // Register restored stats in the shared registry so both left-right @@ -1900,7 +1910,7 @@ impl Snapshotable for Streams { } let items: Slab = stream_entries.into_iter().collect(); - let mut inner = StreamsInner { + let mut inner = Self { index, items, revision: snapshot.revision, @@ -1910,7 +1920,7 @@ impl Snapshotable for Streams { stats_registry, }; inner.recompute_pending_revocations_count(); - Ok(inner.into()) + inner } } diff --git a/core/metadata/src/stm/user.rs b/core/metadata/src/stm/user.rs index c8ea95f9db..cb17efa5c3 100644 --- a/core/metadata/src/stm/user.rs +++ b/core/metadata/src/stm/user.rs @@ -806,10 +806,20 @@ impl Snapshotable for Users { }) } - #[allow(clippy::cast_possible_truncation)] fn from_snapshot( snapshot: Self::Snapshot, ) -> Result { + Ok(UsersInner::inner_from_snapshot(snapshot).into()) + } +} + +impl UsersInner { + /// Build a complete `UsersInner` from a snapshot section. Shared by + /// wrapper construction ([`Snapshotable::from_snapshot`]) and the + /// in-place restore command (state transfer), which absorbs it on both + /// left-right buffers. + #[allow(clippy::cast_possible_truncation)] + pub(crate) fn inner_from_snapshot(snapshot: UsersSnapshot) -> Self { let mut index: AHashMap, UserId> = AHashMap::new(); let mut user_entries: Vec<(usize, User)> = Vec::new(); @@ -869,7 +879,7 @@ impl Snapshotable for Users { .init_permissions_for_user(user_id as UserId, user.permissions.as_deref().cloned()); } - let inner = UsersInner { + Self { index, items, personal_access_tokens, @@ -877,8 +887,7 @@ impl Snapshotable for Users { personal_access_token_expiry_index, permissioner, last_result: None, - }; - Ok(inner.into()) + } } } diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 8d2c3a4bfc..3d7137a0c5 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -1242,7 +1242,7 @@ where // replays and its leader recheck re-routes, whereas a panic // kills the shard and a silent drop wedges the client until its // read timeout. - if consensus.is_follower() || !consensus.is_normal() || consensus.is_syncing() { + if consensus.is_follower() || !consensus.is_normal() || consensus.is_transferring() { emit_partition_diag( tracing::Level::WARN, &PartitionDiagEvent::new( @@ -1322,7 +1322,7 @@ where /// dedup. Buffered `SendMessages` retry commits at fresh offset; consumers /// dedup by message key / content / producer-id+seq. /// - /// Per-iteration `is_primary && is_normal && !is_syncing` asserts inlined + /// Per-iteration `is_primary && is_normal && !is_transferring` asserts inlined /// (closure form's `&consensus` borrow conflicts with `&mut self`). Guards /// against view-change-reset flipping status across `on_replicate` await. /// @@ -1350,7 +1350,7 @@ where "drain_request_queue_into_prepares: status must be normal" ); assert!( - !consensus.is_syncing(), + !consensus.is_transferring(), "drain_request_queue_into_prepares: must not be syncing" ); let prepare = req.message.project(consensus); diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs index 14e7f0fe75..202799a60c 100644 --- a/core/partitions/src/iggy_partitions.rs +++ b/core/partitions/src/iggy_partitions.rs @@ -479,7 +479,7 @@ where let consensus = partition.consensus(); let lagging = consensus.is_follower() || !consensus.is_normal() - || consensus.is_syncing() + || consensus.is_transferring() || consensus.commit_min() < consensus.commit_max(); (partition.nth_oldest_sealed_end_offset(count), lagging) }) diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs index 592fbe8de0..07da826fcb 100644 --- a/core/server-ng/src/bootstrap.rs +++ b/core/server-ng/src/bootstrap.rs @@ -1755,6 +1755,12 @@ fn restore_metadata_consensus( if replica_count > 1 && restored_op > 0 { consensus.init_as_backup(); consensus.begin_view_probe(); + // Restart in a cluster: replace snapshot-shaped metadata state + // (snapshot + client table) from the live primary the probe finds, + // then journal-repair the tail. If the probe exhausts instead -- + // full-cluster bootstrap, nobody live to fetch from -- the election + // fallback clears the stage and this local recovery stands. + consensus.begin_state_transfer_await(); } else { consensus.init(); } diff --git a/core/server-ng/src/dispatch.rs b/core/server-ng/src/dispatch.rs index d02a3d1e90..22fb137c7b 100644 --- a/core/server-ng/src/dispatch.rs +++ b/core/server-ng/src/dispatch.rs @@ -339,7 +339,7 @@ fn submit_auto_commit( .partitions() .with_partition(&namespace, |partition| { let consensus = partition.consensus(); - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { AutoCommitGate::NotPrimary } else if partition.is_auto_commit_offset_covered( applied.kind, diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index 04516b22ec..01badfdccf 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs @@ -19,8 +19,9 @@ use crate::iobuf::{Frozen, Owned}; use iggy_binary_protocol::{ Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, - RequestHeader, RequestPreparesHeader, RequestStartViewHeader, StartViewChangeHeader, - StartViewHeader, + RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, + StateTransferTargetHeader, }; use smallvec::SmallVec; use std::{marker::PhantomData, mem::size_of}; @@ -512,6 +513,11 @@ pub enum MessageBag { RepairPrepare(Message), /// `RepairDone` / `RangeEvicted` (one layout, two commands). RepairRangeReply(Message), + RequestStateTransfer(Message), + StateTransferTarget(Message), + RequestStateChunk(Message), + /// Artifact bytes ride the body (`size` spans header + payload). + StateChunk(Message), } impl MessageBag { @@ -529,6 +535,10 @@ impl MessageBag { Self::RequestPrepares(message) => message.header().command, Self::RepairPrepare(message) => message.header().command(), Self::RepairRangeReply(message) => message.header().command, + Self::RequestStateTransfer(message) => message.header().command, + Self::StateTransferTarget(message) => message.header().command, + Self::RequestStateChunk(message) => message.header().command, + Self::StateChunk(message) => message.header().command, } } @@ -546,6 +556,10 @@ impl MessageBag { Self::RequestPrepares(message) => message.header().size(), Self::RepairPrepare(message) => message.header().size(), Self::RepairRangeReply(message) => message.header().size(), + Self::RequestStateTransfer(message) => message.header().size(), + Self::StateTransferTarget(message) => message.header().size(), + Self::RequestStateChunk(message) => message.header().size(), + Self::StateChunk(message) => message.header().size(), } } @@ -563,6 +577,10 @@ impl MessageBag { Self::RequestPrepares(message) => message.header().operation(), Self::RepairPrepare(message) => message.header().operation(), Self::RepairRangeReply(message) => message.header().operation(), + Self::RequestStateTransfer(message) => message.header().operation(), + Self::StateTransferTarget(message) => message.header().operation(), + Self::RequestStateChunk(message) => message.header().operation(), + Self::StateChunk(message) => message.header().operation(), } } } @@ -608,6 +626,18 @@ where Command2::RepairDone | Command2::RangeEvicted => Ok(Self::RepairRangeReply( value.try_into_typed::()?, )), + Command2::RequestStateTransfer => Ok(Self::RequestStateTransfer( + value.try_into_typed::()?, + )), + Command2::StateTransferTarget => Ok(Self::StateTransferTarget( + value.try_into_typed::()?, + )), + Command2::RequestStateChunk => Ok(Self::RequestStateChunk( + value.try_into_typed::()?, + )), + Command2::StateChunk => Ok(Self::StateChunk( + value.try_into_typed::()?, + )), // Reply / Eviction are server-to-client frames; they do not // appear on the inbound dispatch path. Command2::Reply | Command2::Eviction => { diff --git a/core/shard/Cargo.toml b/core/shard/Cargo.toml index 439221c447..459f8b8b6d 100644 --- a/core/shard/Cargo.toml +++ b/core/shard/Cargo.toml @@ -47,6 +47,7 @@ prometheus-client = { workspace = true } server_common = { path = "../server_common" } thiserror = { workspace = true } tracing = { workspace = true } +twox-hash = { workspace = true } [lints.clippy] enum_glob_use = "deny" diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 27b0b2a10f..a9fe3232e1 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -37,7 +37,9 @@ use futures::FutureExt; use iggy_binary_protocol::{ Command2, CommitHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, StartViewChangeHeader, StartViewHeader, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, STATE_TRANSFER_ARTIFACT_SNAPSHOT, STATE_TRANSFER_ARTIFACT_TABLE, + StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, }; #[cfg(any(test, feature = "simulator"))] use iggy_common::PartitionStats; @@ -737,6 +739,61 @@ struct MetadataRepairSession { idle_ticks: u32, } +/// Chunk size for state-transfer artifact pulls. Lockstep (one in flight), +/// so the bounded per-peer bus queue can never drop a burst tail; the bus +/// message cap is far above this. +const STATE_CHUNK_LEN: u32 = 256 * 1024; + +/// The accepted target descriptor of a state transfer, copied out of the +/// `StateTransferTarget` frame. +#[derive(Debug, Clone, Copy)] +struct StateTransferTargetInfo { + commit_op: u64, + snapshot_len: u64, + snapshot_checksum: u64, + table_frontier: u64, + table_len: u64, + table_checksum: u64, +} + +/// One in-flight metadata state transfer (shard 0 only): a cluster-restart +/// rejoin replacing its snapshot-shaped state (metadata snapshot + client +/// table) from the live primary before tail repair. +#[derive(Debug)] +struct MetadataTransferSession { + nonce: u128, + /// Serving primary; also the stall re-request target. + peer: u8, + /// `None` until the `StateTransferTarget` descriptor is accepted. + target: Option, + /// Snapshot bytes received so far (chunks are sequential, so the length + /// doubles as the next request offset). + snapshot: Vec, + /// Encoded client-table bytes received so far. + table: Vec, + /// Ticks with no frame progress; at [`partitions::REPAIR_RETRY_TICKS`] + /// the missing piece is re-requested. + idle_ticks: u32, +} + +/// A cached serving-side state-transfer offer (shard 0 of the serving +/// primary). Keyed by requester replica id so a rebooted requester's fresh +/// nonce replaces the stale offer; chunks must all come from ONE offer or +/// the artifact checksums cannot hold. +struct ServedStateTransfer { + nonce: u128, + offer: metadata::StateTransferOffer, +} + +/// What `on_request_state_chunk` decided inside its offers borrow; the wire +/// sends run after the borrow drops. +enum ChunkReply { + Chunk(Message), + /// Offer evicted (e.g. the serving process restarted): the requester + /// gets an unavailable descriptor and restarts its session. + UnknownOffer, +} + pub struct IggyShard where B: MessageBus, @@ -765,6 +822,14 @@ where /// the full prefix -- so only the stream identity is tracked. metadata_repair: RefCell>, + /// In-flight metadata state transfer (cluster-restart rejoin); tail + /// repair takes over at install. See [`MetadataTransferSession`]. + metadata_transfer: RefCell>, + + /// Serving-side cache of state-transfer offers, keyed by requester + /// replica id. Bounded by the replica count; replaced per fresh nonce. + metadata_transfer_offers: RefCell>, + /// Handler for inbound [`MetadataSubmit`] frames. Only shard 0 receives /// these (it owns the metadata consensus group); peers send them here /// via [`Self::forward_metadata_submit`]. Defaults to a no-op for the @@ -925,6 +990,8 @@ where reconcile_queue: RefCell::new(VecDeque::new()), pending_partition_frames: RefCell::new(HashMap::new()), metadata_repair: RefCell::new(None), + metadata_transfer: RefCell::new(None), + metadata_transfer_offers: RefCell::new(HashMap::new()), }) } @@ -1132,6 +1199,8 @@ where reconcile_queue: RefCell::new(VecDeque::new()), pending_partition_frames: RefCell::new(HashMap::new()), metadata_repair: RefCell::new(None), + metadata_transfer: RefCell::new(None), + metadata_transfer_offers: RefCell::new(HashMap::new()), } } @@ -1391,7 +1460,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { match MessageBag::try_from(message) { Ok(MessageBag::Request(request)) => { @@ -1432,6 +1504,14 @@ where Ok(MessageBag::RequestPrepares(ref msg)) => self.on_request_prepares(msg).await, Ok(MessageBag::RepairPrepare(msg)) => self.on_repair_prepare(msg).await, Ok(MessageBag::RepairRangeReply(ref msg)) => self.on_repair_range_reply(msg).await, + Ok(MessageBag::RequestStateTransfer(ref msg)) => { + self.on_request_state_transfer(msg).await; + } + Ok(MessageBag::StateTransferTarget(ref msg)) => { + self.on_state_transfer_target(msg).await; + } + Ok(MessageBag::RequestStateChunk(ref msg)) => self.on_request_state_chunk(msg).await, + Ok(MessageBag::StateChunk(ref msg)) => self.on_state_chunk(msg).await, Err(e) => { tracing::warn!(shard = self.id, error = %e, "dropping message with invalid command"); } @@ -1519,7 +1599,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { self.plane.on_request(request).await; } @@ -1538,7 +1621,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { self.plane.on_replicate(prepare).await; } @@ -1557,7 +1643,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { self.plane.on_ack(prepare_ok).await; } @@ -1595,7 +1684,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { debug_assert!(buf.is_empty(), "buf must be empty on entry"); debug_assert!( @@ -1837,6 +1929,30 @@ where { let actions = consensus.handle_start_view(PlaneKind::Metadata, &header); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &actions).await; + // State transfer (cluster-restart rejoin): the adopted view names + // a live primary to fetch snapshot-shaped state from. The commit + // walk and journal repair are deferred until the install lands -- + // walking the pre-transfer STM would apply ops the snapshot + // already contains, and the transfer replaces the table anyway. + if consensus.state_transfer_stage() == consensus::StateTransferStage::AwaitingTarget { + let nonce = iggy_common::random_id::get_uuid(); + *self.metadata_transfer.borrow_mut() = Some(MetadataTransferSession { + nonce, + peer: header.replica, + target: None, + snapshot: Vec::new(), + table: Vec::new(), + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + peer = header.replica, + "restart adopted a live view; requesting metadata state transfer" + ); + self.send_request_state_transfer(consensus, header.replica, nonce) + .await; + return; + } // `dispatch_vsr_actions` deliberately no-ops `CommitJournal` (it // needs the plane); without this the ops a StartView marks // committed stay journaled-but-unapplied forever, because the @@ -1852,36 +1968,8 @@ where // cannot reach (StartView carries numbers, not entries): the // walk above gap-stops. Fill the hole through journal repair // from the announcing primary. - if consensus.is_normal() - && consensus.commit_min() < consensus.commit_max() - && self.metadata_repair.borrow().is_none() - { - let nonce = iggy_common::random_id::get_uuid(); - let to_op = consensus.commit_max(); - let from_op = consensus.commit_min() + 1; - *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { - nonce, - to_op, - peer: header.replica, - idle_ticks: 0, - }); - tracing::info!( - shard = self.id, - from_op, - to_op, - "metadata behind after StartView adoption; requesting repair" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - header.replica, - nonce, - from_op, - to_op, - header.namespace, - ) + self.maybe_request_metadata_repair(consensus, header.replica) .await; - } return; } @@ -1963,7 +2051,15 @@ where && consensus.namespace() == header.namespace { match consensus.handle_commit(&header) { - CommitOutcome::Advanced => planes.0.commit_journal().await, + CommitOutcome::Advanced => { + // Mid-transfer the pre-install STM must not walk: the + // snapshot being installed already contains those ops. + // `commit_max` still advanced inside `handle_commit`, so + // the post-install repair targets the right frontier. + if !consensus.is_transferring() { + planes.0.commit_journal().await; + } + } CommitOutcome::RespondStartView => { respond_start_view::(consensus).await; } @@ -2537,6 +2633,636 @@ where .await; } + /// Start metadata tail journal-repair from `peer` when the commit walk + /// gap-stopped below the known frontier. Shared by `StartView` adoption + /// and the post-install step of a state transfer. + #[allow(clippy::future_not_send)] + async fn maybe_request_metadata_repair

(&self, consensus: &VsrConsensus, peer: u8) + where + B: MessageBus, + P: Pipeline, + { + if consensus.is_normal() + && !consensus.is_transferring() + && consensus.commit_min() < consensus.commit_max() + && self.metadata_repair.borrow().is_none() + { + let nonce = iggy_common::random_id::get_uuid(); + let to_op = consensus.commit_max(); + let from_op = consensus.commit_min() + 1; + *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { + nonce, + to_op, + peer, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + from_op, + to_op, + "metadata behind the group frontier; requesting repair" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + from_op, + to_op, + consensus.namespace(), + ) + .await; + } + } + + #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] + async fn send_request_state_transfer

( + &self, + consensus: &VsrConsensus, + target: u8, + nonce: u128, + ) where + B: MessageBus, + P: Pipeline, + { + let msg = + Message::::new(size_of::()) + .transmute_header(|_, h: &mut RequestStateTransferHeader| { + h.command = Command2::RequestStateTransfer; + h.cluster = consensus.cluster(); + h.replica = consensus.replica(); + h.nonce = nonce; + h.namespace = consensus.namespace(); + h.size = size_of::() as u32; + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + /// Answer a `RequestStateTransfer`: `offer = None` sends + /// `available = 0` (the requester falls back to journal repair or + /// retries elsewhere). + #[allow( + clippy::future_not_send, + clippy::cast_possible_truncation, + clippy::too_many_arguments + )] + async fn send_state_transfer_target( + &self, + cluster: u128, + self_id: u8, + target: u8, + nonce: u128, + namespace: u64, + offer: Option<&metadata::StateTransferOffer>, + ) where + B: MessageBus, + { + let msg = Message::::new(size_of::()) + .transmute_header(|_, h: &mut StateTransferTargetHeader| { + h.command = Command2::StateTransferTarget; + h.cluster = cluster; + h.replica = self_id; + h.nonce = nonce; + h.namespace = namespace; + h.size = size_of::() as u32; + if let Some(offer) = offer { + h.available = 1; + h.commit_op = offer.commit_op; + h.snapshot_seq = offer.snapshot_seq; + h.snapshot_len = offer.snapshot.len() as u64; + h.snapshot_checksum = state_artifact_checksum(&offer.snapshot); + h.table_frontier = offer.table_frontier; + h.table_len = offer.table.len() as u64; + h.table_checksum = state_artifact_checksum(&offer.table); + } + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + #[allow( + clippy::future_not_send, + clippy::cast_possible_truncation, + clippy::too_many_arguments + )] + async fn send_request_state_chunk( + &self, + cluster: u128, + self_id: u8, + target: u8, + nonce: u128, + namespace: u64, + artifact: u8, + offset: u64, + len: u32, + ) where + B: MessageBus, + { + let msg = Message::::new(size_of::()) + .transmute_header(|_, h: &mut RequestStateChunkHeader| { + h.command = Command2::RequestStateChunk; + h.cluster = cluster; + h.replica = self_id; + h.nonce = nonce; + h.namespace = namespace; + h.artifact = artifact; + h.offset = offset; + h.len = len; + h.size = size_of::() as u32; + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + /// Serve one `RequestStateTransfer`: build a fresh offer (or refuse), + /// cache it for the chunk pulls, and answer with the descriptor. + #[allow(clippy::future_not_send)] + async fn on_request_state_transfer(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: StreamsFrontend + + StateMachine< + Input = Message, + Output = metadata::stm::result::ApplyReply, + Error = iggy_common::IggyError, + >, + { + let header = *msg.header(); + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + if consensus.namespace() != header.namespace { + return; + } + let offer = planes.0.state_transfer_offer(); + let cluster = consensus.cluster(); + let self_id = consensus.replica(); + if let Some(offer) = offer { + tracing::info!( + shard = self.id, + requester = header.replica, + commit_op = offer.commit_op, + snapshot_seq = offer.snapshot_seq, + snapshot_len = offer.snapshot.len(), + table_len = offer.table.len(), + "serving metadata state transfer" + ); + self.send_state_transfer_target( + cluster, + self_id, + header.replica, + header.nonce, + header.namespace, + Some(&offer), + ) + .await; + self.metadata_transfer_offers.borrow_mut().insert( + header.replica, + ServedStateTransfer { + nonce: header.nonce, + offer, + }, + ); + } else { + tracing::info!( + shard = self.id, + requester = header.replica, + "cannot serve metadata state transfer (not a caught-up \ + primary, or no snapshot persisted); requester falls back" + ); + self.send_state_transfer_target( + cluster, + self_id, + header.replica, + header.nonce, + header.namespace, + None, + ) + .await; + } + } + + /// Receiver side of the descriptor: accept it and start pulling chunks, + /// or fall back to journal repair when the peer cannot serve. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn on_state_transfer_target(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: StreamsFrontend + + StateMachine< + Input = Message, + Output = metadata::stm::result::ApplyReply, + Error = iggy_common::IggyError, + >, + { + /// Alloc cap per artifact: a corrupt length field must not OOM the + /// shard. Far above any real metadata snapshot or client table. + const ARTIFACT_LEN_MAX: u64 = 1 << 30; + + let header = *msg.header(); + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + if consensus.namespace() != header.namespace { + return; + } + let session_matches = self + .metadata_transfer + .borrow() + .as_ref() + .is_some_and(|session| session.nonce == header.nonce); + if !session_matches { + return; + } + + if header.available == 0 { + // The peer cannot serve. If we have never installed anything the + // local recovery stands; run the deferred commit walk and let + // journal repair cover the gap (a peer that never checkpointed + // retains its full WAL, so repair CAN cover it). + tracing::info!( + shard = self.id, + peer = header.replica, + "state transfer unavailable; falling back to journal repair" + ); + *self.metadata_transfer.borrow_mut() = None; + if consensus.state_transfer_stage() != consensus::StateTransferStage::Idle { + consensus.set_state_transfer_stage(consensus::StateTransferStage::Idle); + } + planes.0.commit_journal().await; + self.maybe_request_metadata_repair(consensus, header.replica) + .await; + return; + } + + if header.snapshot_len > ARTIFACT_LEN_MAX || header.table_len > ARTIFACT_LEN_MAX { + tracing::error!( + shard = self.id, + snapshot_len = header.snapshot_len, + table_len = header.table_len, + "state transfer descriptor exceeds artifact cap; ignoring" + ); + return; + } + + { + let mut session = self.metadata_transfer.borrow_mut(); + let Some(session) = session.as_mut() else { + return; + }; + if session.target.is_some() { + // Duplicate descriptor (stall retry crossed the original). + return; + } + session.target = Some(StateTransferTargetInfo { + commit_op: header.commit_op, + snapshot_len: header.snapshot_len, + snapshot_checksum: header.snapshot_checksum, + table_frontier: header.table_frontier, + table_len: header.table_len, + table_checksum: header.table_checksum, + }); + // Under ARTIFACT_LEN_MAX (checked above), so the casts hold. + #[allow(clippy::cast_possible_truncation)] + { + session.snapshot = Vec::with_capacity(header.snapshot_len as usize); + session.table = Vec::with_capacity(header.table_len as usize); + } + session.idle_ticks = 0; + } + if consensus.state_transfer_stage() == consensus::StateTransferStage::AwaitingTarget { + consensus.set_state_transfer_stage(consensus::StateTransferStage::Fetching); + } + tracing::info!( + shard = self.id, + peer = header.replica, + snapshot_seq = header.snapshot_seq, + snapshot_len = header.snapshot_len, + table_len = header.table_len, + commit_op = header.commit_op, + "state transfer target accepted; fetching" + ); + self.request_pending_state_chunk().await; + } + + /// Ask for the next missing chunk of the in-flight transfer (snapshot + /// first, then table). No-op when nothing is missing or no target is + /// accepted yet; also the stall-retry re-request. + #[allow(clippy::future_not_send)] + async fn request_pending_state_chunk(&self) + where + B: MessageBus, + { + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + let request = { + let session = self.metadata_transfer.borrow(); + session.as_ref().and_then(|session| { + let target = session.target.as_ref()?; + let (artifact, offset, remaining) = + if (session.snapshot.len() as u64) < target.snapshot_len { + ( + STATE_TRANSFER_ARTIFACT_SNAPSHOT, + session.snapshot.len() as u64, + target.snapshot_len - session.snapshot.len() as u64, + ) + } else if (session.table.len() as u64) < target.table_len { + ( + STATE_TRANSFER_ARTIFACT_TABLE, + session.table.len() as u64, + target.table_len - session.table.len() as u64, + ) + } else { + return None; + }; + #[allow(clippy::cast_possible_truncation)] + let len = remaining.min(u64::from(STATE_CHUNK_LEN)) as u32; + Some((session.nonce, session.peer, artifact, offset, len)) + }) + }; + if let Some((nonce, peer, artifact, offset, len)) = request { + self.send_request_state_chunk( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + consensus.namespace(), + artifact, + offset, + len, + ) + .await; + } + } + + /// Serve one chunk out of the cached offer. An unknown nonce (offer + /// evicted, e.g. the serving process restarted) answers with an + /// `available = 0` descriptor so the requester restarts its session. + #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] + async fn on_request_state_chunk(&self, msg: &Message) + where + B: MessageBus, + { + let header = *msg.header(); + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + if consensus.namespace() != header.namespace { + return; + } + let cluster = consensus.cluster(); + let self_id = consensus.replica(); + + // Frame built inside the borrow; every send runs after it drops (a + // RefCell borrow must not cross an await on the shard). + // Out-of-bounds requests are dropped silently inside the block. + let reply = { + let offers = self.metadata_transfer_offers.borrow(); + let served = offers + .get(&header.replica) + .filter(|served| served.nonce == header.nonce); + served.map_or(Some(ChunkReply::UnknownOffer), |served| { + let artifact_bytes = if header.artifact == STATE_TRANSFER_ARTIFACT_SNAPSHOT { + &served.offer.snapshot + } else { + &served.offer.table + }; + let start = header.offset as usize; + let end = start.saturating_add(header.len as usize); + artifact_bytes + .get(start..end.min(artifact_bytes.len())) + .map(|payload| { + let total_size = size_of::() + payload.len(); + let mut chunk = Message::::new(total_size); + chunk.as_mut_slice()[size_of::()..] + .copy_from_slice(payload); + ChunkReply::Chunk(chunk.transmute_header(|_, h: &mut StateChunkHeader| { + h.command = Command2::StateChunk; + h.cluster = cluster; + h.replica = self_id; + h.nonce = header.nonce; + h.namespace = header.namespace; + h.artifact = header.artifact; + h.offset = header.offset; + h.size = total_size as u32; + })) + }) + }) + }; + match reply { + Some(ChunkReply::Chunk(chunk)) => { + let _ = self + .bus + .send_to_replica(header.replica, chunk.into_generic().into_frozen()) + .await; + } + Some(ChunkReply::UnknownOffer) => { + tracing::info!( + shard = self.id, + requester = header.replica, + "state chunk request for an unknown offer; telling requester to restart" + ); + self.send_state_transfer_target( + cluster, + self_id, + header.replica, + header.nonce, + header.namespace, + None, + ) + .await; + } + None => { + tracing::warn!( + shard = self.id, + requester = header.replica, + artifact = header.artifact, + offset = header.offset, + "state chunk request out of artifact bounds; ignoring" + ); + } + } + } + + /// Receive one chunk; on the last one, verify + install + hand the tail + /// to journal repair. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn on_state_chunk(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: StreamsFrontend + + StateMachine< + Input = Message, + Output = metadata::stm::result::ApplyReply, + Error = iggy_common::IggyError, + > + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, + { + let header = *msg.header(); + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + if consensus.namespace() != header.namespace { + return; + } + + let complete = { + let mut session = self.metadata_transfer.borrow_mut(); + let Some(session) = session.as_mut() else { + return; + }; + if session.nonce != header.nonce { + return; + } + let Some(target) = session.target else { + return; + }; + let payload = &msg.as_slice()[size_of::()..header.size as usize]; + let (buf, expected_len) = if header.artifact == STATE_TRANSFER_ARTIFACT_SNAPSHOT { + (&mut session.snapshot, target.snapshot_len) + } else { + (&mut session.table, target.table_len) + }; + // Chunks are pulled sequentially with one in flight; anything + // else is a duplicate or reorder and is dropped (the stall retry + // re-requests from the current frontier). + if header.offset != buf.len() as u64 { + return; + } + if buf.len() as u64 + payload.len() as u64 > expected_len { + tracing::warn!( + shard = self.id, + artifact = header.artifact, + "state chunk overruns the declared artifact length; dropping session" + ); + return; + } + buf.extend_from_slice(payload); + session.idle_ticks = 0; + session.snapshot.len() as u64 == target.snapshot_len + && session.table.len() as u64 == target.table_len + }; + + if !complete { + self.request_pending_state_chunk().await; + return; + } + + // All bytes in: verify, decode, install. + let session = self + .metadata_transfer + .borrow_mut() + .take() + .expect("session checked above"); + let target = session.target.expect("target checked above"); + let peer = session.peer; + + let snapshot_ok = state_artifact_checksum(&session.snapshot) == target.snapshot_checksum; + let table_ok = state_artifact_checksum(&session.table) == target.table_checksum; + let table = if snapshot_ok && table_ok { + match consensus::ClientTable::decode(&session.table, consensus::CLIENTS_TABLE_MAX) { + Ok(table) => Some(table), + Err(error) => { + tracing::error!(shard = self.id, %error, "transferred client table undecodable"); + None + } + } + } else { + tracing::error!( + shard = self.id, + snapshot_ok, + table_ok, + "state transfer artifact checksum mismatch" + ); + None + }; + + let Some(table) = table else { + // Damaged in transit: restart the session from scratch against + // the same peer (fresh nonce; the peer re-offers). + if consensus.state_transfer_stage() == consensus::StateTransferStage::Fetching { + consensus.set_state_transfer_stage(consensus::StateTransferStage::AwaitingTarget); + } + let nonce = iggy_common::random_id::get_uuid(); + *self.metadata_transfer.borrow_mut() = Some(MetadataTransferSession { + nonce, + peer, + target: None, + snapshot: Vec::new(), + table: Vec::new(), + idle_ticks: 0, + }); + self.send_request_state_transfer(consensus, peer, nonce) + .await; + return; + }; + + consensus.set_state_transfer_stage(consensus::StateTransferStage::Installing); + match planes.0.install_state_transfer( + &session.snapshot, + table, + target.table_frontier, + target.commit_op, + ) { + Ok(snapshot_seq) => { + consensus.set_state_transfer_stage(consensus::StateTransferStage::Idle); + tracing::info!( + shard = self.id, + snapshot_seq, + commit_op = target.commit_op, + table_frontier = target.table_frontier, + "metadata state transfer installed; handing tail to journal repair" + ); + // Walk whatever is already walkable, then let repair fetch + // the (snapshot_seq, commit_max] tail. + planes.0.commit_journal().await; + self.maybe_request_metadata_repair(consensus, peer).await; + } + Err(error) => { + tracing::error!( + shard = self.id, + %error, + "state transfer install failed; falling back to journal repair" + ); + consensus.set_state_transfer_stage(consensus::StateTransferStage::Idle); + planes.0.commit_journal().await; + self.maybe_request_metadata_repair(consensus, peer).await; + } + } + } + /// Tick partition consensuses. Loop partitions. No partitions-plane journal. #[allow(clippy::future_not_send)] pub async fn tick_partitions(&self) @@ -2674,7 +3400,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { let metadata = self.plane.metadata(); let Some(ref consensus) = metadata.consensus else { @@ -2700,6 +3429,35 @@ where // nothing is stranded. metadata.resume_stranded_commits().await; + // Stall retry for an in-flight state transfer: descriptor or chunk + // frames are fire-and-forget, so a lost one must not wedge the + // session (and the boot flow behind it) forever. + let transfer_stalled = { + let mut session = self.metadata_transfer.borrow_mut(); + session.as_mut().and_then(|session| { + session.idle_ticks += 1; + if session.idle_ticks < partitions::REPAIR_RETRY_TICKS { + return None; + } + session.idle_ticks = 0; + Some((session.peer, session.nonce, session.target.is_some())) + }) + }; + if let Some((peer, nonce, target_accepted)) = transfer_stalled { + tracing::info!( + shard = self.id, + peer, + target_accepted, + "metadata state transfer stalled; re-requesting" + ); + if target_accepted { + self.request_pending_state_chunk().await; + } else { + self.send_request_state_transfer(consensus, peer, nonce) + .await; + } + } + // Stall retry, mirroring `tick_partitions`: a lost repair frame must // not wedge the session forever. let stalled = { @@ -2771,6 +3529,15 @@ where dispatch_vsr_actions::(consensus, None, &[action]).await; } +/// Artifact-level integrity stamp for state transfer (descriptor checksums; +/// chunks themselves carry none). +fn state_artifact_checksum(bytes: &[u8]) -> u64 { + use std::hash::Hasher; + let mut hasher = twox_hash::XxHash3_64::new(); + hasher.write(bytes); + hasher.finish() +} + /// Re-stamp a stored prepare with the current view before retransmission. /// After a view change the primary re-sends its uncommitted suffix as its /// own prepares (VSR), but the journal keeps the original view stamp and diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index dc0a18e7dc..90c35598a3 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs @@ -89,6 +89,22 @@ fn extract_routing(bag: MessageBag) -> (Operation, u64, Message) let h = *m.header(); (h.operation(), h.namespace, m.into_generic()) } + MessageBag::RequestStateTransfer(m) => { + let h = *m.header(); + (h.operation(), h.namespace, m.into_generic()) + } + MessageBag::StateTransferTarget(m) => { + let h = *m.header(); + (h.operation(), h.namespace, m.into_generic()) + } + MessageBag::RequestStateChunk(m) => { + let h = *m.header(); + (h.operation(), h.namespace, m.into_generic()) + } + MessageBag::StateChunk(m) => { + let h = *m.header(); + (h.operation(), h.namespace, m.into_generic()) + } } } @@ -289,7 +305,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { // Reused across every pump iteration; pre-size to skip the // first-drain reallocation. @@ -420,7 +439,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { match frame { ShardFrame::Consensus { message, .. } => { From 71aca79067319a909adc68a76bfee5604256a526 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 28 Jul 2026 10:42:06 +0200 Subject: [PATCH 05/14] temp --- core/common/src/error/iggy_error.rs | 3 + core/configs/src/server_config/sharding.rs | 28 ++++ core/consensus/src/plane_helpers.rs | 33 +++++ core/integration/tests/server/cg_vsr.rs | 85 ++++++++++++ .../tests/server/concurrent_addition.rs | 12 +- core/message_bus/src/transports/quic.rs | 59 +++++++++ core/metadata/src/impls/metadata.rs | 86 +++++++++++++ core/sdk/src/quic/quic_client.rs | 72 +++++++++++ core/sdk/src/tcp/tcp_client.rs | 121 ++++++++++++++++++ core/sdk/src/vsr.rs | 7 + core/sdk/src/websocket/websocket_client.rs | 4 + core/server-ng/src/auth.rs | 21 +++ core/server-ng/src/responses.rs | 40 ++++++ core/shard/src/lib.rs | 7 + vsr-server-test-status.md | 96 ++++++++++++++ 15 files changed, 672 insertions(+), 2 deletions(-) create mode 100644 core/integration/tests/server/cg_vsr.rs create mode 100644 vsr-server-test-status.md diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs index 4d9d8c0bf7..f2b515c947 100644 --- a/core/common/src/error/iggy_error.rs +++ b/core/common/src/error/iggy_error.rs @@ -122,8 +122,11 @@ pub enum IggyError { InvalidPersonalAccessTokenExpiry = 56, #[error("Request transiently not committed; retry")] TransientNotCommitted = 57, +<<<<<<< Updated upstream #[error("Request transiently not accepted; retry, on any replica")] TransientNotAccepted = 58, +======= +>>>>>>> Stashed changes #[error("Not connected")] NotConnected = 61, #[error("Client shutdown")] diff --git a/core/configs/src/server_config/sharding.rs b/core/configs/src/server_config/sharding.rs index 87a411e1fe..edef0f6a3c 100644 --- a/core/configs/src/server_config/sharding.rs +++ b/core/configs/src/server_config/sharding.rs @@ -20,11 +20,39 @@ use serde::{Deserialize, Serialize}; use super::defaults::SERVER_CONFIG; use configs::ConfigEnv; +<<<<<<< Updated upstream // `CpuAllocation`/`NumaConfig` are pure config types and live in their own // leaf crate so both `configs` and `shard_allocator` can share them without // pulling each other's heavier dependency trees. Re-exported here to keep the // `configs::sharding::*` path stable for existing callers. pub use cpu_allocation::{CpuAllocation, NumaConfig}; +======= +/// Default capacity of the per-shard inter-shard inbox channel. Sized +/// comfortably above the consensus working set, which is roughly +/// `PIPELINE_PREPARE_QUEUE_MAX (= 32) * replica_count * directions` +/// frames in flight per shard, without allowing a runaway producer to +/// eat unbounded memory. Tunable via `[system.sharding] inbox_capacity` +/// in TOML. +/// +/// The capacity must also absorb the worst-case cross-shard client +/// Reply burst. Unlike consensus frames, client Replies have no VSR +/// retransmit path: a Reply lost on full inbox is gone and the client +/// times out. A reasonable lower bound is +/// `max_inflight_client_requests / num_shards` (assuming requests are +/// distributed evenly across owning shards) plus the consensus +/// headroom above. +/// +/// Consensus frames and client-reply forwards share this one channel, +/// so the two headrooms are not independent: a consensus burst or +/// retransmit storm can fill the inbox with consensus frames exactly +/// when a client Reply needs the space. A single `inbox_capacity` knob +/// cannot isolate the two frame classes - size it for the sum of both +/// worst cases occurring together. Watch the drop-site `tracing` logs +/// (and, once a per-shard exporter lands, the `frame_drops_total` +/// `{variant="forward_client_send"}` counter) to detect when the bound +/// is too low in production. +pub const DEFAULT_INBOX_CAPACITY: usize = 1024; +>>>>>>> Stashed changes /// Sharding config for the legacy `core/server`. That server consumes only /// `cpu_allocation` and `pin_cores`; the bus / shutdown / reconcile knobs are diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index c0694bcc4e..f261e8a9cf 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -359,6 +359,7 @@ where .expect("reply buffer must contain a valid reply message") } +<<<<<<< Updated upstream /// Builds a `Reply` carrying only a single-entry result section, no payload: /// the generic `[count=1][index=0][result=code]` rejection frame. /// @@ -374,12 +375,31 @@ where /// `commit` is the primary's current commit position and is informational /// here: a rejection reply is sent, never cached in the `ClientTable`, so it /// is not subject to the `commit_reply` regression order. +======= +/// Builds a `Reply` carrying only a metadata result section, no payload. +/// +/// For a request that could not be committed *right now* (not-caught-up / +/// in-flight / pipeline-full / view-change cancel): the body is +/// `[count=1][index=0][result=code]`. The SDK decodes the nonzero `result` to a +/// typed retryable error (`IggyError::TransientNotCommitted`) and replays the +/// same `request_id` immediately, instead of waiting out its response +/// read-timeout. +/// +/// Stamped from the request header (no prepare exists for an uncommitted op). +/// `commit` is the primary's current commit position and is informational here: +/// a transient reply is sent, never cached in the `ClientTable`, so it is not +/// subject to the `commit_reply` regression order. +>>>>>>> Stashed changes /// /// # Panics /// If the constructed message buffer is not a valid reply. #[must_use] #[allow(clippy::cast_possible_truncation)] +<<<<<<< Updated upstream pub fn build_result_rejection_reply( +======= +pub fn build_transient_reply( +>>>>>>> Stashed changes request_header: &RequestHeader, commit: u64, code: u32, @@ -391,7 +411,13 @@ pub fn build_result_rejection_reply( let total_size = header_size + RESULT_BODY_LEN; let mut buffer = bytes::BytesMut::zeroed(total_size); +<<<<<<< Updated upstream let header = ReplyHeader { +======= + let header = bytemuck::checked::try_from_bytes_mut::(&mut buffer[..header_size]) + .expect("zeroed bytes are valid"); + *header = ReplyHeader { +>>>>>>> Stashed changes cluster: request_header.cluster, size: total_size as u32, view: request_header.view, @@ -400,10 +426,14 @@ pub fn build_result_rejection_reply( replica: request_header.replica, request_checksum: request_header.request_checksum, client: request_header.client, +<<<<<<< Updated upstream // Position-typed like the sibling builders (`build_reply_from_request` // stamps `op: commit` too); inert on this path -- rejections are never // cached -- but keeps the wire field convention for frame inspection. op: commit, +======= + op: request_header.session, +>>>>>>> Stashed changes commit, timestamp: request_header.timestamp, request: request_header.request, @@ -411,7 +441,10 @@ pub fn build_result_rejection_reply( namespace: request_header.namespace, ..Default::default() }; +<<<<<<< Updated upstream buffer[..header_size].copy_from_slice(bytemuck::bytes_of(&header)); +======= +>>>>>>> Stashed changes let body = &mut buffer[header_size..]; body[0..4].copy_from_slice(&1u32.to_le_bytes()); diff --git a/core/integration/tests/server/cg_vsr.rs b/core/integration/tests/server/cg_vsr.rs new file mode 100644 index 0000000000..ecf595a897 --- /dev/null +++ b/core/integration/tests/server/cg_vsr.rs @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Consumer-group scenarios that run against server-ng (vsr): round-robin +//! membership (join) plus group polling (single- and multi-client), where the +//! server picks each member's partition by advancing its round-robin cursor. + +use crate::server::scenarios::{ + consumer_group_auto_commit_reconnection_scenario, + consumer_group_duplicate_name_create_scenario, consumer_group_join_scenario, + consumer_group_new_messages_after_restart_scenario, consumer_group_offset_cleanup_scenario, + consumer_group_with_multiple_clients_polling_messages_scenario, + consumer_group_with_single_client_polling_messages_scenario, +}; +use integration::iggy_harness; + +#[iggy_harness( + test_client_transport = [Tcp, WebSocket, Quic], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn duplicate_name_create_preserves_live_group(harness: &TestHarness) { + consumer_group_duplicate_name_create_scenario::run(harness).await; +} + +#[iggy_harness( + test_client_transport = [Tcp, WebSocket, Quic], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn join(harness: &TestHarness) { + consumer_group_join_scenario::run(harness).await; +} + +#[iggy_harness( + test_client_transport = [Tcp, WebSocket, Quic], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn single_client(harness: &TestHarness) { + consumer_group_with_single_client_polling_messages_scenario::run(harness).await; +} + +#[iggy_harness( + test_client_transport = [Tcp, WebSocket, Quic], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn multiple_clients(harness: &TestHarness) { + consumer_group_with_multiple_clients_polling_messages_scenario::run(harness).await; +} + +#[iggy_harness( + test_client_transport = [Tcp, WebSocket, Quic], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn offset_cleanup(harness: &TestHarness) { + consumer_group_offset_cleanup_scenario::run(harness).await; +} + +#[iggy_harness( + test_client_transport = [Tcp, WebSocket, Quic], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn auto_commit_reconnection(harness: &TestHarness) { + consumer_group_auto_commit_reconnection_scenario::run(harness).await; +} + +#[iggy_harness( + test_client_transport = [Tcp, WebSocket, Quic], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn new_messages_after_restart(harness: &TestHarness) { + consumer_group_new_messages_after_restart_scenario::run(harness).await; +} diff --git a/core/integration/tests/server/concurrent_addition.rs b/core/integration/tests/server/concurrent_addition.rs index ff1a4347e2..e590078f1c 100644 --- a/core/integration/tests/server/concurrent_addition.rs +++ b/core/integration/tests/server/concurrent_addition.rs @@ -37,12 +37,19 @@ use test_case::test_matrix; quic.max_idle_timeout = "500s", quic.keep_alive_interval = "15s" ))] -#[test_matrix( +// vsr excludes HTTP only (server-ng exposes no HTTP listener). +#[cfg_attr(not(feature = "vsr"), test_matrix( [tcp(), http(), quic(), websocket()], [user(), stream(), topic(), partition(), consumer_group()], [hot(), cold()], [barrier_on(), barrier_off()] -)] +))] +#[cfg_attr(feature = "vsr", test_matrix( + [tcp(), quic(), websocket()], + [user(), stream(), topic(), partition(), consumer_group()], + [hot(), cold()], + [barrier_on(), barrier_off()] +))] async fn matrix( harness: &TestHarness, transport: TransportProtocol, @@ -57,6 +64,7 @@ fn tcp() -> TransportProtocol { TransportProtocol::Tcp } +#[cfg(not(feature = "vsr"))] fn http() -> TransportProtocol { TransportProtocol::Http } diff --git a/core/message_bus/src/transports/quic.rs b/core/message_bus/src/transports/quic.rs index 8fdf49e830..1feed7924a 100644 --- a/core/message_bus/src/transports/quic.rs +++ b/core/message_bus/src/transports/quic.rs @@ -33,6 +33,7 @@ //! 2. read exactly one inbound frame from the `RecvStream`, note its //! `request` id, and forward it to [`ActorContext::in_tx`]; //! 3. wait on [`ActorContext::rx`] for the reply whose `request` id matches, +<<<<<<< Updated upstream //! discarding any stale/duplicate reply left by an earlier attempt's bidi //! (the SDK replays an explicit transient with the SAME request id); write //! the matched reply, then `finish()` to deliver pending data + FIN; @@ -55,6 +56,26 @@ //! pending reply with it -- nothing can strand or cross request ids. A //! future pipelining upgrade would additionally run per-bidi handlers //! concurrently (and would then require a real per-op correlation id). +======= +//! discarding any stale/duplicate reply left by an abandoned bidi; on +//! [`REPLY_WAIT_TIMEOUT`] abandon this bidi (the dispatch stayed silent on +//! a transient submit, expecting a replay) and accept the next; write the +//! matched reply, then `finish()` to deliver pending data + FIN; +//! 4. drop the bidi pair and loop to the next `accept_bi`. +//! +//! The `accept_bi` loop is still serial (the iggy SDK serialises per +//! connection via an internal `RwLock`, so one bidi at a time is a semantic +//! match), but the reply wait is now `request_id`-KEYED rather than "next +//! frame on `rx`". This matters because the server stays deliberately silent +//! on a transient submit (it expects the SDK to replay): an unkeyed, +//! untimed wait would block this bidi - and, serial, the whole connection - +//! forever, so the replay's fresh bidi could never be accepted (acute at +//! `max_concurrent_bidi_streams=1`). Keying + the timeout let a transient +//! bidi be abandoned safely: the SDK replays with the SAME request id, so a +//! late reply for the abandoned request matches the replay's bidi and is +//! consumed there, never mis-delivered to a later request. A future +//! pipelining upgrade would additionally run per-bidi handlers concurrently. +>>>>>>> Stashed changes //! //! The handshake itself is driven by [`accept_handshake`] (no bidi //! captured). To avoid a single slow / hostile peer wedging the entire @@ -291,6 +312,7 @@ pub(crate) const DEFAULT_CLOSE_GRACE: Duration = Duration::from_secs(2); /// A replicated submit the server cannot yet commit is answered with silence /// (the server expects the SDK to replay), so without a bound the per-bidi /// wait - and, since `accept_bi` is serial, the whole connection - would wedge: +<<<<<<< Updated upstream /// Backstop for a bidi whose dispatch never produced a reply (a true drop: /// both partition queues full, or a pathologically slow commit). Longer than /// the SDK's whole-request deadline (30s), so the client always times out @@ -300,6 +322,14 @@ pub(crate) const DEFAULT_CLOSE_GRACE: Duration = Duration::from_secs(2); /// the next data-plane op and shift every subsequent reply off by one. /// Closing the connection drops the mailbox and any late reply with it. const REPLY_WAIT_BACKSTOP: Duration = Duration::from_mins(1); +======= +/// the SDK's replay opens a fresh bidi that, at `max_concurrent_bidi_streams=1`, +/// can only be accepted once this one is released. Kept below the SDK's +/// per-attempt retry interval so the slot frees before the client replays; a +/// late reply for the abandoned request carries the same request id and is +/// matched (and consumed) by the replay's bidi, so abandoning early is safe. +const REPLY_WAIT_TIMEOUT: Duration = Duration::from_secs(2); +>>>>>>> Stashed changes /// `Command2` byte offset shared by every consensus header (after the /// fixed checksum/cluster/size/view/release prefix). @@ -430,7 +460,11 @@ impl TransportConn for QuicTransportConn { }; // Request id this bidi is waiting on. Used to match its reply and // to discard stale/duplicate replies left by an abandoned bidi. +<<<<<<< Updated upstream // `None` (unparsable) degrades to accepting the first reply. +======= + // `None` (unparseable) degrades to accepting the first reply. +>>>>>>> Stashed changes let request_id = req .as_slice() .get(..HEADER_SIZE) @@ -444,6 +478,7 @@ impl TransportConn for QuicTransportConn { // Wait for THIS bidi's reply, keyed by request id. The bus `rx` is // a single unkeyed per-connection channel, so filter by id and +<<<<<<< Updated upstream // discard replies belonging to an earlier attempt's bidi (an // explicit-transient replay reuses the SAME request id, so a late // reply matches and is consumed by the replay's bidi - never @@ -453,11 +488,24 @@ impl TransportConn for QuicTransportConn { // pathologically slow. `REPLY_WAIT_BACKSTOP` (see its doc) closes // the whole connection instead of abandoning the bidi, so a late // reply can never strand and shift onto the next data-plane op. +======= + // discard replies belonging to an abandoned earlier bidi (the SDK + // replays with the SAME request id, so a late reply matches and is + // consumed by the replay's bidi - never mis-delivered to a later + // request). On timeout the dispatch produced no reply (a transient + // submit: the server stays silent expecting a replay); abandon this + // bidi and accept the next so the replay's fresh stream is not + // blocked behind us (acute at `max_concurrent_bidi_streams=1`). +>>>>>>> Stashed changes let mut matched: Option = None; let mut connection_gone = false; { let mut reply_deadline = +<<<<<<< Updated upstream std::pin::pin!(compio::time::sleep(REPLY_WAIT_BACKSTOP).fuse()); +======= + std::pin::pin!(compio::time::sleep(REPLY_WAIT_TIMEOUT).fuse()); +>>>>>>> Stashed changes loop { futures::select! { () = shutdown_fut.as_mut() => { @@ -466,8 +514,12 @@ impl TransportConn for QuicTransportConn { break; } () = reply_deadline.as_mut() => { +<<<<<<< Updated upstream warn!(%label, %peer, ?request_id, "quic: no reply within backstop; closing connection so no reply can strand"); connection_gone = true; +======= + debug!(%label, %peer, ?request_id, "quic: no reply within timeout; abandoning bidi for client replay"); +>>>>>>> Stashed changes break; } res = rx.recv().fuse() => if let Ok(m) = res { @@ -496,10 +548,17 @@ impl TransportConn for QuicTransportConn { break; } let Some(first) = matched else { +<<<<<<< Updated upstream // Unreachable in practice: every no-reply path above sets // `connection_gone`. Close the send half defensively. if let Err(e) = send.finish() { debug!(%label, %peer, error = ?e, "quic: send.finish() failed with no reply"); +======= + // Timeout: close our send half so the SDK's read ends, then + // accept the next bidi (the client's replay). + if let Err(e) = send.finish() { + debug!(%label, %peer, error = ?e, "quic: send.finish() failed after reply timeout"); +>>>>>>> Stashed changes } continue; }; diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 7fa76a4d1c..159d4be6da 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -29,7 +29,11 @@ use consensus::{ PipelineEntry, Plane, PlaneIdentity, PlaneKind, PreflightOutcome, Project, ReplicaLogContext, RequestLogEvent, Sequencer, SimEventKind, VsrConsensus, ack_preflight, ack_quorum_reached, apply_preflight_consensus_plane, build_eviction_message, build_reply_message, +<<<<<<< Updated upstream build_reply_message_with, build_result_rejection_reply, emit_sim_event, +======= + build_reply_message_with, build_transient_reply, drain_committable_prefix, emit_sim_event, +>>>>>>> Stashed changes fence_old_prepare_by_commit, is_caught_up_primary, panic_if_hash_chain_would_break_in_same_view, peek_committable_head, pipeline_prepare_common, register_preflight, replicate_preflight, replicate_to_next_in_chain, request_preflight, @@ -785,6 +789,7 @@ where // Durable here but not yet committed, and the primary is retransmitting // it: our original PrepareOk was lost (e.g. the primary's inbox // overflowed under a client burst). Re-forward the tail down the chain +<<<<<<< Updated upstream // so a downstream replica that missed it recovers, then re-ack ONLY // the retransmitted op. The primary's retransmit cycle walks every // un-acked op in the window (`retransmit_targets`), so a lost ack for @@ -793,6 +798,16 @@ where // backups, which can overflow the primary's inbox -- the very failure // this path recovers from. Both downstream and primary are idempotent // on a duplicate (replica, op). +======= + // so a downstream replica that missed it recovers, then re-ack every + // op in our uncommitted prepared suffix - not just this one. Acks are + // tracked per-op (see `ack_quorum_reached`), and the primary's commit + // frontier only advances over a contiguous quorum-acked prefix, so a + // lost ack for a lower op keeps that prefix pinned even after we re-ack + // the tail. Re-acking the whole suffix the backup holds restores + // quorum on the gap. Bounded by the pipeline depth; both downstream + // and primary are idempotent on a duplicate (replica, op). +>>>>>>> Stashed changes #[allow(clippy::cast_possible_truncation)] if journal.handle().header(header.op as usize).is_some() { warn!( @@ -803,10 +818,23 @@ where op = header.op, commit = consensus.commit_max(), operation = ?header.operation, +<<<<<<< Updated upstream "journal already holds prepare, re-forwarding + re-acking it" ); self.replicate(&message).await; self.send_prepare_ok(&header).await; +======= + "journal already holds prepare, re-forwarding + re-acking suffix" + ); + self.replicate(&message).await; + for op in (consensus.commit_max() + 1)..=header.op { + if let Some(suffix_header) = + journal.handle().header(op as usize).map(|header| *header) + { + self.send_prepare_ok(&suffix_header).await; + } + } +>>>>>>> Stashed changes return; } @@ -1603,6 +1631,7 @@ where .as_ref() .expect("submit_request_in_process: consensus only exists on shard 0"); +<<<<<<< Updated upstream // Not-primary is transient: the same request replayed against the // current primary commits fine. Reply with the explicit transient // frame (relayed to the socket by the home shard) so the client @@ -1622,6 +1651,19 @@ where &request_header, consensus.commit_max(), IggyError::TransientNotAccepted.as_code(), +======= + // Not-primary / not-caught-up is transient: the same request replayed + // once a primary is caught up commits fine. Reply with the explicit + // `TransientNotCommitted` frame (relayed to the socket by the home shard) + // so the client replays immediately rather than waiting out its + // read-timeout. The frame keeps the lockstep TCP/WS stream in framing + // sync, so the client resends on the same connection (session intact). + if !is_caught_up_primary(consensus) { + return Ok(build_transient_reply( + &request_header, + consensus.commit_max(), + IggyError::TransientNotCommitted.as_code(), +>>>>>>> Stashed changes ) .into_generic()); } @@ -1631,6 +1673,7 @@ where // routing), so a Replay/Evict/NotReady is returned to the home shard as // the reply -- `handle_client_request` writes it to the originating // socket by transport id, exactly like a fresh commit. Drop (client-bug +<<<<<<< Updated upstream // already-applied / future-epoch) surfaces as Canceled so the home // shard stays silent. match request_preflight( @@ -1641,6 +1684,10 @@ where request, request_checksum, ) { +======= + // stale/gap) surfaces as Canceled so the home shard stays silent. + match request_preflight(consensus, &self.client_table, client_id, session, request) { +>>>>>>> Stashed changes PreflightOutcome::Dispatch => {} PreflightOutcome::Replay(reply) => { return server_common::Message::::try_from( @@ -1657,7 +1704,11 @@ where // In-flight prepare from this client: replaying the same request_id // is absorbed until the original commits, then served from cache. PreflightOutcome::NotReady => { +<<<<<<< Updated upstream return Ok(build_result_rejection_reply( +======= + return Ok(build_transient_reply( +>>>>>>> Stashed changes &request_header, consensus.commit_max(), IggyError::TransientNotCommitted.as_code(), @@ -1667,6 +1718,7 @@ where PreflightOutcome::Drop => return Err(MetadataSubmitError::Canceled), } +<<<<<<< Updated upstream // Prepare queue full: backpressure, not failure. Absorb into the // request queue with this caller's reply subscriber; the commit // path promotes it as slots free up and the await below resolves @@ -1699,6 +1751,16 @@ where ) .into_generic()), }; +======= + // Pipeline full: backpressure, not failure. Tell the client to replay. + if consensus.pipeline().borrow().is_full() { + return Ok(build_transient_reply( + &request_header, + consensus.commit_max(), + IggyError::TransientNotCommitted.as_code(), + ) + .into_generic()); +>>>>>>> Stashed changes } // The acting-user RBAC stamp lives in the shared `prepare_request` @@ -1719,7 +1781,11 @@ where // so reply with the transient frame rather than staying silent. match self.dispatch_prepare_and_await(consensus, prepare).await { Ok(reply) => Ok(reply.into_generic()), +<<<<<<< Updated upstream Err(Canceled) => Ok(build_result_rejection_reply( +======= + Err(Canceled) => Ok(build_transient_reply( +>>>>>>> Stashed changes &request_header, consensus.commit_max(), IggyError::TransientNotCommitted.as_code(), @@ -1809,7 +1875,11 @@ where let Some(consensus) = self.consensus.as_ref() else { return; }; +<<<<<<< Updated upstream if !consensus.is_primary() || !consensus.is_normal() || consensus.is_transferring() { +======= + if !consensus.is_primary() || !consensus.is_normal() || consensus.is_syncing() { +>>>>>>> Stashed changes return; } let Some(journal) = self.journal.as_ref() else { @@ -1842,6 +1912,7 @@ where return; } +<<<<<<< Updated upstream // Interleave push + drain per header instead of push-all-then-drain-once: // each `on_ack` below can promote a full window of buffered requests // (`drain_request_queue_into_prepares`), and every promoted prepare @@ -1874,6 +1945,14 @@ where return false; } for message in loopback.drain(..) { +======= + for header in &headers { + self.send_prepare_ok(header).await; + } + let mut loopback = Vec::new(); + consensus.drain_loopback_into(&mut loopback); + for message in loopback { +>>>>>>> Stashed changes match message.header().command { Command2::PrepareOk => match message.try_into_typed::() { Ok(prepare_ok) => self.on_ack(prepare_ok).await, @@ -1888,6 +1967,7 @@ where ), } } +<<<<<<< Updated upstream true } @@ -2140,6 +2220,12 @@ where /// Promote buffered requests into free prepare slots after a commit /// batch drains. +======= + } + + /// Promote up to `slots_freed` buffered requests into prepares after + /// `on_ack` commits a prefix. +>>>>>>> Stashed changes /// /// # Safety /// Re-preflight per iteration: `commit_journal` may have advanced the diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs index 9a731f6082..6086e52346 100644 --- a/core/sdk/src/quic/quic_client.rs +++ b/core/sdk/src/quic/quic_client.rs @@ -63,6 +63,17 @@ const NAME: &str = "Iggy"; /// under lock per send, every later request on the client. const RESPONSE_READ_TIMEOUT: Duration = Duration::from_secs(30); +<<<<<<< Updated upstream +======= +/// Per-attempt response read timeout for the same-connection transient resend +/// loop (see `send_raw`). A replicated submit the server can't yet commit is +/// answered with silence; rather than wait the full `RESPONSE_READ_TIMEOUT` +/// once, abandon the unanswered bidi after this shorter interval and resend the +/// same request on a fresh stream. Bounded overall by `RESPONSE_READ_TIMEOUT`. +#[cfg(feature = "vsr")] +const TRANSIENT_RETRY_INTERVAL: Duration = Duration::from_secs(3); + +>>>>>>> Stashed changes /// Backoff before replaying a request the server answered with an explicit /// `TransientNotCommitted` frame (not-caught-up / in-flight / pipeline-full / /// view-change cancel). Unlike a silent timeout, this reply arrives promptly, so @@ -177,6 +188,13 @@ impl BinaryTransport for QuicClient { #[cfg(feature = "vsr")] if skip_auto_login { *self.skip_auto_login_once.lock().await = true; + // The replayed login/register must mint a fresh Register: the failed + // attempt already consumed the one-shot register request id and may + // have half-bound the session. + *self + .consensus_session + .lock() + .expect("consensus session mutex poisoned") = ConsensusSession::new(); } let server_address = self.current_server_address.lock().await.to_string(); info!( @@ -723,6 +741,7 @@ impl QuicClient { crate::vsr::encode_request_header(&mut consensus_session, code, &payload)? }; trace!("Sending a QUIC VSR request of size {request_size} with code: {code}"); +<<<<<<< Updated upstream // Same-connection transient resend, gated on the EXPLICIT // `TransientNotCommitted` frame only. The server answers every // transient submit with that frame (it is pre-commit by @@ -736,6 +755,20 @@ impl QuicClient { // back as terminal `ConsumerOffsetNotFound`). A silent deadline // expiry surfaces `Disconnected` and takes the // reconnect path in `send_raw_with_response`, same as TCP. +======= + // Same-connection transient resend. A replicated submit the + // server cannot yet commit is answered with silence - it expects + // the client to replay. The connection and login session are + // intact; only this bidi went unanswered. So abandon it after a + // short read timeout and resend the SAME request header (same + // request id, so the server dedups/commits it idempotently) on a + // fresh bidi, bounded overall by `RESPONSE_READ_TIMEOUT`. This + // needs no reconnect or relogin, so it recovers transients even + // with auto-login disabled. Login/register is one-shot (a replay + // must mint a fresh session), so it is left to the reconnect path + // in `send_raw_with_response` and not resent here. + let allow_resend = !is_login_register_code(code); +>>>>>>> Stashed changes let header_bytes = bytemuck::bytes_of(&request_header); let deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT; loop { @@ -762,14 +795,23 @@ impl QuicClient { if remaining.is_zero() { return Err(IggyError::Disconnected); } +<<<<<<< Updated upstream match QuicClient::handle_response( &mut recv, response_buffer_size as usize, remaining, +======= + let attempt_timeout = remaining.min(TRANSIENT_RETRY_INTERVAL); + match QuicClient::handle_response( + &mut recv, + response_buffer_size as usize, + attempt_timeout, +>>>>>>> Stashed changes ) .await { Ok(reply) => return Ok(reply), +<<<<<<< Updated upstream // `TransientNotCommitted` = the server replied with an // explicit retry frame because it could not commit yet // (not-caught-up / in-flight / pipeline-full / @@ -787,6 +829,36 @@ impl QuicClient { let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); tokio::time::sleep(NOT_READY_RETRY_INTERVAL.min(remaining)).await; +======= + // `Disconnected` = our read timed out (server never + // answered). `EmptyResponse` = the server abandoned the + // bidi (finished it with no body). `TransientNotCommitted` + // = the server replied with an explicit retry frame + // because it could not commit yet (not-caught-up / + // in-flight / pipeline-full / view-change cancel). All + // three mean "resend the same request on a fresh bidi"; + // the session stays intact so no reconnect/relogin needed. + // Login/register resends on `TransientNotCommitted` too + // (nothing committed -> the same Register replays + // idempotently, no fresh session needed); for the silent + // paths login still falls through to the reconnect+relogin + // path in `send_raw_with_response`. + Err( + error @ (IggyError::Disconnected + | IggyError::EmptyResponse + | IggyError::TransientNotCommitted), + ) if (allow_resend || error == IggyError::TransientNotCommitted) + && tokio::time::Instant::now() < deadline => + { + // The explicit frame returns promptly (no read + // timeout elapsed), so pace the replay; the silent + // paths already waited out `attempt_timeout`. + if error == IggyError::TransientNotCommitted { + let remaining = deadline + .saturating_duration_since(tokio::time::Instant::now()); + tokio::time::sleep(NOT_READY_RETRY_INTERVAL.min(remaining)).await; + } +>>>>>>> Stashed changes warn!( "QUIC request code {code} not committed (transient); resending on a new stream" ); diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index f827b9f55b..c21e2f04cd 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -72,6 +72,7 @@ const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_sec #[cfg(feature = "vsr")] const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); +<<<<<<< Updated upstream /// How long a request replays `TransientNotCommitted` on the SAME connection /// before the client re-checks cluster leadership. A node that stopped being /// primary (view change while the connection stayed up) answers transient @@ -81,6 +82,8 @@ const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_ #[cfg(feature = "vsr")] const TRANSIENT_FAILOVER_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); +======= +>>>>>>> Stashed changes /// TCP client for interacting with the Iggy API. /// It requires a valid server address. #[derive(Debug)] @@ -726,6 +729,7 @@ impl TcpClient { } #[cfg(feature = "vsr")] +<<<<<<< Updated upstream { // One overall deadline bounds the request across transient replays // AND leader failovers, matching the previous single-connection @@ -782,10 +786,123 @@ impl TcpClient { return Err(IggyError::Disconnected); } other => return other, +======= + let consensus_session = self.consensus_session.clone(); + // SAFETY: we run code holding the `stream` lock in a task so we can't be cancelled while holding the lock. + let result = tokio::spawn(async move { + let mut stream = stream.lock().await; + if let Some(stream) = stream.as_mut() { + #[cfg(feature = "vsr")] + { + // Encode the request header ONCE: `next_request_id` advances + // here, so a transient replay must reuse the same id for the + // server's dedup. The connection is lockstep (one request in + // flight per client), so a complete reply leaves the stream at + // a clean frame boundary -- a `TransientNotCommitted` answer + // (the server could not commit yet: not-caught-up / in-flight + // / pipeline-full / view-change cancel) lets us resend the + // SAME request on the SAME connection with no reconnect and + // the session intact. Bounded overall by RESPONSE_READ_TIMEOUT. + let (request_header, request_size) = { + let mut consensus_session = consensus_session + .lock() + .expect("consensus session mutex poisoned"); + crate::vsr::encode_request_header(&mut consensus_session, code, &payload)? + }; + trace!("Sending a TCP VSR request of size {request_size} with code: {code}"); + let header_bytes = bytemuck::bytes_of(&request_header); + // One deadline bounds the whole request including transient + // replays, so a slow primary cannot stretch a single call + // beyond RESPONSE_READ_TIMEOUT. + let retry_deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT; + loop { + stream.write(header_bytes).await?; + if !payload.is_empty() { + stream.write(&payload).await?; + } + stream.flush().await?; + trace!("Sent a TCP request with code: {code}, waiting for a response..."); + + let mut response_header = [0u8; iggy_binary_protocol::HEADER_SIZE]; + // `stream.read` delegates to `read_exact`; on success it + // always returns the requested length, so no short-read + // guard is needed here. + // + // Deadline guards against server-side reply loss (e.g. a + // stalled replication quorum that never commits the op): + // the connection is lockstep, so an unanswered read would + // hold the stream lock forever and wedge every later + // request on this client. On expiry drop the stream -- + // a late reply would desync framing for the next request. + // + // One deadline spans BOTH the header and body reads: a + // reply that delivers a header then stalls must not get a + // fresh full timeout for the body. + let header_read = tokio::time::timeout_at( + retry_deadline, + stream.read(&mut response_header), + ) + .await; + let Ok(header_read) = header_read else { + error!( + "Timed out after {RESPONSE_READ_TIMEOUT:?} waiting for VSR response header for TCP request with code: {code}", + ); + return Err(IggyError::Disconnected); + }; + header_read.map_err(|error| { + error!( + "Failed to read VSR response header for TCP request with code: {code}: {error}", + ); + IggyError::Disconnected + })?; + + let response_size = crate::vsr::response_size(&response_header)?; + let body_size = response_size - iggy_binary_protocol::HEADER_SIZE; + let body = if body_size > 0 { + let mut body = BytesMut::with_capacity(body_size); + let body_read = tokio::time::timeout_at( + retry_deadline, + stream.read_buf(&mut body, body_size), + ) + .await; + let Ok(body_read) = body_read else { + error!( + "Timed out after {RESPONSE_READ_TIMEOUT:?} waiting for VSR response body for TCP request with code: {code}", + ); + return Err(IggyError::Disconnected); + }; + body_read.map_err(|error| { + error!( + "Failed to read VSR response body for TCP request with code: {code}: {error}", + ); + IggyError::Disconnected + })?; + body.freeze() + } else { + Bytes::new() + }; + + match crate::vsr::decode_response_split(&response_header, body) { + // Server could not commit yet but answered with a + // complete frame; the lockstep stream is in sync, so + // replay the same request id on this connection after + // a short pause (no reconnect, session intact). + Err(IggyError::TransientNotCommitted) + if tokio::time::Instant::now() < retry_deadline => + { + let remaining = retry_deadline + .saturating_duration_since(tokio::time::Instant::now()); + tokio::time::sleep(NOT_READY_RETRY_INTERVAL.min(remaining)).await; + } + other => return other, + } + } +>>>>>>> Stashed changes } } } +<<<<<<< Updated upstream #[cfg(not(feature = "vsr"))] { let stream = self.stream.clone(); @@ -793,6 +910,10 @@ impl TcpClient { let result = tokio::spawn(async move { let mut stream = stream.lock().await; if let Some(stream) = stream.as_mut() { +======= + #[cfg(not(feature = "vsr"))] + { +>>>>>>> Stashed changes let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; trace!("Sending a TCP request of size {payload_length} with code: {code}"); stream.write(&(payload_length as u32).to_le_bytes()).await?; diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index 671fbd5e9b..9451b9c093 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs @@ -274,12 +274,19 @@ fn split_metadata_result(operation: Operation, body: Bytes) -> Result>>>>>> Stashed changes if !result_framed { return Ok(body); } diff --git a/core/sdk/src/websocket/websocket_client.rs b/core/sdk/src/websocket/websocket_client.rs index 537d68de22..5a8912c6f1 100644 --- a/core/sdk/src/websocket/websocket_client.rs +++ b/core/sdk/src/websocket/websocket_client.rs @@ -774,7 +774,11 @@ impl WebSocketClient { // Server could not commit yet but answered with a complete // frame; the lockstep stream is in sync, so replay the same // request id on this connection after a short pause. +<<<<<<< Updated upstream Err(IggyError::TransientNotCommitted | IggyError::TransientNotAccepted) +======= + Err(IggyError::TransientNotCommitted) +>>>>>>> Stashed changes if tokio::time::Instant::now() < retry_deadline => { let remaining = diff --git a/core/server-ng/src/auth.rs b/core/server-ng/src/auth.rs index 56f4cbea83..027e1b4e2f 100644 --- a/core/server-ng/src/auth.rs +++ b/core/server-ng/src/auth.rs @@ -25,6 +25,7 @@ use crate::bootstrap::{ShellBus, ShellShard}; use crate::dispatch::submit_register_on_owner; use crate::login_register::LoginRegisterError; use crate::responses::{build_empty_reply, build_login_register_reply, current_metadata_commit}; +<<<<<<< Updated upstream use crate::session_manager::{ClientSdkInfo, SessionManager}; use consensus::{MetadataHandle, build_result_rejection_reply}; use iggy_binary_protocol::PrepareHeader; @@ -34,6 +35,13 @@ use iggy_common::defaults::{ }; use iggy_common::{IggyError, IggyTimestamp, PersonalAccessToken, UserStatus}; use journal::{Journal, JournalHandle}; +======= +use crate::session_manager::SessionManager; +use consensus::{MetadataHandle, build_transient_reply}; +use iggy_binary_protocol::RequestHeader; +use iggy_common::{IggyError, IggyTimestamp, PersonalAccessToken, UserStatus}; +use message_bus::MessageBus; +>>>>>>> Stashed changes use metadata::impls::metadata::StreamsFrontend; use server_common::Message; use server_common::crypto; @@ -306,6 +314,7 @@ pub(crate) async fn surface_login_failure( /// same login on the same connection. Only call for transient errors -- see /// [`surface_login_failure`]. #[allow(clippy::future_not_send)] +<<<<<<< Updated upstream async fn send_login_transient_reply( shard: &Rc>, transport_client_id: u128, @@ -324,6 +333,18 @@ async fn send_login_transient_reply( request_header, commit, IggyError::TransientNotAccepted.as_code(), +======= +async fn send_login_transient_reply( + shard: &Rc, + transport_client_id: u128, + request_header: &RequestHeader, +) { + let commit = current_metadata_commit(shard); + let reply = build_transient_reply( + request_header, + commit, + IggyError::TransientNotCommitted.as_code(), +>>>>>>> Stashed changes ); if let Err(error) = shard .bus diff --git a/core/server-ng/src/responses.rs b/core/server-ng/src/responses.rs index 9cf3378ed6..e11d6af546 100644 --- a/core/server-ng/src/responses.rs +++ b/core/server-ng/src/responses.rs @@ -1275,6 +1275,7 @@ pub(crate) fn build_login_register_reply( user_id: u32, ) -> Message { // Result-framed like every metadata reply: a zero result-count (success) +<<<<<<< Updated upstream // followed by the `LoginRegisterResponse` payload. A transient Register // instead ships a `[count=1][index=0][TransientNotCommitted]` frame // (`build_transient_reply`), which the SDK decodes and replays. The matching @@ -1290,11 +1291,28 @@ pub(crate) fn build_login_register_reply( body.extend_from_slice(&[0u8; RESULT_COUNT_LEN]); body.extend_from_slice(&payload); build_reply_from_bytes( +======= + // followed by the `[user_id][session]` payload. A transient Register instead + // ships a `[count=1][index=0][TransientNotCommitted]` frame + // (`build_transient_reply`), which the SDK decodes and replays. The matching + // strip is in the SDK `split_metadata_result` (Register is result-framed). + build_reply_with_body( +>>>>>>> Stashed changes request_header, client_id, session, commit, +<<<<<<< Updated upstream &Bytes::from(body), +======= + RESULT_COUNT_LEN + 12, + |body| { + body[..RESULT_COUNT_LEN].copy_from_slice(&0u32.to_le_bytes()); + body[RESULT_COUNT_LEN..RESULT_COUNT_LEN + 4].copy_from_slice(&user_id.to_le_bytes()); + body[RESULT_COUNT_LEN + 4..RESULT_COUNT_LEN + 12] + .copy_from_slice(&session.to_le_bytes()); + }, +>>>>>>> Stashed changes ) } @@ -1340,6 +1358,14 @@ pub(crate) fn build_raw_pat_reply( return Ok(committed); } let header_len = std::mem::size_of::(); + // A `Reply` whose result section is nonzero is not a successful commit but a + // committed business rejection or a `TransientNotCommitted` retry frame, both + // with no payload and no token to ship. Pass it through so the client decodes + // the typed result (and, for a transient, replays) instead of having a raw + // token grafted onto a rejection body. + if result_code(&committed.as_slice()[header_len..]) != Some(0) { + return Ok(committed); + } let committed_header = bytemuck::checked::try_from_bytes::(&committed.as_slice()[..header_len]) .map_err(|_| IggyError::InvalidFormat)?; @@ -1359,6 +1385,7 @@ pub(crate) fn build_raw_pat_reply( return Ok(committed); } let token = WireName::new(raw.as_str()).map_err(|_| IggyError::InvalidFormat)?; +<<<<<<< Updated upstream let response = RawPersonalAccessTokenResponse { token }; // The SDK strips a leading result section from every metadata reply // (`split_metadata_result`), so the spliced body must carry the success @@ -1367,6 +1394,19 @@ pub(crate) fn build_raw_pat_reply( let mut body = BytesMut::with_capacity(RESULT_COUNT_LEN + response.encoded_size()); body.put_u32_le(0); response.encode(&mut body); +======= + let token_payload = RawPersonalAccessTokenResponse { token }.to_bytes(); + // Metadata reply bodies are framed `[result_count:u32][payload]`; a success + // is `count == 0` followed by the payload (see `ApplyReply::write_reply_body` + // / `result_code`). The committed reply carried an empty payload with this + // same framing, so reproduce it here - prepend the zero result-count rather + // than shipping a bare `RawPersonalAccessTokenResponse`, which the client + // would misread as the result-count and reject as a bogus error. + let mut framed = Vec::with_capacity(RESULT_COUNT_LEN + token_payload.len()); + framed.extend_from_slice(&[0u8; RESULT_COUNT_LEN]); + framed.extend_from_slice(&token_payload); + let body = Bytes::from(framed); +>>>>>>> Stashed changes let reply = build_reply_from_bytes( request_header, request_header.client, diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index a9fe3232e1..dd10aa793e 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -3400,10 +3400,14 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, +<<<<<<< Updated upstream > + StreamsFrontend + metadata::stm::snapshot::RestoreSnapshotInPlace< metadata::stm::snapshot::MetadataSnapshot, >, +======= + > + StreamsFrontend, +>>>>>>> Stashed changes { let metadata = self.plane.metadata(); let Some(ref consensus) = metadata.consensus else { @@ -3420,6 +3424,7 @@ where // forever (commit_min stuck below commit_max). See // `IggyMetadata::repair_primary_self_acks`. metadata.repair_primary_self_acks().await; +<<<<<<< Updated upstream // Backstop for commit work stranded by a canceled `on_ack` driver // (a future dropped at its journal-read or wire-reply await): no @@ -3496,6 +3501,8 @@ where .await; } } +======= +>>>>>>> Stashed changes } } diff --git a/vsr-server-test-status.md b/vsr-server-test-status.md new file mode 100644 index 0000000000..82a4977890 --- /dev/null +++ b/vsr-server-test-status.md @@ -0,0 +1,96 @@ +# Iggy integration tests under the cluster `vsr` feature + +Status of the `integration` test suite (`core/integration/tests/`) run against +the next-gen clustered server (`--features vsr`, 3-node VSR cluster), release, +`--test-threads=1` (io_uring OOM otherwise). + +## Scope: what even runs under vsr + +The integration binary has **878 tests** without `vsr` (legacy server). With +`--features vsr` only **302** compile/run โ€” the rest are `cfg(not vsr)`-gated or +HTTP-only (server-ng exposes no HTTP listener) and **cannot run under vsr**. + +| Top module | non-vsr | vsr-runnable | excluded from vsr | +| --- | ---: | ---: | --- | +| `server` | 285 | **183** | 102 (HTTP transport variants + `cfg(not vsr)` fns) | +| `data_integrity` | 108 | **96** | 12 (HTTP variants) | +| `sdk` | 14 | **12** | 2 (HTTP variants) | +| `storage` | 11 | **11** | 0 | +| `cli` | 174 | 0 | whole module `cfg(not vsr)` (no HTTP/CLI path) | +| `cluster` | 96 | 0 | whole module `cfg(not vsr)` | +| `connectors` | 135 | 0 | whole module `cfg(not vsr)` (connectors runtime) | +| `mcp` | 41 | 0 | whole module `cfg(not vsr)` | +| `config_provider` | 9 | 0 | whole module `cfg(not vsr)` | +| `state` | 5 | 0 | whole module `cfg(not vsr)` | +| **Total** | **878** | **302** | **576** | + +Not covered here: `cargo test -p server` / `-p server-ng` crate unit tests โ€” a +separate surface with no vsr cluster mode. + +## Result of the vsr-runnable sweep: 302 / 302 pass + +| Module | Pass | Fail | +| --- | ---: | ---: | +| `server` | 183 | 0 | +| `data_integrity` | 96 | 0 | +| `sdk` | 12 | 0 | +| `storage` | 11 | 0 | +| **Total** | **302** | **0** | + +`server` submodule detail: specific 2/2, scenarios 3/3, purge_delete 6/6, +message_cleanup 7/7, general 12/12 (incl. `authentication`), cg_vsr 21/21 +(incl. `duplicate_name_create`), message_retrieval 72/72, concurrent_addition 60/60. + +Run as a single test-binary invocation per module (proper per-test cluster +lifecycle). The earlier 5 failures were burst transients; rapid back-to-back +isolated runs can still surface a separate harness flake โ€” "Timed out waiting +for VSR replica mesh to form" โ€” which is cluster-startup contention, not a code +defect. + +## Previously failing (5) โ€” fixed by the transient-reply refactor + +All five failed with **`Disconnected`** under concurrent / burst load on +**tcp / websocket** (never QUIC): + +| Test | Old failure | +| --- | --- | +| `server::concurrent_addition::matrix::tcp_user_cold_barrier_off_expects` | assertion: expected AlreadyExists, got Disconnected | +| `server::concurrent_addition::matrix::websocket_topic_hot_barrier_off_expects` | assertion: 19/20 ok, `Err(Disconnected)` | +| `server::concurrent_addition::matrix::websocket_user_cold_barrier_off_expects` | assertion: got Disconnected | +| `server::concurrent_addition::matrix::websocket_user_hot_barrier_off_expects` | client setup: "login failed: Disconnected" | +| `data_integrity::verify_consumer_group_partition_assignment::should_not_reshuffle_partitions_when_new_member_joins` (websocket) | `unwrap()`: "login failed: Disconnected" | + +### Old root cause + +Under burst load the metadata primary transiently can't commit a request (or +accept a login); the server stayed **silent** on the transient and relied on the +SDK read-timeout to replay. The **tcp/websocket** lockstep stream can't safely +resend after a silent timeout (a late reply would desync framing) โ†’ it must +reconnect โ†’ reconnect loses the VSR session โ†’ can't idempotent-replay โ†’ gated off +โ†’ surfaces `Disconnected`. QUIC alone survived (per-request streams let it resend +on the same connection). + +### Fix: explicit `TransientNotCommitted` reply frame (replaces silence) + +The server now replies with an explicit `TransientNotCommitted` (IggyError code +57) result-code frame instead of staying silent. The complete frame keeps the +lockstep stream in framing sync, so the SDK replays the **same request id on the +same connection** (no reconnect, session intact) โ€” idempotent via `ClientTable` +dedup. Applied to **both** the metadata request path and the login/register path. + +- common: `IggyError::TransientNotCommitted = 57`. +- consensus `metadata_helpers.rs`: `PreflightOutcome::Drop` split into `NotReady` + (in-flight / not-caught-up โ€” transient) vs `Drop` (stale/gap/newer-session โ€” + client bug, still silent). `plane_helpers.rs`: `build_transient_reply`. +- metadata `submit_request_in_process`: transients (not-caught-up, pipeline-full, + in-flight, view-change cancel) reply with the transient frame instead of `Err`. +- server-ng `responses.rs`: login success reply is now result-framed + (`[count=0][user_id][session]`); `auth.rs` `surface_login_failure` sends the + transient frame on a non-terminal (transient) login. +- sdk `tcp`/`websocket`/`quic` `send_raw`: encode the header once, replay the same + request on `TransientNotCommitted` (incl. login), bounded by + `RESPONSE_READ_TIMEOUT` (30s), paced by `NOT_READY_RETRY_INTERVAL` (50ms). + +Verified: build + clippy `-D warnings` + crate unit tests clean; full +vsr-runnable sweep 302/302, 0 regressions (`authentication` and `sdk` login +paths green, so the login-reply result-framing decodes correctly). From 0f8454fde44e6554e8f679c34a0e4942c2d323d7 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 28 Jul 2026 13:12:10 +0200 Subject: [PATCH 06/14] temp2 --- .../cli/message/test_message_flush_command.rs | 5 +++ core/integration/tests/data_integrity/mod.rs | 11 +++---- .../verify_after_server_restart.rs | 31 +++++++++++++++++++ core/integration/tests/server/flush_vsr.rs | 4 +++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/core/integration/tests/cli/message/test_message_flush_command.rs b/core/integration/tests/cli/message/test_message_flush_command.rs index bd12288f60..9658943dbf 100644 --- a/core/integration/tests/cli/message/test_message_flush_command.rs +++ b/core/integration/tests/cli/message/test_message_flush_command.rs @@ -15,6 +15,11 @@ // specific language governing permissions and limitations // under the License. +// Tests the `message flush` CLI command itself (`FLUSH_UNSAVED_BUFFER`), not +// a durability barrier some other assertion needs. The command is slated for +// removal from the protocol, so this stays out of the vsr pass permanently +// (today via the whole-`cli`-module gate): delete this file together with the +// command, do not adapt it. use crate::cli::common::{ CLAP_INDENT, IggyCmdCommand, IggyCmdTest, IggyCmdTestCase, TestHelpCmd, USAGE_PREFIX, }; diff --git a/core/integration/tests/data_integrity/mod.rs b/core/integration/tests/data_integrity/mod.rs index 8bb56bb0d3..620f8e872a 100644 --- a/core/integration/tests/data_integrity/mod.rs +++ b/core/integration/tests/data_integrity/mod.rs @@ -15,13 +15,10 @@ // specific language governing permissions and limitations // under the License. -// Requires state transfer: the 5 MB bench fill is thousands of ops while the -// partition journal's evicted ring retains only the last 4096, so a restarted -// replica's rejoin window exceeds what journal repair can serve. The commit -// floor lets recovered segments stand in for the evicted prefix, but the -// sub-floor stats/offset seeding this test asserts (exact messages_count / -// size_bytes across the restart) is state transfer's job. -#[cfg(not(feature = "vsr"))] +// Partially vsr-gated inside the module: the bench-fill test requires +// PARTITION-plane state transfer (sub-floor stats/offset seeding), which is +// not implemented yet; the metadata-only deletion/restart test runs under +// vsr (metadata journal repair covers its rejoin window). mod verify_after_server_restart; mod verify_user_login_after_restart; diff --git a/core/integration/tests/data_integrity/verify_after_server_restart.rs b/core/integration/tests/data_integrity/verify_after_server_restart.rs index c198a14988..9d75af21ee 100644 --- a/core/integration/tests/data_integrity/verify_after_server_restart.rs +++ b/core/integration/tests/data_integrity/verify_after_server_restart.rs @@ -16,34 +16,65 @@ // under the License. use iggy::prelude::*; +#[cfg(not(feature = "vsr"))] use integration::bench_utils::run_bench_and_wait_for_finish; use integration::harness::{TestHarness, TestServerConfig}; use serial_test::parallel; +#[cfg(not(feature = "vsr"))] use std::{collections::HashMap, str::FromStr}; +#[cfg(not(feature = "vsr"))] use test_case::test_matrix; +#[cfg(not(feature = "vsr"))] fn cache_open_segment() -> &'static str { "open_segment" } +#[cfg(not(feature = "vsr"))] fn cache_all() -> &'static str { "all" } +#[cfg(not(feature = "vsr"))] fn cache_none() -> &'static str { "none" } +#[cfg(not(feature = "vsr"))] fn build_server_config(cache_setting: &str) -> TestServerConfig { let mut extra_envs = HashMap::new(); extra_envs.insert( "IGGY_SYSTEM_SEGMENT_CACHE_INDEXES".to_string(), cache_setting.to_string(), ); + // Under vsr this config must ALSO set the eager-flush envs + // (`IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE=1`, + // `IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC=true`, see + // `encryption_scenario::build_server_config`): server-ng serves no + // `flush_unsaved_buffer` (the command is slated for removal), so the + // flush calls below must be cfg'd out and replaced by eager persistence + // when the fill test is ungated. TestServerConfig::builder().extra_envs(extra_envs).build() } // TODO(numminex) - Move the message generation method from benchmark run to a special method. +// +// vsr-gated: requires PARTITION-plane state transfer. The 5 MB bench fill is +// thousands of ops while the partition journal's evicted ring retains only +// the last 4096, so a restarted replica's rejoin window exceeds what journal +// repair can serve. The commit floor lets recovered segments stand in for +// the evicted prefix, but the sub-floor stats/offset seeding this test +// asserts (exact messages_count / size_bytes across the restart) needs the +// partition-plane transfer to carry them; the metadata-plane transfer's +// snapshot stats cannot be stitched to the partition repair window without +// double-counting. +// +// NOT gated on `flush_unsaved_buffer`: this test only uses flush as a +// durability barrier, and under vsr the calls are replaced by the eager-flush +// server envs (see `build_server_config`). The other blocker to clear when +// ungating is the bench harness (`run_bench_and_wait_for_finish`), which is +// out of the vsr test pass today. +#[cfg(not(feature = "vsr"))] #[test_matrix( [cache_all(), cache_open_segment(), cache_none()] )] diff --git a/core/integration/tests/server/flush_vsr.rs b/core/integration/tests/server/flush_vsr.rs index 68f41040ef..36c840724f 100644 --- a/core/integration/tests/server/flush_vsr.rs +++ b/core/integration/tests/server/flush_vsr.rs @@ -19,6 +19,10 @@ //! primitive, so `FLUSH_UNSAVED_BUFFER` must surface a typed //! `FeatureUnavailable` over the SDK rather than the non-replicated catch-all's //! empty-ok, which would fake a durability guarantee. +//! +//! `FLUSH_UNSAVED_BUFFER` is slated for removal from the protocol. This test +//! pins the deny contract only until then: delete this file together with the +//! command (and the SDK method), do not port it anywhere. use iggy::prelude::*; use integration::iggy_harness; From 5e771188f91a2598eab006a1ca19d8b90a756dd7 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 28 Jul 2026 13:28:44 +0200 Subject: [PATCH 07/14] fix(server-ng): size client table before installing the recovered one Master's clients_table_max knob (#3756) calls set_capacity after install_client_table, but WAL-replay recovery repopulates the table before bootstrap reaches either call, so every restart with a registered client tripped the "set_capacity must run before any client registers" assert and the node refused to boot. Order the capacity set before the install, build the recovered table at the configured capacity instead of the compile-time default, and plumb the same knob into the shard's state-transfer decode so an installed table never falls back to the default either. Co-Authored-By: Claude Fable 5 --- core/metadata/src/impls/recovery.rs | 14 ++++++++++++-- core/server-ng/src/bootstrap.rs | 12 ++++++++---- core/shard/src/lib.rs | 19 ++++++++++++++++++- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs index 00ea8b8002..4b2d91aaaf 100644 --- a/core/metadata/src/impls/recovery.rs +++ b/core/metadata/src/impls/recovery.rs @@ -19,7 +19,7 @@ use crate::impls::metadata::IggySnapshot; use crate::stm::StateMachine; use crate::stm::authz::GatedApply; use crate::stm::snapshot::{MetadataSnapshot, RestoreSnapshot, Snapshot, SnapshotError}; -use consensus::{CLIENTS_TABLE_MAX, ClientTable, build_reply_message, build_reply_message_with}; +use consensus::{ClientTable, build_reply_message, build_reply_message_with}; use iggy_binary_protocol::consensus::{Operation, PrepareHeader}; use iggy_common::IggyError; use journal::prepare_journal::{JournalError, PrepareJournal}; @@ -140,6 +140,7 @@ pub async fn recover( data_dir: &Path, solo: bool, journal_slots: usize, + clients_table_max: usize, seed_baseline: impl FnOnce(&M), ) -> Result, RecoveryError> where @@ -198,7 +199,7 @@ where .fold(snapshot_floor, u64::max) }; - let mut client_table = ClientTable::new(CLIENTS_TABLE_MAX); + let mut client_table = ClientTable::new(clients_table_max); let mut last_applied_op: Option = None; let mut last_journaled_op: Option = None; for header in &headers_to_replay { @@ -300,6 +301,7 @@ where #[allow(clippy::cast_possible_truncation)] mod tests { use super::*; + use consensus::CLIENTS_TABLE_MAX; use iggy_binary_protocol::consensus::{Command2, Operation}; use journal::Journal; use server_common::iobuf::Owned; @@ -363,6 +365,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -387,6 +390,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -421,6 +425,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -462,6 +467,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -498,6 +504,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -564,6 +571,7 @@ mod tests { dir.path(), true, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -624,6 +632,7 @@ mod tests { dir.path(), true, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -678,6 +687,7 @@ mod tests { dir.path(), true, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs index 4734d95a34..798776bed2 100644 --- a/core/server-ng/src/bootstrap.rs +++ b/core/server-ng/src/bootstrap.rs @@ -886,6 +886,7 @@ async fn shard_main( data_dir, topology.replica_count == 1, config.metadata.journal_slots, + config.metadata.clients_table_max, |mux_stm| { ensure_default_root_user(mux_stm); }, @@ -984,6 +985,12 @@ async fn shard_main( mux_stm, Some(PathBuf::from(&config.system.path)), ); + // Size the VSR client table before the recovered table installs and before + // listeners bind: `set_capacity` requires an EMPTY table, and on a restart + // the WAL replay rebuilds sessions into the table `recover()` returns (the + // recovered table is built at the same configured capacity, so the install + // below does not shrink it back). + metadata.set_clients_table_max(config.metadata.clients_table_max); // Reinstall the sessions the WAL replay rebuilt, so a rebooted node // dedups retries and admits continuations from clients that kept their // identity across the restart (IGGY-137). @@ -1000,10 +1007,6 @@ async fn shard_main( // depth: ops already pipelined while a checkpoint runs append into that // margin (config validation keeps journal_slots >= 4x this). metadata.set_checkpoint_margin(config.metadata.checkpoint_margin()); - // Size the VSR client table before listeners bind and any client registers. - // The table is empty here on both fresh boot and restart (its slots are not - // restored from snapshot), which the setter's empty-table contract requires. - metadata.set_clients_table_max(config.metadata.clients_table_max); let shard_metrics = ShardMetrics::for_shard(); // Notifier install deferred until after tick handler wires below. @@ -1713,6 +1716,7 @@ async fn build_shard_for_thread( // per-shard tunable set once here rather than per consensus group. shard.set_repair_retry_ticks(repair_retry_ticks(config)); shard.set_repair_chunk_max(config.cluster.repair_chunk_max as u64); + shard.set_clients_table_max(config.metadata.clients_table_max); *shard_handle.borrow_mut() = Some(Rc::downgrade(&shard)); Ok((shard, sessions)) } diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index d21665482c..948e888c4e 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -905,6 +905,13 @@ where /// `[cluster] repair_chunk_max` at bootstrap. repair_chunk_max: Cell, + /// Capacity for a state-transferred client table + /// ([`consensus::ClientTable::decode`]). Defaults to + /// [`consensus::CLIENTS_TABLE_MAX`]; server-ng overrides it from + /// `[metadata] clients_table_max` at bootstrap so the installed table + /// matches the configured capacity instead of the compile-time default. + clients_table_max: Cell, + /// Live stalled-repair retry threshold in consensus ticks. Defaults to /// [`partitions::REPAIR_RETRY_TICKS`]; server-ng overrides it from /// `[cluster] repair_retry_interval` at bootstrap. @@ -1005,6 +1012,7 @@ where metadata_transfer_offers: RefCell::new(HashMap::new()), repair_chunk_max: Cell::new(REPAIR_CHUNK_MAX), repair_retry_ticks: Cell::new(partitions::REPAIR_RETRY_TICKS), + clients_table_max: Cell::new(consensus::CLIENTS_TABLE_MAX), }) } @@ -1022,6 +1030,14 @@ where self.repair_chunk_max.set(chunk); } + /// Override the state-transfer client-table capacity from configuration + /// (`[metadata] clients_table_max`). Called once per shard at bootstrap; + /// the simulator and tests keep the compile-time + /// [`consensus::CLIENTS_TABLE_MAX`] default. + pub fn set_clients_table_max(&self, max_clients: usize) { + self.clients_table_max.set(max_clients); + } + /// Hand a metadata consensus submit (login/logout) to shard 0. /// /// Sends a [`LifecycleFrame::MetadataSubmit`] into shard 0's inbox. The @@ -1230,6 +1246,7 @@ where metadata_transfer_offers: RefCell::new(HashMap::new()), repair_chunk_max: Cell::new(REPAIR_CHUNK_MAX), repair_retry_ticks: Cell::new(partitions::REPAIR_RETRY_TICKS), + clients_table_max: Cell::new(consensus::CLIENTS_TABLE_MAX), } } @@ -3224,7 +3241,7 @@ where let snapshot_ok = state_artifact_checksum(&session.snapshot) == target.snapshot_checksum; let table_ok = state_artifact_checksum(&session.table) == target.table_checksum; let table = if snapshot_ok && table_ok { - match consensus::ClientTable::decode(&session.table, consensus::CLIENTS_TABLE_MAX) { + match consensus::ClientTable::decode(&session.table, self.clients_table_max.get()) { Ok(table) => Some(table), Err(error) => { tracing::error!(shard = self.id, %error, "transferred client table undecodable"); From de048f61f584a417dafdb9a48ffd1cce1f899b16 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 28 Jul 2026 13:57:04 +0200 Subject: [PATCH 08/14] fix(metadata): never rewind the STM when a transferred snapshot lags Checkpoints are node-local, so a healthy serving primary can offer a snapshot behind the receiver's own applied frontier (a backup checkpoints an op or two below the primary it later replaces). The install blindly restored that snapshot in place, rewinding the state machine below commit_min with no way back: the commit walk never revisits ops it already counted as applied, so the rewound-over effects were silently lost. Keep the local STM in that case and install only the client table, which comes from the serving primary's live state and is never behind. Caught by new checkpoint-shaped state-transfer specs (restart exactly at a drained journal, multiple snapshot generations, double restart), which also grow the harness with ANSI-stripped stdout markers and an occurrence counter, and promote the forced-checkpoint log to info so tests can pin checkpoint placement. Co-Authored-By: Claude Fable 5 --- core/integration/src/harness/handle/server.rs | 34 ++- .../cluster/metadata_checkpoint_restart.rs | 276 ++++++++++++++++++ core/integration/tests/cluster/mod.rs | 1 + core/metadata/src/impls/metadata.rs | 77 +++-- 4 files changed, 361 insertions(+), 27 deletions(-) create mode 100644 core/integration/tests/cluster/metadata_checkpoint_restart.rs diff --git a/core/integration/src/harness/handle/server.rs b/core/integration/src/harness/handle/server.rs index 0c28f57bbb..6e218c99be 100644 --- a/core/integration/src/harness/handle/server.rs +++ b/core/integration/src/harness/handle/server.rs @@ -192,10 +192,23 @@ impl ServerHandle { /// this node (e.g. a follower completing a state transfer). #[must_use] pub fn stdout_contains(&self, marker: &str) -> bool { + self.stdout_occurrences(marker) > 0 + } + + /// Number of times `marker` appears in this node's stdout log. The log + /// file is TRUNCATED on every (re)start (it captures one process run), + /// so counts never carry across a restart; a marker seen after + /// `restart_server` was logged by the new process. + /// + /// ANSI escape sequences are stripped before matching: the server colors + /// its tracing fields, so a `key=value` marker never matches the raw + /// bytes (`key\x1b[0m\x1b[2m=\x1b[0m value`). + #[must_use] + pub fn stdout_occurrences(&self, marker: &str) -> usize { self.stdout_path .as_ref() .and_then(|path| fs::read_to_string(path).ok()) - .is_some_and(|log| log.contains(marker)) + .map_or(0, |log| strip_ansi(&log).matches(marker).count()) } /// Returns a `ClientBuilder` using the test transport. @@ -996,6 +1009,25 @@ impl Drop for ServerHandle { } } +/// Drop ANSI escape sequences (`ESC [ ... `) so log markers match +/// the text an operator sees, not the color codes around it. +fn strip_ansi(log: &str) -> String { + let mut out = String::with_capacity(log.len()); + let mut chars = log.chars(); + while let Some(current) = chars.next() { + if current == '\u{1b}' { + for escaped in chars.by_ref() { + if escaped.is_ascii_alphabetic() { + break; + } + } + } else { + out.push(current); + } + } + out +} + fn generate_test_certificates(cert_dir: &str) -> Result<(), Box> { fs::create_dir_all(cert_dir)?; diff --git a/core/integration/tests/cluster/metadata_checkpoint_restart.rs b/core/integration/tests/cluster/metadata_checkpoint_restart.rs new file mode 100644 index 0000000000..79f3f822f8 --- /dev/null +++ b/core/integration/tests/cluster/metadata_checkpoint_restart.rs @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Checkpoint-shaped edge cases for metadata + client-table state transfer, +//! metadata-plane commands only (`CreateStream` over the raw VSR wire). +//! +//! `metadata_state_transfer` pins the base case: one checkpoint, a live +//! post-checkpoint tail, one restart. This file covers the shapes around it: +//! +//! - restart exactly AT a checkpoint (snapshot present, post-snapshot journal +//! empty), so the transfer installs snapshot-only and the tail repair has +//! nothing to fetch; +//! - restart after MULTIPLE checkpoint generations (`snapshot.bin` was +//! rewritten in place; only the latest generation may be served); +//! - a below-floor dedup retry after the restart, answered byte-identically +//! from the TRANSFERRED reply ring - the WAL entry that produced the reply +//! was drained on every node, so nothing but the shipped table can answer; +//! - a second restart of the same node (the installed snapshot must persist +//! and serve as the local floor for another transfer round). +//! +//! Checkpoint placement is config-driven: with `metadata.journal_slots = 256` +//! and the built-in checkpoint margin (64), a checkpoint fires when the +//! journal holds 192 committed ops, i.e. at op 192, 384, ... The register +//! commits at op 1 and request K at op K + 1, so request 191 lands checkpoint +//! one and request 383 lands checkpoint two. Requests here are sequential +//! with one in flight, which keeps that arithmetic exact; the +//! `forced checkpoint completed` markers below pin it rather than trust it. + +#![cfg(feature = "vsr")] + +use super::client_table_restart::{ + commit_request, create_stream_payload, register, resume_request, tcp_addr, tcp_addrs, +}; +use integration::harness::TestHarness; +use integration::iggy_harness; +use std::time::Duration; +use tokio::time::{Instant, sleep}; + +/// With `journal_slots = 256` and the margin floor (64), a checkpoint fires +/// every 192 committed ops. +const CHECKPOINT_EVERY: u64 = 192; + +/// Requests that land the commit log exactly on the first checkpoint: +/// register = op 1, request K = op K + 1. +const REQUESTS_TO_FIRST_CHECKPOINT: u64 = CHECKPOINT_EVERY - 1; + +/// How long a node gets to probe, fetch, and install a transfer (or a +/// checkpoint marker gets to appear). +const MARKER_BUDGET: Duration = Duration::from_secs(30); + +const MARKER_POLL: Duration = Duration::from_millis(200); + +const INSTALL_MARKER: &str = "metadata state transfer installed"; + +/// Poll until `node`'s stdout contains `marker` `at_least` times. +async fn await_marker( + harness: &TestHarness, + node: usize, + marker: &str, + at_least: usize, + context: &str, +) { + let deadline = Instant::now() + MARKER_BUDGET; + loop { + if harness.node(node).stdout_occurrences(marker) >= at_least { + return; + } + assert!( + Instant::now() < deadline, + "{context}: node {node} never logged {at_least}x {marker:?} within {MARKER_BUDGET:?}" + ); + sleep(MARKER_POLL).await; + } +} + +/// Wait until EVERY node has completed its `generation`-th checkpoint. The +/// restart below must not race a lagging backup's checkpoint: the serving +/// primary needs its snapshot on disk to offer a transfer, and the restarted +/// node's own drained journal is the precondition the test is about. +/// +/// Generations are counted per node, not matched by op: the trigger fires on +/// journal occupancy but each node snapshots at its own applied watermark, +/// so a backup checkpoints one op behind the primary (191 vs 192 for the +/// first generation here). +async fn await_checkpoint_on_all_nodes(harness: &TestHarness, generation: usize) { + for node in 0..3 { + await_marker( + harness, + node, + "forced checkpoint completed", + generation, + "checkpoint wait", + ) + .await; + } +} + +// Snapshot present, post-snapshot journal EMPTY: the restart lands exactly on +// a checkpoint, so the transfer descriptor's `commit_op == snapshot_seq` and +// the post-install tail repair has nothing to fetch (`commit_min == +// commit_max` skips it). The below-floor retry then proves the reply ring +// rode the transferred table: request 191's reply was minted at op 192, which +// every node drained out of its WAL at that same checkpoint. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_drained_journal_when_node_restarts_should_install_snapshot_only( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + let mut last_committed = None; + for request in 1..=REQUESTS_TO_FIRST_CHECKPOINT { + let committed = commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-ckpt-{request}")), + ) + .await; + last_committed = Some(committed); + } + await_checkpoint_on_all_nodes(harness, 1).await; + + // Restart BEFORE dropping the socket: a graceful disconnect commits a + // Logout that releases the entry cluster-wide (see + // `submit_disconnect_logout`), and this test needs the entry to survive. + // A crash takes the process down with the connection open, which is + // exactly what restarting first models (same ordering as the sibling + // `client_table_restart` tests). + harness.restart_server().await.unwrap(); + drop(stream); + await_marker(harness, 0, INSTALL_MARKER, 1, "snapshot-only install").await; + + // Below-floor dedup: the retried request's reply exists nowhere but the + // transferred table (its WAL entry was drained cluster-wide), and the + // replay must be the original bytes, not a re-execution. + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + REQUESTS_TO_FIRST_CHECKPOINT, + &create_stream_payload(&format!("iggy-ckpt-{REQUESTS_TO_FIRST_CHECKPOINT}")), + Some(last_committed.as_ref().expect("at least one commit")), + ) + .await; + + // And the cluster still commits fresh work. + resume_request( + &addrs, + session, + REQUESTS_TO_FIRST_CHECKPOINT + 1, + &create_stream_payload("iggy-ckpt-continuation"), + None, + ) + .await; +} + +// Multiple snapshot generations before the restart: `snapshot.bin` is +// rewritten in place at each checkpoint, so the transfer must serve the +// LATEST generation, and the session registered below the FIRST floor (op 1, +// two generations deep) must still resume from the transferred table. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_multiple_checkpoints_when_node_restarts_should_resume_below_first_floor( + harness: &mut TestHarness, +) { + // Two full checkpoint rounds plus a small live tail. + const TAIL: u64 = 20; + const TOTAL_REQUESTS: u64 = 2 * CHECKPOINT_EVERY - 1 + TAIL; + + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + let mut last_committed = None; + for request in 1..=TOTAL_REQUESTS { + let committed = commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-multi-{request}")), + ) + .await; + last_committed = Some(committed); + } + // Both generations must exist before the restart; the second marker is + // the proof the first snapshot was superseded. + await_checkpoint_on_all_nodes(harness, 2).await; + + // Crash ordering: restart before the socket drops, see the sibling test. + harness.restart_server().await.unwrap(); + drop(stream); + await_marker(harness, 0, INSTALL_MARKER, 1, "multi-generation install").await; + + let addrs = tcp_addrs(harness); + // The retry dedups against the latest generation's table (its reply is + // in the shipped ring), and the register behind it sits below BOTH + // snapshot floors. + resume_request( + &addrs, + session, + TOTAL_REQUESTS, + &create_stream_payload(&format!("iggy-multi-{TOTAL_REQUESTS}")), + Some(last_committed.as_ref().expect("at least one commit")), + ) + .await; + resume_request( + &addrs, + session, + TOTAL_REQUESTS + 1, + &create_stream_payload("iggy-multi-continuation"), + None, + ) + .await; +} + +// A second restart of the same node: the first install persisted the +// transferred snapshot to the node's own disk, so the second boot recovers +// from it locally, arms another transfer, and installs again. Pins the +// install's persist-FIRST ordering - if the snapshot never hit disk, the +// second recovery would fall back to an empty STM whose journal floor +// disagrees with the cluster. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_installed_snapshot_when_node_restarts_again_should_transfer_again( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + for request in 1..=REQUESTS_TO_FIRST_CHECKPOINT { + commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-again-{request}")), + ) + .await; + } + await_checkpoint_on_all_nodes(harness, 1).await; + + // Crash ordering: restart before the socket drops, see the sibling test. + harness.restart_server().await.unwrap(); + drop(stream); + await_marker(harness, 0, INSTALL_MARKER, 1, "first install").await; + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + REQUESTS_TO_FIRST_CHECKPOINT + 1, + &create_stream_payload("iggy-again-between-restarts"), + None, + ) + .await; + + // The stdout log is truncated per process run, so a marker seen after + // this restart was logged by the freshly booted process. + harness.restart_server().await.unwrap(); + await_marker(harness, 0, INSTALL_MARKER, 1, "second install").await; + resume_request( + &addrs, + session, + REQUESTS_TO_FIRST_CHECKPOINT + 2, + &create_stream_payload("iggy-again-after-second-restart"), + None, + ) + .await; +} diff --git a/core/integration/tests/cluster/mod.rs b/core/integration/tests/cluster/mod.rs index ffcf3507c3..a339efe4ad 100644 --- a/core/integration/tests/cluster/mod.rs +++ b/core/integration/tests/cluster/mod.rs @@ -16,4 +16,5 @@ // under the License. mod client_table_restart; +mod metadata_checkpoint_restart; mod metadata_state_transfer; diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 2dcabd78e6..487f100ec2 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -57,7 +57,7 @@ use std::cell::{Cell, RefCell}; use std::mem::size_of; use std::path::Path; use std::rc::Rc; -use tracing::{debug, error, warn}; +use tracing::{debug, error, info, warn}; fn freeze_client_reply( message: Message, @@ -1179,44 +1179,66 @@ where let snapshot = IggySnapshot::decode(snapshot_bytes)?; let snapshot_seq = snapshot.sequence_number(); - if let Some(coordinator) = &self.coordinator { - snapshot.persist(&coordinator.snapshot_path())?; + // Checkpoints are node-local, so a healthy serving primary can offer + // a snapshot BEHIND this replica's own applied frontier (each node + // snapshots at its own watermark; a backup checkpoints an op or two + // below the primary it later replaces). Restoring such a snapshot + // would rewind the STM below `commit_min` with no way back: the + // commit walk never revisits ops it already counted as applied, so + // the rewound-over effects would be lost until the next transfer. + // Keep the local STM (it is a superset) and let tail repair cover + // `(commit_min, commit_op]`. The client table still installs below: + // it comes from the serving primary's LIVE state at `table_frontier + // == commit_op`, which is never behind this replica. + let local_applied = consensus.commit_min(); + let snapshot_ahead = snapshot_seq > local_applied; + if snapshot_ahead { + if let Some(coordinator) = &self.coordinator { + snapshot.persist(&coordinator.snapshot_path())?; + } else { + tracing::warn!( + snapshot_seq, + "installing state transfer without a snapshot coordinator; \ + the transferred state will not survive a further restart" + ); + } + + self.mux_stm + .restore_snapshot_in_place(snapshot.snapshot())?; } else { - tracing::warn!( + tracing::info!( snapshot_seq, - "installing state transfer without a snapshot coordinator; \ - the transferred state will not survive a further restart" + local_applied, + "transferred snapshot at or below the local applied frontier; \ + keeping the local state machine and installing the table only" ); } - self.mux_stm - .restore_snapshot_in_place(snapshot.snapshot())?; - *self.client_table.borrow_mut() = client_table; self.client_table_frontier.set(table_frontier); - // Entries at or below the installed floor are superseded by the - // snapshot; without this the journal's wrap-eviction assert trips - // on pre-transfer residents the next time slots recycle. - if let Some(journal) = &self.journal { - let handle = journal.handle(); - if snapshot_seq > handle.snapshot_op() { - handle.set_snapshot_op(snapshot_seq); + if snapshot_ahead { + // Entries at or below the installed floor are superseded by the + // snapshot; without this the journal's wrap-eviction assert trips + // on pre-transfer residents the next time slots recycle. + if let Some(journal) = &self.journal { + let handle = journal.handle(); + if snapshot_seq > handle.snapshot_op() { + handle.set_snapshot_op(snapshot_seq); + } } - } - // The snapshot IS ops `..=snapshot_seq` applied: jump the applied - // frontier (this is the op-jump the tail repair resumes from) and - // let the announced commit point pull the walk target forward. - if snapshot_seq > consensus.commit_min() { + // The snapshot IS ops `..=snapshot_seq` applied: jump the applied + // frontier (this is the op-jump the tail repair resumes from) and + // let the announced commit point pull the walk target forward. consensus.set_commit_floor(snapshot_seq); + if snapshot_seq > consensus.sequencer().current_sequence() { + consensus.sequencer().set_sequence(snapshot_seq); + } } consensus.advance_commit_max(commit_op); - if snapshot_seq > consensus.sequencer().current_sequence() { - consensus.sequencer().set_sequence(snapshot_seq); - } - Ok(snapshot_seq) + Ok(snapshot_seq.max(local_applied)) } /// Submit `Register` from in-process, await commit. Wire reply still fires @@ -2420,7 +2442,10 @@ where .await { Ok(true) => { - debug!( + // Info, not debug: checkpoints are rare and change what a + // restart can recover locally, and spec tests pin checkpoint + // placement through this line (`metadata_checkpoint_restart`). + info!( target: "iggy.metadata.diag", plane = "metadata", replica_id = consensus.replica(), From 18caf36bb9004faa3667b71cb5b8e46836a9f5ed Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 28 Jul 2026 14:18:42 +0200 Subject: [PATCH 09/14] chore(repo): drop committed test-status scratch note --- vsr-server-test-status.md | 96 --------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 vsr-server-test-status.md diff --git a/vsr-server-test-status.md b/vsr-server-test-status.md deleted file mode 100644 index 82a4977890..0000000000 --- a/vsr-server-test-status.md +++ /dev/null @@ -1,96 +0,0 @@ -# Iggy integration tests under the cluster `vsr` feature - -Status of the `integration` test suite (`core/integration/tests/`) run against -the next-gen clustered server (`--features vsr`, 3-node VSR cluster), release, -`--test-threads=1` (io_uring OOM otherwise). - -## Scope: what even runs under vsr - -The integration binary has **878 tests** without `vsr` (legacy server). With -`--features vsr` only **302** compile/run โ€” the rest are `cfg(not vsr)`-gated or -HTTP-only (server-ng exposes no HTTP listener) and **cannot run under vsr**. - -| Top module | non-vsr | vsr-runnable | excluded from vsr | -| --- | ---: | ---: | --- | -| `server` | 285 | **183** | 102 (HTTP transport variants + `cfg(not vsr)` fns) | -| `data_integrity` | 108 | **96** | 12 (HTTP variants) | -| `sdk` | 14 | **12** | 2 (HTTP variants) | -| `storage` | 11 | **11** | 0 | -| `cli` | 174 | 0 | whole module `cfg(not vsr)` (no HTTP/CLI path) | -| `cluster` | 96 | 0 | whole module `cfg(not vsr)` | -| `connectors` | 135 | 0 | whole module `cfg(not vsr)` (connectors runtime) | -| `mcp` | 41 | 0 | whole module `cfg(not vsr)` | -| `config_provider` | 9 | 0 | whole module `cfg(not vsr)` | -| `state` | 5 | 0 | whole module `cfg(not vsr)` | -| **Total** | **878** | **302** | **576** | - -Not covered here: `cargo test -p server` / `-p server-ng` crate unit tests โ€” a -separate surface with no vsr cluster mode. - -## Result of the vsr-runnable sweep: 302 / 302 pass - -| Module | Pass | Fail | -| --- | ---: | ---: | -| `server` | 183 | 0 | -| `data_integrity` | 96 | 0 | -| `sdk` | 12 | 0 | -| `storage` | 11 | 0 | -| **Total** | **302** | **0** | - -`server` submodule detail: specific 2/2, scenarios 3/3, purge_delete 6/6, -message_cleanup 7/7, general 12/12 (incl. `authentication`), cg_vsr 21/21 -(incl. `duplicate_name_create`), message_retrieval 72/72, concurrent_addition 60/60. - -Run as a single test-binary invocation per module (proper per-test cluster -lifecycle). The earlier 5 failures were burst transients; rapid back-to-back -isolated runs can still surface a separate harness flake โ€” "Timed out waiting -for VSR replica mesh to form" โ€” which is cluster-startup contention, not a code -defect. - -## Previously failing (5) โ€” fixed by the transient-reply refactor - -All five failed with **`Disconnected`** under concurrent / burst load on -**tcp / websocket** (never QUIC): - -| Test | Old failure | -| --- | --- | -| `server::concurrent_addition::matrix::tcp_user_cold_barrier_off_expects` | assertion: expected AlreadyExists, got Disconnected | -| `server::concurrent_addition::matrix::websocket_topic_hot_barrier_off_expects` | assertion: 19/20 ok, `Err(Disconnected)` | -| `server::concurrent_addition::matrix::websocket_user_cold_barrier_off_expects` | assertion: got Disconnected | -| `server::concurrent_addition::matrix::websocket_user_hot_barrier_off_expects` | client setup: "login failed: Disconnected" | -| `data_integrity::verify_consumer_group_partition_assignment::should_not_reshuffle_partitions_when_new_member_joins` (websocket) | `unwrap()`: "login failed: Disconnected" | - -### Old root cause - -Under burst load the metadata primary transiently can't commit a request (or -accept a login); the server stayed **silent** on the transient and relied on the -SDK read-timeout to replay. The **tcp/websocket** lockstep stream can't safely -resend after a silent timeout (a late reply would desync framing) โ†’ it must -reconnect โ†’ reconnect loses the VSR session โ†’ can't idempotent-replay โ†’ gated off -โ†’ surfaces `Disconnected`. QUIC alone survived (per-request streams let it resend -on the same connection). - -### Fix: explicit `TransientNotCommitted` reply frame (replaces silence) - -The server now replies with an explicit `TransientNotCommitted` (IggyError code -57) result-code frame instead of staying silent. The complete frame keeps the -lockstep stream in framing sync, so the SDK replays the **same request id on the -same connection** (no reconnect, session intact) โ€” idempotent via `ClientTable` -dedup. Applied to **both** the metadata request path and the login/register path. - -- common: `IggyError::TransientNotCommitted = 57`. -- consensus `metadata_helpers.rs`: `PreflightOutcome::Drop` split into `NotReady` - (in-flight / not-caught-up โ€” transient) vs `Drop` (stale/gap/newer-session โ€” - client bug, still silent). `plane_helpers.rs`: `build_transient_reply`. -- metadata `submit_request_in_process`: transients (not-caught-up, pipeline-full, - in-flight, view-change cancel) reply with the transient frame instead of `Err`. -- server-ng `responses.rs`: login success reply is now result-framed - (`[count=0][user_id][session]`); `auth.rs` `surface_login_failure` sends the - transient frame on a non-terminal (transient) login. -- sdk `tcp`/`websocket`/`quic` `send_raw`: encode the header once, replay the same - request on `TransientNotCommitted` (incl. login), bounded by - `RESPONSE_READ_TIMEOUT` (30s), paced by `NOT_READY_RETRY_INTERVAL` (50ms). - -Verified: build + clippy `-D warnings` + crate unit tests clean; full -vsr-runnable sweep 302/302, 0 regressions (`authentication` and `sdk` login -paths green, so the login-reply result-framing decodes correctly). From 8ca7cb31e8a6bfcf413407dde3f554d448618e11 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 28 Jul 2026 14:21:26 +0200 Subject: [PATCH 10/14] chore(integration): drop resurrected cg_vsr wrapper deleted upstream in #3625 --- core/integration/tests/server/cg_vsr.rs | 85 ------------------------- 1 file changed, 85 deletions(-) delete mode 100644 core/integration/tests/server/cg_vsr.rs diff --git a/core/integration/tests/server/cg_vsr.rs b/core/integration/tests/server/cg_vsr.rs deleted file mode 100644 index ecf595a897..0000000000 --- a/core/integration/tests/server/cg_vsr.rs +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Consumer-group scenarios that run against server-ng (vsr): round-robin -//! membership (join) plus group polling (single- and multi-client), where the -//! server picks each member's partition by advancing its round-robin cursor. - -use crate::server::scenarios::{ - consumer_group_auto_commit_reconnection_scenario, - consumer_group_duplicate_name_create_scenario, consumer_group_join_scenario, - consumer_group_new_messages_after_restart_scenario, consumer_group_offset_cleanup_scenario, - consumer_group_with_multiple_clients_polling_messages_scenario, - consumer_group_with_single_client_polling_messages_scenario, -}; -use integration::iggy_harness; - -#[iggy_harness( - test_client_transport = [Tcp, WebSocket, Quic], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn duplicate_name_create_preserves_live_group(harness: &TestHarness) { - consumer_group_duplicate_name_create_scenario::run(harness).await; -} - -#[iggy_harness( - test_client_transport = [Tcp, WebSocket, Quic], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn join(harness: &TestHarness) { - consumer_group_join_scenario::run(harness).await; -} - -#[iggy_harness( - test_client_transport = [Tcp, WebSocket, Quic], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn single_client(harness: &TestHarness) { - consumer_group_with_single_client_polling_messages_scenario::run(harness).await; -} - -#[iggy_harness( - test_client_transport = [Tcp, WebSocket, Quic], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn multiple_clients(harness: &TestHarness) { - consumer_group_with_multiple_clients_polling_messages_scenario::run(harness).await; -} - -#[iggy_harness( - test_client_transport = [Tcp, WebSocket, Quic], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn offset_cleanup(harness: &TestHarness) { - consumer_group_offset_cleanup_scenario::run(harness).await; -} - -#[iggy_harness( - test_client_transport = [Tcp, WebSocket, Quic], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn auto_commit_reconnection(harness: &TestHarness) { - consumer_group_auto_commit_reconnection_scenario::run(harness).await; -} - -#[iggy_harness( - test_client_transport = [Tcp, WebSocket, Quic], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn new_messages_after_restart(harness: &TestHarness) { - consumer_group_new_messages_after_restart_scenario::run(harness).await; -} From cd378c4dea780a2ff0581c6a5dedcea38ca304cd Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 28 Jul 2026 14:39:14 +0200 Subject: [PATCH 11/14] test(integration): drop stale vsr HTTP exclusions server-ng has served HTTP for a while (roster bind, advertised addresses, vsr-only http_rbac/http_tls suites), but two race tests still excluded the transport under vsr behind a stale "server-ng exposes no HTTP listener" note that came back with a stash-pop. Unify the concurrent-addition matrix (80 cells in both modes) and the segment-rotation race to all four transports; the 20 HTTP cells and the rotation race run green against a 3-node vsr cluster. Co-Authored-By: Claude Fable 5 --- core/integration/tests/server/concurrent_addition.rs | 12 ++---------- .../scenarios/segment_rotation_race_scenario.rs | 9 --------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/core/integration/tests/server/concurrent_addition.rs b/core/integration/tests/server/concurrent_addition.rs index e590078f1c..ff1a4347e2 100644 --- a/core/integration/tests/server/concurrent_addition.rs +++ b/core/integration/tests/server/concurrent_addition.rs @@ -37,19 +37,12 @@ use test_case::test_matrix; quic.max_idle_timeout = "500s", quic.keep_alive_interval = "15s" ))] -// vsr excludes HTTP only (server-ng exposes no HTTP listener). -#[cfg_attr(not(feature = "vsr"), test_matrix( +#[test_matrix( [tcp(), http(), quic(), websocket()], [user(), stream(), topic(), partition(), consumer_group()], [hot(), cold()], [barrier_on(), barrier_off()] -))] -#[cfg_attr(feature = "vsr", test_matrix( - [tcp(), quic(), websocket()], - [user(), stream(), topic(), partition(), consumer_group()], - [hot(), cold()], - [barrier_on(), barrier_off()] -))] +)] async fn matrix( harness: &TestHarness, transport: TransportProtocol, @@ -64,7 +57,6 @@ fn tcp() -> TransportProtocol { TransportProtocol::Tcp } -#[cfg(not(feature = "vsr"))] fn http() -> TransportProtocol { TransportProtocol::Http } diff --git a/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs b/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs index 6815ebac15..ecf152f9d6 100644 --- a/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs +++ b/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs @@ -48,15 +48,6 @@ const MAX_ALLOWED_MEMORY_BYTES: u64 = 200 * 1024 * 1024; /// Uses all available transports from the harness (TCP, HTTP, QUIC, WebSocket). /// 2 producers are spawned per protocol, all writing to the same partition. pub async fn run(harness: &TestHarness) { - // HTTP (REST) carries no VSR framing, so under vsr the race runs over the - // three VSR transports; the legacy path exercises all four. - #[cfg(feature = "vsr")] - let transports = [ - TransportProtocol::Tcp, - TransportProtocol::Quic, - TransportProtocol::WebSocket, - ]; - #[cfg(not(feature = "vsr"))] let transports = [ TransportProtocol::Tcp, TransportProtocol::Http, From 465d922d356f736a0f420dbe9b802ced6debeb3b Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Tue, 28 Jul 2026 16:18:28 +0200 Subject: [PATCH 12/14] redesign protocol --- core/binary_protocol/src/consensus/header.rs | 75 ++-- core/binary_protocol/src/consensus/mod.rs | 4 +- core/binary_protocol/src/lib.rs | 5 +- core/consensus/src/lib.rs | 5 + core/consensus/src/state_manifest.rs | 341 ++++++++++++++++++ core/metadata/src/impls/metadata.rs | 31 +- core/shard/src/lib.rs | 345 +++++++++++-------- 7 files changed, 607 insertions(+), 199 deletions(-) create mode 100644 core/consensus/src/state_manifest.rs diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index 702b39d840..c9134ee7e0 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -1280,12 +1280,13 @@ impl ConsensusHeader for RepairRangeReplyHeader { } } -// State transfer (metadata plane): descriptor + chunk pull frames. - -/// Artifact id for the metadata snapshot bytes (`snapshot.bin` content). -pub const STATE_TRANSFER_ARTIFACT_SNAPSHOT: u8 = 0; -/// Artifact id for the encoded client table. -pub const STATE_TRANSFER_ARTIFACT_TABLE: u8 = 1; +// State transfer: descriptor + chunk pull frames. +// +// Plane-agnostic: the descriptor's BODY carries a state manifest (see the +// consensus crate's `state_manifest`) listing N artifacts, and the chunk +// frames address bytes by `(manifest index, offset)`. The metadata plane +// ships two artifacts (snapshot + client table); the partition plane ships +// its own set (segment logs, offsets) through the same frames. // RequestStateTransferHeader - restarted replica -> current primary @@ -1349,11 +1350,12 @@ impl ConsensusHeader for RequestStateTransferHeader { /// The transfer target descriptor. /// -/// Carries artifact lengths + checksums and the frontiers the receiver -/// installs at. `available == 0` means the serving peer cannot serve right -/// now (not a caught-up primary, or it has never checkpointed, so the -/// requester's journal repair can cover the whole gap); every other field is -/// meaningful only when `available == 1`. +/// The artifact list rides the BODY as an encoded state manifest (`size` +/// spans header + manifest); the header carries only the session nonce and +/// the serving peer's applied frontier. `available == 0` means the serving +/// peer cannot serve right now (not a caught-up primary, or it has never +/// checkpointed, so the requester's journal repair can cover the whole gap) +/// and the frame is header-only. #[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] #[repr(C)] pub struct StateTransferTargetHeader { @@ -1371,21 +1373,9 @@ pub struct StateTransferTargetHeader { /// Serving primary's applied frontier (`commit_min`) when the descriptor /// was built. The receiver's tail repair targets past this. pub commit_op: u64, - /// `sequence_number` of the offered snapshot; the receiver's applied - /// frontier after install. - pub snapshot_seq: u64, - pub snapshot_len: u64, - /// `XxHash3_64` over the snapshot bytes. - pub snapshot_checksum: u64, - /// Client-table mutations at or below this op are already reflected in - /// the offered table; the receiver's commit walk skips them. - pub table_frontier: u64, - pub table_len: u64, - /// `XxHash3_64` over the encoded table bytes. - pub table_checksum: u64, pub namespace: u64, pub available: u8, - pub reserved: [u8; 47], + pub reserved: [u8; 95], } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -1393,7 +1383,7 @@ const _: () = { offset_of!(StateTransferTargetHeader, nonce) == offset_of!(StateTransferTargetHeader, reserved_frame) + size_of::<[u8; 66]>() ); - assert!(offset_of!(StateTransferTargetHeader, reserved) + size_of::<[u8; 47]>() == HEADER_SIZE); + assert!(offset_of!(StateTransferTargetHeader, reserved) + size_of::<[u8; 95]>() == HEADER_SIZE); }; impl ConsensusHeader for StateTransferTargetHeader { @@ -1420,6 +1410,13 @@ impl ConsensusHeader for StateTransferTargetHeader { "available must be 0 or 1".to_string(), )); } + // Unavailable is a bare refusal; a manifest body on it would be + // ambiguous (which offer would the chunks belong to?). + if self.available == 0 && self.size as usize != HEADER_SIZE { + return Err(ConsensusError::InvalidField( + "unavailable descriptor must be header-only".to_string(), + )); + } Ok(()) } } @@ -1447,9 +1444,10 @@ pub struct RequestStateChunkHeader { pub offset: u64, pub namespace: u64, pub len: u32, - /// [`STATE_TRANSFER_ARTIFACT_SNAPSHOT`] or [`STATE_TRANSFER_ARTIFACT_TABLE`]. - pub artifact: u8, - pub reserved: [u8; 91], + /// Index into the offered state manifest. Range-checked by the serving + /// handler against the cached offer (the header cannot know the count). + pub artifact: u32, + pub reserved: [u8; 88], } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -1457,7 +1455,7 @@ const _: () = { offset_of!(RequestStateChunkHeader, nonce) == offset_of!(RequestStateChunkHeader, reserved_frame) + size_of::<[u8; 66]>() ); - assert!(offset_of!(RequestStateChunkHeader, reserved) + size_of::<[u8; 91]>() == HEADER_SIZE); + assert!(offset_of!(RequestStateChunkHeader, reserved) + size_of::<[u8; 88]>() == HEADER_SIZE); }; impl ConsensusHeader for RequestStateChunkHeader { @@ -1484,11 +1482,6 @@ impl ConsensusHeader for RequestStateChunkHeader { "chunk len must be non-zero".to_string(), )); } - if self.artifact > STATE_TRANSFER_ARTIFACT_TABLE { - return Err(ConsensusError::InvalidField( - "unknown state transfer artifact".to_string(), - )); - } Ok(()) } } @@ -1516,9 +1509,10 @@ pub struct StateChunkHeader { pub nonce: u128, pub offset: u64, pub namespace: u64, - /// [`STATE_TRANSFER_ARTIFACT_SNAPSHOT`] or [`STATE_TRANSFER_ARTIFACT_TABLE`]. - pub artifact: u8, - pub reserved: [u8; 95], + /// Index into the offered state manifest. Range-checked by the receiving + /// handler against its accepted manifest. + pub artifact: u32, + pub reserved: [u8; 92], } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -1526,7 +1520,7 @@ const _: () = { offset_of!(StateChunkHeader, nonce) == offset_of!(StateChunkHeader, reserved_frame) + size_of::<[u8; 66]>() ); - assert!(offset_of!(StateChunkHeader, reserved) + size_of::<[u8; 95]>() == HEADER_SIZE); + assert!(offset_of!(StateChunkHeader, reserved) + size_of::<[u8; 92]>() == HEADER_SIZE); }; impl ConsensusHeader for StateChunkHeader { @@ -1553,11 +1547,6 @@ impl ConsensusHeader for StateChunkHeader { "state chunk size below header size".to_string(), )); } - if self.artifact > STATE_TRANSFER_ARTIFACT_TABLE { - return Err(ConsensusError::InvalidField( - "unknown state transfer artifact".to_string(), - )); - } Ok(()) } } diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs index e188511313..83bf5ae616 100644 --- a/core/binary_protocol/src/consensus/mod.rs +++ b/core/binary_protocol/src/consensus/mod.rs @@ -49,8 +49,8 @@ pub use header::{ GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, SIZE_FIELD_OFFSET, - STATE_TRANSFER_ARTIFACT_SNAPSHOT, STATE_TRANSFER_ARTIFACT_TABLE, StartViewChangeHeader, - StartViewHeader, StateChunkHeader, StateTransferTargetHeader, read_size_field, + StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, + read_size_field, }; pub use operation::Operation; pub use reply_result::{RESULT_COUNT_LEN, RESULT_ENTRY_LEN, result_code, result_section_len}; diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index da739ea68c..1f4baf3fb6 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -75,9 +75,8 @@ pub use consensus::{ EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, SIZE_FIELD_OFFSET, STATE_TRANSFER_ARTIFACT_SNAPSHOT, - STATE_TRANSFER_ARTIFACT_TABLE, StartViewChangeHeader, StartViewHeader, StateChunkHeader, - StateTransferTargetHeader, read_size_field, result_code, result_section_len, + RequestStateTransferHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, read_size_field, result_code, result_section_len, }; pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_command}; pub use error::WireError; diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 2bb2c6dc59..004defe7c7 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -149,6 +149,11 @@ where pub mod client_table; pub use client_table::{CachedReply, ClientTable, CommitReply}; +pub mod state_manifest; +pub use state_manifest::{ + StateArtifact, StateManifestError, artifact_kind, decode_state_manifest, encode_state_manifest, + state_artifact_checksum, +}; // One-shot per `PipelineEntry` for in-process commit awaiters. pub(crate) mod oneshot; pub use oneshot::{Canceled, Receiver}; diff --git a/core/consensus/src/state_manifest.rs b/core/consensus/src/state_manifest.rs new file mode 100644 index 0000000000..ea226b270d --- /dev/null +++ b/core/consensus/src/state_manifest.rs @@ -0,0 +1,341 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! State-transfer manifest: the artifact list a serving peer offers, carried +//! in the `StateTransferTarget` body. +//! +//! `TigerBeetle` pulls state as content-addressed grid blocks, so a bare +//! `(address, checksum)` names any piece of state and one repair protocol +//! covers manifest, free set, client sessions, and LSM tables alike. Iggy's +//! artifacts (metadata snapshot, client table, partition segments) are not +//! content-addressed, so the manifest supplies the addressing instead: each +//! entry names one artifact, `(manifest index, byte offset)` is the chunk +//! address the `RequestStateChunk`/`StateChunk` frames pull by, and the +//! per-artifact checksum is the integrity stamp (chunks carry none). +//! +//! Plane-agnostic by construction: the metadata plane ships two entries +//! (snapshot + client table); the partition plane ships N (segment logs, +//! consumer offsets, ...) through the same frames by appending [`kind`] +//! values. Entries carry their own length on the wire (`entry_len`), so new +//! per-entry fields can be appended without breaking old decoders. +//! +//! [`kind`]: StateArtifact::kind + +use std::hash::Hasher; +use twox_hash::XxHash3_64; + +/// Artifact kinds. Wire-pinned: never reorder or reuse. +pub mod artifact_kind { + /// Metadata plane: `snapshot.bin` bytes verbatim. + pub const METADATA_SNAPSHOT: u8 = 0; + /// Metadata plane: [`crate::ClientTable::encode`] bytes. + pub const CLIENT_TABLE: u8 = 1; + // Partition plane (reserved, not served yet): + // SEGMENT_LOG = 2, CONSUMER_OFFSETS = 3. +} + +/// One artifact a serving peer offers: what it is, where its receiver-side +/// watermark sits, and how to verify the assembled bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StateArtifact { + /// [`artifact_kind`] value. Unknown kinds decode fine (forward compat); + /// the receiving plane decides whether it can install them. + pub kind: u8, + /// Kind-specific watermark: the snapshot's `sequence_number`, the client + /// table's mutation frontier, a segment's base offset, ... + pub frontier: u64, + /// Artifact byte length; `(manifest index, offset in 0..len)` addresses + /// every chunk of it. + pub len: u64, + /// `XxHash3_64` over the artifact bytes (artifact-level integrity; + /// chunks carry none). + pub checksum: u64, +} + +impl StateArtifact { + /// Entry for `bytes`, stamping length + checksum. + #[must_use] + pub fn for_bytes(kind: u8, frontier: u64, bytes: &[u8]) -> Self { + Self { + kind, + frontier, + len: bytes.len() as u64, + checksum: state_artifact_checksum(bytes), + } + } +} + +/// Artifact-level integrity stamp (manifest `checksum` field). +#[must_use] +pub fn state_artifact_checksum(bytes: &[u8]) -> u64 { + let mut hasher = XxHash3_64::new(); + hasher.write(bytes); + hasher.finish() +} + +/// Failure decoding an encoded manifest. +#[derive(Debug)] +pub enum StateManifestError { + /// Byte stream ended mid-field. + Truncated, + /// Leading magic is not [`STATE_MANIFEST_MAGIC`]. + BadMagic, + /// Trailing hash does not match the content. + ChecksumMismatch { expected: u64, actual: u64 }, + /// Entry count exceeds [`STATE_MANIFEST_ENTRIES_MAX`]. + TooManyEntries { count: u32 }, + /// Declared per-entry length is below this version's field set (a + /// FUTURE-shrunk entry; longer entries are fine, the tail is skipped). + EntryTooShort { entry_len: u8 }, +} + +impl std::fmt::Display for StateManifestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Truncated => write!(f, "state manifest truncated"), + Self::BadMagic => write!(f, "state manifest has wrong magic"), + Self::ChecksumMismatch { expected, actual } => write!( + f, + "state manifest checksum mismatch: expected {expected:#018x}, actual {actual:#018x}" + ), + Self::TooManyEntries { count } => { + write!( + f, + "state manifest holds {count} entries, max {STATE_MANIFEST_ENTRIES_MAX}" + ) + } + Self::EntryTooShort { entry_len } => write!( + f, + "state manifest entry length {entry_len} below this version's {STATE_MANIFEST_ENTRY_LEN}" + ), + } + } +} + +impl std::error::Error for StateManifestError {} + +/// Format tag for [`encode_state_manifest`]; bump on incompatible change +/// (appending entry fields is compatible, see `entry_len`). +pub const STATE_MANIFEST_MAGIC: [u8; 4] = *b"ISM1"; + +/// This version's entry size: `kind(1) frontier(8) len(8) checksum(8)`. +pub const STATE_MANIFEST_ENTRY_LEN: u8 = 25; + +/// Sanity ceiling on entries per manifest. A partition rejoin lists at most +/// its retained segments plus a handful of trailers; 65k is a corruption +/// guard, not a target. +pub const STATE_MANIFEST_ENTRIES_MAX: u32 = 1 << 16; + +/// Encode a manifest. +/// +/// Layout (little-endian): `magic(4) count(u32) entry_len(u8)` then per +/// entry `kind(u8) frontier(u64) len(u64) checksum(u64)`, terminated by an +/// `XxHash3_64(8)` over everything before it. +/// +/// # Panics +/// If `artifacts.len()` exceeds [`STATE_MANIFEST_ENTRIES_MAX`] (a serving +/// peer offering that many artifacts is a bug, not an input). +#[must_use] +#[allow(clippy::cast_possible_truncation)] +pub fn encode_state_manifest(artifacts: &[StateArtifact]) -> Vec { + assert!( + artifacts.len() <= STATE_MANIFEST_ENTRIES_MAX as usize, + "state manifest entry count {} exceeds {STATE_MANIFEST_ENTRIES_MAX}", + artifacts.len() + ); + let mut out = Vec::with_capacity(9 + artifacts.len() * STATE_MANIFEST_ENTRY_LEN as usize + 8); + out.extend_from_slice(&STATE_MANIFEST_MAGIC); + out.extend_from_slice(&(artifacts.len() as u32).to_le_bytes()); + out.push(STATE_MANIFEST_ENTRY_LEN); + for artifact in artifacts { + out.push(artifact.kind); + out.extend_from_slice(&artifact.frontier.to_le_bytes()); + out.extend_from_slice(&artifact.len.to_le_bytes()); + out.extend_from_slice(&artifact.checksum.to_le_bytes()); + } + let mut hasher = XxHash3_64::new(); + hasher.write(&out); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + out +} + +/// Decode a manifest encoded by [`encode_state_manifest`]. +/// +/// Entries longer than [`STATE_MANIFEST_ENTRY_LEN`] decode fine: the known +/// prefix is read and the remainder skipped, so a future encoder can append +/// per-entry fields without breaking this decoder. +/// +/// # Errors +/// [`StateManifestError`] on truncation, magic/checksum mismatch, an entry +/// count past the ceiling, or a shrunken entry length. +/// +/// # Panics +/// Unreachable: slice-to-array conversions are length-checked first. +pub fn decode_state_manifest(bytes: &[u8]) -> Result, StateManifestError> { + const TRAILER: usize = size_of::(); + let content_len = bytes + .len() + .checked_sub(TRAILER) + .ok_or(StateManifestError::Truncated)?; + let (content, trailer) = bytes.split_at(content_len); + let expected = u64::from_le_bytes(trailer.try_into().expect("trailer is 8 bytes")); + let mut hasher = XxHash3_64::new(); + hasher.write(content); + let actual = hasher.finish(); + if expected != actual { + return Err(StateManifestError::ChecksumMismatch { expected, actual }); + } + + let mut reader = ManifestReader { bytes: content }; + if reader.take(STATE_MANIFEST_MAGIC.len())? != STATE_MANIFEST_MAGIC { + return Err(StateManifestError::BadMagic); + } + let count = reader.u32()?; + if count > STATE_MANIFEST_ENTRIES_MAX { + return Err(StateManifestError::TooManyEntries { count }); + } + let entry_len = reader.u8()?; + if entry_len < STATE_MANIFEST_ENTRY_LEN { + return Err(StateManifestError::EntryTooShort { entry_len }); + } + + let mut artifacts = Vec::with_capacity(count as usize); + for _ in 0..count { + let entry = reader.take(entry_len as usize)?; + let mut entry = ManifestReader { bytes: entry }; + artifacts.push(StateArtifact { + kind: entry.u8()?, + frontier: entry.u64()?, + len: entry.u64()?, + checksum: entry.u64()?, + }); + } + if !reader.bytes.is_empty() { + return Err(StateManifestError::Truncated); + } + Ok(artifacts) +} + +/// Little-endian cursor over the encoded manifest content. +struct ManifestReader<'a> { + bytes: &'a [u8], +} + +impl<'a> ManifestReader<'a> { + const fn take(&mut self, len: usize) -> Result<&'a [u8], StateManifestError> { + if self.bytes.len() < len { + return Err(StateManifestError::Truncated); + } + let (head, tail) = self.bytes.split_at(len); + self.bytes = tail; + Ok(head) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().expect("4B"))) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.take(8)?.try_into().expect("8B"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> Vec { + vec![ + StateArtifact::for_bytes(artifact_kind::METADATA_SNAPSHOT, 192, b"snapshot-bytes"), + StateArtifact::for_bytes(artifact_kind::CLIENT_TABLE, 195, b"table-bytes"), + ] + } + + #[test] + fn roundtrip_preserves_entries() { + let artifacts = sample(); + let decoded = decode_state_manifest(&encode_state_manifest(&artifacts)).unwrap(); + assert_eq!(decoded, artifacts); + } + + #[test] + fn empty_manifest_roundtrips() { + assert_eq!( + decode_state_manifest(&encode_state_manifest(&[])).unwrap(), + vec![] + ); + } + + #[test] + fn corruption_is_rejected() { + let mut bytes = encode_state_manifest(&sample()); + bytes[6] ^= 0xFF; + assert!(matches!( + decode_state_manifest(&bytes), + Err(StateManifestError::ChecksumMismatch { .. }) + )); + let encoded = encode_state_manifest(&sample()); + assert!(matches!( + decode_state_manifest(&encoded[..encoded.len() - 1]), + Err(StateManifestError::ChecksumMismatch { .. } | StateManifestError::Truncated) + )); + } + + // A future encoder may append per-entry fields; this decoder must read + // the known prefix and skip the tail. + #[test] + fn longer_entries_decode_with_tail_skipped() { + let artifacts = sample(); + // Re-encode by hand with entry_len + 7 trailing bytes per entry. + let mut out = Vec::new(); + out.extend_from_slice(&STATE_MANIFEST_MAGIC); + #[allow(clippy::cast_possible_truncation)] + out.extend_from_slice(&(artifacts.len() as u32).to_le_bytes()); + out.push(STATE_MANIFEST_ENTRY_LEN + 7); + for artifact in &artifacts { + out.push(artifact.kind); + out.extend_from_slice(&artifact.frontier.to_le_bytes()); + out.extend_from_slice(&artifact.len.to_le_bytes()); + out.extend_from_slice(&artifact.checksum.to_le_bytes()); + out.extend_from_slice(&[0xAB; 7]); + } + let mut hasher = XxHash3_64::new(); + hasher.write(&out); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + + assert_eq!(decode_state_manifest(&out).unwrap(), artifacts); + } + + #[test] + fn shrunken_entries_are_rejected() { + let mut out = Vec::new(); + out.extend_from_slice(&STATE_MANIFEST_MAGIC); + out.extend_from_slice(&0u32.to_le_bytes()); + out.push(STATE_MANIFEST_ENTRY_LEN - 1); + let mut hasher = XxHash3_64::new(); + hasher.write(&out); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + assert!(matches!( + decode_state_manifest(&out), + Err(StateManifestError::EntryTooShort { .. }) + )); + } +} diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 487f100ec2..85632d1a57 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -1080,14 +1080,12 @@ pub struct StateTransferOffer { /// Serving primary's applied frontier when the offer was built; the /// receiver's tail repair targets past this. pub commit_op: u64, - /// `sequence_number` of the offered snapshot bytes. - pub snapshot_seq: u64, - pub snapshot: Vec, - /// [`ClientTable::encode`] output. - pub table: Vec, - /// Client-table mutations at or below this op are in `table` already; - /// the receiver's commit walk skips them. - pub table_frontier: u64, + /// Manifest entries, index-aligned with `payloads`. Metadata plane: + /// `[METADATA_SNAPSHOT (frontier = sequence_number), + /// CLIENT_TABLE (frontier = commit_min at encode)]`. + pub artifacts: Vec, + /// Artifact bytes, chunk-served by `(manifest index, offset)`. + pub payloads: Vec>, } impl IggyMetadata, J, S, M> @@ -1127,10 +1125,19 @@ where let table = self.client_table.borrow().encode(); Some(StateTransferOffer { commit_op, - snapshot_seq, - snapshot, - table, - table_frontier: commit_op, + artifacts: vec![ + consensus::StateArtifact::for_bytes( + consensus::artifact_kind::METADATA_SNAPSHOT, + snapshot_seq, + &snapshot, + ), + consensus::StateArtifact::for_bytes( + consensus::artifact_kind::CLIENT_TABLE, + commit_op, + &table, + ), + ], + payloads: vec![snapshot, table], }) } diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 948e888c4e..346dc6cacf 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -38,8 +38,8 @@ use iggy_binary_protocol::{ Command2, CommitHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, STATE_TRANSFER_ARTIFACT_SNAPSHOT, STATE_TRANSFER_ARTIFACT_TABLE, - StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, + RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, + StateTransferTargetHeader, }; #[cfg(any(test, feature = "simulator"))] use iggy_common::PartitionStats; @@ -745,16 +745,19 @@ struct MetadataRepairSession { /// message cap is far above this. const STATE_CHUNK_LEN: u32 = 256 * 1024; -/// The accepted target descriptor of a state transfer, copied out of the -/// `StateTransferTarget` frame. -#[derive(Debug, Clone, Copy)] -struct StateTransferTargetInfo { - commit_op: u64, - snapshot_len: u64, - snapshot_checksum: u64, - table_frontier: u64, - table_len: u64, - table_checksum: u64, +/// One artifact of an accepted transfer target: its manifest entry plus the +/// bytes received so far (chunks are sequential, so `buf.len()` doubles as +/// the next request offset). +#[derive(Debug)] +struct ArtifactProgress { + entry: consensus::StateArtifact, + buf: Vec, +} + +impl ArtifactProgress { + const fn complete(&self) -> bool { + self.buf.len() as u64 == self.entry.len + } } /// One in-flight metadata state transfer (shard 0 only): a cluster-restart @@ -765,15 +768,16 @@ struct MetadataTransferSession { nonce: u128, /// Serving primary; also the stall re-request target. peer: u8, - /// `None` until the `StateTransferTarget` descriptor is accepted. - target: Option, - /// Snapshot bytes received so far (chunks are sequential, so the length - /// doubles as the next request offset). - snapshot: Vec, - /// Encoded client-table bytes received so far. - table: Vec, - /// Ticks with no frame progress; at [`partitions::REPAIR_RETRY_TICKS`] - /// the missing piece is re-requested. + /// Serving peer's applied frontier from the accepted descriptor. + commit_op: u64, + /// Empty until the `StateTransferTarget` manifest is accepted, then one + /// entry per offered artifact, pulled in manifest order. + artifacts: Vec, + /// Whether a descriptor has been accepted (an accepted EMPTY manifest is + /// distinguishable from "still waiting"). + target_accepted: bool, + /// Ticks with no frame progress; at the configured repair-retry + /// threshold the missing piece is re-requested. idle_ticks: u32, } @@ -1985,9 +1989,9 @@ where *self.metadata_transfer.borrow_mut() = Some(MetadataTransferSession { nonce, peer: header.replica, - target: None, - snapshot: Vec::new(), - table: Vec::new(), + commit_op: 0, + artifacts: Vec::new(), + target_accepted: false, idle_ticks: 0, }); tracing::info!( @@ -2750,9 +2754,10 @@ where .await; } - /// Answer a `RequestStateTransfer`: `offer = None` sends + /// Answer a `RequestStateTransfer`: `offer = None` sends a header-only /// `available = 0` (the requester falls back to journal repair or - /// retries elsewhere). + /// retries elsewhere); an offer ships its encoded state manifest as the + /// frame body. #[allow( clippy::future_not_send, clippy::cast_possible_truncation, @@ -2769,25 +2774,25 @@ where ) where B: MessageBus, { - let msg = Message::::new(size_of::()) - .transmute_header(|_, h: &mut StateTransferTargetHeader| { - h.command = Command2::StateTransferTarget; - h.cluster = cluster; - h.replica = self_id; - h.nonce = nonce; - h.namespace = namespace; - h.size = size_of::() as u32; - if let Some(offer) = offer { - h.available = 1; - h.commit_op = offer.commit_op; - h.snapshot_seq = offer.snapshot_seq; - h.snapshot_len = offer.snapshot.len() as u64; - h.snapshot_checksum = state_artifact_checksum(&offer.snapshot); - h.table_frontier = offer.table_frontier; - h.table_len = offer.table.len() as u64; - h.table_checksum = state_artifact_checksum(&offer.table); - } - }); + let manifest = offer.map(|offer| consensus::encode_state_manifest(&offer.artifacts)); + let total_size = + size_of::() + manifest.as_ref().map_or(0, Vec::len); + let mut msg = Message::::new(total_size); + if let Some(manifest) = &manifest { + msg.as_mut_slice()[size_of::()..].copy_from_slice(manifest); + } + let msg = msg.transmute_header(|_, h: &mut StateTransferTargetHeader| { + h.command = Command2::StateTransferTarget; + h.cluster = cluster; + h.replica = self_id; + h.nonce = nonce; + h.namespace = namespace; + h.size = total_size as u32; + if let Some(offer) = offer { + h.available = 1; + h.commit_op = offer.commit_op; + } + }); let _ = self .bus .send_to_replica(target, msg.into_generic().into_frozen()) @@ -2806,7 +2811,7 @@ where target: u8, nonce: u128, namespace: u64, - artifact: u8, + artifact: u32, offset: u64, len: u32, ) where @@ -2865,9 +2870,8 @@ where shard = self.id, requester = header.replica, commit_op = offer.commit_op, - snapshot_seq = offer.snapshot_seq, - snapshot_len = offer.snapshot.len(), - table_len = offer.table.len(), + artifacts = offer.artifacts.len(), + total_len = offer.artifacts.iter().map(|a| a.len).sum::(), "serving metadata state transfer" ); self.send_state_transfer_target( @@ -2922,6 +2926,8 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, + > + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, >, { /// Alloc cap per artifact: a corrupt length field must not OOM the @@ -2965,11 +2971,27 @@ where return; } - if header.snapshot_len > ARTIFACT_LEN_MAX || header.table_len > ARTIFACT_LEN_MAX { + // The manifest rides the body; a well-formed available=1 descriptor + // always carries one (an empty manifest still encodes its envelope). + let manifest = match consensus::decode_state_manifest( + &msg.as_slice()[size_of::()..header.size as usize], + ) { + Ok(manifest) => manifest, + Err(error) => { + tracing::error!( + shard = self.id, + peer = header.replica, + %error, + "state transfer descriptor manifest undecodable; ignoring" + ); + return; + } + }; + if let Some(oversized) = manifest.iter().find(|entry| entry.len > ARTIFACT_LEN_MAX) { tracing::error!( shard = self.id, - snapshot_len = header.snapshot_len, - table_len = header.table_len, + kind = oversized.kind, + len = oversized.len, "state transfer descriptor exceeds artifact cap; ignoring" ); return; @@ -2980,23 +3002,22 @@ where let Some(session) = session.as_mut() else { return; }; - if session.target.is_some() { + if session.target_accepted { // Duplicate descriptor (stall retry crossed the original). return; } - session.target = Some(StateTransferTargetInfo { - commit_op: header.commit_op, - snapshot_len: header.snapshot_len, - snapshot_checksum: header.snapshot_checksum, - table_frontier: header.table_frontier, - table_len: header.table_len, - table_checksum: header.table_checksum, - }); + session.target_accepted = true; + session.commit_op = header.commit_op; // Under ARTIFACT_LEN_MAX (checked above), so the casts hold. #[allow(clippy::cast_possible_truncation)] { - session.snapshot = Vec::with_capacity(header.snapshot_len as usize); - session.table = Vec::with_capacity(header.table_len as usize); + session.artifacts = manifest + .iter() + .map(|&entry| ArtifactProgress { + entry, + buf: Vec::with_capacity(entry.len as usize), + }) + .collect(); } session.idle_ticks = 0; } @@ -3006,18 +3027,17 @@ where tracing::info!( shard = self.id, peer = header.replica, - snapshot_seq = header.snapshot_seq, - snapshot_len = header.snapshot_len, - table_len = header.table_len, + artifacts = manifest.len(), + total_len = manifest.iter().map(|entry| entry.len).sum::(), commit_op = header.commit_op, "state transfer target accepted; fetching" ); - self.request_pending_state_chunk().await; + self.on_transfer_progress().await; } - /// Ask for the next missing chunk of the in-flight transfer (snapshot - /// first, then table). No-op when nothing is missing or no target is - /// accepted yet; also the stall-retry re-request. + /// Ask for the next missing chunk of the in-flight transfer (artifacts + /// pulled in manifest order). No-op when nothing is missing or no + /// manifest is accepted yet; also the stall-retry re-request. #[allow(clippy::future_not_send)] async fn request_pending_state_chunk(&self) where @@ -3030,26 +3050,20 @@ where let request = { let session = self.metadata_transfer.borrow(); session.as_ref().and_then(|session| { - let target = session.target.as_ref()?; - let (artifact, offset, remaining) = - if (session.snapshot.len() as u64) < target.snapshot_len { - ( - STATE_TRANSFER_ARTIFACT_SNAPSHOT, - session.snapshot.len() as u64, - target.snapshot_len - session.snapshot.len() as u64, - ) - } else if (session.table.len() as u64) < target.table_len { - ( - STATE_TRANSFER_ARTIFACT_TABLE, - session.table.len() as u64, - target.table_len - session.table.len() as u64, - ) - } else { - return None; - }; + if !session.target_accepted { + return None; + } + let (index, artifact) = session + .artifacts + .iter() + .enumerate() + .find(|(_, artifact)| !artifact.complete())?; + let offset = artifact.buf.len() as u64; + let remaining = artifact.entry.len - offset; #[allow(clippy::cast_possible_truncation)] let len = remaining.min(u64::from(STATE_CHUNK_LEN)) as u32; - Some((session.nonce, session.peer, artifact, offset, len)) + #[allow(clippy::cast_possible_truncation)] + Some((session.nonce, session.peer, index as u32, offset, len)) }) }; if let Some((nonce, peer, artifact, offset, len)) = request { @@ -3095,11 +3109,9 @@ where .get(&header.replica) .filter(|served| served.nonce == header.nonce); served.map_or(Some(ChunkReply::UnknownOffer), |served| { - let artifact_bytes = if header.artifact == STATE_TRANSFER_ARTIFACT_SNAPSHOT { - &served.offer.snapshot - } else { - &served.offer.table - }; + // Manifest-index addressing: an index past the offer is a + // requester bug (or a stale frame) and is dropped below. + let artifact_bytes = served.offer.payloads.get(header.artifact as usize)?; let start = header.offset as usize; let end = start.saturating_add(header.len as usize); artifact_bytes @@ -3187,43 +3199,75 @@ where return; } - let complete = { + { let mut session = self.metadata_transfer.borrow_mut(); let Some(session) = session.as_mut() else { return; }; - if session.nonce != header.nonce { + if session.nonce != header.nonce || !session.target_accepted { return; } - let Some(target) = session.target else { + let Some(artifact) = session.artifacts.get_mut(header.artifact as usize) else { return; }; let payload = &msg.as_slice()[size_of::()..header.size as usize]; - let (buf, expected_len) = if header.artifact == STATE_TRANSFER_ARTIFACT_SNAPSHOT { - (&mut session.snapshot, target.snapshot_len) - } else { - (&mut session.table, target.table_len) - }; // Chunks are pulled sequentially with one in flight; anything // else is a duplicate or reorder and is dropped (the stall retry // re-requests from the current frontier). - if header.offset != buf.len() as u64 { + if header.offset != artifact.buf.len() as u64 { return; } - if buf.len() as u64 + payload.len() as u64 > expected_len { + if artifact.buf.len() as u64 + payload.len() as u64 > artifact.entry.len { tracing::warn!( shard = self.id, artifact = header.artifact, - "state chunk overruns the declared artifact length; dropping session" + "state chunk overruns the declared artifact length; dropping frame" ); return; } - buf.extend_from_slice(payload); + artifact.buf.extend_from_slice(payload); session.idle_ticks = 0; - session.snapshot.len() as u64 == target.snapshot_len - && session.table.len() as u64 == target.table_len - }; + } + self.on_transfer_progress().await; + } + /// Drive the in-flight transfer forward: request the next missing chunk, + /// or - once every artifact is complete - verify, decode, and install. + /// Shared by descriptor acceptance and chunk arrival, so a manifest whose + /// artifacts are already complete (all empty) installs without waiting + /// for a chunk that will never come. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn on_transfer_progress(&self) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: StreamsFrontend + + StateMachine< + Input = Message, + Output = metadata::stm::result::ApplyReply, + Error = iggy_common::IggyError, + > + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, + { + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + let complete = { + let session = self.metadata_transfer.borrow(); + match session.as_ref() { + Some(session) if session.target_accepted => { + session.artifacts.iter().all(ArtifactProgress::complete) + } + _ => return, + } + }; if !complete { self.request_pending_state_chunk().await; return; @@ -3235,14 +3279,50 @@ where .borrow_mut() .take() .expect("session checked above"); - let target = session.target.expect("target checked above"); let peer = session.peer; + let commit_op = session.commit_op; + + // Per-artifact integrity, then pick the pieces this plane installs. + // Unknown kinds are refused rather than skipped: an artifact the + // serving peer thought worth shipping but this receiver cannot + // install would otherwise be silently dropped. + let mut snapshot: Option> = None; + let mut table: Option<(Vec, u64)> = None; + let mut damaged = false; + for (index, artifact) in session.artifacts.into_iter().enumerate() { + let actual = consensus::state_artifact_checksum(&artifact.buf); + if actual != artifact.entry.checksum { + tracing::error!( + shard = self.id, + artifact = index, + kind = artifact.entry.kind, + "state transfer artifact checksum mismatch" + ); + damaged = true; + break; + } + match artifact.entry.kind { + consensus::artifact_kind::METADATA_SNAPSHOT => snapshot = Some(artifact.buf), + consensus::artifact_kind::CLIENT_TABLE => { + table = Some((artifact.buf, artifact.entry.frontier)); + } + kind => { + tracing::error!( + shard = self.id, + kind, + "state transfer manifest carries a kind this plane cannot install" + ); + damaged = true; + break; + } + } + } - let snapshot_ok = state_artifact_checksum(&session.snapshot) == target.snapshot_checksum; - let table_ok = state_artifact_checksum(&session.table) == target.table_checksum; - let table = if snapshot_ok && table_ok { - match consensus::ClientTable::decode(&session.table, self.clients_table_max.get()) { - Ok(table) => Some(table), + let decoded = if damaged { + None + } else if let (Some(snapshot), Some((table_bytes, table_frontier))) = (snapshot, table) { + match consensus::ClientTable::decode(&table_bytes, self.clients_table_max.get()) { + Ok(table) => Some((snapshot, table, table_frontier)), Err(error) => { tracing::error!(shard = self.id, %error, "transferred client table undecodable"); None @@ -3251,14 +3331,12 @@ where } else { tracing::error!( shard = self.id, - snapshot_ok, - table_ok, - "state transfer artifact checksum mismatch" + "state transfer manifest is missing the snapshot or client table artifact" ); None }; - let Some(table) = table else { + let Some((snapshot, table, table_frontier)) = decoded else { // Damaged in transit: restart the session from scratch against // the same peer (fresh nonce; the peer re-offers). if consensus.state_transfer_stage() == consensus::StateTransferStage::Fetching { @@ -3268,9 +3346,9 @@ where *self.metadata_transfer.borrow_mut() = Some(MetadataTransferSession { nonce, peer, - target: None, - snapshot: Vec::new(), - table: Vec::new(), + commit_op: 0, + artifacts: Vec::new(), + target_accepted: false, idle_ticks: 0, }); self.send_request_state_transfer(consensus, peer, nonce) @@ -3279,19 +3357,17 @@ where }; consensus.set_state_transfer_stage(consensus::StateTransferStage::Installing); - match planes.0.install_state_transfer( - &session.snapshot, - table, - target.table_frontier, - target.commit_op, - ) { + match planes + .0 + .install_state_transfer(&snapshot, table, table_frontier, commit_op) + { Ok(snapshot_seq) => { consensus.set_state_transfer_stage(consensus::StateTransferStage::Idle); tracing::info!( shard = self.id, snapshot_seq, - commit_op = target.commit_op, - table_frontier = target.table_frontier, + commit_op, + table_frontier, "metadata state transfer installed; handing tail to journal repair" ); // Walk whatever is already walkable, then let repair fetch @@ -3490,7 +3566,7 @@ where return None; } session.idle_ticks = 0; - Some((session.peer, session.nonce, session.target.is_some())) + Some((session.peer, session.nonce, session.target_accepted)) }) }; if let Some((peer, nonce, target_accepted)) = transfer_stalled { @@ -3580,15 +3656,6 @@ where dispatch_vsr_actions::(consensus, None, &[action]).await; } -/// Artifact-level integrity stamp for state transfer (descriptor checksums; -/// chunks themselves carry none). -fn state_artifact_checksum(bytes: &[u8]) -> u64 { - use std::hash::Hasher; - let mut hasher = twox_hash::XxHash3_64::new(); - hasher.write(bytes); - hasher.finish() -} - /// Re-stamp a stored prepare with the current view before retransmission. /// After a view change the primary re-sends its uncommitted suffix as its /// own prepares (VSR), but the journal keeps the original view stamp and From 515dd5a524a233c37a7defcb05b9906e0d3d3174 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Fri, 31 Jul 2026 10:52:23 +0200 Subject: [PATCH 13/14] arm state transfer during new node lag --- core/integration/src/harness/handle/server.rs | 29 +++++++ .../src/harness/orchestrator/harness.rs | 13 ++++ .../tests/cluster/metadata_state_transfer.rs | 69 ++++++++++++++++ core/shard/src/lib.rs | 78 ++++++++++++++++--- 4 files changed, 178 insertions(+), 11 deletions(-) diff --git a/core/integration/src/harness/handle/server.rs b/core/integration/src/harness/handle/server.rs index 6e218c99be..7d6fefbc12 100644 --- a/core/integration/src/harness/handle/server.rs +++ b/core/integration/src/harness/handle/server.rs @@ -999,6 +999,35 @@ impl Restartable for ServerHandle { } } +impl ServerHandle { + /// Stop this node, delete its data directory, then start it again -- a + /// node that rejoins the cluster with no on-disk history, as a fresh + /// operator-provisioned replacement or a wiped disk does. + /// + /// Distinct from [`Restartable::restart`], which preserves the WAL and so + /// exercises the recovery-with-history path. The empty data directory is + /// what forces the state-transfer join: local recovery has nothing, and + /// the committed prefix the node missed no longer exists as WAL entries on + /// the peers that checkpointed it. + pub fn restart_from_clean_slate(&mut self) -> Result<(), TestBinaryError> { + let cleanup = self.config.cleanup; + self.config.cleanup = false; + self.stop_dependents()?; + self.stop()?; + + let data_path = self.data_path(); + if data_path.exists() { + fs::remove_dir_all(&data_path).map_err(|source| TestBinaryError::FileSystemError { + path: data_path, + source, + })?; + } + + self.config.cleanup = cleanup; + self.start() + } +} + impl Drop for ServerHandle { fn drop(&mut self) { // Reap the child before `port_reserver` drops: `Child::drop` detaches diff --git a/core/integration/src/harness/orchestrator/harness.rs b/core/integration/src/harness/orchestrator/harness.rs index df1d545e57..5d44c73173 100644 --- a/core/integration/src/harness/orchestrator/harness.rs +++ b/core/integration/src/harness/orchestrator/harness.rs @@ -330,6 +330,19 @@ impl TestHarness { Ok(()) } + /// Restart node `index` with its data directory wiped, so it rejoins the + /// still-live cluster as a fresh replica with no local history. The other + /// nodes stay up throughout (they hold quorum and keep committing), which + /// is the late-joiner / provisioned-replacement scenario rather than a + /// full-cluster restart. Clients are left connected to their own nodes. + pub fn restart_node_from_clean_slate(&mut self, index: usize) -> Result<(), TestBinaryError> { + let server = self + .servers + .get_mut(index) + .ok_or(TestBinaryError::MissingServer)?; + server.restart_from_clean_slate() + } + /// Restart EVERY node and reconnect all clients: the full-cluster /// restart path, where no settled primary survives to answer the rejoin /// probes and the replicas must fall back to an election among their diff --git a/core/integration/tests/cluster/metadata_state_transfer.rs b/core/integration/tests/cluster/metadata_state_transfer.rs index a2dcfb7be2..fd6962ea6d 100644 --- a/core/integration/tests/cluster/metadata_state_transfer.rs +++ b/core/integration/tests/cluster/metadata_state_transfer.rs @@ -104,3 +104,72 @@ async fn given_checkpointed_cluster_when_node_restarts_should_state_transfer_met sleep(MARKER_POLL).await; } } + +/// A node that joins a live, already-checkpointed cluster with NO local +/// history must state-transfer, exactly like the restart-with-WAL case above. +/// +/// This is the shape the user cares about beyond a plain restart: a fresh +/// operator-provisioned replacement, or the third node of a cluster whose +/// first two formed quorum and committed past a checkpoint before it arrived. +/// It has no WAL to probe from, so the restart-arms-transfer boot path does +/// not apply -- it joins as a plain view-0 backup, learns the frontier from +/// the primary's heartbeats, discovers via journal repair that the gap sits +/// below the retained floor, and converts THAT into a state transfer. The +/// same `RangeEvicted`-to-transfer path serves the restart, the late join, +/// and a partition heal; only the way each reaches repair differs. +/// +/// Node 2 (a follower) is wiped rather than node 0 so the client keeps its +/// primary and the cluster never loses quorum -- a genuine late join, not a +/// restart of the whole cluster. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_checkpointed_cluster_when_fresh_node_joins_late_should_state_transfer_metadata( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + for request in 1..=OPS_BEFORE_RESTART { + commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-latejoin-{request}")), + ) + .await; + } + + // Wipe a follower and bring it back with an empty data directory while the + // other two keep quorum. Its missing prefix (register at op 1 included) was + // checkpointed away on every survivor, so nothing but a transfer can seed + // its metadata state and client table. + harness.restart_node_from_clean_slate(2).unwrap(); + + let deadline = Instant::now() + TRANSFER_BUDGET; + loop { + if harness + .node(2) + .stdout_contains("metadata state transfer installed") + { + break; + } + assert!( + Instant::now() < deadline, + "fresh late-joining node never completed the metadata state transfer \ + within {TRANSFER_BUDGET:?}; it must convert a repair-floor eviction \ + into a transfer, not wedge below the floor" + ); + sleep(MARKER_POLL).await; + } + + // Functional: the cluster still commits with the rejoined node present, and + // the original session (registered below every floor) continues. + drop(stream); + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + OPS_BEFORE_RESTART + 1, + &create_stream_payload("iggy-latejoin-continuation"), + None, + ) + .await; +} diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 346dc6cacf..edb5e1eb08 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -2108,6 +2108,25 @@ where // the post-install repair targets the right frontier. if !consensus.is_transferring() { planes.0.commit_journal().await; + // A heartbeat is the only signal a behind-but-same-view + // replica gets that the frontier moved: it advances + // `commit_max`, but the walk above cannot cross a gap in + // its own WAL (a late joiner missed the ops below the + // primary's active window; the primary only retransmits + // uncommitted ops, never the committed prefix). Without + // this, such a replica learns it is behind and does + // nothing about it -- metadata repair is otherwise only + // rooted at StartView adoption, which a same-view + // late joiner never sees. Request repair from the + // primary; if it has checkpointed past the gap the + // repair floor evicts and the handler above converts to + // state transfer. Idempotent: `maybe_request_metadata_repair` + // no-ops when caught up, already transferring, or a + // session is live, so a caught-up replica and a + // cold-start node (commit_max == commit_min == 0) both + // skip it. + self.maybe_request_metadata_repair(consensus, header.replica) + .await; } } CommitOutcome::RespondStartView => { @@ -2500,17 +2519,54 @@ where } } Command2::RangeEvicted => { - // The metadata WAL retains everything above the snapshot - // floor; an evicted range means the peer compacted past - // this replica's gap -- bulk snapshot transfer (phase 3) - // territory. Surface loudly; the divergence backstop - // still guards the walk. - tracing::warn!( - shard = self.id, - retained_from = header.op, - "metadata repair range evicted on the serving peer; \ - snapshot transfer required" - ); + // Journal repair cannot close this gap: the serving peer + // compacted past it, so the ops this replica is missing no + // longer exist as WAL entries anywhere. This is the one + // authoritative "repair is impossible" signal, and it is + // shape-identical for every way a replica falls behind a + // checkpoint -- a fresh node joining an already-checkpointed + // cluster, a node whose partition healed after the quorum + // moved on, or a restart whose gap sits below the floor. All + // three convert here to state transfer against the peer that + // just announced the eviction (it has the checkpoint by + // definition), which replaces the snapshot-shaped state + // wholesale rather than replaying ops that are gone. + // + // Drop the repair session and arm the transfer only from + // `Idle`: a transfer already in flight owns the stage, and + // its own post-install tail repair can legitimately hit + // `RangeEvicted` again if the primary checkpointed mid + // transfer -- that reraises through the same path, and each + // round lifts the local floor, so it converges. + if consensus.state_transfer_stage() == consensus::StateTransferStage::Idle { + *self.metadata_repair.borrow_mut() = None; + consensus.begin_state_transfer_await(); + let nonce = iggy_common::random_id::get_uuid(); + *self.metadata_transfer.borrow_mut() = Some(MetadataTransferSession { + nonce, + peer: header.replica, + commit_op: 0, + artifacts: Vec::new(), + target_accepted: false, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + peer = header.replica, + retained_from = header.op, + local_commit = consensus.commit_min(), + "metadata repair floor evicted; converting to state transfer" + ); + self.send_request_state_transfer(consensus, header.replica, nonce) + .await; + } else { + tracing::debug!( + shard = self.id, + retained_from = header.op, + stage = ?consensus.state_transfer_stage(), + "metadata repair range evicted while a transfer is already in flight" + ); + } } _ => {} } From ea6c607e606a3fb1e13f26af6994777b2e5040d1 Mon Sep 17 00:00:00 2001 From: Grzegorz Koszyk Date: Fri, 31 Jul 2026 11:33:25 +0200 Subject: [PATCH 14/14] peer feetching during stale view restart --- core/consensus/src/impls.rs | 214 ++++++++++++++++++ core/consensus/src/plane_helpers.rs | 4 + .../src/harness/orchestrator/harness.rs | 13 ++ .../tests/cluster/metadata_state_transfer.rs | 90 ++++++++ 4 files changed, 321 insertions(+) diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index 6905b83bf9..29a10fa8a9 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -850,6 +850,16 @@ where /// the probe-exhausted election fallback (full-cluster bootstrap). state_transfer_stage: Cell, + /// Highest view seen on inbound traffic that this replica could not + /// process because the view was ahead of its own (a dropped newer-view + /// prepare or heartbeat). Proof the cluster elected past this replica + /// rather than that the primary died, which is why the heartbeat-timeout + /// handler probes to catch up instead of starting a futile election (see + /// [`Self::handle_normal_heartbeat_timeout`]). Monotone `max`; a value at + /// or below `view` is stale and inert because the catch-up guard is a + /// strict `>`. + observed_newer_view: Cell, + /// Highest op number that has been locally executed (state machine applied, /// client table updated). Advances one-by-one in `commit_journal` (backup) /// and `on_ack` (primary). On a normal primary, `commit_min == commit_max`. @@ -976,6 +986,7 @@ impl> VsrConsensus { ceded_primaryship: Cell::new(false), status: Cell::new(Status::Recovering), state_transfer_stage: Cell::new(StateTransferStage::Idle), + observed_newer_view: Cell::new(0), sequencer: LocalSequencer::new(0), commit_min: Cell::new(0), commit_max: Cell::new(0), @@ -1656,6 +1667,30 @@ impl> VsrConsensus { return Vec::new(); } + // Distinguish "primary died" from "primary moved to a view I missed". + // If newer-view traffic has been arriving (a partition that healed + // across an election, or a fresh/rejoined node that never saw the + // election), the primary is alive and the timer only fired because a + // matching-view heartbeat never reset it. Electing would be futile -- + // a lagging replica cannot win -- and would drag a healthy cluster + // through a needless view change. Probe instead: the current primary + // answers a `RequestStartView` with its live `StartView` regardless of + // the probe's view stamp, and adopting it routes into journal repair + // (and state transfer, if the gap fell below the peer's floor). The + // probe re-broadcasts on its own timer, so no action is emitted here; + // `Recovering` also makes this timeout inert until the probe resolves. + if self.observed_newer_view.get() > self.view.get() { + tracing::info!( + replica = self.replica, + namespace_raw = self.namespace, + view = self.view.get(), + observed_newer_view = self.observed_newer_view.get(), + "heartbeat timed out behind a newer view; probing to catch up" + ); + self.begin_view_probe(); + return Vec::new(); + } + self.start_election(plane, ViewChangeReason::NormalHeartbeatTimeout) } @@ -1898,6 +1933,26 @@ impl> VsrConsensus { return Vec::new(); } + // A primary-by-index that has seen a newer view is stale: the cluster + // elected past it while it was gone, and it booted primary-by-index at + // an old view (a fresh or rejoined node that missed the election). It + // has no `NormalHeartbeat` timeout to catch this -- it IS the heartbeat + // sender -- so convert here instead of advertising a commit point no + // peer will accept. The probe solicits the current `StartView`, which + // demotes this replica to a backup and routes into repair / state + // transfer. `begin_view_probe` stops this timer. + if self.observed_newer_view.get() > self.view.get() { + tracing::info!( + replica = self.replica, + namespace_raw = self.namespace, + view = self.view.get(), + observed_newer_view = self.observed_newer_view.get(), + "stale primary-by-index behind a newer view; probing to catch up" + ); + self.begin_view_probe(); + return Vec::new(); + } + self.timeouts.borrow_mut().reset(TimeoutKind::CommitMessage); // Don't advertise a commit point we haven't locally executed yet. @@ -2237,6 +2292,18 @@ impl> VsrConsensus { self.set_state_transfer_stage(StateTransferStage::AwaitingTarget); } + /// Record that a message stamped with `view` arrived that this replica + /// could not process because the view is ahead of its own. Monotone: only + /// a value strictly above the current record moves it. Called from the two + /// ingress paths that drop newer-view traffic (`handle_commit` and + /// `replicate_preflight`); read by the heartbeat-timeout handler to choose + /// catch-up over election. + pub fn observe_newer_view(&self, view: u32) { + if view > self.observed_newer_view.get() { + self.observed_newer_view.set(view); + } + } + #[must_use] pub const fn state_transfer_stage(&self) -> StateTransferStage { self.state_transfer_stage.get() @@ -2497,6 +2564,12 @@ impl> VsrConsensus { { return CommitOutcome::RespondStartView; } + // A "primary" hearing a NEWER view is stale-by-index: the cluster + // elected past it. Record so its heartbeat-send timer converts to + // a probe (see `handle_commit_message_timeout`). + if header.view > self.view.get() { + self.observe_newer_view(header.view); + } return CommitOutcome::Accepted; } @@ -2505,6 +2578,13 @@ impl> VsrConsensus { } if header.view != self.view.get() { + // A heartbeat from a newer view is proof the cluster elected past + // this replica. Record it so the heartbeat-timeout handler probes + // to catch up instead of electing (this same heartbeat cannot + // reset the timer -- the reset below is gated on a matching view). + if header.view > self.view.get() { + self.observe_newer_view(header.view); + } return CommitOutcome::Accepted; } @@ -3344,4 +3424,138 @@ mod state_transfer_stage_tests { consensus.set_state_transfer_stage(StateTransferStage::Idle); assert!(!consensus.is_transferring()); } + + // A backup left behind on VIEW (partition healed across an election, or a + // fresh node that never saw it) keeps getting newer-view heartbeats it + // cannot process -- they never reset its heartbeat timer, so the timer + // fires. The primary is alive, so it must PROBE to adopt the current view + // (which routes into repair / state transfer), not elect: a lagging + // replica cannot win, and electing would drag a healthy cluster through a + // needless view change. + #[test] + fn heartbeat_timeout_behind_a_newer_view_probes_not_elects() { + // Replica 1 is a backup at view 0 (primary_index(0) == 0). + let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), 0); + + // A newer-view frame arrived and was dropped by the ingress path. + consensus.observe_newer_view(3); + + let actions = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + + assert_eq!( + consensus.status(), + Status::Recovering, + "must enter the probe (Recovering), not an election" + ); + assert_eq!(consensus.view(), 0, "probing must not bump the view"); + assert!( + !actions + .iter() + .any(|action| matches!(action, VsrAction::SendStartViewChange { .. })), + "must not broadcast an election SVC" + ); + } + + // No newer view seen means the primary is presumed dead, so the timeout + // must still elect -- the split must not swallow the ordinary case. + #[test] + fn heartbeat_timeout_with_no_newer_view_still_elects() { + let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + + let actions = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + + assert_eq!( + consensus.status(), + Status::ViewChange, + "a silent primary must still trigger an election" + ); + assert_eq!(consensus.view(), 1, "election advances the view"); + assert!( + actions + .iter() + .any(|action| matches!(action, VsrAction::SendStartViewChange { .. })), + "election must broadcast its SVC" + ); + } + + // A recorded view at or below the current one is stale (already caught up) + // and must not divert a genuine election, since the guard is a strict `>`. + #[test] + fn stale_observed_view_does_not_divert_election() { + let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + consensus.observe_newer_view(0); + + let _ = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + + assert_eq!( + consensus.status(), + Status::ViewChange, + "a stale observed view must not suppress the election" + ); + } + + // A fresh or rejoined node that missed an election boots primary-by-index + // at a stale view (replica 0 is primary_index(0)). It has no heartbeat + // timeout -- it IS the heartbeat sender -- so its heartbeat-SEND timer must + // convert to a probe once it observes the newer view, or it wedges forever + // advertising a commit point no peer accepts. + #[test] + fn stale_primary_by_index_probes_on_its_heartbeat_send_timer() { + // Replica 0 is primary-by-index at view 0. + let consensus = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + assert!(consensus.is_primary()); + assert_eq!(consensus.status(), Status::Normal); + + // A prepare/heartbeat from the real primary at a newer view was seen. + consensus.observe_newer_view(2); + + let actions = consensus.handle_commit_message_timeout(); + + assert_eq!( + consensus.status(), + Status::Recovering, + "a stale primary-by-index must probe, not keep heartbeating" + ); + assert!( + !actions + .iter() + .any(|action| matches!(action, VsrAction::SendCommit { .. })), + "a stale primary must not advertise a commit point" + ); + } + + // A genuine current primary (no newer view seen) keeps heartbeating. + #[test] + fn current_primary_keeps_heartbeating() { + let consensus = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + + let actions = consensus.handle_commit_message_timeout(); + + assert_eq!(consensus.status(), Status::Normal, "must stay primary"); + assert!( + actions + .iter() + .any(|action| matches!(action, VsrAction::SendCommit { .. })), + "a healthy primary must heartbeat" + ); + } + + // `observe_newer_view` is monotone: an older stamp cannot lower it. + #[test] + fn observe_newer_view_keeps_the_max() { + let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + consensus.observe_newer_view(5); + consensus.observe_newer_view(2); + // Timeout at view 0 with the record still 5 must probe. + let _ = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + assert_eq!(consensus.status(), Status::Recovering); + } } diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index 3df29523b8..cb56d54da6 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -145,6 +145,10 @@ where } if header.view > consensus.view() { + // Dropped, but recorded: a newer-view prepare is proof the cluster + // moved past this replica, so the heartbeat-timeout handler probes to + // catch up rather than starting a futile election. + consensus.observe_newer_view(header.view); return Err(IgnoreReason::NewerView); } diff --git a/core/integration/src/harness/orchestrator/harness.rs b/core/integration/src/harness/orchestrator/harness.rs index 5d44c73173..7f14cc4184 100644 --- a/core/integration/src/harness/orchestrator/harness.rs +++ b/core/integration/src/harness/orchestrator/harness.rs @@ -343,6 +343,19 @@ impl TestHarness { server.restart_from_clean_slate() } + /// Stop node `index` and leave it down. The surviving nodes keep quorum + /// (in a 3-node cluster, 2 of 3), and if the stopped node was the primary + /// they elect a new one, advancing the view. Pairs with + /// [`Self::restart_node_from_clean_slate`] to model a node that misses an + /// election entirely and rejoins at a stale view. + pub fn stop_node(&mut self, index: usize) -> Result<(), TestBinaryError> { + let server = self + .servers + .get_mut(index) + .ok_or(TestBinaryError::MissingServer)?; + server.stop() + } + /// Restart EVERY node and reconnect all clients: the full-cluster /// restart path, where no settled primary survives to answer the rejoin /// probes and the replicas must fall back to an election among their diff --git a/core/integration/tests/cluster/metadata_state_transfer.rs b/core/integration/tests/cluster/metadata_state_transfer.rs index fd6962ea6d..ba353ee21c 100644 --- a/core/integration/tests/cluster/metadata_state_transfer.rs +++ b/core/integration/tests/cluster/metadata_state_transfer.rs @@ -173,3 +173,93 @@ async fn given_checkpointed_cluster_when_fresh_node_joins_late_should_state_tran ) .await; } + +/// A node that rejoins at a STALE VIEW must probe to catch up, not wedge. +/// +/// The late-join test above rejoins a follower while the cluster is still at +/// view 0, so it adopts the frontier from same-view heartbeats. This one forces +/// the harder case: kill the view-0 primary so the survivors elect past it, +/// then rejoin that same node (replica 0) with an empty disk. It boots +/// primary-by-index at view 0 -- so it has NO heartbeat timeout to notice it is +/// behind, and thinks it is the primary of a view the cluster has abandoned. +/// +/// It must observe the newer-view traffic it keeps dropping, convert its own +/// heartbeat-SEND timer into a view probe, adopt the current view from the +/// live primary's `StartView`, and from there follow the same repair -> +/// `RangeEvicted` -> transfer path. Without that it advertises a commit point +/// no peer accepts, forever. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_election_past_a_node_when_it_rejoins_stale_should_probe_then_state_transfer( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + for request in 1..=OPS_BEFORE_RESTART { + commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-stale-{request}")), + ) + .await; + } + drop(stream); + + // Kill the view-0 primary (replica 0). The survivors (1, 2) hold quorum and + // elect a new primary at a higher view, leaving replica 0 behind on view. + harness.stop_node(0).unwrap(); + + // Confirm the cluster recovered to a live primary at the new view before + // rejoining: one continuation must commit against a survivor. The stopped + // node's address is skipped by the round-robin. + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + OPS_BEFORE_RESTART + 1, + &create_stream_payload("iggy-stale-failover"), + None, + ) + .await; + + // Rejoin replica 0 with an empty disk. It boots primary-by-index at view 0 + // while the cluster sits at a higher view -- the stale-primary case. + harness.restart_node_from_clean_slate(0).unwrap(); + + // It must first PROBE (its heartbeat-send timer converting, since it has no + // heartbeat-receive timer as a "primary"), then complete the transfer. The + // probe marker distinguishes this path from the same-view backstop. + let deadline = Instant::now() + TRANSFER_BUDGET; + let mut probed = false; + loop { + probed = probed || harness.node(0).stdout_contains("probing to catch up"); + if harness + .node(0) + .stdout_contains("metadata state transfer installed") + { + assert!( + probed, + "the rejoined node transferred without first probing; the \ + stale-view path must reach the transfer through a view probe" + ); + break; + } + assert!( + Instant::now() < deadline, + "node that rejoined at a stale view never state-transferred within \ + {TRANSFER_BUDGET:?}; a stale primary-by-index must convert its \ + heartbeat-send timer into a probe rather than wedge" + ); + sleep(MARKER_POLL).await; + } + + // Functional: the session continues cluster-wide with the node rejoined. + resume_request( + &addrs, + session, + OPS_BEFORE_RESTART + 2, + &create_stream_payload("iggy-stale-continuation"), + None, + ) + .await; +}