diff --git a/Cargo.lock b/Cargo.lock index 7591cdbf30..f336698105 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3227,6 +3227,7 @@ dependencies = [ "rand_xoshiro", "server_common", "tracing", + "twox-hash", ] [[package]] @@ -12018,6 +12019,7 @@ dependencies = [ "server_common", "thiserror 2.0.19", "tracing", + "twox-hash", ] [[package]] diff --git a/core/binary_protocol/src/consensus/command.rs b/core/binary_protocol/src/consensus/command.rs index d2d0ada7cc..6dcf1298ae 100644 --- a/core/binary_protocol/src/consensus/command.rs +++ b/core/binary_protocol/src/consensus/command.rs @@ -59,6 +59,17 @@ pub enum Command2 { RepairPrepare = 19, RepairDone = 20, RangeEvicted = 21, + + // State transfer (metadata plane): a restarted replica replaces its + // snapshot-shaped state (metadata snapshot + client table) from the + // current primary, then journal repair covers the tail. Pull-based: + // the requester asks for the target descriptor, then fetches each + // artifact in bounded chunks (per-peer bus queues drop overruns + // silently, so push cannot work). + RequestStateTransfer = 22, + StateTransferTarget = 23, + RequestStateChunk = 24, + StateChunk = 25, } // SAFETY: Command2 is #[repr(u8)] with no padding bytes. @@ -69,7 +80,7 @@ unsafe impl CheckedBitPattern for Command2 { type Bits = u8; fn is_valid_bit_pattern(bits: &u8) -> bool { - *bits <= Self::RangeEvicted as u8 + *bits <= Self::StateChunk as u8 } } @@ -90,8 +101,8 @@ mod tests { #[test] fn replica_auth_commands_are_valid_bit_patterns() { - // Locks the is_valid_bit_pattern bump: 14..=21 parse, 22 still rejects. - for command in 14u8..=21 { + // Locks the is_valid_bit_pattern bump: 14..=25 parse, 26 still rejects. + for command in 14u8..=25 { let mut buf: AVec> = AVec::new(16); buf.resize(256, 0); buf[60] = command; @@ -99,7 +110,7 @@ mod tests { } let mut buf: AVec> = AVec::new(16); buf.resize(256, 0); - buf[60] = 22; + buf[60] = 26; assert!(bytemuck::checked::try_from_bytes::(&buf).is_err()); } } diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index b819037a74..c9134ee7e0 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -1280,6 +1280,277 @@ impl ConsensusHeader for RepairRangeReplyHeader { } } +// State transfer: descriptor + chunk pull frames. +// +// Plane-agnostic: the descriptor's BODY carries a state manifest (see the +// consensus crate's `state_manifest`) listing N artifacts, and the chunk +// frames address bytes by `(manifest index, offset)`. The metadata plane +// ships two artifacts (snapshot + client table); the partition plane ships +// its own set (segment logs, offsets) through the same frames. + +// RequestStateTransferHeader - restarted replica -> current primary + +/// Ask the current primary for a state-transfer target descriptor. +/// +/// Answered with `StateTransferTarget`. Sent by a restarted replica after it +/// adopts a view from a live primary; `nonce` correlates the whole transfer +/// session. +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[repr(C)] +pub struct RequestStateTransferHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub nonce: u128, + pub namespace: u64, + pub reserved: [u8; 104], +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!( + offset_of!(RequestStateTransferHeader, nonce) + == offset_of!(RequestStateTransferHeader, reserved_frame) + size_of::<[u8; 66]>() + ); + assert!( + offset_of!(RequestStateTransferHeader, reserved) + size_of::<[u8; 104]>() == HEADER_SIZE + ); +}; + +impl ConsensusHeader for RequestStateTransferHeader { + const COMMAND: Command2 = Command2::RequestStateTransfer; + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::RequestStateTransfer { + return Err(ConsensusError::InvalidCommand { + expected: Command2::RequestStateTransfer, + found: self.command, + }); + } + Ok(()) + } +} + +// StateTransferTargetHeader - serving primary -> requester + +/// The transfer target descriptor. +/// +/// The artifact list rides the BODY as an encoded state manifest (`size` +/// spans header + manifest); the header carries only the session nonce and +/// the serving peer's applied frontier. `available == 0` means the serving +/// peer cannot serve right now (not a caught-up primary, or it has never +/// checkpointed, so the requester's journal repair can cover the whole gap) +/// and the frame is header-only. +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[repr(C)] +pub struct StateTransferTargetHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub nonce: u128, + /// Serving primary's applied frontier (`commit_min`) when the descriptor + /// was built. The receiver's tail repair targets past this. + pub commit_op: u64, + pub namespace: u64, + pub available: u8, + pub reserved: [u8; 95], +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!( + offset_of!(StateTransferTargetHeader, nonce) + == offset_of!(StateTransferTargetHeader, reserved_frame) + size_of::<[u8; 66]>() + ); + assert!(offset_of!(StateTransferTargetHeader, reserved) + size_of::<[u8; 95]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for StateTransferTargetHeader { + const COMMAND: Command2 = Command2::StateTransferTarget; + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::StateTransferTarget { + return Err(ConsensusError::InvalidCommand { + expected: Command2::StateTransferTarget, + found: self.command, + }); + } + if self.available > 1 { + return Err(ConsensusError::InvalidField( + "available must be 0 or 1".to_string(), + )); + } + // Unavailable is a bare refusal; a manifest body on it would be + // ambiguous (which offer would the chunks belong to?). + if self.available == 0 && self.size as usize != HEADER_SIZE { + return Err(ConsensusError::InvalidField( + "unavailable descriptor must be header-only".to_string(), + )); + } + Ok(()) + } +} + +// RequestStateChunkHeader - requester -> serving primary + +/// Pull one bounded chunk of an artifact. +/// +/// Lockstep per artifact: the requester keeps at most one chunk in flight, +/// so the bounded per-peer bus queue can never drop a burst tail. +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[repr(C)] +pub struct RequestStateChunkHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub nonce: u128, + pub offset: u64, + pub namespace: u64, + pub len: u32, + /// Index into the offered state manifest. Range-checked by the serving + /// handler against the cached offer (the header cannot know the count). + pub artifact: u32, + pub reserved: [u8; 88], +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!( + offset_of!(RequestStateChunkHeader, nonce) + == offset_of!(RequestStateChunkHeader, reserved_frame) + size_of::<[u8; 66]>() + ); + assert!(offset_of!(RequestStateChunkHeader, reserved) + size_of::<[u8; 88]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for RequestStateChunkHeader { + const COMMAND: Command2 = Command2::RequestStateChunk; + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::RequestStateChunk { + return Err(ConsensusError::InvalidCommand { + expected: Command2::RequestStateChunk, + found: self.command, + }); + } + if self.len == 0 { + return Err(ConsensusError::InvalidField( + "chunk len must be non-zero".to_string(), + )); + } + Ok(()) + } +} + +// StateChunkHeader - serving primary -> requester (carries payload) + +/// One chunk of artifact bytes at `offset`. +/// +/// The payload rides the body (`size` spans header + payload). Transit +/// integrity is checked at the artifact level (descriptor checksums), not +/// per chunk. +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[repr(C)] +pub struct StateChunkHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub nonce: u128, + pub offset: u64, + pub namespace: u64, + /// Index into the offered state manifest. Range-checked by the receiving + /// handler against its accepted manifest. + pub artifact: u32, + pub reserved: [u8; 92], +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + assert!( + offset_of!(StateChunkHeader, nonce) + == offset_of!(StateChunkHeader, reserved_frame) + size_of::<[u8; 66]>() + ); + assert!(offset_of!(StateChunkHeader, reserved) + size_of::<[u8; 92]>() == HEADER_SIZE); +}; + +impl ConsensusHeader for StateChunkHeader { + const COMMAND: Command2 = Command2::StateChunk; + fn operation(&self) -> Operation { + Operation::Reserved + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::StateChunk { + return Err(ConsensusError::InvalidCommand { + expected: Command2::StateChunk, + found: self.command, + }); + } + if (self.size as usize) < HEADER_SIZE { + return Err(ConsensusError::InvalidField( + "state chunk size below header size".to_string(), + )); + } + Ok(()) + } +} + // Tests #[cfg(test)] diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs index 594d1ed989..83bf5ae616 100644 --- a/core/binary_protocol/src/consensus/mod.rs +++ b/core/binary_protocol/src/consensus/mod.rs @@ -48,7 +48,8 @@ pub use header::{ CommitHeader, ConsensusHeader, DoViewChangeHeader, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, - RequestStartViewHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, SIZE_FIELD_OFFSET, + StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, read_size_field, }; pub use operation::Operation; diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index 1d2010f3f6..1f4baf3fb6 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -74,8 +74,9 @@ pub use consensus::{ Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, - StartViewHeader, read_size_field, result_code, result_section_len, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, read_size_field, result_code, result_section_len, }; pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_command}; pub use error::WireError; diff --git a/core/consensus/Cargo.toml b/core/consensus/Cargo.toml index a612dca77f..5aec679f58 100644 --- a/core/consensus/Cargo.toml +++ b/core/consensus/Cargo.toml @@ -41,6 +41,7 @@ rand = { workspace = true } rand_xoshiro = { workspace = true } server_common = { workspace = true } tracing = { workspace = true } +twox-hash = { workspace = true } [dev-dependencies] futures = { workspace = true } diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 1011a5ae8c..dda3847376 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -15,11 +15,16 @@ // specific language governing permissions and limitations // under the License. -use iggy_binary_protocol::ReplyHeader; -use server_common::{MESSAGE_ALIGN, Message, iobuf::Frozen}; +use iggy_binary_protocol::{GenericHeader, ReplyHeader}; +use server_common::{ + MESSAGE_ALIGN, Message, + iobuf::{Frozen, Owned}, +}; use std::collections::{HashMap, VecDeque}; +use std::hash::Hasher; use std::mem::size_of; use tracing::trace; +use twox_hash::XxHash3_64; /// Refcounted wrapper around a committed reply. /// @@ -204,10 +209,12 @@ pub enum CommitReply { /// `commit_register` in the apply path, so every replica of the group /// derives an identical table from the committed log. /// -/// ## Known gaps +/// ## Serialization /// -/// - **Serialization**: encode/decode for rejoin slice-fetch and state -/// transfer TODO (IGGY-137). +/// [`Self::encode`] / [`Self::decode`] carry the table across state transfer +/// (a cluster-restart rejoin replaces its table with the primary's copy). +/// Deterministic: slot-order walk over apply-derived state, so every +/// caught-up replica encodes identical bytes. #[derive(Debug)] pub struct ClientTable { /// `None` = free slot. Deterministic iteration for eviction + serialization. @@ -635,6 +642,200 @@ impl ClientTable { } } +/// Failure decoding an encoded client table (state transfer install path). +#[derive(Debug)] +pub enum ClientTableCodecError { + /// Byte stream ended mid-field. + Truncated, + /// Leading magic is not [`CLIENT_TABLE_MAGIC`]. + BadMagic, + /// Trailing hash does not match the content. + ChecksumMismatch { expected: u64, actual: u64 }, + /// Encoded entry count exceeds this table's capacity. + TooManyEntries { count: u32, max: usize }, + /// A cached reply's bytes do not parse as a valid reply message. + InvalidReply, + /// An entry carries an empty reply ring (violates the never-empty + /// invariant registration establishes). + EmptyRing, +} + +impl std::fmt::Display for ClientTableCodecError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Truncated => write!(f, "encoded client table truncated"), + Self::BadMagic => write!(f, "encoded client table has wrong magic"), + Self::ChecksumMismatch { expected, actual } => write!( + f, + "client table checksum mismatch: expected {expected:#018x}, actual {actual:#018x}" + ), + Self::TooManyEntries { count, max } => { + write!(f, "encoded client table holds {count} entries, max {max}") + } + Self::InvalidReply => write!(f, "encoded client table holds an invalid cached reply"), + Self::EmptyRing => write!(f, "encoded client table entry has an empty reply ring"), + } + } +} + +impl std::error::Error for ClientTableCodecError {} + +/// Format tag for [`ClientTable::encode`]; bump on layout change. +pub const CLIENT_TABLE_MAGIC: [u8; 4] = *b"ICT1"; + +impl ClientTable { + /// Encode the table for state transfer. + /// + /// Layout (all little-endian): `magic(4) count(u32)` then per entry in + /// slot order `client(u128) epoch(u64) user_id(u32) watermark(u64) + /// watermark_checksum(u128) ring_len(u8) [reply_len(u32) reply_bytes]*`, + /// terminated by an `XxHash3_64(8)` over everything before it. + /// + /// Slot order makes the bytes deterministic across caught-up replicas: + /// slots are apply-derived state. + #[must_use] + #[allow(clippy::cast_possible_truncation)] + pub fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(64 * self.index.len() + 16); + out.extend_from_slice(&CLIENT_TABLE_MAGIC); + out.extend_from_slice(&(self.index.len() as u32).to_le_bytes()); + for (slot_idx, slot) in self.slots.iter().enumerate() { + let Some(entry) = slot else { continue }; + debug_assert_eq!(self.index.get(&entry.client_id), Some(&slot_idx)); + out.extend_from_slice(&entry.client_id.to_le_bytes()); + out.extend_from_slice(&entry.epoch.to_le_bytes()); + out.extend_from_slice(&entry.user_id.to_le_bytes()); + out.extend_from_slice(&entry.watermark.to_le_bytes()); + out.extend_from_slice(&entry.watermark_checksum.to_le_bytes()); + out.push(entry.ring.len() as u8); + for reply in &entry.ring { + let bytes = reply.bytes.as_slice(); + out.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + out.extend_from_slice(bytes); + } + } + let mut hasher = XxHash3_64::new(); + hasher.write(&out); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + out + } + + /// Decode a table encoded by [`Self::encode`] into a fresh table of + /// `max_clients` capacity. + /// + /// The denormalized `latest_commit` is rebuilt from the decoded ring's + /// back, not trusted from the wire, so it cannot drift from the ring. + /// + /// # Errors + /// [`ClientTableCodecError`] on truncation, magic/checksum mismatch, + /// capacity overflow, or an undecodable cached reply. + /// + /// # Panics + /// Unreachable: slice-to-array conversions are length-checked first. + pub fn decode(bytes: &[u8], max_clients: usize) -> Result { + const TRAILER: usize = size_of::(); + let content_len = bytes + .len() + .checked_sub(TRAILER) + .ok_or(ClientTableCodecError::Truncated)?; + let (content, trailer) = bytes.split_at(content_len); + let expected = u64::from_le_bytes(trailer.try_into().expect("trailer is 8 bytes")); + let mut hasher = XxHash3_64::new(); + hasher.write(content); + let actual = hasher.finish(); + if expected != actual { + return Err(ClientTableCodecError::ChecksumMismatch { expected, actual }); + } + + let mut reader = CodecReader { bytes: content }; + if reader.take(CLIENT_TABLE_MAGIC.len())? != CLIENT_TABLE_MAGIC { + return Err(ClientTableCodecError::BadMagic); + } + let count = reader.u32()?; + if count as usize > max_clients { + return Err(ClientTableCodecError::TooManyEntries { + count, + max: max_clients, + }); + } + + let mut table = Self::new(max_clients); + for slot_idx in 0..count as usize { + let client_id = reader.u128()?; + let epoch = reader.u64()?; + let user_id = reader.u32()?; + let watermark = reader.u64()?; + let watermark_checksum = reader.u128()?; + let ring_len = reader.u8()?; + if ring_len == 0 { + return Err(ClientTableCodecError::EmptyRing); + } + let mut ring = VecDeque::with_capacity(REPLY_RING_CAPACITY); + for _ in 0..ring_len { + let reply_len = reader.u32()? as usize; + let reply_bytes = reader.take(reply_len)?; + let owned = Owned::::copy_from_slice(reply_bytes); + let message = Message::::try_from(owned) + .map_err(|_| ClientTableCodecError::InvalidReply)? + .try_into_typed::() + .map_err(|_| ClientTableCodecError::InvalidReply)?; + ring.push_back(CachedReply::from_message(message)); + } + let latest_commit = ring + .back() + .expect("ring_len checked non-zero above") + .header() + .commit; + table.slots[slot_idx] = Some(ClientEntry { + epoch, + user_id, + watermark, + watermark_checksum, + ring, + client_id, + latest_commit, + }); + table.index.insert(client_id, slot_idx); + } + if !reader.bytes.is_empty() { + return Err(ClientTableCodecError::Truncated); + } + Ok(table) + } +} + +/// Little-endian cursor over the encoded table content. +struct CodecReader<'a> { + bytes: &'a [u8], +} + +impl<'a> CodecReader<'a> { + const fn take(&mut self, len: usize) -> Result<&'a [u8], ClientTableCodecError> { + if self.bytes.len() < len { + return Err(ClientTableCodecError::Truncated); + } + let (head, tail) = self.bytes.split_at(len); + self.bytes = tail; + Ok(head) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().expect("4B"))) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.take(8)?.try_into().expect("8B"))) + } + + fn u128(&mut self) -> Result { + Ok(u128::from_le_bytes(self.take(16)?.try_into().expect("16B"))) + } +} + impl ClientEntry { /// Latest committed reply (register or app op). /// @@ -675,6 +876,7 @@ mod tests { /// assert on it (see `register_stores_user_id` for the accessor check). const TEST_USER_ID: u32 = 7; + #[allow(clippy::cast_possible_truncation)] fn make_register_reply(client: u128, commit: u64) -> Message { let header_size = std::mem::size_of::(); let mut msg = Message::::new(header_size); @@ -686,6 +888,8 @@ mod tests { client, request: REGISTER_REQUEST_ID, commit, + // Real size so codec-roundtripped replies re-parse. + size: header_size as u32, command: Command2::Reply, operation: Operation::Register, ..ReplyHeader::default() @@ -697,6 +901,7 @@ mod tests { make_reply_with_checksum(client, request, commit, 0) } + #[allow(clippy::cast_possible_truncation)] fn make_reply_with_checksum( client: u128, request: u64, @@ -714,6 +919,8 @@ mod tests { request, commit, request_checksum, + // Real size so codec-roundtripped replies re-parse. + size: header_size as u32, command: Command2::Reply, operation: Operation::SendMessages, ..ReplyHeader::default() @@ -1197,4 +1404,98 @@ mod tests { RequestStatus::Fenced { .. } )); } + + // Codec tests + + // State transfer ships the table as bytes; the decoded table must answer + // every dedup question identically to the original. + #[test] + fn encode_decode_roundtrip_preserves_dedup_state() { + let mut table = ClientTable::new(10); + table.commit_register(1, 11, make_register_reply(1, 10)); + table.commit_reply(1, make_reply_with_checksum(1, 1, 11, 0xAA)); + table.commit_reply(1, make_reply_for(1, 2, 12)); + // Rebind at op 20: fence moves to 20, watermark preserved. + table.commit_register(1, 22, make_register_reply(1, 20)); + table.commit_register(2, 33, make_register_reply(2, 30)); + + let encoded = table.encode(); + let decoded = ClientTable::decode(&encoded, 10).expect("roundtrip decodes"); + + assert_eq!(decoded.count(), 2); + assert_eq!(decoded.get_epoch(1), Some(20)); + assert_eq!(decoded.get_user_id(1), Some(22)); + assert_eq!(decoded.get_watermark(1), Some(2)); + assert_eq!(decoded.get_epoch(2), Some(30)); + match decoded.check_request(1, 20, 2, 0) { + RequestStatus::Duplicate(cached) => { + assert_eq!(cached.header().request, 2, "latest reply survives"); + assert_eq!(cached.header().commit, 12, "original bytes survive"); + } + other => panic!("expected Duplicate, got {other:?}"), + } + assert!(matches!( + decoded.check_request(1, 20, 1, 0xBB), + RequestStatus::ChecksumMismatch { request: 1 } + )); + assert!(matches!( + decoded.check_request(1, 10, 1, 0), + RequestStatus::Fenced { .. } + )); + + // Deterministic bytes: encoding the decoded table reproduces them. + assert_eq!(decoded.encode(), encoded); + } + + // The denormalized `latest_commit` is rebuilt from the ring on decode, so + // eviction ranks a transferred table exactly like the original. + #[test] + fn decode_rebuilds_eviction_ranking() { + let mut table = ClientTable::new(2); + table.commit_register(100, TEST_USER_ID, make_register_reply(100, 10)); + table.commit_register(200, TEST_USER_ID, make_register_reply(200, 20)); + table.commit_reply(100, make_reply_for(100, 1, 30)); + + let mut decoded = ClientTable::decode(&table.encode(), 2).expect("roundtrip decodes"); + // 200 (latest commit 20) is older than 100 (latest commit 30). + decoded.commit_register(300, TEST_USER_ID, make_register_reply(300, 40)); + assert!(decoded.get_reply(100).is_some()); + assert!(decoded.get_reply(200).is_none(), "200 was the oldest"); + assert!(decoded.get_reply(300).is_some()); + } + + #[test] + fn decode_rejects_corruption() { + let mut table = ClientTable::new(4); + table.commit_register(1, TEST_USER_ID, make_register_reply(1, 10)); + let encoded = table.encode(); + + // Flipped content byte -> checksum mismatch. + let mut flipped = encoded.clone(); + flipped[8] ^= 0xFF; + assert!(matches!( + ClientTable::decode(&flipped, 4), + Err(ClientTableCodecError::ChecksumMismatch { .. }) + )); + + // Truncation. + assert!(matches!( + ClientTable::decode(&encoded[..encoded.len() - 1], 4), + Err(ClientTableCodecError::ChecksumMismatch { .. } | ClientTableCodecError::Truncated) + )); + + // Capacity overflow. + assert!(matches!( + ClientTable::decode(&encoded, 0), + Err(ClientTableCodecError::TooManyEntries { .. }) + )); + + let empty = ClientTable::new(4).encode(); + assert_eq!( + ClientTable::decode(&empty, 4) + .expect("empty table decodes") + .count(), + 0 + ); + } } diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index f969cb0224..29a10fa8a9 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -661,6 +661,47 @@ pub enum Status { Recovering, } +/// State-transfer progress, orthogonal to [`Status`]. +/// +/// A transferring replica stays a normal protocol participant (it adopts +/// views, votes, answers probes) but withholds `PrepareOk` and ignores live +/// prepares (`replicate_preflight` / `send_prepare_ok` gate on +/// [`Consensus::is_transferring`]) -- it must not vouch for state it is +/// about to replace. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StateTransferStage { + /// No transfer in progress. + Idle, + /// Restarted into a cluster: waiting for the view probe to find a live + /// primary to fetch from. The probe-exhausted election fallback proves + /// full-cluster bootstrap instead and abandons the transfer (local + /// recovery is then authoritative). + AwaitingTarget, + /// Target descriptor accepted; pulling artifact chunks. + Fetching, + /// Artifacts verified; installing. + Installing, +} + +impl StateTransferStage { + /// Legal stage transitions; [`VsrConsensus::set_state_transfer_stage`] + /// asserts against this so a mis-sequenced handler fails loudly instead + /// of corrupting the install. + #[must_use] + pub const fn valid_transition(from: Self, to: Self) -> bool { + matches!( + (from, to), + (Self::Idle, Self::AwaitingTarget) + | (Self::AwaitingTarget, Self::Fetching | Self::Idle) + | ( + Self::Fetching, + Self::Installing | Self::AwaitingTarget | Self::Idle + ) + | (Self::Installing, Self::Idle) + ) + } +} + /// What a received `Commit` heartbeat did, so the caller knows whether to /// drain the journal or correct a stale peer. #[derive(Debug, Clone, Copy)] @@ -803,6 +844,21 @@ where /// legitimately (`StartView` adoption, DVC completion). ceded_primaryship: Cell, status: Cell, + /// See [`StateTransferStage`]. Set to `AwaitingTarget` by a cluster + /// restart boot (`begin_state_transfer_await`), driven through + /// `Fetching`/`Installing` by the shard's transfer session, cleared by + /// the probe-exhausted election fallback (full-cluster bootstrap). + state_transfer_stage: Cell, + + /// Highest view seen on inbound traffic that this replica could not + /// process because the view was ahead of its own (a dropped newer-view + /// prepare or heartbeat). Proof the cluster elected past this replica + /// rather than that the primary died, which is why the heartbeat-timeout + /// handler probes to catch up instead of starting a futile election (see + /// [`Self::handle_normal_heartbeat_timeout`]). Monotone `max`; a value at + /// or below `view` is stale and inert because the catch-up guard is a + /// strict `>`. + observed_newer_view: Cell, /// Highest op number that has been locally executed (state machine applied, /// client table updated). Advances one-by-one in `commit_journal` (backup) @@ -929,6 +985,8 @@ impl> VsrConsensus { recovery_deadline: Cell::new(Duration::ZERO), ceded_primaryship: Cell::new(false), status: Cell::new(Status::Recovering), + state_transfer_stage: Cell::new(StateTransferStage::Idle), + observed_newer_view: Cell::new(0), sequencer: LocalSequencer::new(0), commit_min: Cell::new(0), commit_max: Cell::new(0), @@ -1528,6 +1586,18 @@ impl> VsrConsensus { let attempts = self.probe_attempts.get() + 1; self.probe_attempts.set(attempts); if attempts >= self.probe_attempts_max.get() { + // Nobody answered: full-cluster bootstrap, so there + // is no live primary to fetch state from. Local + // recovery is authoritative; abandon the transfer + // and elect on recovered logs. + if self.state_transfer_stage.get() != StateTransferStage::Idle { + tracing::info!( + replica = self.replica, + namespace_raw = self.namespace, + "view probe exhausted; abandoning state transfer (cluster bootstrap)" + ); + self.set_state_transfer_stage(StateTransferStage::Idle); + } self.finish_view_probe(); actions.extend( self.start_election(plane, ViewChangeReason::ViewProbeUnanswered), @@ -1597,6 +1667,30 @@ impl> VsrConsensus { return Vec::new(); } + // Distinguish "primary died" from "primary moved to a view I missed". + // If newer-view traffic has been arriving (a partition that healed + // across an election, or a fresh/rejoined node that never saw the + // election), the primary is alive and the timer only fired because a + // matching-view heartbeat never reset it. Electing would be futile -- + // a lagging replica cannot win -- and would drag a healthy cluster + // through a needless view change. Probe instead: the current primary + // answers a `RequestStartView` with its live `StartView` regardless of + // the probe's view stamp, and adopting it routes into journal repair + // (and state transfer, if the gap fell below the peer's floor). The + // probe re-broadcasts on its own timer, so no action is emitted here; + // `Recovering` also makes this timeout inert until the probe resolves. + if self.observed_newer_view.get() > self.view.get() { + tracing::info!( + replica = self.replica, + namespace_raw = self.namespace, + view = self.view.get(), + observed_newer_view = self.observed_newer_view.get(), + "heartbeat timed out behind a newer view; probing to catch up" + ); + self.begin_view_probe(); + return Vec::new(); + } + self.start_election(plane, ViewChangeReason::NormalHeartbeatTimeout) } @@ -1839,6 +1933,26 @@ impl> VsrConsensus { return Vec::new(); } + // A primary-by-index that has seen a newer view is stale: the cluster + // elected past it while it was gone, and it booted primary-by-index at + // an old view (a fresh or rejoined node that missed the election). It + // has no `NormalHeartbeat` timeout to catch this -- it IS the heartbeat + // sender -- so convert here instead of advertising a commit point no + // peer will accept. The probe solicits the current `StartView`, which + // demotes this replica to a backup and routes into repair / state + // transfer. `begin_view_probe` stops this timer. + if self.observed_newer_view.get() > self.view.get() { + tracing::info!( + replica = self.replica, + namespace_raw = self.namespace, + view = self.view.get(), + observed_newer_view = self.observed_newer_view.get(), + "stale primary-by-index behind a newer view; probing to catch up" + ); + self.begin_view_probe(); + return Vec::new(); + } + self.timeouts.borrow_mut().reset(TimeoutKind::CommitMessage); // Don't advertise a commit point we haven't locally executed yet. @@ -2168,6 +2282,52 @@ impl> VsrConsensus { timeouts.start(TimeoutKind::RequestStartViewMessage); } + /// Arm state transfer for a cluster restart: the replica will replace + /// its snapshot-shaped state from the live primary the view probe finds + /// (the shard's transfer session drives the stage from there). If the + /// probe exhausts instead -- full-cluster bootstrap, nobody to fetch + /// from -- the election fallback clears the stage and local recovery + /// stands. + pub fn begin_state_transfer_await(&self) { + self.set_state_transfer_stage(StateTransferStage::AwaitingTarget); + } + + /// Record that a message stamped with `view` arrived that this replica + /// could not process because the view is ahead of its own. Monotone: only + /// a value strictly above the current record moves it. Called from the two + /// ingress paths that drop newer-view traffic (`handle_commit` and + /// `replicate_preflight`); read by the heartbeat-timeout handler to choose + /// catch-up over election. + pub fn observe_newer_view(&self, view: u32) { + if view > self.observed_newer_view.get() { + self.observed_newer_view.set(view); + } + } + + #[must_use] + pub const fn state_transfer_stage(&self) -> StateTransferStage { + self.state_transfer_stage.get() + } + + /// # Panics + /// On an illegal stage transition (see + /// [`StateTransferStage::valid_transition`]). + pub fn set_state_transfer_stage(&self, to: StateTransferStage) { + let from = self.state_transfer_stage.get(); + assert!( + StateTransferStage::valid_transition(from, to), + "state transfer stage transition {from:?} -> {to:?} is illegal" + ); + tracing::info!( + replica = self.replica, + namespace_raw = self.namespace, + ?from, + ?to, + "state transfer stage" + ); + self.state_transfer_stage.set(to); + } + /// Peer side of the probe (sent by a Recovering replica at boot, or by /// a `ViewChange` backup whose copy of the concluding `StartView` was /// lost). Only the current view's PRIMARY answers, with a `StartView`; @@ -2404,6 +2564,12 @@ impl> VsrConsensus { { return CommitOutcome::RespondStartView; } + // A "primary" hearing a NEWER view is stale-by-index: the cluster + // elected past it. Record so its heartbeat-send timer converts to + // a probe (see `handle_commit_message_timeout`). + if header.view > self.view.get() { + self.observe_newer_view(header.view); + } return CommitOutcome::Accepted; } @@ -2412,6 +2578,13 @@ impl> VsrConsensus { } if header.view != self.view.get() { + // A heartbeat from a newer view is proof the cluster elected past + // this replica. Record it so the heartbeat-timeout handler probes + // to catch up instead of electing (this same heartbeat cannot + // reset the timer -- the reset below is gated on a matching view). + if header.view > self.view.get() { + self.observe_newer_view(header.view); + } return CommitOutcome::Accepted; } @@ -2578,9 +2751,9 @@ impl> VsrConsensus { } // Ignore if syncing - if self.is_syncing() { + if self.is_transferring() { return PrepareOkOutcome::Ignored { - reason: IgnoreReason::Syncing, + reason: IgnoreReason::StateTransfer, }; } @@ -2800,9 +2973,8 @@ where self.status() == Status::Normal } - fn is_syncing(&self) -> bool { - // TODO: for now return false. we have to add syncing related setup to VsrConsensus to make this work. - false + fn is_transferring(&self) -> bool { + self.state_transfer_stage.get() != StateTransferStage::Idle } } @@ -3174,3 +3346,216 @@ mod timestamp_clamp_tests { ); } } + +#[cfg(test)] +mod state_transfer_stage_tests { + use super::*; + + #[test] + fn stage_transitions_follow_the_machine() { + use StateTransferStage::{AwaitingTarget, Fetching, Idle, Installing}; + let legal = [ + (Idle, AwaitingTarget), + (AwaitingTarget, Fetching), + (AwaitingTarget, Idle), + (Fetching, Installing), + (Fetching, AwaitingTarget), + (Fetching, Idle), + (Installing, Idle), + ]; + for (from, to) in legal { + assert!( + StateTransferStage::valid_transition(from, to), + "{from:?} -> {to:?} must be legal" + ); + } + let illegal = [ + (Idle, Fetching), + (Idle, Installing), + (AwaitingTarget, Installing), + (Installing, Fetching), + (Installing, AwaitingTarget), + (Idle, Idle), + ]; + for (from, to) in illegal { + assert!( + !StateTransferStage::valid_transition(from, to), + "{from:?} -> {to:?} must be illegal" + ); + } + } + + /// Bus stub: stage plumbing never touches the wire. + struct StageNoopBus; + + impl MessageBus for StageNoopBus { + async fn send_to_client( + &self, + _client_id: u128, + _data: server_common::iobuf::Frozen<{ server_common::MESSAGE_ALIGN }>, + ) -> Result<(), message_bus::SendError> { + Ok(()) + } + + async fn send_to_replica( + &self, + _replica: u8, + _data: server_common::iobuf::Frozen<{ server_common::MESSAGE_ALIGN }>, + ) -> Result<(), message_bus::SendError> { + Ok(()) + } + + fn track_background(&self, _handle: message_bus::JoinHandle<()>) {} + fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {} + fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {} + fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {} + } + + #[test] + fn is_transferring_tracks_stage() { + let consensus = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + assert!(!consensus.is_transferring()); + consensus.begin_state_transfer_await(); + assert!(consensus.is_transferring()); + consensus.set_state_transfer_stage(StateTransferStage::Fetching); + consensus.set_state_transfer_stage(StateTransferStage::Installing); + assert!(consensus.is_transferring()); + consensus.set_state_transfer_stage(StateTransferStage::Idle); + assert!(!consensus.is_transferring()); + } + + // A backup left behind on VIEW (partition healed across an election, or a + // fresh node that never saw it) keeps getting newer-view heartbeats it + // cannot process -- they never reset its heartbeat timer, so the timer + // fires. The primary is alive, so it must PROBE to adopt the current view + // (which routes into repair / state transfer), not elect: a lagging + // replica cannot win, and electing would drag a healthy cluster through a + // needless view change. + #[test] + fn heartbeat_timeout_behind_a_newer_view_probes_not_elects() { + // Replica 1 is a backup at view 0 (primary_index(0) == 0). + let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.view(), 0); + + // A newer-view frame arrived and was dropped by the ingress path. + consensus.observe_newer_view(3); + + let actions = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + + assert_eq!( + consensus.status(), + Status::Recovering, + "must enter the probe (Recovering), not an election" + ); + assert_eq!(consensus.view(), 0, "probing must not bump the view"); + assert!( + !actions + .iter() + .any(|action| matches!(action, VsrAction::SendStartViewChange { .. })), + "must not broadcast an election SVC" + ); + } + + // No newer view seen means the primary is presumed dead, so the timeout + // must still elect -- the split must not swallow the ordinary case. + #[test] + fn heartbeat_timeout_with_no_newer_view_still_elects() { + let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + + let actions = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + + assert_eq!( + consensus.status(), + Status::ViewChange, + "a silent primary must still trigger an election" + ); + assert_eq!(consensus.view(), 1, "election advances the view"); + assert!( + actions + .iter() + .any(|action| matches!(action, VsrAction::SendStartViewChange { .. })), + "election must broadcast its SVC" + ); + } + + // A recorded view at or below the current one is stale (already caught up) + // and must not divert a genuine election, since the guard is a strict `>`. + #[test] + fn stale_observed_view_does_not_divert_election() { + let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + consensus.observe_newer_view(0); + + let _ = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + + assert_eq!( + consensus.status(), + Status::ViewChange, + "a stale observed view must not suppress the election" + ); + } + + // A fresh or rejoined node that missed an election boots primary-by-index + // at a stale view (replica 0 is primary_index(0)). It has no heartbeat + // timeout -- it IS the heartbeat sender -- so its heartbeat-SEND timer must + // convert to a probe once it observes the newer view, or it wedges forever + // advertising a commit point no peer accepts. + #[test] + fn stale_primary_by_index_probes_on_its_heartbeat_send_timer() { + // Replica 0 is primary-by-index at view 0. + let consensus = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + assert!(consensus.is_primary()); + assert_eq!(consensus.status(), Status::Normal); + + // A prepare/heartbeat from the real primary at a newer view was seen. + consensus.observe_newer_view(2); + + let actions = consensus.handle_commit_message_timeout(); + + assert_eq!( + consensus.status(), + Status::Recovering, + "a stale primary-by-index must probe, not keep heartbeating" + ); + assert!( + !actions + .iter() + .any(|action| matches!(action, VsrAction::SendCommit { .. })), + "a stale primary must not advertise a commit point" + ); + } + + // A genuine current primary (no newer view seen) keeps heartbeating. + #[test] + fn current_primary_keeps_heartbeating() { + let consensus = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + + let actions = consensus.handle_commit_message_timeout(); + + assert_eq!(consensus.status(), Status::Normal, "must stay primary"); + assert!( + actions + .iter() + .any(|action| matches!(action, VsrAction::SendCommit { .. })), + "a healthy primary must heartbeat" + ); + } + + // `observe_newer_view` is monotone: an older stamp cannot lower it. + #[test] + fn observe_newer_view_keeps_the_max() { + let consensus = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + consensus.observe_newer_view(5); + consensus.observe_newer_view(2); + // Timeout at view 0 with the record still 5 must probe. + let _ = consensus.handle_normal_heartbeat_timeout(PlaneKind::Metadata); + assert_eq!(consensus.status(), Status::Recovering); + } +} diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 3937eb0e6f..004defe7c7 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -114,7 +114,7 @@ pub trait Consensus: Sized { fn is_follower(&self) -> bool; fn is_normal(&self) -> bool; - fn is_syncing(&self) -> bool; + fn is_transferring(&self) -> bool; } /// Shared consensus lifecycle interface for control/data planes. @@ -149,6 +149,11 @@ where pub mod client_table; pub use client_table::{CachedReply, ClientTable, CommitReply}; +pub mod state_manifest; +pub use state_manifest::{ + StateArtifact, StateManifestError, artifact_kind, decode_state_manifest, encode_state_manifest, + state_artifact_checksum, +}; // One-shot per `PipelineEntry` for in-process commit awaiters. pub(crate) mod oneshot; pub use oneshot::{Canceled, Receiver}; diff --git a/core/consensus/src/metadata_helpers.rs b/core/consensus/src/metadata_helpers.rs index 065de77991..329ca18d94 100644 --- a/core/consensus/src/metadata_helpers.rs +++ b/core/consensus/src/metadata_helpers.rs @@ -125,7 +125,7 @@ where request, is_primary = consensus.is_primary(), is_normal = consensus.is_normal(), - is_syncing = consensus.is_syncing(), + is_transferring = consensus.is_transferring(), commit_min = consensus.commit_min(), commit_max = consensus.commit_max(), "request_preflight: not caught up, not ready" @@ -281,7 +281,7 @@ where client_id, is_primary = consensus.is_primary(), is_normal = consensus.is_normal(), - is_syncing = consensus.is_syncing(), + is_transferring = consensus.is_transferring(), commit_min = consensus.commit_min(), commit_max = consensus.commit_max(), "register_preflight: not caught up, drop" @@ -432,9 +432,10 @@ fn build_eviction_from_header(header: EvictionHeader) -> Message /// `false` -> caller silent-drops; client retry lands on peer or here /// post-catch-up. /// -/// `is_syncing` is currently hardcoded `false` (inert clause); safety rests -/// on `commit_min == commit_max`, correlated (a syncing replica also has -/// `commit_min < commit_max`). Do not weaken commit-equality. +/// `is_transferring` is real (state transfer replaces snapshot-shaped state +/// on a cluster-restart rejoin), but safety still rests on +/// `commit_min == commit_max` -- a transferring replica also fails that. Do +/// not weaken commit-equality. pub fn is_caught_up_primary(consensus: &VsrConsensus) -> bool where B: MessageBus, @@ -443,7 +444,7 @@ where consensus.is_primary() && !consensus.has_ceded_primaryship() && consensus.is_normal() - && !consensus.is_syncing() + && !consensus.is_transferring() && consensus.commit_min() == consensus.commit_max() // Recovery re-pipelines the WAL's prepared-but-uncommitted suffix; // those ops were acked to clients before the restart, so admitting @@ -845,9 +846,8 @@ mod tests { assert!(sends.is_empty(), "silent drop until catch-up"); } - // Direct gate test. is_syncing is hardcoded false today; clause is - // inert. Test exercises primary, normal, commit_min == commit_max. - // Extends naturally when is_syncing becomes real. + // Direct gate test: primary, normal, commit_min == commit_max, and not + // mid-state-transfer. #[test] fn is_caught_up_primary_gate_states() { // Primary, normal, equal commits -> true. @@ -855,10 +855,17 @@ mod tests { primary.init(); assert!(primary.is_primary()); assert!(primary.is_normal()); - assert!(!primary.is_syncing(), "is_syncing inert; pin"); + assert!(!primary.is_transferring()); assert_eq!(primary.commit_min(), primary.commit_max()); assert!(is_caught_up_primary(&primary)); + // Mid-transfer -> false, even with equal commits. + primary.begin_state_transfer_await(); + assert!(primary.is_transferring()); + assert!(!is_caught_up_primary(&primary)); + primary.set_state_transfer_stage(crate::StateTransferStage::Idle); + assert!(is_caught_up_primary(&primary)); + // commit_min < commit_max -> false. primary.advance_commit_max(5); assert_ne!(primary.commit_min(), primary.commit_max()); diff --git a/core/consensus/src/observability.rs b/core/consensus/src/observability.rs index 03a9b1b088..4cd041bc4f 100644 --- a/core/consensus/src/observability.rs +++ b/core/consensus/src/observability.rs @@ -88,7 +88,7 @@ impl ViewChangeReason { pub enum IgnoreReason { NotPrimary, NotNormal, - Syncing, + StateTransfer, NewerView, OlderView, OldPrepare, @@ -106,7 +106,7 @@ impl IgnoreReason { match self { Self::NotPrimary => "not_primary", Self::NotNormal => "not_normal", - Self::Syncing => "syncing", + Self::StateTransfer => "state_transfer", Self::NewerView => "newer_view", Self::OlderView => "older_view", Self::OldPrepare => "old_prepare", diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index f9b36e5546..cb56d54da6 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -40,7 +40,10 @@ pub async fn pipeline_prepare_common( { assert!(!consensus.is_follower(), "on_request: primary only"); assert!(consensus.is_normal(), "on_request: status must be normal"); - assert!(!consensus.is_syncing(), "on_request: must not be syncing"); + assert!( + !consensus.is_transferring(), + "on_request: must not be syncing" + ); consensus.verify_pipeline(); consensus.pipeline_message(plane, &prepare); @@ -131,8 +134,8 @@ where { assert_eq!(header.command, Command2::Prepare); - if consensus.is_syncing() { - return Err(IgnoreReason::Syncing); + if consensus.is_transferring() { + return Err(IgnoreReason::StateTransfer); } let current_op = consensus.sequencer().current_sequence(); @@ -142,6 +145,10 @@ where } if header.view > consensus.view() { + // Dropped, but recorded: a newer-view prepare is proof the cluster + // moved past this replica, so the heartbeat-timeout handler probes to + // catch up rather than starting a futile election. + consensus.observe_newer_view(header.view); return Err(IgnoreReason::NewerView); } @@ -540,7 +547,7 @@ pub async fn send_prepare_ok( return; } - if consensus.is_syncing() { + if consensus.is_transferring() { return; } diff --git a/core/consensus/src/state_manifest.rs b/core/consensus/src/state_manifest.rs new file mode 100644 index 0000000000..ea226b270d --- /dev/null +++ b/core/consensus/src/state_manifest.rs @@ -0,0 +1,341 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! State-transfer manifest: the artifact list a serving peer offers, carried +//! in the `StateTransferTarget` body. +//! +//! `TigerBeetle` pulls state as content-addressed grid blocks, so a bare +//! `(address, checksum)` names any piece of state and one repair protocol +//! covers manifest, free set, client sessions, and LSM tables alike. Iggy's +//! artifacts (metadata snapshot, client table, partition segments) are not +//! content-addressed, so the manifest supplies the addressing instead: each +//! entry names one artifact, `(manifest index, byte offset)` is the chunk +//! address the `RequestStateChunk`/`StateChunk` frames pull by, and the +//! per-artifact checksum is the integrity stamp (chunks carry none). +//! +//! Plane-agnostic by construction: the metadata plane ships two entries +//! (snapshot + client table); the partition plane ships N (segment logs, +//! consumer offsets, ...) through the same frames by appending [`kind`] +//! values. Entries carry their own length on the wire (`entry_len`), so new +//! per-entry fields can be appended without breaking old decoders. +//! +//! [`kind`]: StateArtifact::kind + +use std::hash::Hasher; +use twox_hash::XxHash3_64; + +/// Artifact kinds. Wire-pinned: never reorder or reuse. +pub mod artifact_kind { + /// Metadata plane: `snapshot.bin` bytes verbatim. + pub const METADATA_SNAPSHOT: u8 = 0; + /// Metadata plane: [`crate::ClientTable::encode`] bytes. + pub const CLIENT_TABLE: u8 = 1; + // Partition plane (reserved, not served yet): + // SEGMENT_LOG = 2, CONSUMER_OFFSETS = 3. +} + +/// One artifact a serving peer offers: what it is, where its receiver-side +/// watermark sits, and how to verify the assembled bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StateArtifact { + /// [`artifact_kind`] value. Unknown kinds decode fine (forward compat); + /// the receiving plane decides whether it can install them. + pub kind: u8, + /// Kind-specific watermark: the snapshot's `sequence_number`, the client + /// table's mutation frontier, a segment's base offset, ... + pub frontier: u64, + /// Artifact byte length; `(manifest index, offset in 0..len)` addresses + /// every chunk of it. + pub len: u64, + /// `XxHash3_64` over the artifact bytes (artifact-level integrity; + /// chunks carry none). + pub checksum: u64, +} + +impl StateArtifact { + /// Entry for `bytes`, stamping length + checksum. + #[must_use] + pub fn for_bytes(kind: u8, frontier: u64, bytes: &[u8]) -> Self { + Self { + kind, + frontier, + len: bytes.len() as u64, + checksum: state_artifact_checksum(bytes), + } + } +} + +/// Artifact-level integrity stamp (manifest `checksum` field). +#[must_use] +pub fn state_artifact_checksum(bytes: &[u8]) -> u64 { + let mut hasher = XxHash3_64::new(); + hasher.write(bytes); + hasher.finish() +} + +/// Failure decoding an encoded manifest. +#[derive(Debug)] +pub enum StateManifestError { + /// Byte stream ended mid-field. + Truncated, + /// Leading magic is not [`STATE_MANIFEST_MAGIC`]. + BadMagic, + /// Trailing hash does not match the content. + ChecksumMismatch { expected: u64, actual: u64 }, + /// Entry count exceeds [`STATE_MANIFEST_ENTRIES_MAX`]. + TooManyEntries { count: u32 }, + /// Declared per-entry length is below this version's field set (a + /// FUTURE-shrunk entry; longer entries are fine, the tail is skipped). + EntryTooShort { entry_len: u8 }, +} + +impl std::fmt::Display for StateManifestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Truncated => write!(f, "state manifest truncated"), + Self::BadMagic => write!(f, "state manifest has wrong magic"), + Self::ChecksumMismatch { expected, actual } => write!( + f, + "state manifest checksum mismatch: expected {expected:#018x}, actual {actual:#018x}" + ), + Self::TooManyEntries { count } => { + write!( + f, + "state manifest holds {count} entries, max {STATE_MANIFEST_ENTRIES_MAX}" + ) + } + Self::EntryTooShort { entry_len } => write!( + f, + "state manifest entry length {entry_len} below this version's {STATE_MANIFEST_ENTRY_LEN}" + ), + } + } +} + +impl std::error::Error for StateManifestError {} + +/// Format tag for [`encode_state_manifest`]; bump on incompatible change +/// (appending entry fields is compatible, see `entry_len`). +pub const STATE_MANIFEST_MAGIC: [u8; 4] = *b"ISM1"; + +/// This version's entry size: `kind(1) frontier(8) len(8) checksum(8)`. +pub const STATE_MANIFEST_ENTRY_LEN: u8 = 25; + +/// Sanity ceiling on entries per manifest. A partition rejoin lists at most +/// its retained segments plus a handful of trailers; 65k is a corruption +/// guard, not a target. +pub const STATE_MANIFEST_ENTRIES_MAX: u32 = 1 << 16; + +/// Encode a manifest. +/// +/// Layout (little-endian): `magic(4) count(u32) entry_len(u8)` then per +/// entry `kind(u8) frontier(u64) len(u64) checksum(u64)`, terminated by an +/// `XxHash3_64(8)` over everything before it. +/// +/// # Panics +/// If `artifacts.len()` exceeds [`STATE_MANIFEST_ENTRIES_MAX`] (a serving +/// peer offering that many artifacts is a bug, not an input). +#[must_use] +#[allow(clippy::cast_possible_truncation)] +pub fn encode_state_manifest(artifacts: &[StateArtifact]) -> Vec { + assert!( + artifacts.len() <= STATE_MANIFEST_ENTRIES_MAX as usize, + "state manifest entry count {} exceeds {STATE_MANIFEST_ENTRIES_MAX}", + artifacts.len() + ); + let mut out = Vec::with_capacity(9 + artifacts.len() * STATE_MANIFEST_ENTRY_LEN as usize + 8); + out.extend_from_slice(&STATE_MANIFEST_MAGIC); + out.extend_from_slice(&(artifacts.len() as u32).to_le_bytes()); + out.push(STATE_MANIFEST_ENTRY_LEN); + for artifact in artifacts { + out.push(artifact.kind); + out.extend_from_slice(&artifact.frontier.to_le_bytes()); + out.extend_from_slice(&artifact.len.to_le_bytes()); + out.extend_from_slice(&artifact.checksum.to_le_bytes()); + } + let mut hasher = XxHash3_64::new(); + hasher.write(&out); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + out +} + +/// Decode a manifest encoded by [`encode_state_manifest`]. +/// +/// Entries longer than [`STATE_MANIFEST_ENTRY_LEN`] decode fine: the known +/// prefix is read and the remainder skipped, so a future encoder can append +/// per-entry fields without breaking this decoder. +/// +/// # Errors +/// [`StateManifestError`] on truncation, magic/checksum mismatch, an entry +/// count past the ceiling, or a shrunken entry length. +/// +/// # Panics +/// Unreachable: slice-to-array conversions are length-checked first. +pub fn decode_state_manifest(bytes: &[u8]) -> Result, StateManifestError> { + const TRAILER: usize = size_of::(); + let content_len = bytes + .len() + .checked_sub(TRAILER) + .ok_or(StateManifestError::Truncated)?; + let (content, trailer) = bytes.split_at(content_len); + let expected = u64::from_le_bytes(trailer.try_into().expect("trailer is 8 bytes")); + let mut hasher = XxHash3_64::new(); + hasher.write(content); + let actual = hasher.finish(); + if expected != actual { + return Err(StateManifestError::ChecksumMismatch { expected, actual }); + } + + let mut reader = ManifestReader { bytes: content }; + if reader.take(STATE_MANIFEST_MAGIC.len())? != STATE_MANIFEST_MAGIC { + return Err(StateManifestError::BadMagic); + } + let count = reader.u32()?; + if count > STATE_MANIFEST_ENTRIES_MAX { + return Err(StateManifestError::TooManyEntries { count }); + } + let entry_len = reader.u8()?; + if entry_len < STATE_MANIFEST_ENTRY_LEN { + return Err(StateManifestError::EntryTooShort { entry_len }); + } + + let mut artifacts = Vec::with_capacity(count as usize); + for _ in 0..count { + let entry = reader.take(entry_len as usize)?; + let mut entry = ManifestReader { bytes: entry }; + artifacts.push(StateArtifact { + kind: entry.u8()?, + frontier: entry.u64()?, + len: entry.u64()?, + checksum: entry.u64()?, + }); + } + if !reader.bytes.is_empty() { + return Err(StateManifestError::Truncated); + } + Ok(artifacts) +} + +/// Little-endian cursor over the encoded manifest content. +struct ManifestReader<'a> { + bytes: &'a [u8], +} + +impl<'a> ManifestReader<'a> { + const fn take(&mut self, len: usize) -> Result<&'a [u8], StateManifestError> { + if self.bytes.len() < len { + return Err(StateManifestError::Truncated); + } + let (head, tail) = self.bytes.split_at(len); + self.bytes = tail; + Ok(head) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().expect("4B"))) + } + + fn u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.take(8)?.try_into().expect("8B"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> Vec { + vec![ + StateArtifact::for_bytes(artifact_kind::METADATA_SNAPSHOT, 192, b"snapshot-bytes"), + StateArtifact::for_bytes(artifact_kind::CLIENT_TABLE, 195, b"table-bytes"), + ] + } + + #[test] + fn roundtrip_preserves_entries() { + let artifacts = sample(); + let decoded = decode_state_manifest(&encode_state_manifest(&artifacts)).unwrap(); + assert_eq!(decoded, artifacts); + } + + #[test] + fn empty_manifest_roundtrips() { + assert_eq!( + decode_state_manifest(&encode_state_manifest(&[])).unwrap(), + vec![] + ); + } + + #[test] + fn corruption_is_rejected() { + let mut bytes = encode_state_manifest(&sample()); + bytes[6] ^= 0xFF; + assert!(matches!( + decode_state_manifest(&bytes), + Err(StateManifestError::ChecksumMismatch { .. }) + )); + let encoded = encode_state_manifest(&sample()); + assert!(matches!( + decode_state_manifest(&encoded[..encoded.len() - 1]), + Err(StateManifestError::ChecksumMismatch { .. } | StateManifestError::Truncated) + )); + } + + // A future encoder may append per-entry fields; this decoder must read + // the known prefix and skip the tail. + #[test] + fn longer_entries_decode_with_tail_skipped() { + let artifacts = sample(); + // Re-encode by hand with entry_len + 7 trailing bytes per entry. + let mut out = Vec::new(); + out.extend_from_slice(&STATE_MANIFEST_MAGIC); + #[allow(clippy::cast_possible_truncation)] + out.extend_from_slice(&(artifacts.len() as u32).to_le_bytes()); + out.push(STATE_MANIFEST_ENTRY_LEN + 7); + for artifact in &artifacts { + out.push(artifact.kind); + out.extend_from_slice(&artifact.frontier.to_le_bytes()); + out.extend_from_slice(&artifact.len.to_le_bytes()); + out.extend_from_slice(&artifact.checksum.to_le_bytes()); + out.extend_from_slice(&[0xAB; 7]); + } + let mut hasher = XxHash3_64::new(); + hasher.write(&out); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + + assert_eq!(decode_state_manifest(&out).unwrap(), artifacts); + } + + #[test] + fn shrunken_entries_are_rejected() { + let mut out = Vec::new(); + out.extend_from_slice(&STATE_MANIFEST_MAGIC); + out.extend_from_slice(&0u32.to_le_bytes()); + out.push(STATE_MANIFEST_ENTRY_LEN - 1); + let mut hasher = XxHash3_64::new(); + hasher.write(&out); + out.extend_from_slice(&hasher.finish().to_le_bytes()); + assert!(matches!( + decode_state_manifest(&out), + Err(StateManifestError::EntryTooShort { .. }) + )); + } +} diff --git a/core/integration/src/harness/config/resolve.rs b/core/integration/src/harness/config/resolve.rs index 6a82c29655..99d89f4d30 100644 --- a/core/integration/src/harness/config/resolve.rs +++ b/core/integration/src/harness/config/resolve.rs @@ -52,7 +52,12 @@ pub fn resolve_config_paths( }; let mapping = ServerConfig::find_by_config_path(resolved_path) - .or_else(|| ServerConfig::find_by_config_path(&format!("system.{}", resolved_path))); + .or_else(|| ServerConfig::find_by_config_path(&format!("system.{}", resolved_path))) + // Fields that exist only in the next-gen server's config (e.g. + // `metadata.*`, `cluster.*`, `message_bus.*`). Env names share + // the `IGGY_` prefix, so the resolved variable reaches whichever + // binary the harness spawns; the legacy server ignores unknowns. + .or_else(|| configs::server_ng::ServerNgConfig::find_by_config_path(resolved_path)); match mapping { Some(m) => { diff --git a/core/integration/src/harness/handle/server.rs b/core/integration/src/harness/handle/server.rs index 62913f95a7..7d6fefbc12 100644 --- a/core/integration/src/harness/handle/server.rs +++ b/core/integration/src/harness/handle/server.rs @@ -184,10 +184,31 @@ impl ServerHandle { /// a late joiner that misses early ops. #[must_use] pub fn replica_mesh_complete(&self) -> bool { + self.stdout_contains("replica mesh complete") + } + + /// True once this node's stdout log contains `marker`. Spec tests use + /// this to pin server-side behavior that has no client-visible effect on + /// this node (e.g. a follower completing a state transfer). + #[must_use] + pub fn stdout_contains(&self, marker: &str) -> bool { + self.stdout_occurrences(marker) > 0 + } + + /// Number of times `marker` appears in this node's stdout log. The log + /// file is TRUNCATED on every (re)start (it captures one process run), + /// so counts never carry across a restart; a marker seen after + /// `restart_server` was logged by the new process. + /// + /// ANSI escape sequences are stripped before matching: the server colors + /// its tracing fields, so a `key=value` marker never matches the raw + /// bytes (`key\x1b[0m\x1b[2m=\x1b[0m value`). + #[must_use] + pub fn stdout_occurrences(&self, marker: &str) -> usize { self.stdout_path .as_ref() .and_then(|path| fs::read_to_string(path).ok()) - .is_some_and(|log| log.contains("replica mesh complete")) + .map_or(0, |log| strip_ansi(&log).matches(marker).count()) } /// Returns a `ClientBuilder` using the test transport. @@ -978,6 +999,35 @@ impl Restartable for ServerHandle { } } +impl ServerHandle { + /// Stop this node, delete its data directory, then start it again -- a + /// node that rejoins the cluster with no on-disk history, as a fresh + /// operator-provisioned replacement or a wiped disk does. + /// + /// Distinct from [`Restartable::restart`], which preserves the WAL and so + /// exercises the recovery-with-history path. The empty data directory is + /// what forces the state-transfer join: local recovery has nothing, and + /// the committed prefix the node missed no longer exists as WAL entries on + /// the peers that checkpointed it. + pub fn restart_from_clean_slate(&mut self) -> Result<(), TestBinaryError> { + let cleanup = self.config.cleanup; + self.config.cleanup = false; + self.stop_dependents()?; + self.stop()?; + + let data_path = self.data_path(); + if data_path.exists() { + fs::remove_dir_all(&data_path).map_err(|source| TestBinaryError::FileSystemError { + path: data_path, + source, + })?; + } + + self.config.cleanup = cleanup; + self.start() + } +} + impl Drop for ServerHandle { fn drop(&mut self) { // Reap the child before `port_reserver` drops: `Child::drop` detaches @@ -988,6 +1038,25 @@ impl Drop for ServerHandle { } } +/// Drop ANSI escape sequences (`ESC [ ... `) so log markers match +/// the text an operator sees, not the color codes around it. +fn strip_ansi(log: &str) -> String { + let mut out = String::with_capacity(log.len()); + let mut chars = log.chars(); + while let Some(current) = chars.next() { + if current == '\u{1b}' { + for escaped in chars.by_ref() { + if escaped.is_ascii_alphabetic() { + break; + } + } + } else { + out.push(current); + } + } + out +} + fn generate_test_certificates(cert_dir: &str) -> Result<(), Box> { fs::create_dir_all(cert_dir)?; diff --git a/core/integration/src/harness/orchestrator/harness.rs b/core/integration/src/harness/orchestrator/harness.rs index df1d545e57..7f14cc4184 100644 --- a/core/integration/src/harness/orchestrator/harness.rs +++ b/core/integration/src/harness/orchestrator/harness.rs @@ -330,6 +330,32 @@ impl TestHarness { Ok(()) } + /// Restart node `index` with its data directory wiped, so it rejoins the + /// still-live cluster as a fresh replica with no local history. The other + /// nodes stay up throughout (they hold quorum and keep committing), which + /// is the late-joiner / provisioned-replacement scenario rather than a + /// full-cluster restart. Clients are left connected to their own nodes. + pub fn restart_node_from_clean_slate(&mut self, index: usize) -> Result<(), TestBinaryError> { + let server = self + .servers + .get_mut(index) + .ok_or(TestBinaryError::MissingServer)?; + server.restart_from_clean_slate() + } + + /// Stop node `index` and leave it down. The surviving nodes keep quorum + /// (in a 3-node cluster, 2 of 3), and if the stopped node was the primary + /// they elect a new one, advancing the view. Pairs with + /// [`Self::restart_node_from_clean_slate`] to model a node that misses an + /// election entirely and rejoins at a stale view. + pub fn stop_node(&mut self, index: usize) -> Result<(), TestBinaryError> { + let server = self + .servers + .get_mut(index) + .ok_or(TestBinaryError::MissingServer)?; + server.stop() + } + /// Restart EVERY node and reconnect all clients: the full-cluster /// restart path, where no settled primary survives to answer the rejoin /// probes and the replicas must fall back to an election among their diff --git a/core/integration/tests/cli/message/test_message_flush_command.rs b/core/integration/tests/cli/message/test_message_flush_command.rs index bd12288f60..9658943dbf 100644 --- a/core/integration/tests/cli/message/test_message_flush_command.rs +++ b/core/integration/tests/cli/message/test_message_flush_command.rs @@ -15,6 +15,11 @@ // specific language governing permissions and limitations // under the License. +// Tests the `message flush` CLI command itself (`FLUSH_UNSAVED_BUFFER`), not +// a durability barrier some other assertion needs. The command is slated for +// removal from the protocol, so this stays out of the vsr pass permanently +// (today via the whole-`cli`-module gate): delete this file together with the +// command, do not adapt it. use crate::cli::common::{ CLAP_INDENT, IggyCmdCommand, IggyCmdTest, IggyCmdTestCase, TestHelpCmd, USAGE_PREFIX, }; diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs index 750e83ce20..4e572eb5c4 100644 --- a/core/integration/tests/cluster/client_table_restart.rs +++ b/core/integration/tests/cluster/client_table_restart.rs @@ -236,7 +236,7 @@ async fn given_live_session_when_unauthenticated_peer_presents_it_should_refuse_ .await; } -fn tcp_addr(harness: &TestHarness) -> SocketAddr { +pub(super) fn tcp_addr(harness: &TestHarness) -> SocketAddr { harness .server() .tcp_addr() @@ -247,7 +247,7 @@ fn tcp_addr(harness: &TestHarness) -> SocketAddr { /// way a leader-aware SDK re-routes: after a node restart in a cluster the /// primary may be any survivor, and only the primary commits (or answers /// dedup for) replicated requests. -fn tcp_addrs(harness: &TestHarness) -> Vec { +pub(super) fn tcp_addrs(harness: &TestHarness) -> Vec { (0..harness.cluster_size()) .map(|node| { harness @@ -258,7 +258,7 @@ fn tcp_addrs(harness: &TestHarness) -> Vec { .collect() } -fn create_stream_payload(name: &str) -> Bytes { +pub(super) fn create_stream_payload(name: &str) -> Bytes { CreateStreamRequest { name: WireName::new(name).unwrap(), } @@ -291,7 +291,7 @@ fn request_header( /// so the pre-restart request must reuse the returned stream. Replays on /// transient rejections: right after boot the single node may not have /// elected itself yet. -async fn register(addr: SocketAddr) -> (TcpStream, u64) { +pub(super) async fn register(addr: SocketAddr) -> (TcpStream, u64) { let mut stream = TcpStream::connect(addr).await.unwrap(); let deadline = Instant::now() + COMMIT_BUDGET; loop { @@ -338,7 +338,7 @@ async fn login_on(stream: &mut TcpStream) -> Option { /// Send one replicated metadata request on the registered connection and /// require a committed success within `COMMIT_BUDGET`. Returns the committed /// reply so a later replay can be compared against it byte for byte. -async fn commit_request( +pub(super) async fn commit_request( stream: &mut TcpStream, session: u64, request: u64, @@ -373,7 +373,7 @@ async fn commit_request( /// Every attempt uses a fresh connection, both because the old one died with /// the node and so an unanswered frame cannot desync the next attempt. Panics /// with the last observed failure mode when the budget runs out. -async fn resume_request( +pub(super) async fn resume_request( addrs: &[SocketAddr], session: u64, request: u64, @@ -513,7 +513,7 @@ enum Verdict { /// The header is boxed to keep it off the stack in `Verdict`/`Exchange`: at /// `HEADER_SIZE` inline it dwarfs every other variant. #[derive(Debug)] -struct CommittedReply { +pub(super) struct CommittedReply { header: Box<[u8; HEADER_SIZE]>, payload: Bytes, } diff --git a/core/integration/tests/cluster/metadata_checkpoint_restart.rs b/core/integration/tests/cluster/metadata_checkpoint_restart.rs new file mode 100644 index 0000000000..79f3f822f8 --- /dev/null +++ b/core/integration/tests/cluster/metadata_checkpoint_restart.rs @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Checkpoint-shaped edge cases for metadata + client-table state transfer, +//! metadata-plane commands only (`CreateStream` over the raw VSR wire). +//! +//! `metadata_state_transfer` pins the base case: one checkpoint, a live +//! post-checkpoint tail, one restart. This file covers the shapes around it: +//! +//! - restart exactly AT a checkpoint (snapshot present, post-snapshot journal +//! empty), so the transfer installs snapshot-only and the tail repair has +//! nothing to fetch; +//! - restart after MULTIPLE checkpoint generations (`snapshot.bin` was +//! rewritten in place; only the latest generation may be served); +//! - a below-floor dedup retry after the restart, answered byte-identically +//! from the TRANSFERRED reply ring - the WAL entry that produced the reply +//! was drained on every node, so nothing but the shipped table can answer; +//! - a second restart of the same node (the installed snapshot must persist +//! and serve as the local floor for another transfer round). +//! +//! Checkpoint placement is config-driven: with `metadata.journal_slots = 256` +//! and the built-in checkpoint margin (64), a checkpoint fires when the +//! journal holds 192 committed ops, i.e. at op 192, 384, ... The register +//! commits at op 1 and request K at op K + 1, so request 191 lands checkpoint +//! one and request 383 lands checkpoint two. Requests here are sequential +//! with one in flight, which keeps that arithmetic exact; the +//! `forced checkpoint completed` markers below pin it rather than trust it. + +#![cfg(feature = "vsr")] + +use super::client_table_restart::{ + commit_request, create_stream_payload, register, resume_request, tcp_addr, tcp_addrs, +}; +use integration::harness::TestHarness; +use integration::iggy_harness; +use std::time::Duration; +use tokio::time::{Instant, sleep}; + +/// With `journal_slots = 256` and the margin floor (64), a checkpoint fires +/// every 192 committed ops. +const CHECKPOINT_EVERY: u64 = 192; + +/// Requests that land the commit log exactly on the first checkpoint: +/// register = op 1, request K = op K + 1. +const REQUESTS_TO_FIRST_CHECKPOINT: u64 = CHECKPOINT_EVERY - 1; + +/// How long a node gets to probe, fetch, and install a transfer (or a +/// checkpoint marker gets to appear). +const MARKER_BUDGET: Duration = Duration::from_secs(30); + +const MARKER_POLL: Duration = Duration::from_millis(200); + +const INSTALL_MARKER: &str = "metadata state transfer installed"; + +/// Poll until `node`'s stdout contains `marker` `at_least` times. +async fn await_marker( + harness: &TestHarness, + node: usize, + marker: &str, + at_least: usize, + context: &str, +) { + let deadline = Instant::now() + MARKER_BUDGET; + loop { + if harness.node(node).stdout_occurrences(marker) >= at_least { + return; + } + assert!( + Instant::now() < deadline, + "{context}: node {node} never logged {at_least}x {marker:?} within {MARKER_BUDGET:?}" + ); + sleep(MARKER_POLL).await; + } +} + +/// Wait until EVERY node has completed its `generation`-th checkpoint. The +/// restart below must not race a lagging backup's checkpoint: the serving +/// primary needs its snapshot on disk to offer a transfer, and the restarted +/// node's own drained journal is the precondition the test is about. +/// +/// Generations are counted per node, not matched by op: the trigger fires on +/// journal occupancy but each node snapshots at its own applied watermark, +/// so a backup checkpoints one op behind the primary (191 vs 192 for the +/// first generation here). +async fn await_checkpoint_on_all_nodes(harness: &TestHarness, generation: usize) { + for node in 0..3 { + await_marker( + harness, + node, + "forced checkpoint completed", + generation, + "checkpoint wait", + ) + .await; + } +} + +// Snapshot present, post-snapshot journal EMPTY: the restart lands exactly on +// a checkpoint, so the transfer descriptor's `commit_op == snapshot_seq` and +// the post-install tail repair has nothing to fetch (`commit_min == +// commit_max` skips it). The below-floor retry then proves the reply ring +// rode the transferred table: request 191's reply was minted at op 192, which +// every node drained out of its WAL at that same checkpoint. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_drained_journal_when_node_restarts_should_install_snapshot_only( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + let mut last_committed = None; + for request in 1..=REQUESTS_TO_FIRST_CHECKPOINT { + let committed = commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-ckpt-{request}")), + ) + .await; + last_committed = Some(committed); + } + await_checkpoint_on_all_nodes(harness, 1).await; + + // Restart BEFORE dropping the socket: a graceful disconnect commits a + // Logout that releases the entry cluster-wide (see + // `submit_disconnect_logout`), and this test needs the entry to survive. + // A crash takes the process down with the connection open, which is + // exactly what restarting first models (same ordering as the sibling + // `client_table_restart` tests). + harness.restart_server().await.unwrap(); + drop(stream); + await_marker(harness, 0, INSTALL_MARKER, 1, "snapshot-only install").await; + + // Below-floor dedup: the retried request's reply exists nowhere but the + // transferred table (its WAL entry was drained cluster-wide), and the + // replay must be the original bytes, not a re-execution. + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + REQUESTS_TO_FIRST_CHECKPOINT, + &create_stream_payload(&format!("iggy-ckpt-{REQUESTS_TO_FIRST_CHECKPOINT}")), + Some(last_committed.as_ref().expect("at least one commit")), + ) + .await; + + // And the cluster still commits fresh work. + resume_request( + &addrs, + session, + REQUESTS_TO_FIRST_CHECKPOINT + 1, + &create_stream_payload("iggy-ckpt-continuation"), + None, + ) + .await; +} + +// Multiple snapshot generations before the restart: `snapshot.bin` is +// rewritten in place at each checkpoint, so the transfer must serve the +// LATEST generation, and the session registered below the FIRST floor (op 1, +// two generations deep) must still resume from the transferred table. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_multiple_checkpoints_when_node_restarts_should_resume_below_first_floor( + harness: &mut TestHarness, +) { + // Two full checkpoint rounds plus a small live tail. + const TAIL: u64 = 20; + const TOTAL_REQUESTS: u64 = 2 * CHECKPOINT_EVERY - 1 + TAIL; + + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + let mut last_committed = None; + for request in 1..=TOTAL_REQUESTS { + let committed = commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-multi-{request}")), + ) + .await; + last_committed = Some(committed); + } + // Both generations must exist before the restart; the second marker is + // the proof the first snapshot was superseded. + await_checkpoint_on_all_nodes(harness, 2).await; + + // Crash ordering: restart before the socket drops, see the sibling test. + harness.restart_server().await.unwrap(); + drop(stream); + await_marker(harness, 0, INSTALL_MARKER, 1, "multi-generation install").await; + + let addrs = tcp_addrs(harness); + // The retry dedups against the latest generation's table (its reply is + // in the shipped ring), and the register behind it sits below BOTH + // snapshot floors. + resume_request( + &addrs, + session, + TOTAL_REQUESTS, + &create_stream_payload(&format!("iggy-multi-{TOTAL_REQUESTS}")), + Some(last_committed.as_ref().expect("at least one commit")), + ) + .await; + resume_request( + &addrs, + session, + TOTAL_REQUESTS + 1, + &create_stream_payload("iggy-multi-continuation"), + None, + ) + .await; +} + +// A second restart of the same node: the first install persisted the +// transferred snapshot to the node's own disk, so the second boot recovers +// from it locally, arms another transfer, and installs again. Pins the +// install's persist-FIRST ordering - if the snapshot never hit disk, the +// second recovery would fall back to an empty STM whose journal floor +// disagrees with the cluster. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_installed_snapshot_when_node_restarts_again_should_transfer_again( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + for request in 1..=REQUESTS_TO_FIRST_CHECKPOINT { + commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-again-{request}")), + ) + .await; + } + await_checkpoint_on_all_nodes(harness, 1).await; + + // Crash ordering: restart before the socket drops, see the sibling test. + harness.restart_server().await.unwrap(); + drop(stream); + await_marker(harness, 0, INSTALL_MARKER, 1, "first install").await; + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + REQUESTS_TO_FIRST_CHECKPOINT + 1, + &create_stream_payload("iggy-again-between-restarts"), + None, + ) + .await; + + // The stdout log is truncated per process run, so a marker seen after + // this restart was logged by the freshly booted process. + harness.restart_server().await.unwrap(); + await_marker(harness, 0, INSTALL_MARKER, 1, "second install").await; + resume_request( + &addrs, + session, + REQUESTS_TO_FIRST_CHECKPOINT + 2, + &create_stream_payload("iggy-again-after-second-restart"), + None, + ) + .await; +} diff --git a/core/integration/tests/cluster/metadata_state_transfer.rs b/core/integration/tests/cluster/metadata_state_transfer.rs new file mode 100644 index 0000000000..ba353ee21c --- /dev/null +++ b/core/integration/tests/cluster/metadata_state_transfer.rs @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Spec test for metadata state transfer: a node that restarts into a live +//! cluster replaces its snapshot-shaped state (metadata snapshot + client +//! table) from the current primary instead of relying on its own WAL. +//! +//! The scenario forces the interesting case: enough committed metadata ops +//! to trip a checkpoint on every node (`journal_slots` shrunk below), which +//! drains the early ops -- including the client's register -- out of every +//! WAL. A restarted node's local recovery can then neither replay those ops +//! nor journal-repair them (the serving peers evicted them too); only the +//! transferred snapshot + table carry that history. +//! +//! The transferred node is a follower afterwards, so its installed state has +//! no client-visible surface to assert against (followers neither commit nor +//! serve resume lookups). The install is pinned via its log marker; the +//! functional assert (post-restart continuation commits cluster-wide) rides +//! on top. + +#![cfg(feature = "vsr")] + +use super::client_table_restart::{ + commit_request, create_stream_payload, register, resume_request, tcp_addr, tcp_addrs, +}; +use integration::iggy_harness; +use std::time::Duration; +use tokio::time::{Instant, sleep}; + +/// Committed ops before the restart. `journal_slots = 256` with the built-in +/// checkpoint margin (64) forces a checkpoint at ~192 committed ops, so this +/// guarantees at least one checkpoint+drain on every node, pushing the +/// register (op 1) below every snapshot floor. +const OPS_BEFORE_RESTART: u64 = 220; + +/// How long the restarted follower gets to probe, fetch, and install. +const TRANSFER_BUDGET: Duration = Duration::from_secs(30); + +const MARKER_POLL: Duration = Duration::from_millis(200); + +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_checkpointed_cluster_when_node_restarts_should_state_transfer_metadata( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + for request in 1..=OPS_BEFORE_RESTART { + commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-transfer-{request}")), + ) + .await; + } + drop(stream); + + harness.restart_server().await.unwrap(); + + // Functional: the session (registered below every snapshot floor by now) + // still continues cluster-wide. + let addrs = tcp_addrs(harness); + // A continuation is a fresh op, so there is no cached reply to compare. + resume_request( + &addrs, + session, + OPS_BEFORE_RESTART + 1, + &create_stream_payload("iggy-transfer-continuation"), + None, + ) + .await; + + // The restarted node itself: it must have entered state transfer, + // fetched the primary's snapshot + client table, and installed them. + // Its own WAL no longer holds the pre-checkpoint history, so nothing + // short of the transfer can restore that state on it. + let deadline = Instant::now() + TRANSFER_BUDGET; + loop { + if harness + .node(0) + .stdout_contains("metadata state transfer installed") + { + break; + } + assert!( + Instant::now() < deadline, + "restarted node never completed the metadata state transfer \ + within {TRANSFER_BUDGET:?}" + ); + sleep(MARKER_POLL).await; + } +} + +/// A node that joins a live, already-checkpointed cluster with NO local +/// history must state-transfer, exactly like the restart-with-WAL case above. +/// +/// This is the shape the user cares about beyond a plain restart: a fresh +/// operator-provisioned replacement, or the third node of a cluster whose +/// first two formed quorum and committed past a checkpoint before it arrived. +/// It has no WAL to probe from, so the restart-arms-transfer boot path does +/// not apply -- it joins as a plain view-0 backup, learns the frontier from +/// the primary's heartbeats, discovers via journal repair that the gap sits +/// below the retained floor, and converts THAT into a state transfer. The +/// same `RangeEvicted`-to-transfer path serves the restart, the late join, +/// and a partition heal; only the way each reaches repair differs. +/// +/// Node 2 (a follower) is wiped rather than node 0 so the client keeps its +/// primary and the cluster never loses quorum -- a genuine late join, not a +/// restart of the whole cluster. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_checkpointed_cluster_when_fresh_node_joins_late_should_state_transfer_metadata( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + for request in 1..=OPS_BEFORE_RESTART { + commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-latejoin-{request}")), + ) + .await; + } + + // Wipe a follower and bring it back with an empty data directory while the + // other two keep quorum. Its missing prefix (register at op 1 included) was + // checkpointed away on every survivor, so nothing but a transfer can seed + // its metadata state and client table. + harness.restart_node_from_clean_slate(2).unwrap(); + + let deadline = Instant::now() + TRANSFER_BUDGET; + loop { + if harness + .node(2) + .stdout_contains("metadata state transfer installed") + { + break; + } + assert!( + Instant::now() < deadline, + "fresh late-joining node never completed the metadata state transfer \ + within {TRANSFER_BUDGET:?}; it must convert a repair-floor eviction \ + into a transfer, not wedge below the floor" + ); + sleep(MARKER_POLL).await; + } + + // Functional: the cluster still commits with the rejoined node present, and + // the original session (registered below every floor) continues. + drop(stream); + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + OPS_BEFORE_RESTART + 1, + &create_stream_payload("iggy-latejoin-continuation"), + None, + ) + .await; +} + +/// A node that rejoins at a STALE VIEW must probe to catch up, not wedge. +/// +/// The late-join test above rejoins a follower while the cluster is still at +/// view 0, so it adopts the frontier from same-view heartbeats. This one forces +/// the harder case: kill the view-0 primary so the survivors elect past it, +/// then rejoin that same node (replica 0) with an empty disk. It boots +/// primary-by-index at view 0 -- so it has NO heartbeat timeout to notice it is +/// behind, and thinks it is the primary of a view the cluster has abandoned. +/// +/// It must observe the newer-view traffic it keeps dropping, convert its own +/// heartbeat-SEND timer into a view probe, adopt the current view from the +/// live primary's `StartView`, and from there follow the same repair -> +/// `RangeEvicted` -> transfer path. Without that it advertises a commit point +/// no peer accepts, forever. +#[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] +async fn given_election_past_a_node_when_it_rejoins_stale_should_probe_then_state_transfer( + harness: &mut TestHarness, +) { + let addr = tcp_addr(harness); + let (mut stream, session) = register(addr).await; + for request in 1..=OPS_BEFORE_RESTART { + commit_request( + &mut stream, + session, + request, + &create_stream_payload(&format!("iggy-stale-{request}")), + ) + .await; + } + drop(stream); + + // Kill the view-0 primary (replica 0). The survivors (1, 2) hold quorum and + // elect a new primary at a higher view, leaving replica 0 behind on view. + harness.stop_node(0).unwrap(); + + // Confirm the cluster recovered to a live primary at the new view before + // rejoining: one continuation must commit against a survivor. The stopped + // node's address is skipped by the round-robin. + let addrs = tcp_addrs(harness); + resume_request( + &addrs, + session, + OPS_BEFORE_RESTART + 1, + &create_stream_payload("iggy-stale-failover"), + None, + ) + .await; + + // Rejoin replica 0 with an empty disk. It boots primary-by-index at view 0 + // while the cluster sits at a higher view -- the stale-primary case. + harness.restart_node_from_clean_slate(0).unwrap(); + + // It must first PROBE (its heartbeat-send timer converting, since it has no + // heartbeat-receive timer as a "primary"), then complete the transfer. The + // probe marker distinguishes this path from the same-view backstop. + let deadline = Instant::now() + TRANSFER_BUDGET; + let mut probed = false; + loop { + probed = probed || harness.node(0).stdout_contains("probing to catch up"); + if harness + .node(0) + .stdout_contains("metadata state transfer installed") + { + assert!( + probed, + "the rejoined node transferred without first probing; the \ + stale-view path must reach the transfer through a view probe" + ); + break; + } + assert!( + Instant::now() < deadline, + "node that rejoined at a stale view never state-transferred within \ + {TRANSFER_BUDGET:?}; a stale primary-by-index must convert its \ + heartbeat-send timer into a probe rather than wedge" + ); + sleep(MARKER_POLL).await; + } + + // Functional: the session continues cluster-wide with the node rejoined. + resume_request( + &addrs, + session, + OPS_BEFORE_RESTART + 2, + &create_stream_payload("iggy-stale-continuation"), + None, + ) + .await; +} diff --git a/core/integration/tests/cluster/mod.rs b/core/integration/tests/cluster/mod.rs index 93c631bd00..a339efe4ad 100644 --- a/core/integration/tests/cluster/mod.rs +++ b/core/integration/tests/cluster/mod.rs @@ -16,3 +16,5 @@ // under the License. mod client_table_restart; +mod metadata_checkpoint_restart; +mod metadata_state_transfer; diff --git a/core/integration/tests/data_integrity/mod.rs b/core/integration/tests/data_integrity/mod.rs index 8bb56bb0d3..620f8e872a 100644 --- a/core/integration/tests/data_integrity/mod.rs +++ b/core/integration/tests/data_integrity/mod.rs @@ -15,13 +15,10 @@ // specific language governing permissions and limitations // under the License. -// Requires state transfer: the 5 MB bench fill is thousands of ops while the -// partition journal's evicted ring retains only the last 4096, so a restarted -// replica's rejoin window exceeds what journal repair can serve. The commit -// floor lets recovered segments stand in for the evicted prefix, but the -// sub-floor stats/offset seeding this test asserts (exact messages_count / -// size_bytes across the restart) is state transfer's job. -#[cfg(not(feature = "vsr"))] +// Partially vsr-gated inside the module: the bench-fill test requires +// PARTITION-plane state transfer (sub-floor stats/offset seeding), which is +// not implemented yet; the metadata-only deletion/restart test runs under +// vsr (metadata journal repair covers its rejoin window). mod verify_after_server_restart; mod verify_user_login_after_restart; diff --git a/core/integration/tests/data_integrity/verify_after_server_restart.rs b/core/integration/tests/data_integrity/verify_after_server_restart.rs index c198a14988..9d75af21ee 100644 --- a/core/integration/tests/data_integrity/verify_after_server_restart.rs +++ b/core/integration/tests/data_integrity/verify_after_server_restart.rs @@ -16,34 +16,65 @@ // under the License. use iggy::prelude::*; +#[cfg(not(feature = "vsr"))] use integration::bench_utils::run_bench_and_wait_for_finish; use integration::harness::{TestHarness, TestServerConfig}; use serial_test::parallel; +#[cfg(not(feature = "vsr"))] use std::{collections::HashMap, str::FromStr}; +#[cfg(not(feature = "vsr"))] use test_case::test_matrix; +#[cfg(not(feature = "vsr"))] fn cache_open_segment() -> &'static str { "open_segment" } +#[cfg(not(feature = "vsr"))] fn cache_all() -> &'static str { "all" } +#[cfg(not(feature = "vsr"))] fn cache_none() -> &'static str { "none" } +#[cfg(not(feature = "vsr"))] fn build_server_config(cache_setting: &str) -> TestServerConfig { let mut extra_envs = HashMap::new(); extra_envs.insert( "IGGY_SYSTEM_SEGMENT_CACHE_INDEXES".to_string(), cache_setting.to_string(), ); + // Under vsr this config must ALSO set the eager-flush envs + // (`IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE=1`, + // `IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC=true`, see + // `encryption_scenario::build_server_config`): server-ng serves no + // `flush_unsaved_buffer` (the command is slated for removal), so the + // flush calls below must be cfg'd out and replaced by eager persistence + // when the fill test is ungated. TestServerConfig::builder().extra_envs(extra_envs).build() } // TODO(numminex) - Move the message generation method from benchmark run to a special method. +// +// vsr-gated: requires PARTITION-plane state transfer. The 5 MB bench fill is +// thousands of ops while the partition journal's evicted ring retains only +// the last 4096, so a restarted replica's rejoin window exceeds what journal +// repair can serve. The commit floor lets recovered segments stand in for +// the evicted prefix, but the sub-floor stats/offset seeding this test +// asserts (exact messages_count / size_bytes across the restart) needs the +// partition-plane transfer to carry them; the metadata-plane transfer's +// snapshot stats cannot be stitched to the partition repair window without +// double-counting. +// +// NOT gated on `flush_unsaved_buffer`: this test only uses flush as a +// durability barrier, and under vsr the calls are replaced by the eager-flush +// server envs (see `build_server_config`). The other blocker to clear when +// ungating is the bench harness (`run_bench_and_wait_for_finish`), which is +// out of the vsr test pass today. +#[cfg(not(feature = "vsr"))] #[test_matrix( [cache_all(), cache_open_segment(), cache_none()] )] diff --git a/core/integration/tests/server/flush_vsr.rs b/core/integration/tests/server/flush_vsr.rs index 68f41040ef..36c840724f 100644 --- a/core/integration/tests/server/flush_vsr.rs +++ b/core/integration/tests/server/flush_vsr.rs @@ -19,6 +19,10 @@ //! primitive, so `FLUSH_UNSAVED_BUFFER` must surface a typed //! `FeatureUnavailable` over the SDK rather than the non-replicated catch-all's //! empty-ok, which would fake a durability guarantee. +//! +//! `FLUSH_UNSAVED_BUFFER` is slated for removal from the protocol. This test +//! pins the deny contract only until then: delete this file together with the +//! command (and the SDK method), do not port it anywhere. use iggy::prelude::*; use integration::iggy_harness; diff --git a/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs b/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs index 6815ebac15..ecf152f9d6 100644 --- a/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs +++ b/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs @@ -48,15 +48,6 @@ const MAX_ALLOWED_MEMORY_BYTES: u64 = 200 * 1024 * 1024; /// Uses all available transports from the harness (TCP, HTTP, QUIC, WebSocket). /// 2 producers are spawned per protocol, all writing to the same partition. pub async fn run(harness: &TestHarness) { - // HTTP (REST) carries no VSR framing, so under vsr the race runs over the - // three VSR transports; the legacy path exercises all four. - #[cfg(feature = "vsr")] - let transports = [ - TransportProtocol::Tcp, - TransportProtocol::Quic, - TransportProtocol::WebSocket, - ]; - #[cfg(not(feature = "vsr"))] let transports = [ TransportProtocol::Tcp, TransportProtocol::Http, diff --git a/core/journal/src/lib.rs b/core/journal/src/lib.rs index 629a2e4ec7..691b3e30eb 100644 --- a/core/journal/src/lib.rs +++ b/core/journal/src/lib.rs @@ -58,6 +58,17 @@ where ) -> impl Future>> { async { Ok(Vec::new()) } } + + /// Snapshot watermark: entries at or below it are evictable. `0` for + /// journals without snapshot bookkeeping. + fn snapshot_op(&self) -> u64 { + 0 + } + + /// Advance the snapshot watermark (see [`Self::snapshot_op`]). No-op for + /// journals without snapshot bookkeeping. State transfer uses this to + /// mark pre-transfer residents superseded by the installed snapshot. + fn set_snapshot_op(&self, _op: u64) {} } // TODO: Move to other crate. diff --git a/core/journal/src/prepare_journal.rs b/core/journal/src/prepare_journal.rs index 802b36d819..674b94e209 100644 --- a/core/journal/src/prepare_journal.rs +++ b/core/journal/src/prepare_journal.rs @@ -413,6 +413,11 @@ impl PrepareJournal { self.last_op.get() } + /// Current snapshot watermark: entries at or below it are evictable. + pub const fn snapshot_op(&self) -> u64 { + self.snapshot_op.get() + } + /// Advance the snapshot watermark. The caller must ensure `op` is /// monotonically increasing and corresponds to a durable snapshot. /// @@ -524,6 +529,14 @@ impl Journal for PrepareJournal { type Entry = Message; type HeaderRef<'a> = Ref<'a, PrepareHeader>; + fn snapshot_op(&self) -> u64 { + Self::snapshot_op(self) + } + + fn set_snapshot_op(&self, op: u64) { + Self::set_snapshot_op(self, op); + } + fn header(&self, idx: usize) -> Option> { let headers = self.headers.borrow(); Ref::filter_map(headers, |h| { diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 5d0ae06797..85632d1a57 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -18,7 +18,9 @@ use crate::MuxStateMachine; use crate::stm::authz::gated_apply; use crate::stm::consumer_group::CompleteConsumerGroupRevocationRequest; -use crate::stm::snapshot::{FillSnapshot, MetadataSnapshot, Snapshot, SnapshotError}; +use crate::stm::snapshot::{ + FillSnapshot, MetadataSnapshot, RestoreSnapshotInPlace, Snapshot, SnapshotError, +}; use crate::stm::stream::{Streams, TruncatePartitionRequest}; use crate::stm::user::{DeletePersonalAccessTokenRequest, Users}; use crate::stm::{ConsensusGroupAllocator, StateMachine}; @@ -55,7 +57,7 @@ use std::cell::{Cell, RefCell}; use std::mem::size_of; use std::path::Path; use std::rc::Rc; -use tracing::{debug, error, warn}; +use tracing::{debug, error, info, warn}; fn freeze_client_reply( message: Message, @@ -244,6 +246,16 @@ impl SnapshotCoordinator { /// /// # Errors /// Returns `SnapshotError` if snapshotting, persistence, or drain fails. + /// On-disk location of the persisted snapshot; also the artifact state + /// transfer serves and installs. + #[must_use] + pub fn snapshot_path(&self) -> std::path::PathBuf { + self.data_dir.join(super::METADATA_DIR).join("snapshot.bin") + } + + /// # Errors + /// Returns `SnapshotError` if snapshotting, persistence, or the journal + /// drain fails. #[allow(clippy::future_not_send)] pub async fn checkpoint( &self, @@ -256,7 +268,7 @@ impl SnapshotCoordinator { J: JournalHandle, { let snapshot = (self.create_snapshot)(stm, last_op, created_at)?; - let path = self.data_dir.join(super::METADATA_DIR).join("snapshot.bin"); + let path = self.snapshot_path(); snapshot.persist(&path)?; let _ = journal @@ -543,6 +555,11 @@ pub struct IggyMetadata { /// set from server config at bootstrap ([`Self::set_default_message_expiry`]). /// Defaults to never-expire, matching the shipped server config. default_message_expiry: Cell, + /// Client-table mutations at or below this op are already reflected in a + /// state-transferred table, so the tail-repair commit walk must skip + /// them (re-running `commit_register` would double-bump epochs). `0` + /// outside state transfer (no op is skipped). Monotone per install. + client_table_frontier: Cell, } impl IggyMetadata @@ -576,6 +593,7 @@ where commit_notifier: RefCell::new(None), default_max_topic_size: Cell::new(u64::MAX), default_message_expiry: Cell::new(u64::MAX), + client_table_frontier: Cell::new(0), } } } @@ -591,7 +609,9 @@ impl IggyMetadata { /// Install the client table rebuilt by WAL-replay recovery /// ([`crate::impls::recovery::recover`]). Boot-time only, on the owning /// shard, before it serves traffic - replacing a live table would drop - /// committed session state. + /// committed session state. (State transfer replaces a LIVE table via + /// [`IggyMetadata::install_state_transfer`], which also stamps the + /// frontier.) /// /// Refuses (leaving the live table in place) when the current table /// already holds sessions, which means a client registered before recovery @@ -618,6 +638,14 @@ impl IggyMetadata { true } + /// Client-table mutations at or below the frontier are already in the + /// state-transferred table; the commit walk skips them (re-running + /// `commit_register` would double-bump epochs). STM effects still apply + /// -- the frontier fences the TABLE only. + const fn client_table_mutation_allowed(&self, op: u64) -> bool { + op > self.client_table_frontier.get() + } + /// Install the resolved byte value used for `MaxTopicSize::ServerDefault`. /// Server-ng bootstrap calls this with `system.topic.max_size` on every /// shard (responses read it too); only shard 0's copy feeds admission. @@ -1042,6 +1070,24 @@ where } } +/// One state-transfer serving payload. +/// +/// The on-disk snapshot bytes plus the live client table, both +/// frontier-stamped. Built by [`IggyMetadata::state_transfer_offer`] on the +/// serving primary; the shard caches it per requester and serves chunks +/// from it. +pub struct StateTransferOffer { + /// Serving primary's applied frontier when the offer was built; the + /// receiver's tail repair targets past this. + pub commit_op: u64, + /// Manifest entries, index-aligned with `payloads`. Metadata plane: + /// `[METADATA_SNAPSHOT (frontier = sequence_number), + /// CLIENT_TABLE (frontier = commit_min at encode)]`. + pub artifacts: Vec, + /// Artifact bytes, chunk-served by `(manifest index, offset)`. + pub payloads: Vec>, +} + impl IggyMetadata, J, S, M> where B: MessageBus, @@ -1054,6 +1100,154 @@ where Error = iggy_common::IggyError, >, { + /// Build a state-transfer offer for a restarted peer, or `None` when + /// this replica cannot serve one right now: not a caught-up primary + /// (table reads would not be authoritative), or no snapshot has ever + /// been persisted (the WAL then still holds the full history and the + /// requester's journal repair covers the whole gap). + /// + /// The snapshot is served as the on-disk bytes verbatim -- possibly + /// stale, which costs nothing: the receiver journal-repairs + /// `(snapshot_seq, commit_max]` afterwards through the existing repair + /// machinery. The table is encoded live at this instant; both frontier + /// stamps read `commit_min` inside one synchronous region, so they are + /// mutually consistent. + #[must_use] + pub fn state_transfer_offer(&self) -> Option { + let consensus = self.consensus.as_ref()?; + if !is_caught_up_primary(consensus) { + return None; + } + let coordinator = self.coordinator.as_ref()?; + let snapshot = std::fs::read(coordinator.snapshot_path()).ok()?; + let snapshot_seq = IggySnapshot::decode(&snapshot).ok()?.sequence_number(); + let commit_op = consensus.commit_min(); + let table = self.client_table.borrow().encode(); + Some(StateTransferOffer { + commit_op, + artifacts: vec![ + consensus::StateArtifact::for_bytes( + consensus::artifact_kind::METADATA_SNAPSHOT, + snapshot_seq, + &snapshot, + ), + consensus::StateArtifact::for_bytes( + consensus::artifact_kind::CLIENT_TABLE, + commit_op, + &table, + ), + ], + payloads: vec![snapshot, table], + }) + } + + /// Install a fetched state transfer: persist + restore the snapshot, + /// replace the client table, and jump the commit state to the snapshot + /// floor so the tail repair takes over from there. + /// + /// Ordering: persist FIRST (a crash mid-install must reboot from the + /// transferred state, not the pre-transfer one), then the in-place STM + /// restore (readers observe it on their next read), then the table + + /// frontier, then journal/commit bookkeeping. + /// + /// The partition plane is deliberately untouched: partitions load + /// whatever their disks hold at boot and repair through their own + /// consensus groups. Topology changes the snapshot carries below the + /// receiver's old frontier (topics created/deleted while it was down) + /// fire no commit notifier -- convergence rests on the partition + /// reconciler's periodic full diff against the committed STM, which + /// reads the restored state on its next tick. + /// + /// Returns the installed snapshot sequence (the receiver's new applied + /// frontier). + /// + /// # Errors + /// [`SnapshotError`] when the snapshot bytes do not decode, the persist + /// fails, or the in-place restore is rejected. + /// + /// # Panics + /// If called on a shard without consensus (state transfer is a shard-0 + /// concern). + pub fn install_state_transfer( + &self, + snapshot_bytes: &[u8], + client_table: ClientTable, + table_frontier: u64, + commit_op: u64, + ) -> Result + where + M: RestoreSnapshotInPlace, + { + let consensus = self + .consensus + .as_ref() + .expect("install_state_transfer: consensus only exists on shard 0"); + + let snapshot = IggySnapshot::decode(snapshot_bytes)?; + let snapshot_seq = snapshot.sequence_number(); + + // Checkpoints are node-local, so a healthy serving primary can offer + // a snapshot BEHIND this replica's own applied frontier (each node + // snapshots at its own watermark; a backup checkpoints an op or two + // below the primary it later replaces). Restoring such a snapshot + // would rewind the STM below `commit_min` with no way back: the + // commit walk never revisits ops it already counted as applied, so + // the rewound-over effects would be lost until the next transfer. + // Keep the local STM (it is a superset) and let tail repair cover + // `(commit_min, commit_op]`. The client table still installs below: + // it comes from the serving primary's LIVE state at `table_frontier + // == commit_op`, which is never behind this replica. + let local_applied = consensus.commit_min(); + let snapshot_ahead = snapshot_seq > local_applied; + if snapshot_ahead { + if let Some(coordinator) = &self.coordinator { + snapshot.persist(&coordinator.snapshot_path())?; + } else { + tracing::warn!( + snapshot_seq, + "installing state transfer without a snapshot coordinator; \ + the transferred state will not survive a further restart" + ); + } + + self.mux_stm + .restore_snapshot_in_place(snapshot.snapshot())?; + } else { + tracing::info!( + snapshot_seq, + local_applied, + "transferred snapshot at or below the local applied frontier; \ + keeping the local state machine and installing the table only" + ); + } + + *self.client_table.borrow_mut() = client_table; + self.client_table_frontier.set(table_frontier); + + if snapshot_ahead { + // Entries at or below the installed floor are superseded by the + // snapshot; without this the journal's wrap-eviction assert trips + // on pre-transfer residents the next time slots recycle. + if let Some(journal) = &self.journal { + let handle = journal.handle(); + if snapshot_seq > handle.snapshot_op() { + handle.set_snapshot_op(snapshot_seq); + } + } + + // The snapshot IS ops `..=snapshot_seq` applied: jump the applied + // frontier (this is the op-jump the tail repair resumes from) and + // let the announced commit point pull the walk target forward. + consensus.set_commit_floor(snapshot_seq); + if snapshot_seq > consensus.sequencer().current_sequence() { + consensus.sequencer().set_sequence(snapshot_seq); + } + } + consensus.advance_commit_max(commit_op); + + Ok(snapshot_seq.max(local_applied)) + } + /// Submit `Register` from in-process, await commit. Wire reply still fires /// via `message_bus.send_to_client`; subscriber is additive. /// @@ -1099,7 +1293,7 @@ where // Wrong node: waiting or queueing cannot fix that, the client must // re-route to the primary. - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { return Err(MetadataSubmitError::NotPrimary); } @@ -1344,7 +1538,7 @@ where // logout-vs-logout race. It simply pipelines behind the in-flight // batch and commits with it, so a one-shot client's session // teardown is latency, never an error. - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { return Err(MetadataSubmitError::NotPrimary); } @@ -1458,7 +1652,7 @@ where // client submits compete for. if !is_caught_up_primary(consensus) { return Err( - if consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing() { + if consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring() { MetadataSubmitError::NotCaughtUp } else { MetadataSubmitError::NotPrimary @@ -1540,7 +1734,7 @@ where // deletion on its next sweep. if !is_caught_up_primary(consensus) { return Err( - if consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing() { + if consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring() { MetadataSubmitError::NotCaughtUp } else { MetadataSubmitError::NotPrimary @@ -1637,7 +1831,7 @@ where // simply pipelines behind the in-flight batch (the wire path has // always done this); the register-specific invariant is guarded in // `submit_register_in_process` / `register_preflight`. - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { return Ok(build_result_rejection_reply( &request_header, consensus.commit_max(), @@ -1807,7 +2001,7 @@ where let Some(consensus) = self.consensus.as_ref() else { return; }; - if !consensus.is_primary() || !consensus.is_normal() || consensus.is_syncing() { + if !consensus.is_primary() || !consensus.is_normal() || consensus.is_transferring() { return; } let Some(journal) = self.journal.as_ref() else { @@ -1975,21 +2169,28 @@ where let reply = if prepare_header.operation == Operation::Register { // Register: commit_register creates session, no SM. let reply = build_reply_message(&prepare_header, &bytes::Bytes::new()); - self.client_table.borrow_mut().commit_register( - prepare_header.client, - prepare_header.user_id, - reply.clone(), - ); + if self.client_table_mutation_allowed(prepare_header.op) { + self.client_table.borrow_mut().commit_register( + prepare_header.client, + prepare_header.user_id, + reply.clone(), + ); + } reply } else if prepare_header.operation == Operation::Logout { // Logout unregisters the VSR client session on every replica. let reply = build_reply_message(&prepare_header, &bytes::Bytes::new()); - self.client_table - .borrow_mut() - .remove_client(prepare_header.client); + if self.client_table_mutation_allowed(prepare_header.op) { + self.client_table + .borrow_mut() + .remove_client(prepare_header.client); + } // Drop the disconnected client from every consumer group it // joined and rebalance. Deterministic side-effect of the - // Logout commit, applied identically on every replica. + // Logout commit, applied identically on every replica. Runs + // regardless of the table frontier: the STM was restored at + // the snapshot floor, so ops above it still owe their STM + // effects even where the table already reflects them. self.mux_stm.streams().remove_consumer_group_member( prepare_header.client, iggy_common::IggyTimestamp::from(prepare_header.timestamp), @@ -2013,12 +2214,16 @@ where build_reply_message_with(&prepare_header, apply.reply_body_len(), |dst| { apply.write_reply_body(dst); }); - // Best-effort cache; the wire reply ships either way. - let outcome = self - .client_table - .borrow_mut() - .commit_reply(prepare_header.client, reply.clone()); - log_commit_reply_outcome(outcome, prepare_header.client, prepare_header.op); + // Best-effort cache; the wire reply ships either way. Ops at + // or below the state-transfer frontier are already reflected + // in the transferred table and are skipped. + if self.client_table_mutation_allowed(prepare_header.op) { + let outcome = self + .client_table + .borrow_mut() + .commit_reply(prepare_header.client, reply.clone()); + log_commit_reply_outcome(outcome, prepare_header.client, prepare_header.op); + } reply }; consensus.advance_commit_min(prepare_header.op); @@ -2099,7 +2304,7 @@ where let Some(consensus) = self.consensus.as_ref() else { return; }; - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { return; } let stranded_commits = consensus.commit_min() < consensus.commit_max(); @@ -2189,7 +2394,10 @@ where // at enqueue time resolves on this prepare's commit. assert!(!consensus.is_follower(), "promotion: primary only"); assert!(consensus.is_normal(), "promotion: status must be normal"); - assert!(!consensus.is_syncing(), "promotion: must not be syncing"); + assert!( + !consensus.is_transferring(), + "promotion: must not be syncing" + ); consensus.verify_pipeline(); match reply_sender { Some(sender) => { @@ -2241,7 +2449,10 @@ where .await { Ok(true) => { - debug!( + // Info, not debug: checkpoints are rare and change what a + // restart can recover locally, and spec tests pin checkpoint + // placement through this line (`metadata_checkpoint_restart`). + info!( target: "iggy.metadata.diag", plane = "metadata", replica_id = consensus.replica(), @@ -2494,16 +2705,22 @@ where // mutation and counter bump. if header.operation == Operation::Register { // Register: commit_register creates session, no SM. - let reply = build_reply_message(&header, &bytes::Bytes::new()); - self.client_table.borrow_mut().commit_register( - header.client, - header.user_id, - reply, - ); + if self.client_table_mutation_allowed(header.op) { + let reply = build_reply_message(&header, &bytes::Bytes::new()); + self.client_table.borrow_mut().commit_register( + header.client, + header.user_id, + reply, + ); + } } else if header.operation == Operation::Logout { - self.client_table.borrow_mut().remove_client(header.client); + if self.client_table_mutation_allowed(header.op) { + self.client_table.borrow_mut().remove_client(header.client); + } // Mirror the on_ack path: drop the disconnected client from - // every consumer group it joined and rebalance. + // every consumer group it joined and rebalance. Not fenced by + // the table frontier: the STM was restored at the snapshot + // floor and still owes this effect. self.mux_stm.streams().remove_consumer_group_member( header.client, iggy_common::IggyTimestamp::from(header.timestamp), @@ -2526,11 +2743,15 @@ where // Best-effort cache; WAL replay may carry a reply for a // later-evicted client, and replica-local eviction makes a // stale-request replay reachable. Both are skips, not faults. - let outcome = self - .client_table - .borrow_mut() - .commit_reply(header.client, reply); - log_commit_reply_outcome(outcome, header.client, op); + // Ops at or below the state-transfer frontier are already + // reflected in the transferred table and are skipped. + if self.client_table_mutation_allowed(header.op) { + let outcome = self + .client_table + .borrow_mut() + .commit_reply(header.client, reply); + log_commit_reply_outcome(outcome, header.client, op); + } } consensus.advance_commit_min(op); debug!("commit_journal: committed op={op}"); diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs index 00ea8b8002..4b2d91aaaf 100644 --- a/core/metadata/src/impls/recovery.rs +++ b/core/metadata/src/impls/recovery.rs @@ -19,7 +19,7 @@ use crate::impls::metadata::IggySnapshot; use crate::stm::StateMachine; use crate::stm::authz::GatedApply; use crate::stm::snapshot::{MetadataSnapshot, RestoreSnapshot, Snapshot, SnapshotError}; -use consensus::{CLIENTS_TABLE_MAX, ClientTable, build_reply_message, build_reply_message_with}; +use consensus::{ClientTable, build_reply_message, build_reply_message_with}; use iggy_binary_protocol::consensus::{Operation, PrepareHeader}; use iggy_common::IggyError; use journal::prepare_journal::{JournalError, PrepareJournal}; @@ -140,6 +140,7 @@ pub async fn recover( data_dir: &Path, solo: bool, journal_slots: usize, + clients_table_max: usize, seed_baseline: impl FnOnce(&M), ) -> Result, RecoveryError> where @@ -198,7 +199,7 @@ where .fold(snapshot_floor, u64::max) }; - let mut client_table = ClientTable::new(CLIENTS_TABLE_MAX); + let mut client_table = ClientTable::new(clients_table_max); let mut last_applied_op: Option = None; let mut last_journaled_op: Option = None; for header in &headers_to_replay { @@ -300,6 +301,7 @@ where #[allow(clippy::cast_possible_truncation)] mod tests { use super::*; + use consensus::CLIENTS_TABLE_MAX; use iggy_binary_protocol::consensus::{Command2, Operation}; use journal::Journal; use server_common::iobuf::Owned; @@ -363,6 +365,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -387,6 +390,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -421,6 +425,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -462,6 +467,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -498,6 +504,7 @@ mod tests { dir.path(), false, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -564,6 +571,7 @@ mod tests { dir.path(), true, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -624,6 +632,7 @@ mod tests { dir.path(), true, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await @@ -678,6 +687,7 @@ mod tests { dir.path(), true, journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, |_| {}, ) .await diff --git a/core/metadata/src/lib.rs b/core/metadata/src/lib.rs index 9f7af7df5d..9dd4b29375 100644 --- a/core/metadata/src/lib.rs +++ b/core/metadata/src/lib.rs @@ -22,7 +22,9 @@ pub mod permissioner; pub mod stm; // Re-export IggyMetadata for use in other modules -pub use impls::metadata::{BoundSession, CommitNotifier, IggyMetadata, MetadataSubmitError}; +pub use impls::metadata::{ + BoundSession, CommitNotifier, IggyMetadata, MetadataSubmitError, StateTransferOffer, +}; // Re-export MuxStateMachine for use in other modules pub use stm::mux::MuxStateMachine; diff --git a/core/metadata/src/stm/mod.rs b/core/metadata/src/stm/mod.rs index 5fc0c10d1d..0763f44017 100644 --- a/core/metadata/src/stm/mod.rs +++ b/core/metadata/src/stm/mod.rs @@ -396,6 +396,12 @@ macro_rules! collect_handlers { $( $operation([<$operation Request>], ::iggy_common::IggyTimestamp), )* + /// Replace the whole state from a snapshot section, in place. + /// Never parsed off the wire (state transfer installs it via + /// `RestoreSnapshotInPlace`); absorbed on both left-right + /// buffers like any op, which is what makes an in-place + /// restore sound under the double-apply contract. + RestoreSnapshot([<$state Snapshot>]), } impl $crate::stm::Command for [<$state Inner>] { @@ -434,6 +440,10 @@ macro_rules! collect_handlers { $crate::stm::StateHandler::apply(payload, self, *ts) }, )* + [<$state Command>]::RestoreSnapshot(snapshot) => { + *self = Self::inner_from_snapshot(snapshot.clone()); + $crate::stm::result::ApplyReply::default() + }, }); } } diff --git a/core/metadata/src/stm/mux.rs b/core/metadata/src/stm/mux.rs index 864d40e2fa..027b4bcaf7 100644 --- a/core/metadata/src/stm/mux.rs +++ b/core/metadata/src/stm/mux.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::stm::snapshot::{FillSnapshot, RestoreSnapshot, SnapshotError}; +use crate::stm::snapshot::{FillSnapshot, RestoreSnapshot, RestoreSnapshotInPlace, SnapshotError}; use iggy_binary_protocol::PrepareHeader; use iggy_common::Either; use iggy_common::variadic; @@ -188,6 +188,26 @@ where } } +impl RestoreSnapshotInPlace for variadic!(Head, ...Tail) +where + Head: RestoreSnapshotInPlace, + Tail: RestoreSnapshotInPlace, +{ + fn restore_snapshot_in_place(&self, snapshot: &SnapshotData) -> Result<(), SnapshotError> { + self.0.restore_snapshot_in_place(snapshot)?; + self.1.restore_snapshot_in_place(snapshot) + } +} + +impl RestoreSnapshotInPlace for MuxStateMachine +where + T: StateMachine + RestoreSnapshotInPlace, +{ + fn restore_snapshot_in_place(&self, snapshot: &SnapshotData) -> Result<(), SnapshotError> { + self.inner.restore_snapshot_in_place(snapshot) + } +} + impl FillSnapshot for MuxStateMachine where T: StateMachine + FillSnapshot, diff --git a/core/metadata/src/stm/snapshot.rs b/core/metadata/src/stm/snapshot.rs index afe9a70810..274ea56c33 100644 --- a/core/metadata/src/stm/snapshot.rs +++ b/core/metadata/src/stm/snapshot.rs @@ -243,6 +243,27 @@ pub trait RestoreSnapshot: Sized { fn restore_snapshot(snapshot: &S) -> Result; } +/// Restore a LIVE state machine from a snapshot without reconstructing it. +/// +/// [`RestoreSnapshot`] builds a fresh instance -- boot-time only, because a +/// running system shares read handles (left-right factories) across shards +/// that a swap would orphan. This variant replaces the state THROUGH the +/// existing write handle (an absorbed `RestoreSnapshot` command), so every +/// reader observes the restored state on its next read. State transfer +/// installs with this. +#[allow(clippy::missing_errors_doc)] +pub trait RestoreSnapshotInPlace { + /// Replace this state machine's contents from the snapshot. + fn restore_snapshot_in_place(&self, snapshot: &S) -> Result<(), SnapshotError>; +} + +/// Base case for the recursive tuple pattern - unit type terminates the recursion. +impl RestoreSnapshotInPlace for () { + fn restore_snapshot_in_place(&self, _snapshot: &S) -> Result<(), SnapshotError> { + Ok(()) + } +} + /// Base case for the recursive tuple pattern - unit type terminates the recursion. impl FillSnapshot for () { fn fill_snapshot(&self, _snapshot: &mut S) -> Result<(), SnapshotError> { @@ -298,6 +319,32 @@ macro_rules! impl_fill_restore { Self::from_snapshot(snap) } } + + impl $crate::stm::snapshot::RestoreSnapshotInPlace<$crate::stm::snapshot::MetadataSnapshot> + for $wrapper + { + fn restore_snapshot_in_place( + &self, + snapshot: &$crate::stm::snapshot::MetadataSnapshot, + ) -> Result<(), $crate::stm::snapshot::SnapshotError> { + use serde::de::Error as _; + use $crate::stm::snapshot::SnapshotError; + paste::paste! { + let snap = snapshot.$field.clone().ok_or_else(|| { + SnapshotError::Deserialize(rmp_serde::decode::Error::custom( + format_args!("Snapshot Restore Error: {}", stringify!($field)), + )) + })?; + self.inner + .try_apply([<$wrapper Command>]::RestoreSnapshot(snap)) + .map_err(|err| { + SnapshotError::Io(std::io::Error::other(format!( + "in-place restore on a reader-only state machine: {err}" + ))) + }) + } + } + } }; } diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs index 046d9ca0fa..c490d7390c 100644 --- a/core/metadata/src/stm/stream.rs +++ b/core/metadata/src/stm/stream.rs @@ -1808,6 +1808,16 @@ impl Snapshotable for Streams { fn from_snapshot( snapshot: Self::Snapshot, ) -> Result { + Ok(StreamsInner::inner_from_snapshot(snapshot).into()) + } +} + +impl StreamsInner { + /// Build a complete `StreamsInner` from a snapshot section. Shared by + /// wrapper construction ([`Snapshotable::from_snapshot`]) and the + /// in-place restore command (state transfer), which absorbs it on both + /// left-right buffers. + pub(crate) fn inner_from_snapshot(snapshot: StreamsSnapshot) -> Self { let mut index: AHashMap, usize> = AHashMap::new(); let mut stream_entries: Vec<(usize, Stream)> = Vec::new(); // Register restored stats in the shared registry so both left-right @@ -1898,7 +1908,7 @@ impl Snapshotable for Streams { } let items: Slab = stream_entries.into_iter().collect(); - let mut inner = StreamsInner { + let mut inner = Self { index, items, revision: snapshot.revision, @@ -1908,7 +1918,7 @@ impl Snapshotable for Streams { stats_registry, }; inner.recompute_pending_revocations_count(); - Ok(inner.into()) + inner } } diff --git a/core/metadata/src/stm/user.rs b/core/metadata/src/stm/user.rs index c8ea95f9db..cb17efa5c3 100644 --- a/core/metadata/src/stm/user.rs +++ b/core/metadata/src/stm/user.rs @@ -806,10 +806,20 @@ impl Snapshotable for Users { }) } - #[allow(clippy::cast_possible_truncation)] fn from_snapshot( snapshot: Self::Snapshot, ) -> Result { + Ok(UsersInner::inner_from_snapshot(snapshot).into()) + } +} + +impl UsersInner { + /// Build a complete `UsersInner` from a snapshot section. Shared by + /// wrapper construction ([`Snapshotable::from_snapshot`]) and the + /// in-place restore command (state transfer), which absorbs it on both + /// left-right buffers. + #[allow(clippy::cast_possible_truncation)] + pub(crate) fn inner_from_snapshot(snapshot: UsersSnapshot) -> Self { let mut index: AHashMap, UserId> = AHashMap::new(); let mut user_entries: Vec<(usize, User)> = Vec::new(); @@ -869,7 +879,7 @@ impl Snapshotable for Users { .init_permissions_for_user(user_id as UserId, user.permissions.as_deref().cloned()); } - let inner = UsersInner { + Self { index, items, personal_access_tokens, @@ -877,8 +887,7 @@ impl Snapshotable for Users { personal_access_token_expiry_index, permissioner, last_result: None, - }; - Ok(inner.into()) + } } } diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 8d2c3a4bfc..3d7137a0c5 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -1242,7 +1242,7 @@ where // replays and its leader recheck re-routes, whereas a panic // kills the shard and a silent drop wedges the client until its // read timeout. - if consensus.is_follower() || !consensus.is_normal() || consensus.is_syncing() { + if consensus.is_follower() || !consensus.is_normal() || consensus.is_transferring() { emit_partition_diag( tracing::Level::WARN, &PartitionDiagEvent::new( @@ -1322,7 +1322,7 @@ where /// dedup. Buffered `SendMessages` retry commits at fresh offset; consumers /// dedup by message key / content / producer-id+seq. /// - /// Per-iteration `is_primary && is_normal && !is_syncing` asserts inlined + /// Per-iteration `is_primary && is_normal && !is_transferring` asserts inlined /// (closure form's `&consensus` borrow conflicts with `&mut self`). Guards /// against view-change-reset flipping status across `on_replicate` await. /// @@ -1350,7 +1350,7 @@ where "drain_request_queue_into_prepares: status must be normal" ); assert!( - !consensus.is_syncing(), + !consensus.is_transferring(), "drain_request_queue_into_prepares: must not be syncing" ); let prepare = req.message.project(consensus); diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs index 14e7f0fe75..202799a60c 100644 --- a/core/partitions/src/iggy_partitions.rs +++ b/core/partitions/src/iggy_partitions.rs @@ -479,7 +479,7 @@ where let consensus = partition.consensus(); let lagging = consensus.is_follower() || !consensus.is_normal() - || consensus.is_syncing() + || consensus.is_transferring() || consensus.commit_min() < consensus.commit_max(); (partition.nth_oldest_sealed_end_offset(count), lagging) }) diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs index 9a731f6082..c053d28643 100644 --- a/core/sdk/src/quic/quic_client.rs +++ b/core/sdk/src/quic/quic_client.rs @@ -177,6 +177,13 @@ impl BinaryTransport for QuicClient { #[cfg(feature = "vsr")] if skip_auto_login { *self.skip_auto_login_once.lock().await = true; + // The replayed login/register must mint a fresh Register: the failed + // attempt already consumed the one-shot register request id and may + // have half-bound the session. + *self + .consensus_session + .lock() + .expect("consensus session mutex poisoned") = ConsensusSession::new(); } let server_address = self.current_server_address.lock().await.to_string(); info!( diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs index f30a186f01..798776bed2 100644 --- a/core/server-ng/src/bootstrap.rs +++ b/core/server-ng/src/bootstrap.rs @@ -886,6 +886,7 @@ async fn shard_main( data_dir, topology.replica_count == 1, config.metadata.journal_slots, + config.metadata.clients_table_max, |mux_stm| { ensure_default_root_user(mux_stm); }, @@ -984,6 +985,12 @@ async fn shard_main( mux_stm, Some(PathBuf::from(&config.system.path)), ); + // Size the VSR client table before the recovered table installs and before + // listeners bind: `set_capacity` requires an EMPTY table, and on a restart + // the WAL replay rebuilds sessions into the table `recover()` returns (the + // recovered table is built at the same configured capacity, so the install + // below does not shrink it back). + metadata.set_clients_table_max(config.metadata.clients_table_max); // Reinstall the sessions the WAL replay rebuilt, so a rebooted node // dedups retries and admits continuations from clients that kept their // identity across the restart (IGGY-137). @@ -1000,10 +1007,6 @@ async fn shard_main( // depth: ops already pipelined while a checkpoint runs append into that // margin (config validation keeps journal_slots >= 4x this). metadata.set_checkpoint_margin(config.metadata.checkpoint_margin()); - // Size the VSR client table before listeners bind and any client registers. - // The table is empty here on both fresh boot and restart (its slots are not - // restored from snapshot), which the setter's empty-table contract requires. - metadata.set_clients_table_max(config.metadata.clients_table_max); let shard_metrics = ShardMetrics::for_shard(); // Notifier install deferred until after tick handler wires below. @@ -1713,6 +1716,7 @@ async fn build_shard_for_thread( // per-shard tunable set once here rather than per consensus group. shard.set_repair_retry_ticks(repair_retry_ticks(config)); shard.set_repair_chunk_max(config.cluster.repair_chunk_max as u64); + shard.set_clients_table_max(config.metadata.clients_table_max); *shard_handle.borrow_mut() = Some(Rc::downgrade(&shard)); Ok((shard, sessions)) } @@ -1912,6 +1916,12 @@ fn restore_metadata_consensus( if replica_count > 1 && restored_op > 0 { consensus.init_as_backup(); consensus.begin_view_probe(); + // Restart in a cluster: replace snapshot-shaped metadata state + // (snapshot + client table) from the live primary the probe finds, + // then journal-repair the tail. If the probe exhausts instead -- + // full-cluster bootstrap, nobody live to fetch from -- the election + // fallback clears the stage and this local recovery stands. + consensus.begin_state_transfer_await(); } else { consensus.init(); } diff --git a/core/server-ng/src/dispatch.rs b/core/server-ng/src/dispatch.rs index 3c7b7cf6b3..e32523c9db 100644 --- a/core/server-ng/src/dispatch.rs +++ b/core/server-ng/src/dispatch.rs @@ -339,7 +339,7 @@ fn submit_auto_commit( .partitions() .with_partition(&namespace, |partition| { let consensus = partition.consensus(); - if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_syncing()) { + if !(consensus.is_primary() && consensus.is_normal() && !consensus.is_transferring()) { AutoCommitGate::NotPrimary } else if partition.is_auto_commit_offset_covered( applied.kind, diff --git a/core/server-ng/src/responses.rs b/core/server-ng/src/responses.rs index 9cf3378ed6..047816bfaa 100644 --- a/core/server-ng/src/responses.rs +++ b/core/server-ng/src/responses.rs @@ -1340,6 +1340,14 @@ pub(crate) fn build_raw_pat_reply( return Ok(committed); } let header_len = std::mem::size_of::(); + // A `Reply` whose result section is nonzero is not a successful commit but a + // committed business rejection or a `TransientNotCommitted` retry frame, both + // with no payload and no token to ship. Pass it through so the client decodes + // the typed result (and, for a transient, replays) instead of having a raw + // token grafted onto a rejection body. + if result_code(&committed.as_slice()[header_len..]) != Some(0) { + return Ok(committed); + } let committed_header = bytemuck::checked::try_from_bytes::(&committed.as_slice()[..header_len]) .map_err(|_| IggyError::InvalidFormat)?; diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index e495291eb1..4fe65638be 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs @@ -19,8 +19,9 @@ use crate::iobuf::{Frozen, Owned}; use iggy_binary_protocol::{ Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, - RequestHeader, RequestPreparesHeader, RequestStartViewHeader, StartViewChangeHeader, - StartViewHeader, + RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, + StateTransferTargetHeader, }; use smallvec::SmallVec; use std::{marker::PhantomData, mem::size_of}; @@ -512,6 +513,11 @@ pub enum MessageBag { RepairPrepare(Message), /// `RepairDone` / `RangeEvicted` (one layout, two commands). RepairRangeReply(Message), + RequestStateTransfer(Message), + StateTransferTarget(Message), + RequestStateChunk(Message), + /// Artifact bytes ride the body (`size` spans header + payload). + StateChunk(Message), } impl MessageBag { @@ -529,6 +535,10 @@ impl MessageBag { Self::RequestPrepares(message) => message.header().command, Self::RepairPrepare(message) => message.header().command(), Self::RepairRangeReply(message) => message.header().command, + Self::RequestStateTransfer(message) => message.header().command, + Self::StateTransferTarget(message) => message.header().command, + Self::RequestStateChunk(message) => message.header().command, + Self::StateChunk(message) => message.header().command, } } @@ -546,6 +556,10 @@ impl MessageBag { Self::RequestPrepares(message) => message.header().size(), Self::RepairPrepare(message) => message.header().size(), Self::RepairRangeReply(message) => message.header().size(), + Self::RequestStateTransfer(message) => message.header().size(), + Self::StateTransferTarget(message) => message.header().size(), + Self::RequestStateChunk(message) => message.header().size(), + Self::StateChunk(message) => message.header().size(), } } @@ -563,6 +577,10 @@ impl MessageBag { Self::RequestPrepares(message) => message.header().operation(), Self::RepairPrepare(message) => message.header().operation(), Self::RepairRangeReply(message) => message.header().operation(), + Self::RequestStateTransfer(message) => message.header().operation(), + Self::StateTransferTarget(message) => message.header().operation(), + Self::RequestStateChunk(message) => message.header().operation(), + Self::StateChunk(message) => message.header().operation(), } } } @@ -608,6 +626,18 @@ where Command2::RepairDone | Command2::RangeEvicted => Ok(Self::RepairRangeReply( value.try_into_typed::()?, )), + Command2::RequestStateTransfer => Ok(Self::RequestStateTransfer( + value.try_into_typed::()?, + )), + Command2::StateTransferTarget => Ok(Self::StateTransferTarget( + value.try_into_typed::()?, + )), + Command2::RequestStateChunk => Ok(Self::RequestStateChunk( + value.try_into_typed::()?, + )), + Command2::StateChunk => Ok(Self::StateChunk( + value.try_into_typed::()?, + )), // Reply / Eviction are server-to-client frames; they do not // appear on the inbound dispatch path. Command2::Reply | Command2::Eviction => { diff --git a/core/shard/Cargo.toml b/core/shard/Cargo.toml index 439221c447..459f8b8b6d 100644 --- a/core/shard/Cargo.toml +++ b/core/shard/Cargo.toml @@ -47,6 +47,7 @@ prometheus-client = { workspace = true } server_common = { path = "../server_common" } thiserror = { workspace = true } tracing = { workspace = true } +twox-hash = { workspace = true } [lints.clippy] enum_glob_use = "deny" diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index db33fc92b3..edb5e1eb08 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -37,7 +37,9 @@ use futures::FutureExt; use iggy_binary_protocol::{ Command2, CommitHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, StartViewChangeHeader, StartViewHeader, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, + StateTransferTargetHeader, }; #[cfg(any(test, feature = "simulator"))] use iggy_common::PartitionStats; @@ -738,6 +740,65 @@ struct MetadataRepairSession { idle_ticks: u32, } +/// Chunk size for state-transfer artifact pulls. Lockstep (one in flight), +/// so the bounded per-peer bus queue can never drop a burst tail; the bus +/// message cap is far above this. +const STATE_CHUNK_LEN: u32 = 256 * 1024; + +/// One artifact of an accepted transfer target: its manifest entry plus the +/// bytes received so far (chunks are sequential, so `buf.len()` doubles as +/// the next request offset). +#[derive(Debug)] +struct ArtifactProgress { + entry: consensus::StateArtifact, + buf: Vec, +} + +impl ArtifactProgress { + const fn complete(&self) -> bool { + self.buf.len() as u64 == self.entry.len + } +} + +/// One in-flight metadata state transfer (shard 0 only): a cluster-restart +/// rejoin replacing its snapshot-shaped state (metadata snapshot + client +/// table) from the live primary before tail repair. +#[derive(Debug)] +struct MetadataTransferSession { + nonce: u128, + /// Serving primary; also the stall re-request target. + peer: u8, + /// Serving peer's applied frontier from the accepted descriptor. + commit_op: u64, + /// Empty until the `StateTransferTarget` manifest is accepted, then one + /// entry per offered artifact, pulled in manifest order. + artifacts: Vec, + /// Whether a descriptor has been accepted (an accepted EMPTY manifest is + /// distinguishable from "still waiting"). + target_accepted: bool, + /// Ticks with no frame progress; at the configured repair-retry + /// threshold the missing piece is re-requested. + idle_ticks: u32, +} + +/// A cached serving-side state-transfer offer (shard 0 of the serving +/// primary). Keyed by requester replica id so a rebooted requester's fresh +/// nonce replaces the stale offer; chunks must all come from ONE offer or +/// the artifact checksums cannot hold. +struct ServedStateTransfer { + nonce: u128, + offer: metadata::StateTransferOffer, +} + +/// What `on_request_state_chunk` decided inside its offers borrow; the wire +/// sends run after the borrow drops. +enum ChunkReply { + Chunk(Message), + /// Offer evicted (e.g. the serving process restarted): the requester + /// gets an unavailable descriptor and restarts its session. + UnknownOffer, +} + pub struct IggyShard where B: MessageBus, @@ -766,6 +827,14 @@ where /// the full prefix -- so only the stream identity is tracked. metadata_repair: RefCell>, + /// In-flight metadata state transfer (cluster-restart rejoin); tail + /// repair takes over at install. See [`MetadataTransferSession`]. + metadata_transfer: RefCell>, + + /// Serving-side cache of state-transfer offers, keyed by requester + /// replica id. Bounded by the replica count; replaced per fresh nonce. + metadata_transfer_offers: RefCell>, + /// Handler for inbound [`MetadataSubmit`] frames. Only shard 0 receives /// these (it owns the metadata consensus group); peers send them here /// via [`Self::forward_metadata_submit`]. Defaults to a no-op for the @@ -840,6 +909,13 @@ where /// `[cluster] repair_chunk_max` at bootstrap. repair_chunk_max: Cell, + /// Capacity for a state-transferred client table + /// ([`consensus::ClientTable::decode`]). Defaults to + /// [`consensus::CLIENTS_TABLE_MAX`]; server-ng overrides it from + /// `[metadata] clients_table_max` at bootstrap so the installed table + /// matches the configured capacity instead of the compile-time default. + clients_table_max: Cell, + /// Live stalled-repair retry threshold in consensus ticks. Defaults to /// [`partitions::REPAIR_RETRY_TICKS`]; server-ng overrides it from /// `[cluster] repair_retry_interval` at bootstrap. @@ -936,8 +1012,11 @@ where reconcile_queue: RefCell::new(VecDeque::new()), pending_partition_frames: RefCell::new(HashMap::new()), metadata_repair: RefCell::new(None), + metadata_transfer: RefCell::new(None), + metadata_transfer_offers: RefCell::new(HashMap::new()), repair_chunk_max: Cell::new(REPAIR_CHUNK_MAX), repair_retry_ticks: Cell::new(partitions::REPAIR_RETRY_TICKS), + clients_table_max: Cell::new(consensus::CLIENTS_TABLE_MAX), }) } @@ -955,6 +1034,14 @@ where self.repair_chunk_max.set(chunk); } + /// Override the state-transfer client-table capacity from configuration + /// (`[metadata] clients_table_max`). Called once per shard at bootstrap; + /// the simulator and tests keep the compile-time + /// [`consensus::CLIENTS_TABLE_MAX`] default. + pub fn set_clients_table_max(&self, max_clients: usize) { + self.clients_table_max.set(max_clients); + } + /// Hand a metadata consensus submit (login/logout) to shard 0. /// /// Sends a [`LifecycleFrame::MetadataSubmit`] into shard 0's inbox. The @@ -1159,8 +1246,11 @@ where reconcile_queue: RefCell::new(VecDeque::new()), pending_partition_frames: RefCell::new(HashMap::new()), metadata_repair: RefCell::new(None), + metadata_transfer: RefCell::new(None), + metadata_transfer_offers: RefCell::new(HashMap::new()), repair_chunk_max: Cell::new(REPAIR_CHUNK_MAX), repair_retry_ticks: Cell::new(partitions::REPAIR_RETRY_TICKS), + clients_table_max: Cell::new(consensus::CLIENTS_TABLE_MAX), } } @@ -1420,7 +1510,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { match MessageBag::try_from(message) { Ok(MessageBag::Request(request)) => { @@ -1461,6 +1554,14 @@ where Ok(MessageBag::RequestPrepares(ref msg)) => self.on_request_prepares(msg).await, Ok(MessageBag::RepairPrepare(msg)) => self.on_repair_prepare(msg).await, Ok(MessageBag::RepairRangeReply(ref msg)) => self.on_repair_range_reply(msg).await, + Ok(MessageBag::RequestStateTransfer(ref msg)) => { + self.on_request_state_transfer(msg).await; + } + Ok(MessageBag::StateTransferTarget(ref msg)) => { + self.on_state_transfer_target(msg).await; + } + Ok(MessageBag::RequestStateChunk(ref msg)) => self.on_request_state_chunk(msg).await, + Ok(MessageBag::StateChunk(ref msg)) => self.on_state_chunk(msg).await, Err(e) => { tracing::warn!(shard = self.id, error = %e, "dropping message with invalid command"); } @@ -1548,7 +1649,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { self.plane.on_request(request).await; } @@ -1567,7 +1671,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { self.plane.on_replicate(prepare).await; } @@ -1586,7 +1693,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { self.plane.on_ack(prepare_ok).await; } @@ -1624,7 +1734,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { debug_assert!(buf.is_empty(), "buf must be empty on entry"); debug_assert!( @@ -1866,6 +1979,30 @@ where { let actions = consensus.handle_start_view(PlaneKind::Metadata, &header); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &actions).await; + // State transfer (cluster-restart rejoin): the adopted view names + // a live primary to fetch snapshot-shaped state from. The commit + // walk and journal repair are deferred until the install lands -- + // walking the pre-transfer STM would apply ops the snapshot + // already contains, and the transfer replaces the table anyway. + if consensus.state_transfer_stage() == consensus::StateTransferStage::AwaitingTarget { + let nonce = iggy_common::random_id::get_uuid(); + *self.metadata_transfer.borrow_mut() = Some(MetadataTransferSession { + nonce, + peer: header.replica, + commit_op: 0, + artifacts: Vec::new(), + target_accepted: false, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + peer = header.replica, + "restart adopted a live view; requesting metadata state transfer" + ); + self.send_request_state_transfer(consensus, header.replica, nonce) + .await; + return; + } // `dispatch_vsr_actions` deliberately no-ops `CommitJournal` (it // needs the plane); without this the ops a StartView marks // committed stay journaled-but-unapplied forever, because the @@ -1881,36 +2018,8 @@ where // cannot reach (StartView carries numbers, not entries): the // walk above gap-stops. Fill the hole through journal repair // from the announcing primary. - if consensus.is_normal() - && consensus.commit_min() < consensus.commit_max() - && self.metadata_repair.borrow().is_none() - { - let nonce = iggy_common::random_id::get_uuid(); - let to_op = consensus.commit_max(); - let from_op = consensus.commit_min() + 1; - *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { - nonce, - to_op, - peer: header.replica, - idle_ticks: 0, - }); - tracing::info!( - shard = self.id, - from_op, - to_op, - "metadata behind after StartView adoption; requesting repair" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - header.replica, - nonce, - from_op, - to_op, - header.namespace, - ) + self.maybe_request_metadata_repair(consensus, header.replica) .await; - } return; } @@ -1992,7 +2101,34 @@ where && consensus.namespace() == header.namespace { match consensus.handle_commit(&header) { - CommitOutcome::Advanced => planes.0.commit_journal().await, + CommitOutcome::Advanced => { + // Mid-transfer the pre-install STM must not walk: the + // snapshot being installed already contains those ops. + // `commit_max` still advanced inside `handle_commit`, so + // the post-install repair targets the right frontier. + if !consensus.is_transferring() { + planes.0.commit_journal().await; + // A heartbeat is the only signal a behind-but-same-view + // replica gets that the frontier moved: it advances + // `commit_max`, but the walk above cannot cross a gap in + // its own WAL (a late joiner missed the ops below the + // primary's active window; the primary only retransmits + // uncommitted ops, never the committed prefix). Without + // this, such a replica learns it is behind and does + // nothing about it -- metadata repair is otherwise only + // rooted at StartView adoption, which a same-view + // late joiner never sees. Request repair from the + // primary; if it has checkpointed past the gap the + // repair floor evicts and the handler above converts to + // state transfer. Idempotent: `maybe_request_metadata_repair` + // no-ops when caught up, already transferring, or a + // session is live, so a caught-up replica and a + // cold-start node (commit_max == commit_min == 0) both + // skip it. + self.maybe_request_metadata_repair(consensus, header.replica) + .await; + } + } CommitOutcome::RespondStartView => { respond_start_view::(consensus).await; } @@ -2383,17 +2519,54 @@ where } } Command2::RangeEvicted => { - // The metadata WAL retains everything above the snapshot - // floor; an evicted range means the peer compacted past - // this replica's gap -- bulk snapshot transfer (phase 3) - // territory. Surface loudly; the divergence backstop - // still guards the walk. - tracing::warn!( - shard = self.id, - retained_from = header.op, - "metadata repair range evicted on the serving peer; \ - snapshot transfer required" - ); + // Journal repair cannot close this gap: the serving peer + // compacted past it, so the ops this replica is missing no + // longer exist as WAL entries anywhere. This is the one + // authoritative "repair is impossible" signal, and it is + // shape-identical for every way a replica falls behind a + // checkpoint -- a fresh node joining an already-checkpointed + // cluster, a node whose partition healed after the quorum + // moved on, or a restart whose gap sits below the floor. All + // three convert here to state transfer against the peer that + // just announced the eviction (it has the checkpoint by + // definition), which replaces the snapshot-shaped state + // wholesale rather than replaying ops that are gone. + // + // Drop the repair session and arm the transfer only from + // `Idle`: a transfer already in flight owns the stage, and + // its own post-install tail repair can legitimately hit + // `RangeEvicted` again if the primary checkpointed mid + // transfer -- that reraises through the same path, and each + // round lifts the local floor, so it converges. + if consensus.state_transfer_stage() == consensus::StateTransferStage::Idle { + *self.metadata_repair.borrow_mut() = None; + consensus.begin_state_transfer_await(); + let nonce = iggy_common::random_id::get_uuid(); + *self.metadata_transfer.borrow_mut() = Some(MetadataTransferSession { + nonce, + peer: header.replica, + commit_op: 0, + artifacts: Vec::new(), + target_accepted: false, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + peer = header.replica, + retained_from = header.op, + local_commit = consensus.commit_min(), + "metadata repair floor evicted; converting to state transfer" + ); + self.send_request_state_transfer(consensus, header.replica, nonce) + .await; + } else { + tracing::debug!( + shard = self.id, + retained_from = header.op, + stage = ?consensus.state_transfer_stage(), + "metadata repair range evicted while a transfer is already in flight" + ); + } } _ => {} } @@ -2569,6 +2742,708 @@ where .await; } + /// Start metadata tail journal-repair from `peer` when the commit walk + /// gap-stopped below the known frontier. Shared by `StartView` adoption + /// and the post-install step of a state transfer. + #[allow(clippy::future_not_send)] + async fn maybe_request_metadata_repair

(&self, consensus: &VsrConsensus, peer: u8) + where + B: MessageBus, + P: Pipeline, + { + if consensus.is_normal() + && !consensus.is_transferring() + && consensus.commit_min() < consensus.commit_max() + && self.metadata_repair.borrow().is_none() + { + let nonce = iggy_common::random_id::get_uuid(); + let to_op = consensus.commit_max(); + let from_op = consensus.commit_min() + 1; + *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { + nonce, + to_op, + peer, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + from_op, + to_op, + "metadata behind the group frontier; requesting repair" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + from_op, + to_op, + consensus.namespace(), + ) + .await; + } + } + + #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] + async fn send_request_state_transfer

( + &self, + consensus: &VsrConsensus, + target: u8, + nonce: u128, + ) where + B: MessageBus, + P: Pipeline, + { + let msg = + Message::::new(size_of::()) + .transmute_header(|_, h: &mut RequestStateTransferHeader| { + h.command = Command2::RequestStateTransfer; + h.cluster = consensus.cluster(); + h.replica = consensus.replica(); + h.nonce = nonce; + h.namespace = consensus.namespace(); + h.size = size_of::() as u32; + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + /// Answer a `RequestStateTransfer`: `offer = None` sends a header-only + /// `available = 0` (the requester falls back to journal repair or + /// retries elsewhere); an offer ships its encoded state manifest as the + /// frame body. + #[allow( + clippy::future_not_send, + clippy::cast_possible_truncation, + clippy::too_many_arguments + )] + async fn send_state_transfer_target( + &self, + cluster: u128, + self_id: u8, + target: u8, + nonce: u128, + namespace: u64, + offer: Option<&metadata::StateTransferOffer>, + ) where + B: MessageBus, + { + let manifest = offer.map(|offer| consensus::encode_state_manifest(&offer.artifacts)); + let total_size = + size_of::() + manifest.as_ref().map_or(0, Vec::len); + let mut msg = Message::::new(total_size); + if let Some(manifest) = &manifest { + msg.as_mut_slice()[size_of::()..].copy_from_slice(manifest); + } + let msg = msg.transmute_header(|_, h: &mut StateTransferTargetHeader| { + h.command = Command2::StateTransferTarget; + h.cluster = cluster; + h.replica = self_id; + h.nonce = nonce; + h.namespace = namespace; + h.size = total_size as u32; + if let Some(offer) = offer { + h.available = 1; + h.commit_op = offer.commit_op; + } + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + #[allow( + clippy::future_not_send, + clippy::cast_possible_truncation, + clippy::too_many_arguments + )] + async fn send_request_state_chunk( + &self, + cluster: u128, + self_id: u8, + target: u8, + nonce: u128, + namespace: u64, + artifact: u32, + offset: u64, + len: u32, + ) where + B: MessageBus, + { + let msg = Message::::new(size_of::()) + .transmute_header(|_, h: &mut RequestStateChunkHeader| { + h.command = Command2::RequestStateChunk; + h.cluster = cluster; + h.replica = self_id; + h.nonce = nonce; + h.namespace = namespace; + h.artifact = artifact; + h.offset = offset; + h.len = len; + h.size = size_of::() as u32; + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + /// Serve one `RequestStateTransfer`: build a fresh offer (or refuse), + /// cache it for the chunk pulls, and answer with the descriptor. + #[allow(clippy::future_not_send)] + async fn on_request_state_transfer(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: StreamsFrontend + + StateMachine< + Input = Message, + Output = metadata::stm::result::ApplyReply, + Error = iggy_common::IggyError, + >, + { + let header = *msg.header(); + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + if consensus.namespace() != header.namespace { + return; + } + let offer = planes.0.state_transfer_offer(); + let cluster = consensus.cluster(); + let self_id = consensus.replica(); + if let Some(offer) = offer { + tracing::info!( + shard = self.id, + requester = header.replica, + commit_op = offer.commit_op, + artifacts = offer.artifacts.len(), + total_len = offer.artifacts.iter().map(|a| a.len).sum::(), + "serving metadata state transfer" + ); + self.send_state_transfer_target( + cluster, + self_id, + header.replica, + header.nonce, + header.namespace, + Some(&offer), + ) + .await; + self.metadata_transfer_offers.borrow_mut().insert( + header.replica, + ServedStateTransfer { + nonce: header.nonce, + offer, + }, + ); + } else { + tracing::info!( + shard = self.id, + requester = header.replica, + "cannot serve metadata state transfer (not a caught-up \ + primary, or no snapshot persisted); requester falls back" + ); + self.send_state_transfer_target( + cluster, + self_id, + header.replica, + header.nonce, + header.namespace, + None, + ) + .await; + } + } + + /// Receiver side of the descriptor: accept it and start pulling chunks, + /// or fall back to journal repair when the peer cannot serve. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn on_state_transfer_target(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: StreamsFrontend + + StateMachine< + Input = Message, + Output = metadata::stm::result::ApplyReply, + Error = iggy_common::IggyError, + > + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, + { + /// Alloc cap per artifact: a corrupt length field must not OOM the + /// shard. Far above any real metadata snapshot or client table. + const ARTIFACT_LEN_MAX: u64 = 1 << 30; + + let header = *msg.header(); + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + if consensus.namespace() != header.namespace { + return; + } + let session_matches = self + .metadata_transfer + .borrow() + .as_ref() + .is_some_and(|session| session.nonce == header.nonce); + if !session_matches { + return; + } + + if header.available == 0 { + // The peer cannot serve. If we have never installed anything the + // local recovery stands; run the deferred commit walk and let + // journal repair cover the gap (a peer that never checkpointed + // retains its full WAL, so repair CAN cover it). + tracing::info!( + shard = self.id, + peer = header.replica, + "state transfer unavailable; falling back to journal repair" + ); + *self.metadata_transfer.borrow_mut() = None; + if consensus.state_transfer_stage() != consensus::StateTransferStage::Idle { + consensus.set_state_transfer_stage(consensus::StateTransferStage::Idle); + } + planes.0.commit_journal().await; + self.maybe_request_metadata_repair(consensus, header.replica) + .await; + return; + } + + // The manifest rides the body; a well-formed available=1 descriptor + // always carries one (an empty manifest still encodes its envelope). + let manifest = match consensus::decode_state_manifest( + &msg.as_slice()[size_of::()..header.size as usize], + ) { + Ok(manifest) => manifest, + Err(error) => { + tracing::error!( + shard = self.id, + peer = header.replica, + %error, + "state transfer descriptor manifest undecodable; ignoring" + ); + return; + } + }; + if let Some(oversized) = manifest.iter().find(|entry| entry.len > ARTIFACT_LEN_MAX) { + tracing::error!( + shard = self.id, + kind = oversized.kind, + len = oversized.len, + "state transfer descriptor exceeds artifact cap; ignoring" + ); + return; + } + + { + let mut session = self.metadata_transfer.borrow_mut(); + let Some(session) = session.as_mut() else { + return; + }; + if session.target_accepted { + // Duplicate descriptor (stall retry crossed the original). + return; + } + session.target_accepted = true; + session.commit_op = header.commit_op; + // Under ARTIFACT_LEN_MAX (checked above), so the casts hold. + #[allow(clippy::cast_possible_truncation)] + { + session.artifacts = manifest + .iter() + .map(|&entry| ArtifactProgress { + entry, + buf: Vec::with_capacity(entry.len as usize), + }) + .collect(); + } + session.idle_ticks = 0; + } + if consensus.state_transfer_stage() == consensus::StateTransferStage::AwaitingTarget { + consensus.set_state_transfer_stage(consensus::StateTransferStage::Fetching); + } + tracing::info!( + shard = self.id, + peer = header.replica, + artifacts = manifest.len(), + total_len = manifest.iter().map(|entry| entry.len).sum::(), + commit_op = header.commit_op, + "state transfer target accepted; fetching" + ); + self.on_transfer_progress().await; + } + + /// Ask for the next missing chunk of the in-flight transfer (artifacts + /// pulled in manifest order). No-op when nothing is missing or no + /// manifest is accepted yet; also the stall-retry re-request. + #[allow(clippy::future_not_send)] + async fn request_pending_state_chunk(&self) + where + B: MessageBus, + { + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + let request = { + let session = self.metadata_transfer.borrow(); + session.as_ref().and_then(|session| { + if !session.target_accepted { + return None; + } + let (index, artifact) = session + .artifacts + .iter() + .enumerate() + .find(|(_, artifact)| !artifact.complete())?; + let offset = artifact.buf.len() as u64; + let remaining = artifact.entry.len - offset; + #[allow(clippy::cast_possible_truncation)] + let len = remaining.min(u64::from(STATE_CHUNK_LEN)) as u32; + #[allow(clippy::cast_possible_truncation)] + Some((session.nonce, session.peer, index as u32, offset, len)) + }) + }; + if let Some((nonce, peer, artifact, offset, len)) = request { + self.send_request_state_chunk( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + consensus.namespace(), + artifact, + offset, + len, + ) + .await; + } + } + + /// Serve one chunk out of the cached offer. An unknown nonce (offer + /// evicted, e.g. the serving process restarted) answers with an + /// `available = 0` descriptor so the requester restarts its session. + #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] + async fn on_request_state_chunk(&self, msg: &Message) + where + B: MessageBus, + { + let header = *msg.header(); + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + if consensus.namespace() != header.namespace { + return; + } + let cluster = consensus.cluster(); + let self_id = consensus.replica(); + + // Frame built inside the borrow; every send runs after it drops (a + // RefCell borrow must not cross an await on the shard). + // Out-of-bounds requests are dropped silently inside the block. + let reply = { + let offers = self.metadata_transfer_offers.borrow(); + let served = offers + .get(&header.replica) + .filter(|served| served.nonce == header.nonce); + served.map_or(Some(ChunkReply::UnknownOffer), |served| { + // Manifest-index addressing: an index past the offer is a + // requester bug (or a stale frame) and is dropped below. + let artifact_bytes = served.offer.payloads.get(header.artifact as usize)?; + let start = header.offset as usize; + let end = start.saturating_add(header.len as usize); + artifact_bytes + .get(start..end.min(artifact_bytes.len())) + .map(|payload| { + let total_size = size_of::() + payload.len(); + let mut chunk = Message::::new(total_size); + chunk.as_mut_slice()[size_of::()..] + .copy_from_slice(payload); + ChunkReply::Chunk(chunk.transmute_header(|_, h: &mut StateChunkHeader| { + h.command = Command2::StateChunk; + h.cluster = cluster; + h.replica = self_id; + h.nonce = header.nonce; + h.namespace = header.namespace; + h.artifact = header.artifact; + h.offset = header.offset; + h.size = total_size as u32; + })) + }) + }) + }; + match reply { + Some(ChunkReply::Chunk(chunk)) => { + let _ = self + .bus + .send_to_replica(header.replica, chunk.into_generic().into_frozen()) + .await; + } + Some(ChunkReply::UnknownOffer) => { + tracing::info!( + shard = self.id, + requester = header.replica, + "state chunk request for an unknown offer; telling requester to restart" + ); + self.send_state_transfer_target( + cluster, + self_id, + header.replica, + header.nonce, + header.namespace, + None, + ) + .await; + } + None => { + tracing::warn!( + shard = self.id, + requester = header.replica, + artifact = header.artifact, + offset = header.offset, + "state chunk request out of artifact bounds; ignoring" + ); + } + } + } + + /// Receive one chunk; on the last one, verify + install + hand the tail + /// to journal repair. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn on_state_chunk(&self, msg: &Message) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: StreamsFrontend + + StateMachine< + Input = Message, + Output = metadata::stm::result::ApplyReply, + Error = iggy_common::IggyError, + > + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, + { + let header = *msg.header(); + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + if consensus.namespace() != header.namespace { + return; + } + + { + let mut session = self.metadata_transfer.borrow_mut(); + let Some(session) = session.as_mut() else { + return; + }; + if session.nonce != header.nonce || !session.target_accepted { + return; + } + let Some(artifact) = session.artifacts.get_mut(header.artifact as usize) else { + return; + }; + let payload = &msg.as_slice()[size_of::()..header.size as usize]; + // Chunks are pulled sequentially with one in flight; anything + // else is a duplicate or reorder and is dropped (the stall retry + // re-requests from the current frontier). + if header.offset != artifact.buf.len() as u64 { + return; + } + if artifact.buf.len() as u64 + payload.len() as u64 > artifact.entry.len { + tracing::warn!( + shard = self.id, + artifact = header.artifact, + "state chunk overruns the declared artifact length; dropping frame" + ); + return; + } + artifact.buf.extend_from_slice(payload); + session.idle_ticks = 0; + } + self.on_transfer_progress().await; + } + + /// Drive the in-flight transfer forward: request the next missing chunk, + /// or - once every artifact is complete - verify, decode, and install. + /// Shared by descriptor acceptance and chunk arrival, so a manifest whose + /// artifacts are already complete (all empty) installs without waiting + /// for a chunk that will never come. + #[allow(clippy::future_not_send, clippy::too_many_lines)] + async fn on_transfer_progress(&self) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: StreamsFrontend + + StateMachine< + Input = Message, + Output = metadata::stm::result::ApplyReply, + Error = iggy_common::IggyError, + > + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, + { + let planes = self.plane.inner(); + let Some(ref consensus) = planes.0.consensus else { + return; + }; + let complete = { + let session = self.metadata_transfer.borrow(); + match session.as_ref() { + Some(session) if session.target_accepted => { + session.artifacts.iter().all(ArtifactProgress::complete) + } + _ => return, + } + }; + if !complete { + self.request_pending_state_chunk().await; + return; + } + + // All bytes in: verify, decode, install. + let session = self + .metadata_transfer + .borrow_mut() + .take() + .expect("session checked above"); + let peer = session.peer; + let commit_op = session.commit_op; + + // Per-artifact integrity, then pick the pieces this plane installs. + // Unknown kinds are refused rather than skipped: an artifact the + // serving peer thought worth shipping but this receiver cannot + // install would otherwise be silently dropped. + let mut snapshot: Option> = None; + let mut table: Option<(Vec, u64)> = None; + let mut damaged = false; + for (index, artifact) in session.artifacts.into_iter().enumerate() { + let actual = consensus::state_artifact_checksum(&artifact.buf); + if actual != artifact.entry.checksum { + tracing::error!( + shard = self.id, + artifact = index, + kind = artifact.entry.kind, + "state transfer artifact checksum mismatch" + ); + damaged = true; + break; + } + match artifact.entry.kind { + consensus::artifact_kind::METADATA_SNAPSHOT => snapshot = Some(artifact.buf), + consensus::artifact_kind::CLIENT_TABLE => { + table = Some((artifact.buf, artifact.entry.frontier)); + } + kind => { + tracing::error!( + shard = self.id, + kind, + "state transfer manifest carries a kind this plane cannot install" + ); + damaged = true; + break; + } + } + } + + let decoded = if damaged { + None + } else if let (Some(snapshot), Some((table_bytes, table_frontier))) = (snapshot, table) { + match consensus::ClientTable::decode(&table_bytes, self.clients_table_max.get()) { + Ok(table) => Some((snapshot, table, table_frontier)), + Err(error) => { + tracing::error!(shard = self.id, %error, "transferred client table undecodable"); + None + } + } + } else { + tracing::error!( + shard = self.id, + "state transfer manifest is missing the snapshot or client table artifact" + ); + None + }; + + let Some((snapshot, table, table_frontier)) = decoded else { + // Damaged in transit: restart the session from scratch against + // the same peer (fresh nonce; the peer re-offers). + if consensus.state_transfer_stage() == consensus::StateTransferStage::Fetching { + consensus.set_state_transfer_stage(consensus::StateTransferStage::AwaitingTarget); + } + let nonce = iggy_common::random_id::get_uuid(); + *self.metadata_transfer.borrow_mut() = Some(MetadataTransferSession { + nonce, + peer, + commit_op: 0, + artifacts: Vec::new(), + target_accepted: false, + idle_ticks: 0, + }); + self.send_request_state_transfer(consensus, peer, nonce) + .await; + return; + }; + + consensus.set_state_transfer_stage(consensus::StateTransferStage::Installing); + match planes + .0 + .install_state_transfer(&snapshot, table, table_frontier, commit_op) + { + Ok(snapshot_seq) => { + consensus.set_state_transfer_stage(consensus::StateTransferStage::Idle); + tracing::info!( + shard = self.id, + snapshot_seq, + commit_op, + table_frontier, + "metadata state transfer installed; handing tail to journal repair" + ); + // Walk whatever is already walkable, then let repair fetch + // the (snapshot_seq, commit_max] tail. + planes.0.commit_journal().await; + self.maybe_request_metadata_repair(consensus, peer).await; + } + Err(error) => { + tracing::error!( + shard = self.id, + %error, + "state transfer install failed; falling back to journal repair" + ); + consensus.set_state_transfer_stage(consensus::StateTransferStage::Idle); + planes.0.commit_journal().await; + self.maybe_request_metadata_repair(consensus, peer).await; + } + } + } + /// Tick partition consensuses. Loop partitions. No partitions-plane journal. #[allow(clippy::future_not_send)] pub async fn tick_partitions(&self) @@ -2707,7 +3582,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { let metadata = self.plane.metadata(); let Some(ref consensus) = metadata.consensus else { @@ -2733,6 +3611,35 @@ where // nothing is stranded. metadata.resume_stranded_commits().await; + // Stall retry for an in-flight state transfer: descriptor or chunk + // frames are fire-and-forget, so a lost one must not wedge the + // session (and the boot flow behind it) forever. + let transfer_stalled = { + let mut session = self.metadata_transfer.borrow_mut(); + session.as_mut().and_then(|session| { + session.idle_ticks += 1; + if session.idle_ticks < self.repair_retry_ticks.get() { + return None; + } + session.idle_ticks = 0; + Some((session.peer, session.nonce, session.target_accepted)) + }) + }; + if let Some((peer, nonce, target_accepted)) = transfer_stalled { + tracing::info!( + shard = self.id, + peer, + target_accepted, + "metadata state transfer stalled; re-requesting" + ); + if target_accepted { + self.request_pending_state_chunk().await; + } else { + self.send_request_state_transfer(consensus, peer, nonce) + .await; + } + } + // Stall retry, mirroring `tick_partitions`: a lost repair frame must // not wedge the session forever. let repair_retry_ticks = self.repair_retry_ticks.get(); diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index dc0a18e7dc..90c35598a3 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs @@ -89,6 +89,22 @@ fn extract_routing(bag: MessageBag) -> (Operation, u64, Message) let h = *m.header(); (h.operation(), h.namespace, m.into_generic()) } + MessageBag::RequestStateTransfer(m) => { + let h = *m.header(); + (h.operation(), h.namespace, m.into_generic()) + } + MessageBag::StateTransferTarget(m) => { + let h = *m.header(); + (h.operation(), h.namespace, m.into_generic()) + } + MessageBag::RequestStateChunk(m) => { + let h = *m.header(); + (h.operation(), h.namespace, m.into_generic()) + } + MessageBag::StateChunk(m) => { + let h = *m.header(); + (h.operation(), h.namespace, m.into_generic()) + } } } @@ -289,7 +305,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { // Reused across every pump iteration; pre-size to skip the // first-drain reallocation. @@ -420,7 +439,10 @@ where Input = Message, Output = metadata::stm::result::ApplyReply, Error = iggy_common::IggyError, - > + StreamsFrontend, + > + StreamsFrontend + + metadata::stm::snapshot::RestoreSnapshotInPlace< + metadata::stm::snapshot::MetadataSnapshot, + >, { match frame { ShardFrame::Consensus { message, .. } => {