From 9c7cddc4fb2a15d844378aa369c30b873e55fb66 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 21 Jul 2026 18:36:14 +0100 Subject: [PATCH 1/7] feat[layout]: extract a `LayoutStrategyEncodingValidator` from the flat layout to allow composing encoding validation with different layout strategies This will be used to adding edition encoding filtering Signed-off-by: Joe Isaacs --- vortex-cuda/src/layout.rs | 21 -------- vortex-file/src/strategy.rs | 12 +++-- vortex-layout/src/layouts/flat/writer.rs | 64 ++++++++-------------- vortex-layout/src/strategy.rs | 68 +++++++++++++++++++++++- 4 files changed, 96 insertions(+), 69 deletions(-) diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index 1b1d37cd14d..6bd81b59f03 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -14,7 +14,6 @@ use futures::FutureExt; use futures::StreamExt; use futures::future::BoxFuture; use vortex::array::ArrayContext; -use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::DeserializeMetadata; @@ -26,8 +25,6 @@ use vortex::array::expr::Expression; use vortex::array::expr::stats::Precision; use vortex::array::expr::stats::Stat; use vortex::array::expr::stats::StatsProvider; -use vortex::array::normalize::NormalizeOptions; -use vortex::array::normalize::Operation; use vortex::array::serde::SerializeOptions; use vortex::array::serde::SerializedArray; use vortex::array::stats::StatsSetRef; @@ -68,7 +65,6 @@ use vortex::session::VortexSession; use vortex::session::registry::CachedId; use vortex::session::registry::ReadContext; use vortex::utils::aliases::hash_map::HashMap; -use vortex::utils::aliases::hash_set::HashSet; /// A buffer inlined into layout metadata for host-side access. #[derive(Clone, prost::Message)] @@ -403,8 +399,6 @@ pub struct CudaFlatLayoutStrategy { pub include_padding: bool, /// Maximum length of variable length statistics. pub max_variable_length_statistics_size: usize, - /// Optional set of allowed array encodings for normalization. - pub allowed_encodings: Option>, } impl Default for CudaFlatLayoutStrategy { @@ -412,7 +406,6 @@ impl Default for CudaFlatLayoutStrategy { Self { include_padding: true, max_variable_length_statistics_size: 64, - allowed_encodings: None, } } } @@ -427,11 +420,6 @@ impl CudaFlatLayoutStrategy { self.max_variable_length_statistics_size = size; self } - - pub fn with_allow_encodings(mut self, allow_encodings: HashSet) -> Self { - self.allowed_encodings = Some(allow_encodings); - self - } } fn truncate_scalar_stat Option<(Scalar, bool)>>( @@ -508,15 +496,6 @@ impl LayoutStrategy for CudaFlatLayoutStrategy { _ => {} } - let chunk = if let Some(allowed) = &options.allowed_encodings { - chunk.normalize(&mut NormalizeOptions { - allowed, - operation: Operation::Error, - })? - } else { - chunk - }; - // Scan for constant array buffers before serialization (while data is still on host). let host_buffers = extract_constant_buffers(&chunk); diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index 3cedbc1d68c..763db758cb7 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -43,6 +43,7 @@ use vortex_fastlanes::FoR; use vortex_fastlanes::RLE; use vortex_fsst::FSST; use vortex_layout::LayoutStrategy; +use vortex_layout::LayoutStrategyEncodingValidator; use vortex_layout::layouts::buffered::BufferedStrategy; use vortex_layout::layouts::chunked::writer::ChunkedLayoutStrategy; use vortex_layout::layouts::collect::CollectStrategy; @@ -217,8 +218,8 @@ impl WriteStrategyBuilder { /// Override the allowed array encodings for normalization. /// - /// The flat leaf writer uses this set when deciding whether an existing encoded array can be - /// written as-is or must be normalized before serialization. + /// The configured flat leaf strategy is wrapped in a [`LayoutStrategyEncodingValidator`] + /// that recursively checks every chunk before passing it to the leaf writer. pub fn with_allow_encodings(mut self, allow_encodings: HashSet) -> Self { self.allow_encodings = Some(allow_encodings); self @@ -262,11 +263,14 @@ impl WriteStrategyBuilder { pub fn build(self) -> Arc { let flat: Arc = if let Some(flat) = self.flat_strategy { flat - } else if let Some(allow_encodings) = self.allow_encodings { - Arc::new(FlatLayoutStrategy::default().with_allow_encodings(allow_encodings)) } else { Arc::new(FlatLayoutStrategy::default()) }; + let flat: Arc = if let Some(allow_encodings) = self.allow_encodings { + Arc::new(LayoutStrategyEncodingValidator::new(flat, allow_encodings)) + } else { + flat + }; // 7. for each chunk create a flat layout let chunked = ChunkedLayoutStrategy::new(Arc::clone(&flat)); diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index f6b6d5acf02..55c07275301 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -4,13 +4,10 @@ use async_trait::async_trait; use futures::StreamExt; use vortex_array::ArrayContext; -use vortex_array::ArrayId; use vortex_array::dtype::DType; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; use vortex_array::expr::stats::StatsProvider; -use vortex_array::normalize::NormalizeOptions; -use vortex_array::normalize::Operation; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarTruncation; use vortex_array::scalar::lower_bound; @@ -24,7 +21,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_session::VortexSession; use vortex_session::registry::ReadContext; -use vortex_utils::aliases::hash_set::HashSet; use crate::IntoLayout; use crate::LayoutRef; @@ -43,9 +39,6 @@ pub struct FlatLayoutStrategy { pub include_padding: bool, /// Maximum length of variable length statistics pub max_variable_length_statistics_size: usize, - /// Optional set of allowed array encodings for normalization. - /// If None, then all are allowed. - pub allowed_encodings: Option>, } impl Default for FlatLayoutStrategy { @@ -53,7 +46,6 @@ impl Default for FlatLayoutStrategy { Self { include_padding: true, max_variable_length_statistics_size: 64, - allowed_encodings: None, } } } @@ -70,12 +62,6 @@ impl FlatLayoutStrategy { self.max_variable_length_statistics_size = size; self } - - /// Set the allowed array encodings for normalization. - pub fn with_allow_encodings(mut self, allow_encodings: HashSet) -> Self { - self.allowed_encodings = Some(allow_encodings); - self - } } fn truncate_scalar_stat Option<(Scalar, bool)>>( @@ -158,15 +144,6 @@ impl LayoutStrategy for FlatLayoutStrategy { _ => {} } - let chunk = if let Some(allowed) = &self.allowed_encodings { - chunk.normalize(&mut NormalizeOptions { - allowed, - operation: Operation::Error, - })? - } else { - chunk - }; - let buffers = chunk.serialize( &ctx, session, @@ -239,6 +216,7 @@ mod tests { use vortex_utils::aliases::hash_set::HashSet; use crate::LayoutStrategy; + use crate::LayoutStrategyEncodingValidator; use crate::layouts::flat::writer::FlatLayoutStrategy; use crate::segments::TestSegments; use crate::sequence::SequenceId; @@ -449,16 +427,16 @@ mod tests { let (ptr, eof) = SequenceId::root().split(); // Disallow all encodings so filter arrays fail normalization immediately. let allowed = HashSet::default(); - let layout = FlatLayoutStrategy::default() - .with_allow_encodings(allowed) - .write_stream( - ctx, - Arc::::clone(&segments), - filter.into_array().to_array_stream().sequenced(ptr), - eof, - &session, - ) - .await; + let layout = + LayoutStrategyEncodingValidator::new(FlatLayoutStrategy::default(), allowed) + .write_stream( + ctx, + Arc::::clone(&segments), + filter.into_array().to_array_stream().sequenced(ptr), + eof, + &session, + ) + .await; (layout, segments) }; @@ -491,16 +469,16 @@ mod tests { // Only allow the dict encoding; canonical primitive children remain permitted. let mut allowed = HashSet::default(); allowed.insert(Dict.id()); - let layout = FlatLayoutStrategy::default() - .with_allow_encodings(allowed) - .write_stream( - ctx, - Arc::::clone(&segments), - dict.into_array().to_array_stream().sequenced(ptr), - eof, - &session, - ) - .await; + let layout = + LayoutStrategyEncodingValidator::new(FlatLayoutStrategy::default(), allowed) + .write_stream( + ctx, + Arc::::clone(&segments), + dict.into_array().to_array_stream().sequenced(ptr), + eof, + &session, + ) + .await; (layout, segments) }; diff --git a/vortex-layout/src/strategy.rs b/vortex-layout/src/strategy.rs index 4fcd24dd02f..f9f91d9d82f 100644 --- a/vortex-layout/src/strategy.rs +++ b/vortex-layout/src/strategy.rs @@ -1,15 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::sync::Arc; + use async_trait::async_trait; +use futures::StreamExt; use vortex_array::ArrayContext; +use vortex_array::ArrayId; +use vortex_array::normalize::NormalizeOptions; +use vortex_array::normalize::Operation; use vortex_error::VortexResult; use vortex_session::VortexSession; +use vortex_utils::aliases::hash_set::HashSet; use crate::LayoutRef; use crate::segments::SegmentSinkRef; use crate::sequence::SendableSequentialStream; use crate::sequence::SequencePointer; +use crate::sequence::SequentialStreamAdapter; +use crate::sequence::SequentialStreamExt; // [layout writer] /// Writes an ordered array stream into a layout tree and segment sink. @@ -71,8 +80,65 @@ pub trait LayoutStrategy: 'static + Send + Sync { } } +/// A layout strategy wrapper that rejects arrays containing encodings outside an allow-list. +/// +/// Canonical encodings are always permitted. Every chunk is recursively validated before it is +/// passed to the wrapped strategy. +#[derive(Clone)] +pub struct LayoutStrategyEncodingValidator { + child: Arc, + allowed_encodings: Arc>, +} + +impl LayoutStrategyEncodingValidator { + /// Creates a validator around `child` using the supplied encoding allow-list. + pub fn new(child: S, allowed_encodings: HashSet) -> Self { + Self { + child: Arc::new(child), + allowed_encodings: Arc::new(allowed_encodings), + } + } +} + +#[async_trait] +impl LayoutStrategy for LayoutStrategyEncodingValidator { + async fn write_stream( + &self, + ctx: ArrayContext, + segment_sink: SegmentSinkRef, + stream: SendableSequentialStream, + eof: SequencePointer, + session: &VortexSession, + ) -> VortexResult { + let dtype = stream.dtype().clone(); + let allowed_encodings = Arc::clone(&self.allowed_encodings); + let stream = stream.map(move |chunk| { + let (sequence_id, chunk) = chunk?; + let chunk = chunk.normalize(&mut NormalizeOptions { + allowed: &allowed_encodings, + operation: Operation::Error, + })?; + Ok((sequence_id, chunk)) + }); + + self.child + .write_stream( + ctx, + segment_sink, + SequentialStreamAdapter::new(dtype, stream).sendable(), + eof, + session, + ) + .await + } + + fn buffered_bytes(&self) -> u64 { + self.child.buffered_bytes() + } +} + #[async_trait] -impl LayoutStrategy for std::sync::Arc { +impl LayoutStrategy for Arc { async fn write_stream( &self, ctx: ArrayContext, From a3c358938b26163be3354fdc3553a5f17a1c6011 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 21 Jul 2026 15:02:11 +0100 Subject: [PATCH 2/7] new produced_encodings Signed-off-by: Joe Isaacs --- Cargo.lock | 1 + docs/specs/editions.md | 11 ++++++++--- vortex-btrblocks/src/schemes/binary/zstd.rs | 6 ++++++ vortex-btrblocks/src/schemes/binary/zstd_buffers.rs | 6 ++++++ vortex-btrblocks/src/schemes/decimal.rs | 6 ++++++ vortex-btrblocks/src/schemes/float/alp.rs | 10 ++++++++++ vortex-btrblocks/src/schemes/float/alprd.rs | 6 ++++++ vortex-btrblocks/src/schemes/float/pco.rs | 6 ++++++ vortex-btrblocks/src/schemes/float/rle.rs | 7 +++++++ vortex-btrblocks/src/schemes/float/sparse.rs | 6 ++++++ vortex-btrblocks/src/schemes/integer/bitpacking.rs | 10 ++++++++++ vortex-btrblocks/src/schemes/integer/delta.rs | 6 ++++++ vortex-btrblocks/src/schemes/integer/for_.rs | 6 ++++++ vortex-btrblocks/src/schemes/integer/pco.rs | 6 ++++++ vortex-btrblocks/src/schemes/integer/rle.rs | 6 ++++++ vortex-btrblocks/src/schemes/integer/runend.rs | 6 ++++++ vortex-btrblocks/src/schemes/integer/sequence.rs | 7 +++++++ vortex-btrblocks/src/schemes/integer/sparse.rs | 7 +++++++ vortex-btrblocks/src/schemes/integer/zigzag.rs | 6 ++++++ vortex-btrblocks/src/schemes/string/fsst.rs | 7 +++++++ vortex-btrblocks/src/schemes/string/onpair.rs | 6 ++++++ vortex-btrblocks/src/schemes/string/sparse.rs | 6 ++++++ vortex-btrblocks/src/schemes/string/zstd.rs | 6 ++++++ vortex-btrblocks/src/schemes/string/zstd_buffers.rs | 6 ++++++ vortex-btrblocks/src/schemes/temporal.rs | 6 ++++++ vortex-compressor/src/builtins/dict/binary.rs | 7 +++++++ vortex-compressor/src/builtins/dict/float.rs | 7 +++++++ vortex-compressor/src/builtins/dict/integer.rs | 7 +++++++ vortex-compressor/src/builtins/dict/string.rs | 7 +++++++ vortex-edition/src/lib.rs | 12 ++++++------ vortex-file/Cargo.toml | 1 + vortex-file/src/lib.rs | 4 ++-- vortex/src/lib.rs | 2 +- 33 files changed, 197 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab869e730b9..21483fc8000 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10185,6 +10185,7 @@ dependencies = [ "vortex-bytebool", "vortex-datetime-parts", "vortex-decimal-byte-parts", + "vortex-edition", "vortex-error", "vortex-fastlanes", "vortex-flatbuffers", diff --git a/docs/specs/editions.md b/docs/specs/editions.md index d76bf954f1d..6dd87cd250d 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -37,6 +37,10 @@ Every file you write carries the read-forever guarantee. If a file would contain outside the targeted edition, the write fails immediately; edition violations never surface as someone else's read error later. +The enabled editions are stored on the writer's Vortex session. Registering an edition makes +its declaration available to the session; enabling it separately allows the writer to emit its +encodings. Enabling another edition from the same family replaces the earlier selection. + Two knobs exist when the default is not what you want: - **Pin an older edition** when files must stay readable by deployments running older Vortex. @@ -45,9 +49,10 @@ Two knobs exist when the default is not what you want: example spatial encodings) possible later. A writer targets at most one edition per family and may emit any encoding in their union; each encoding belongs to exactly one family. -You can also opt out of editions entirely to write custom or experimental encodings. Doing so -is an explicit choice that gives up the standardization guarantee — only readers that know your -encodings can read those files. +Lower-level sessions without an enabled-editions store opt out of editions entirely and can write +custom or experimental encodings. A raw `with_allow_encodings` writer policy is another explicit +opt-out. Either choice gives up the standardization guarantee — only readers that know those +encodings can read the files. ## How editions change diff --git a/vortex-btrblocks/src/schemes/binary/zstd.rs b/vortex-btrblocks/src/schemes/binary/zstd.rs index 83c09d24e94..c8f3cf6132f 100644 --- a/vortex-btrblocks/src/schemes/binary/zstd.rs +++ b/vortex-btrblocks/src/schemes/binary/zstd.rs @@ -3,10 +3,12 @@ //! Zstd compression for binary arrays. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_error::VortexResult; @@ -29,6 +31,10 @@ impl Scheme for ZstdScheme { canonical.dtype().is_binary() } + fn produced_encodings(&self) -> Option> { + Some(vec![vortex_zstd::Zstd.id()]) + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, diff --git a/vortex-btrblocks/src/schemes/binary/zstd_buffers.rs b/vortex-btrblocks/src/schemes/binary/zstd_buffers.rs index a84672ef860..95a3c0c4531 100644 --- a/vortex-btrblocks/src/schemes/binary/zstd_buffers.rs +++ b/vortex-btrblocks/src/schemes/binary/zstd_buffers.rs @@ -3,10 +3,12 @@ //! Zstd buffer-level binary compression preserving array layout for GPU decompression. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_error::VortexResult; @@ -29,6 +31,10 @@ impl Scheme for ZstdBuffersScheme { canonical.dtype().is_binary() } + fn produced_encodings(&self) -> Option> { + Some(vec![vortex_zstd::ZstdBuffers.id()]) + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, diff --git a/vortex-btrblocks/src/schemes/decimal.rs b/vortex-btrblocks/src/schemes/decimal.rs index 20c5cd80a3c..de8c97c4495 100644 --- a/vortex-btrblocks/src/schemes/decimal.rs +++ b/vortex-btrblocks/src/schemes/decimal.rs @@ -3,10 +3,12 @@ //! Decimal compression scheme using byte-part decomposition. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::decimal::narrowed_decimal; @@ -38,6 +40,10 @@ impl Scheme for DecimalScheme { matches!(canonical, Canonical::Decimal(_)) } + fn produced_encodings(&self) -> Option> { + Some(vec![DecimalByteParts.id()]) + } + /// Children: primitive=0. fn num_children(&self) -> usize { 1 diff --git a/vortex-btrblocks/src/schemes/float/alp.rs b/vortex-btrblocks/src/schemes/float/alp.rs index b8a26abf348..e330f17ce82 100644 --- a/vortex-btrblocks/src/schemes/float/alp.rs +++ b/vortex-btrblocks/src/schemes/float/alp.rs @@ -7,10 +7,12 @@ use vortex_alp::ALP; use vortex_alp::ALPArrayExt; use vortex_alp::ALPArraySlotsExt; use vortex_alp::alp_encode; +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::Patched; use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -40,6 +42,14 @@ impl Scheme for ALPScheme { canonical.dtype().is_float() } + fn produced_encodings(&self) -> Option> { + let mut encodings = vec![ALP.id()]; + if use_experimental_patches() { + encodings.push(Patched.id()); + } + Some(encodings) + } + /// Children: encoded_ints=0. fn num_children(&self) -> usize { 1 diff --git a/vortex-btrblocks/src/schemes/float/alprd.rs b/vortex-btrblocks/src/schemes/float/alprd.rs index 662a47a1d9c..28427d14eb0 100644 --- a/vortex-btrblocks/src/schemes/float/alprd.rs +++ b/vortex-btrblocks/src/schemes/float/alprd.rs @@ -6,10 +6,12 @@ use vortex_alp::ALPRDArrayExt; use vortex_alp::ALPRDArrayOwnedExt; use vortex_alp::RDEncoder; +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::PType; use vortex_compressor::scheme::CompressionEstimate; @@ -37,6 +39,10 @@ impl Scheme for ALPRDScheme { canonical.dtype().is_float() } + fn produced_encodings(&self) -> Option> { + Some(vec![vortex_alp::ALPRD.id()]) + } + fn expected_compression_ratio( &self, data: &ArrayAndStats, diff --git a/vortex-btrblocks/src/schemes/float/pco.rs b/vortex-btrblocks/src/schemes/float/pco.rs index dc9a96133d5..ac1c426f92c 100644 --- a/vortex-btrblocks/src/schemes/float/pco.rs +++ b/vortex-btrblocks/src/schemes/float/pco.rs @@ -3,10 +3,12 @@ //! Pco (pcodec) float compression. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_error::VortexResult; @@ -29,6 +31,10 @@ impl Scheme for PcoScheme { canonical.dtype().is_float() } + fn produced_encodings(&self) -> Option> { + Some(vec![vortex_pco::Pco.id()]) + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, diff --git a/vortex-btrblocks/src/schemes/float/rle.rs b/vortex-btrblocks/src/schemes/float/rle.rs index c2683ed131c..04ae5cfc8cf 100644 --- a/vortex-btrblocks/src/schemes/float/rle.rs +++ b/vortex-btrblocks/src/schemes/float/rle.rs @@ -3,15 +3,18 @@ //! Run-length float encoding. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; +use vortex_array::VTable; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::DescendantExclusion; use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; +use vortex_fastlanes::RLE; use crate::ArrayAndStats; use crate::CascadingCompressor; @@ -35,6 +38,10 @@ impl Scheme for FloatRLEScheme { canonical.dtype().is_float() } + fn produced_encodings(&self) -> Option> { + Some(vec![RLE.id()]) + } + /// Children: values=0, indices=1, offsets=2. fn num_children(&self) -> usize { 3 diff --git a/vortex-btrblocks/src/schemes/float/sparse.rs b/vortex-btrblocks/src/schemes/float/sparse.rs index 42d09e323c5..5f3e1a2d52d 100644 --- a/vortex-btrblocks/src/schemes/float/sparse.rs +++ b/vortex-btrblocks/src/schemes/float/sparse.rs @@ -3,10 +3,12 @@ //! Sparse encoding for null-dominated float arrays. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_compressor::scheme::ChildSelection; @@ -39,6 +41,10 @@ impl Scheme for NullDominatedSparseScheme { canonical.dtype().is_float() } + fn produced_encodings(&self) -> Option> { + Some(vec![Sparse.id()]) + } + /// Children: indices=0. fn num_children(&self) -> usize { 1 diff --git a/vortex-btrblocks/src/schemes/integer/bitpacking.rs b/vortex-btrblocks/src/schemes/integer/bitpacking.rs index 2424402649e..ea1c14a7b1b 100644 --- a/vortex-btrblocks/src/schemes/integer/bitpacking.rs +++ b/vortex-btrblocks/src/schemes/integer/bitpacking.rs @@ -3,10 +3,12 @@ //! BitPacking integer encoding. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::Patched; use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -38,6 +40,14 @@ impl Scheme for BitPackingScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + let mut encodings = vec![BitPacked.id()]; + if use_experimental_patches() { + encodings.push(Patched.id()); + } + Some(encodings) + } + fn expected_compression_ratio( &self, data: &ArrayAndStats, diff --git a/vortex-btrblocks/src/schemes/integer/delta.rs b/vortex-btrblocks/src/schemes/integer/delta.rs index ed087f5d845..6fb0dc6a0ad 100644 --- a/vortex-btrblocks/src/schemes/integer/delta.rs +++ b/vortex-btrblocks/src/schemes/integer/delta.rs @@ -3,10 +3,12 @@ //! FastLanes Delta integer encoding. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; @@ -78,6 +80,10 @@ impl Scheme for DeltaScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![Delta.id()]) + } + fn num_children(&self) -> usize { 2 } diff --git a/vortex-btrblocks/src/schemes/integer/for_.rs b/vortex-btrblocks/src/schemes/integer/for_.rs index 832ed4819fd..421d7fd08c6 100644 --- a/vortex-btrblocks/src/schemes/integer/for_.rs +++ b/vortex-btrblocks/src/schemes/integer/for_.rs @@ -3,10 +3,12 @@ //! Frame of Reference integer encoding. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; @@ -41,6 +43,10 @@ impl Scheme for FoRScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![FoR.id()]) + } + /// Dict codes always start at 0, so FoR (which subtracts the min) is a no-op. fn ancestor_exclusions(&self) -> Vec { vec![ diff --git a/vortex-btrblocks/src/schemes/integer/pco.rs b/vortex-btrblocks/src/schemes/integer/pco.rs index d7f182f588e..b3041fa4ca9 100644 --- a/vortex-btrblocks/src/schemes/integer/pco.rs +++ b/vortex-btrblocks/src/schemes/integer/pco.rs @@ -3,10 +3,12 @@ //! Pco (pcodec) integer compression. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_compressor::scheme::EstimateVerdict; @@ -30,6 +32,10 @@ impl Scheme for PcoScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![vortex_pco::Pco.id()]) + } + fn expected_compression_ratio( &self, data: &ArrayAndStats, diff --git a/vortex-btrblocks/src/schemes/integer/rle.rs b/vortex-btrblocks/src/schemes/integer/rle.rs index 5bc4e5296f1..0cd1c7a47bc 100644 --- a/vortex-btrblocks/src/schemes/integer/rle.rs +++ b/vortex-btrblocks/src/schemes/integer/rle.rs @@ -3,10 +3,12 @@ //! Run-length integer encoding and shared RLE compression helpers. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_compressor::scheme::AncestorExclusion; @@ -156,6 +158,10 @@ impl Scheme for IntRLEScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![RLE.id()]) + } + /// Children: values=0, indices=1, offsets=2. fn num_children(&self) -> usize { 3 diff --git a/vortex-btrblocks/src/schemes/integer/runend.rs b/vortex-btrblocks/src/schemes/integer/runend.rs index 175f33e7406..79e83896404 100644 --- a/vortex-btrblocks/src/schemes/integer/runend.rs +++ b/vortex-btrblocks/src/schemes/integer/runend.rs @@ -3,10 +3,12 @@ //! Run-end integer encoding. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; @@ -46,6 +48,10 @@ impl Scheme for RunEndScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![RunEnd.id()]) + } + /// Children: values=0, ends=1. fn num_children(&self) -> usize { 2 diff --git a/vortex-btrblocks/src/schemes/integer/sequence.rs b/vortex-btrblocks/src/schemes/integer/sequence.rs index fda9995b510..0628cb52c1a 100644 --- a/vortex-btrblocks/src/schemes/integer/sequence.rs +++ b/vortex-btrblocks/src/schemes/integer/sequence.rs @@ -3,9 +3,11 @@ //! Sequence integer encoding for sequential patterns. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; +use vortex_array::VTable; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::IntDictScheme; @@ -19,6 +21,7 @@ use vortex_compressor::scheme::EstimateVerdict; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; +use vortex_sequence::Sequence; use vortex_sequence::sequence_encode; use crate::ArrayAndStats; @@ -40,6 +43,10 @@ impl Scheme for SequenceScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![Sequence.id()]) + } + /// Sequence encoding on dictionary codes just adds a layer of indirection without compressing /// the data. Dict codes are compact integers that benefit from BitPacking or FoR, not from /// sequence detection. diff --git a/vortex-btrblocks/src/schemes/integer/sparse.rs b/vortex-btrblocks/src/schemes/integer/sparse.rs index e843d9d7999..63fff9f883d 100644 --- a/vortex-btrblocks/src/schemes/integer/sparse.rs +++ b/vortex-btrblocks/src/schemes/integer/sparse.rs @@ -3,10 +3,13 @@ //! Sparse integer encoding for single-value-dominated arrays. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -43,6 +46,10 @@ impl Scheme for SparseScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![Sparse.id(), Constant.id()]) + } + fn stats_options(&self) -> GenerateStatsOptions { GenerateStatsOptions { count_distinct_values: true, diff --git a/vortex-btrblocks/src/schemes/integer/zigzag.rs b/vortex-btrblocks/src/schemes/integer/zigzag.rs index 959889dabac..95383aea664 100644 --- a/vortex-btrblocks/src/schemes/integer/zigzag.rs +++ b/vortex-btrblocks/src/schemes/integer/zigzag.rs @@ -3,10 +3,12 @@ //! ZigZag integer encoding for signed integers. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; use vortex_compressor::builtins::BinaryDictScheme; use vortex_compressor::builtins::FloatDictScheme; @@ -44,6 +46,10 @@ impl Scheme for ZigZagScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![ZigZag.id()]) + } + /// Children: encoded=0. fn num_children(&self) -> usize { 1 diff --git a/vortex-btrblocks/src/schemes/string/fsst.rs b/vortex-btrblocks/src/schemes/string/fsst.rs index bd5bd010396..31c207aca24 100644 --- a/vortex-btrblocks/src/schemes/string/fsst.rs +++ b/vortex-btrblocks/src/schemes/string/fsst.rs @@ -3,11 +3,14 @@ //! FSST (Fast Static Symbol Table) string compression. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::VarBin; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::arrays::varbin::VarBinArrayExt; @@ -43,6 +46,10 @@ impl Scheme for FSSTScheme { canonical.dtype().is_utf8() } + fn produced_encodings(&self) -> Option> { + Some(vec![FSST.id(), VarBin.id()]) + } + /// Children: lengths=0, code_offsets=1. fn num_children(&self) -> usize { 2 diff --git a/vortex-btrblocks/src/schemes/string/onpair.rs b/vortex-btrblocks/src/schemes/string/onpair.rs index 8c7b0561502..d4625823670 100644 --- a/vortex-btrblocks/src/schemes/string/onpair.rs +++ b/vortex-btrblocks/src/schemes/string/onpair.rs @@ -3,10 +3,12 @@ //! OnPair short-string compression (dict-12). +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_compressor::scheme::CompressionEstimate; @@ -47,6 +49,10 @@ impl Scheme for OnPairScheme { canonical.dtype().is_utf8() } + fn produced_encodings(&self) -> Option> { + Some(vec![OnPair.id()]) + } + /// 4 primitive slot children flow through the cascading compressor: /// `dict_offsets` (u32 → typically `FoR`/`BitPacked`), `codes` (u16 → /// usually `FastLanes::BitPacked` after scheme selection), diff --git a/vortex-btrblocks/src/schemes/string/sparse.rs b/vortex-btrblocks/src/schemes/string/sparse.rs index 750dd978030..1461a39f250 100644 --- a/vortex-btrblocks/src/schemes/string/sparse.rs +++ b/vortex-btrblocks/src/schemes/string/sparse.rs @@ -3,10 +3,12 @@ //! Sparse encoding for null-dominated string arrays. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_compressor::scheme::ChildSelection; @@ -40,6 +42,10 @@ impl Scheme for NullDominatedSparseScheme { canonical.dtype().is_utf8() } + fn produced_encodings(&self) -> Option> { + Some(vec![Sparse.id()]) + } + /// Children: indices=0. fn num_children(&self) -> usize { 1 diff --git a/vortex-btrblocks/src/schemes/string/zstd.rs b/vortex-btrblocks/src/schemes/string/zstd.rs index 177acfa1543..957ec87f2d6 100644 --- a/vortex-btrblocks/src/schemes/string/zstd.rs +++ b/vortex-btrblocks/src/schemes/string/zstd.rs @@ -3,10 +3,12 @@ //! Zstd string compression without dictionaries (nvCOMP compatible). +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_error::VortexResult; @@ -29,6 +31,10 @@ impl Scheme for ZstdScheme { canonical.dtype().is_utf8() } + fn produced_encodings(&self) -> Option> { + Some(vec![vortex_zstd::Zstd.id()]) + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, diff --git a/vortex-btrblocks/src/schemes/string/zstd_buffers.rs b/vortex-btrblocks/src/schemes/string/zstd_buffers.rs index a8d16cc837f..6e2be309e8d 100644 --- a/vortex-btrblocks/src/schemes/string/zstd_buffers.rs +++ b/vortex-btrblocks/src/schemes/string/zstd_buffers.rs @@ -3,10 +3,12 @@ //! Zstd buffer-level string compression preserving array layout for GPU decompression. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::DeferredEstimate; use vortex_error::VortexResult; @@ -29,6 +31,10 @@ impl Scheme for ZstdBuffersScheme { canonical.dtype().is_utf8() } + fn produced_encodings(&self) -> Option> { + Some(vec![vortex_zstd::ZstdBuffers.id()]) + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index 6989b6a2aba..bcf297b8dfd 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -3,10 +3,12 @@ //! Temporal compression scheme using datetime-part decomposition. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; @@ -53,6 +55,10 @@ impl Scheme for TemporalScheme { ) } + fn produced_encodings(&self) -> Option> { + Some(vec![DateTimeParts.id()]) + } + /// Children: days=0, seconds=1, subseconds=2. fn num_children(&self) -> usize { 3 diff --git a/vortex-compressor/src/builtins/dict/binary.rs b/vortex-compressor/src/builtins/dict/binary.rs index 72b5f01141a..3530bfe1adc 100644 --- a/vortex-compressor/src/builtins/dict/binary.rs +++ b/vortex-compressor/src/builtins/dict/binary.rs @@ -6,10 +6,13 @@ //! Vortex encoders must always produce unsigned integer codes; signed codes are only accepted //! for external compatibility. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Dict; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArrayExt; @@ -45,6 +48,10 @@ impl Scheme for BinaryDictScheme { canonical.dtype().is_binary() } + fn produced_encodings(&self) -> Option> { + Some(vec![Dict.id()]) + } + fn stats_options(&self) -> GenerateStatsOptions { GenerateStatsOptions { count_distinct_values: true, diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index 074d3ce5e07..f6a41c5938f 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -6,11 +6,14 @@ //! Vortex encoders must always produce unsigned integer codes; signed codes are only accepted for //! external compatibility. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Dict; use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -51,6 +54,10 @@ impl Scheme for FloatDictScheme { canonical.dtype().is_float() } + fn produced_encodings(&self) -> Option> { + Some(vec![Dict.id()]) + } + fn stats_options(&self) -> GenerateStatsOptions { GenerateStatsOptions { count_distinct_values: true, diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 4e91bace3ac..2c76a7e5522 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -6,11 +6,14 @@ //! Vortex encoders must always produce unsigned integer codes; signed codes are only accepted //! for external compatibility. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Dict; use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -46,6 +49,10 @@ impl Scheme for IntDictScheme { canonical.dtype().is_int() } + fn produced_encodings(&self) -> Option> { + Some(vec![Dict.id()]) + } + fn stats_options(&self) -> GenerateStatsOptions { GenerateStatsOptions { count_distinct_values: true, diff --git a/vortex-compressor/src/builtins/dict/string.rs b/vortex-compressor/src/builtins/dict/string.rs index 64ed469dde9..33f8b457bdd 100644 --- a/vortex-compressor/src/builtins/dict/string.rs +++ b/vortex-compressor/src/builtins/dict/string.rs @@ -6,10 +6,13 @@ //! Vortex encoders must always produce unsigned integer codes; signed codes are only accepted //! for external compatibility. +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::VTable; +use vortex_array::arrays::Dict; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArrayExt; @@ -45,6 +48,10 @@ impl Scheme for StringDictScheme { canonical.dtype().is_utf8() } + fn produced_encodings(&self) -> Option> { + Some(vec![Dict.id()]) + } + fn stats_options(&self) -> GenerateStatsOptions { GenerateStatsOptions { count_distinct_values: true, diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs index c4e341450d8..a8eaf8e8287 100644 --- a/vortex-edition/src/lib.rs +++ b/vortex-edition/src/lib.rs @@ -4,8 +4,8 @@ //! Definitions of Vortex *editions*: named, frozen sets of encodings that a writer may put in //! a file, carrying a forever read-compatibility guarantee. //! -//! Editions live on the session, like encodings do: [`EditionSession`] is the session -//! variable holding the edition registry, populated at initialization time. Declarations +//! Editions live on the session, like encodings do: [`EditionSession`] holds the registered +//! editions and [`EnabledEditions`] selects which of them a writer may emit. Declarations //! are plain constants — an [`EditionId`] plus an [`Edition`] record, and one //! [`EditionInclusion`] per encoding stating that it is a member of an edition *and every //! later edition of the same family*. Any crate can register declarations into a session, @@ -18,9 +18,8 @@ //! [`test_harness::validate_edition`] validates one edition's constraints — call it once in //! the `#[cfg(test)]` module of each edition definition. //! -//! This crate defines only the types and the session variable; the first-party edition -//! declarations live in the public `vortex` crate (`vortex::editions`), which seeds them -//! into the default session. See the published spec at +//! The first-party edition declarations live in the public `vortex` crate, which registers +//! and enables them on the default session. See the published spec at //! . mod session; @@ -36,6 +35,7 @@ use std::fmt::Formatter; pub use session::EditionSession; pub use session::EditionSessionExt; +pub use session::EnabledEditions; use vortex_session::registry::Id; /// The identifier of an edition, e.g. `core2026.07.0`. @@ -43,7 +43,7 @@ use vortex_session::registry::Id; /// The `family` names an independently versioned, additive group of encodings (`core` is the /// set the default writer emits). The date components record when the edition was frozen and /// order editions chronologically *within* a family; there is no ordering across families. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct EditionId { /// The edition family, e.g. `core`. pub family: &'static str, diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index 347bd0fc69e..5992ff06434 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -39,6 +39,7 @@ vortex-bytebool = { workspace = true } vortex-datetime-parts = { workspace = true } vortex-decimal-byte-parts = { workspace = true } +vortex-edition = { workspace = true } vortex-error = { workspace = true } vortex-fastlanes = { workspace = true } vortex-flatbuffers = { workspace = true, features = ["file"] } diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 4ece7d9cdab..73c6c7d9c7d 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -160,8 +160,8 @@ mod forever_constant { /// Register the default encodings use in Vortex files with the provided session. /// -/// NOTE: this function will be changed in the future to encapsulate logic for using different -/// Vortex "Editions" that may support different sets of encodings. +/// Registration covers reading: a session can decode every encoding registered here. The +/// writer is gated separately by the editions enabled on its session. pub fn register_default_encodings(session: &VortexSession) { vortex_bytebool::initialize(session); vortex_fsst::initialize(session); diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 1af8484f230..2f5c3e59aac 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -145,7 +145,6 @@ pub mod compressor { pub use vortex_btrblocks::SchemeId; } -/// Logical Vortex data types. /// Vortex editions: named, frozen sets of encodings with a read-compatibility guarantee. pub mod editions; @@ -309,6 +308,7 @@ impl VortexSessionDefault for VortexSession { .with::(); vortex_arrow::initialize(&session); editions::register_default_editions(&session); + editions::enable_default_editions(&session); // `MultiFileSession` holds a `moka` cache whose clock reads `std::time::Instant::now()` // when constructed. `Instant` is unsupported on `wasm32` and panics with "time not From 2d7ea6f8b5ffb698e015539332c23508b487eab5 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 21 Jul 2026 15:19:08 +0100 Subject: [PATCH 3/7] Require schemes to declare produced encodings Signed-off-by: Joe Isaacs --- vortex-btrblocks/src/builder.rs | 35 ++++++++++++++++++ vortex-btrblocks/src/schemes/binary/zstd.rs | 4 +- .../src/schemes/binary/zstd_buffers.rs | 4 +- vortex-btrblocks/src/schemes/decimal.rs | 4 +- vortex-btrblocks/src/schemes/float/alp.rs | 4 +- vortex-btrblocks/src/schemes/float/alprd.rs | 4 +- vortex-btrblocks/src/schemes/float/pco.rs | 4 +- vortex-btrblocks/src/schemes/float/rle.rs | 4 +- vortex-btrblocks/src/schemes/float/sparse.rs | 4 +- .../src/schemes/integer/bitpacking.rs | 4 +- vortex-btrblocks/src/schemes/integer/delta.rs | 4 +- vortex-btrblocks/src/schemes/integer/for_.rs | 4 +- vortex-btrblocks/src/schemes/integer/pco.rs | 4 +- vortex-btrblocks/src/schemes/integer/rle.rs | 4 +- .../src/schemes/integer/runend.rs | 4 +- .../src/schemes/integer/sequence.rs | 4 +- .../src/schemes/integer/sparse.rs | 4 +- .../src/schemes/integer/zigzag.rs | 4 +- vortex-btrblocks/src/schemes/string/fsst.rs | 4 +- vortex-btrblocks/src/schemes/string/onpair.rs | 4 +- vortex-btrblocks/src/schemes/string/sparse.rs | 4 +- vortex-btrblocks/src/schemes/string/zstd.rs | 4 +- .../src/schemes/string/zstd_buffers.rs | 4 +- vortex-btrblocks/src/schemes/temporal.rs | 4 +- vortex-compressor/src/builtins/dict/binary.rs | 4 +- vortex-compressor/src/builtins/dict/float.rs | 4 +- .../src/builtins/dict/integer.rs | 4 +- vortex-compressor/src/builtins/dict/string.rs | 4 +- vortex-compressor/src/compressor/tests.rs | 37 +++++++++++++++++++ vortex-compressor/src/scheme/mod.rs | 8 ++++ vortex-tensor/src/encodings/l2_denorm.rs | 7 ++++ 31 files changed, 141 insertions(+), 54 deletions(-) diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index e51fac9a9d5..749c131ed05 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -3,6 +3,7 @@ //! Builder for configuring `BtrBlocksCompressor` instances. +use vortex_array::ArrayId; use vortex_utils::aliases::hash_set::HashSet; use crate::BtrBlocksCompressor; @@ -208,6 +209,16 @@ impl BtrBlocksCompressorBuilder { self } + /// Retains only schemes whose produced encodings all belong to `allowed`. + /// + /// The file writer uses this to restrict compression to the encodings of its configured + /// editions. + pub fn retain_allowed_encodings(mut self, allowed: &HashSet) -> Self { + self.schemes + .retain(|s| s.produced_encodings().iter().all(|id| allowed.contains(id))); + self + } + /// Builds the configured [`BtrBlocksCompressor`]. pub fn build(self) -> BtrBlocksCompressor { BtrBlocksCompressor(CascadingCompressor::new(self.schemes)) @@ -216,6 +227,9 @@ impl BtrBlocksCompressorBuilder { #[cfg(test)] mod tests { + use vortex_array::VTable; + use vortex_fastlanes::FoR; + use super::*; #[test] @@ -230,6 +244,27 @@ mod tests { assert_eq!(builder.schemes.len(), ALL_SCHEMES.len()); } + #[test] + fn retain_allowed_encodings_filters_schemes() { + let allowed: HashSet = [FoR.id()].into_iter().collect(); + let builder = BtrBlocksCompressorBuilder::default().retain_allowed_encodings(&allowed); + assert_eq!(builder.schemes.len(), 1); + assert_eq!(builder.schemes[0].id(), integer::FoRScheme.id()); + + let none = BtrBlocksCompressorBuilder::default().retain_allowed_encodings(&HashSet::new()); + assert!(none.schemes.is_empty()); + } + + #[test] + fn retaining_all_declared_outputs_keeps_every_scheme() { + let allowed: HashSet = ALL_SCHEMES + .iter() + .flat_map(|scheme| scheme.produced_encodings()) + .collect(); + let builder = BtrBlocksCompressorBuilder::default().retain_allowed_encodings(&allowed); + assert_eq!(builder.schemes.len(), ALL_SCHEMES.len()); + } + #[test] fn cuda_compatible_excludes_alprd() { let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible(); diff --git a/vortex-btrblocks/src/schemes/binary/zstd.rs b/vortex-btrblocks/src/schemes/binary/zstd.rs index c8f3cf6132f..d652e344db2 100644 --- a/vortex-btrblocks/src/schemes/binary/zstd.rs +++ b/vortex-btrblocks/src/schemes/binary/zstd.rs @@ -31,8 +31,8 @@ impl Scheme for ZstdScheme { canonical.dtype().is_binary() } - fn produced_encodings(&self) -> Option> { - Some(vec![vortex_zstd::Zstd.id()]) + fn produced_encodings(&self) -> Vec { + vec![vortex_zstd::Zstd.id()] } fn expected_compression_ratio( diff --git a/vortex-btrblocks/src/schemes/binary/zstd_buffers.rs b/vortex-btrblocks/src/schemes/binary/zstd_buffers.rs index 95a3c0c4531..3f06d65b061 100644 --- a/vortex-btrblocks/src/schemes/binary/zstd_buffers.rs +++ b/vortex-btrblocks/src/schemes/binary/zstd_buffers.rs @@ -31,8 +31,8 @@ impl Scheme for ZstdBuffersScheme { canonical.dtype().is_binary() } - fn produced_encodings(&self) -> Option> { - Some(vec![vortex_zstd::ZstdBuffers.id()]) + fn produced_encodings(&self) -> Vec { + vec![vortex_zstd::ZstdBuffers.id()] } fn expected_compression_ratio( diff --git a/vortex-btrblocks/src/schemes/decimal.rs b/vortex-btrblocks/src/schemes/decimal.rs index de8c97c4495..1dff2171f60 100644 --- a/vortex-btrblocks/src/schemes/decimal.rs +++ b/vortex-btrblocks/src/schemes/decimal.rs @@ -40,8 +40,8 @@ impl Scheme for DecimalScheme { matches!(canonical, Canonical::Decimal(_)) } - fn produced_encodings(&self) -> Option> { - Some(vec![DecimalByteParts.id()]) + fn produced_encodings(&self) -> Vec { + vec![DecimalByteParts.id()] } /// Children: primitive=0. diff --git a/vortex-btrblocks/src/schemes/float/alp.rs b/vortex-btrblocks/src/schemes/float/alp.rs index e330f17ce82..f9fc7066bf4 100644 --- a/vortex-btrblocks/src/schemes/float/alp.rs +++ b/vortex-btrblocks/src/schemes/float/alp.rs @@ -42,12 +42,12 @@ impl Scheme for ALPScheme { canonical.dtype().is_float() } - fn produced_encodings(&self) -> Option> { + fn produced_encodings(&self) -> Vec { let mut encodings = vec![ALP.id()]; if use_experimental_patches() { encodings.push(Patched.id()); } - Some(encodings) + encodings } /// Children: encoded_ints=0. diff --git a/vortex-btrblocks/src/schemes/float/alprd.rs b/vortex-btrblocks/src/schemes/float/alprd.rs index 28427d14eb0..4b208b8ead1 100644 --- a/vortex-btrblocks/src/schemes/float/alprd.rs +++ b/vortex-btrblocks/src/schemes/float/alprd.rs @@ -39,8 +39,8 @@ impl Scheme for ALPRDScheme { canonical.dtype().is_float() } - fn produced_encodings(&self) -> Option> { - Some(vec![vortex_alp::ALPRD.id()]) + fn produced_encodings(&self) -> Vec { + vec![vortex_alp::ALPRD.id()] } fn expected_compression_ratio( diff --git a/vortex-btrblocks/src/schemes/float/pco.rs b/vortex-btrblocks/src/schemes/float/pco.rs index ac1c426f92c..416668c2fd0 100644 --- a/vortex-btrblocks/src/schemes/float/pco.rs +++ b/vortex-btrblocks/src/schemes/float/pco.rs @@ -31,8 +31,8 @@ impl Scheme for PcoScheme { canonical.dtype().is_float() } - fn produced_encodings(&self) -> Option> { - Some(vec![vortex_pco::Pco.id()]) + fn produced_encodings(&self) -> Vec { + vec![vortex_pco::Pco.id()] } fn expected_compression_ratio( diff --git a/vortex-btrblocks/src/schemes/float/rle.rs b/vortex-btrblocks/src/schemes/float/rle.rs index 04ae5cfc8cf..71158b9dc3b 100644 --- a/vortex-btrblocks/src/schemes/float/rle.rs +++ b/vortex-btrblocks/src/schemes/float/rle.rs @@ -38,8 +38,8 @@ impl Scheme for FloatRLEScheme { canonical.dtype().is_float() } - fn produced_encodings(&self) -> Option> { - Some(vec![RLE.id()]) + fn produced_encodings(&self) -> Vec { + vec![RLE.id()] } /// Children: values=0, indices=1, offsets=2. diff --git a/vortex-btrblocks/src/schemes/float/sparse.rs b/vortex-btrblocks/src/schemes/float/sparse.rs index 5f3e1a2d52d..3d9c25b18e4 100644 --- a/vortex-btrblocks/src/schemes/float/sparse.rs +++ b/vortex-btrblocks/src/schemes/float/sparse.rs @@ -41,8 +41,8 @@ impl Scheme for NullDominatedSparseScheme { canonical.dtype().is_float() } - fn produced_encodings(&self) -> Option> { - Some(vec![Sparse.id()]) + fn produced_encodings(&self) -> Vec { + vec![Sparse.id()] } /// Children: indices=0. diff --git a/vortex-btrblocks/src/schemes/integer/bitpacking.rs b/vortex-btrblocks/src/schemes/integer/bitpacking.rs index ea1c14a7b1b..5ac7d0e4078 100644 --- a/vortex-btrblocks/src/schemes/integer/bitpacking.rs +++ b/vortex-btrblocks/src/schemes/integer/bitpacking.rs @@ -40,12 +40,12 @@ impl Scheme for BitPackingScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { + fn produced_encodings(&self) -> Vec { let mut encodings = vec![BitPacked.id()]; if use_experimental_patches() { encodings.push(Patched.id()); } - Some(encodings) + encodings } fn expected_compression_ratio( diff --git a/vortex-btrblocks/src/schemes/integer/delta.rs b/vortex-btrblocks/src/schemes/integer/delta.rs index 6fb0dc6a0ad..f0288ef9436 100644 --- a/vortex-btrblocks/src/schemes/integer/delta.rs +++ b/vortex-btrblocks/src/schemes/integer/delta.rs @@ -80,8 +80,8 @@ impl Scheme for DeltaScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![Delta.id()]) + fn produced_encodings(&self) -> Vec { + vec![Delta.id()] } fn num_children(&self) -> usize { diff --git a/vortex-btrblocks/src/schemes/integer/for_.rs b/vortex-btrblocks/src/schemes/integer/for_.rs index 421d7fd08c6..2014582ddcf 100644 --- a/vortex-btrblocks/src/schemes/integer/for_.rs +++ b/vortex-btrblocks/src/schemes/integer/for_.rs @@ -43,8 +43,8 @@ impl Scheme for FoRScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![FoR.id()]) + fn produced_encodings(&self) -> Vec { + vec![FoR.id()] } /// Dict codes always start at 0, so FoR (which subtracts the min) is a no-op. diff --git a/vortex-btrblocks/src/schemes/integer/pco.rs b/vortex-btrblocks/src/schemes/integer/pco.rs index b3041fa4ca9..675a112d44f 100644 --- a/vortex-btrblocks/src/schemes/integer/pco.rs +++ b/vortex-btrblocks/src/schemes/integer/pco.rs @@ -32,8 +32,8 @@ impl Scheme for PcoScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![vortex_pco::Pco.id()]) + fn produced_encodings(&self) -> Vec { + vec![vortex_pco::Pco.id()] } fn expected_compression_ratio( diff --git a/vortex-btrblocks/src/schemes/integer/rle.rs b/vortex-btrblocks/src/schemes/integer/rle.rs index 0cd1c7a47bc..778ba5690e4 100644 --- a/vortex-btrblocks/src/schemes/integer/rle.rs +++ b/vortex-btrblocks/src/schemes/integer/rle.rs @@ -158,8 +158,8 @@ impl Scheme for IntRLEScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![RLE.id()]) + fn produced_encodings(&self) -> Vec { + vec![RLE.id()] } /// Children: values=0, indices=1, offsets=2. diff --git a/vortex-btrblocks/src/schemes/integer/runend.rs b/vortex-btrblocks/src/schemes/integer/runend.rs index 79e83896404..6a97f7ec37d 100644 --- a/vortex-btrblocks/src/schemes/integer/runend.rs +++ b/vortex-btrblocks/src/schemes/integer/runend.rs @@ -48,8 +48,8 @@ impl Scheme for RunEndScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![RunEnd.id()]) + fn produced_encodings(&self) -> Vec { + vec![RunEnd.id()] } /// Children: values=0, ends=1. diff --git a/vortex-btrblocks/src/schemes/integer/sequence.rs b/vortex-btrblocks/src/schemes/integer/sequence.rs index 0628cb52c1a..edcefb99fc2 100644 --- a/vortex-btrblocks/src/schemes/integer/sequence.rs +++ b/vortex-btrblocks/src/schemes/integer/sequence.rs @@ -43,8 +43,8 @@ impl Scheme for SequenceScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![Sequence.id()]) + fn produced_encodings(&self) -> Vec { + vec![Sequence.id()] } /// Sequence encoding on dictionary codes just adds a layer of indirection without compressing diff --git a/vortex-btrblocks/src/schemes/integer/sparse.rs b/vortex-btrblocks/src/schemes/integer/sparse.rs index 63fff9f883d..429ff5c1a31 100644 --- a/vortex-btrblocks/src/schemes/integer/sparse.rs +++ b/vortex-btrblocks/src/schemes/integer/sparse.rs @@ -46,8 +46,8 @@ impl Scheme for SparseScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![Sparse.id(), Constant.id()]) + fn produced_encodings(&self) -> Vec { + vec![Sparse.id(), Constant.id()] } fn stats_options(&self) -> GenerateStatsOptions { diff --git a/vortex-btrblocks/src/schemes/integer/zigzag.rs b/vortex-btrblocks/src/schemes/integer/zigzag.rs index 95383aea664..6082fcc541c 100644 --- a/vortex-btrblocks/src/schemes/integer/zigzag.rs +++ b/vortex-btrblocks/src/schemes/integer/zigzag.rs @@ -46,8 +46,8 @@ impl Scheme for ZigZagScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![ZigZag.id()]) + fn produced_encodings(&self) -> Vec { + vec![ZigZag.id()] } /// Children: encoded=0. diff --git a/vortex-btrblocks/src/schemes/string/fsst.rs b/vortex-btrblocks/src/schemes/string/fsst.rs index 31c207aca24..b2e91b8e9a8 100644 --- a/vortex-btrblocks/src/schemes/string/fsst.rs +++ b/vortex-btrblocks/src/schemes/string/fsst.rs @@ -46,8 +46,8 @@ impl Scheme for FSSTScheme { canonical.dtype().is_utf8() } - fn produced_encodings(&self) -> Option> { - Some(vec![FSST.id(), VarBin.id()]) + fn produced_encodings(&self) -> Vec { + vec![FSST.id(), VarBin.id()] } /// Children: lengths=0, code_offsets=1. diff --git a/vortex-btrblocks/src/schemes/string/onpair.rs b/vortex-btrblocks/src/schemes/string/onpair.rs index d4625823670..a771593efc2 100644 --- a/vortex-btrblocks/src/schemes/string/onpair.rs +++ b/vortex-btrblocks/src/schemes/string/onpair.rs @@ -49,8 +49,8 @@ impl Scheme for OnPairScheme { canonical.dtype().is_utf8() } - fn produced_encodings(&self) -> Option> { - Some(vec![OnPair.id()]) + fn produced_encodings(&self) -> Vec { + vec![OnPair.id()] } /// 4 primitive slot children flow through the cascading compressor: diff --git a/vortex-btrblocks/src/schemes/string/sparse.rs b/vortex-btrblocks/src/schemes/string/sparse.rs index 1461a39f250..8620c366f77 100644 --- a/vortex-btrblocks/src/schemes/string/sparse.rs +++ b/vortex-btrblocks/src/schemes/string/sparse.rs @@ -42,8 +42,8 @@ impl Scheme for NullDominatedSparseScheme { canonical.dtype().is_utf8() } - fn produced_encodings(&self) -> Option> { - Some(vec![Sparse.id()]) + fn produced_encodings(&self) -> Vec { + vec![Sparse.id()] } /// Children: indices=0. diff --git a/vortex-btrblocks/src/schemes/string/zstd.rs b/vortex-btrblocks/src/schemes/string/zstd.rs index 957ec87f2d6..84e8860d626 100644 --- a/vortex-btrblocks/src/schemes/string/zstd.rs +++ b/vortex-btrblocks/src/schemes/string/zstd.rs @@ -31,8 +31,8 @@ impl Scheme for ZstdScheme { canonical.dtype().is_utf8() } - fn produced_encodings(&self) -> Option> { - Some(vec![vortex_zstd::Zstd.id()]) + fn produced_encodings(&self) -> Vec { + vec![vortex_zstd::Zstd.id()] } fn expected_compression_ratio( diff --git a/vortex-btrblocks/src/schemes/string/zstd_buffers.rs b/vortex-btrblocks/src/schemes/string/zstd_buffers.rs index 6e2be309e8d..cf691c70fcb 100644 --- a/vortex-btrblocks/src/schemes/string/zstd_buffers.rs +++ b/vortex-btrblocks/src/schemes/string/zstd_buffers.rs @@ -31,8 +31,8 @@ impl Scheme for ZstdBuffersScheme { canonical.dtype().is_utf8() } - fn produced_encodings(&self) -> Option> { - Some(vec![vortex_zstd::ZstdBuffers.id()]) + fn produced_encodings(&self) -> Vec { + vec![vortex_zstd::ZstdBuffers.id()] } fn expected_compression_ratio( diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index bcf297b8dfd..79748b69450 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -55,8 +55,8 @@ impl Scheme for TemporalScheme { ) } - fn produced_encodings(&self) -> Option> { - Some(vec![DateTimeParts.id()]) + fn produced_encodings(&self) -> Vec { + vec![DateTimeParts.id()] } /// Children: days=0, seconds=1, subseconds=2. diff --git a/vortex-compressor/src/builtins/dict/binary.rs b/vortex-compressor/src/builtins/dict/binary.rs index 3530bfe1adc..c407d0251c6 100644 --- a/vortex-compressor/src/builtins/dict/binary.rs +++ b/vortex-compressor/src/builtins/dict/binary.rs @@ -48,8 +48,8 @@ impl Scheme for BinaryDictScheme { canonical.dtype().is_binary() } - fn produced_encodings(&self) -> Option> { - Some(vec![Dict.id()]) + fn produced_encodings(&self) -> Vec { + vec![Dict.id()] } fn stats_options(&self) -> GenerateStatsOptions { diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index f6a41c5938f..f962c3ff967 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -54,8 +54,8 @@ impl Scheme for FloatDictScheme { canonical.dtype().is_float() } - fn produced_encodings(&self) -> Option> { - Some(vec![Dict.id()]) + fn produced_encodings(&self) -> Vec { + vec![Dict.id()] } fn stats_options(&self) -> GenerateStatsOptions { diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 2c76a7e5522..27a17ef94ad 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -49,8 +49,8 @@ impl Scheme for IntDictScheme { canonical.dtype().is_int() } - fn produced_encodings(&self) -> Option> { - Some(vec![Dict.id()]) + fn produced_encodings(&self) -> Vec { + vec![Dict.id()] } fn stats_options(&self) -> GenerateStatsOptions { diff --git a/vortex-compressor/src/builtins/dict/string.rs b/vortex-compressor/src/builtins/dict/string.rs index 33f8b457bdd..f5cbcd54d89 100644 --- a/vortex-compressor/src/builtins/dict/string.rs +++ b/vortex-compressor/src/builtins/dict/string.rs @@ -48,8 +48,8 @@ impl Scheme for StringDictScheme { canonical.dtype().is_utf8() } - fn produced_encodings(&self) -> Option> { - Some(vec![Dict.id()]) + fn produced_encodings(&self) -> Vec { + vec![Dict.id()] } fn stats_options(&self) -> GenerateStatsOptions { diff --git a/vortex-compressor/src/compressor/tests.rs b/vortex-compressor/src/compressor/tests.rs index 08eafa369de..b23152d7ad3 100644 --- a/vortex-compressor/src/compressor/tests.rs +++ b/vortex-compressor/src/compressor/tests.rs @@ -4,6 +4,7 @@ use std::sync::LazyLock; use parking_lot::Mutex; +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; @@ -63,6 +64,10 @@ impl Scheme for DirectRatioScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, @@ -95,6 +100,10 @@ impl Scheme for ImmediateAlwaysUseScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, @@ -127,6 +136,10 @@ impl Scheme for CallbackAlwaysUseScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, @@ -161,6 +174,10 @@ impl Scheme for CallbackSkipScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, @@ -195,6 +212,10 @@ impl Scheme for CallbackRatioScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, @@ -229,6 +250,10 @@ impl Scheme for HugeRatioScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, @@ -261,6 +286,10 @@ impl Scheme for ZeroBytesSamplingScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, @@ -470,6 +499,10 @@ impl Scheme for ThresholdObservingScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, @@ -507,6 +540,10 @@ impl Scheme for CallbackMatchingRatioScheme { matches_integer_primitive(canonical) } + fn produced_encodings(&self) -> Vec { + Vec::new() + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, diff --git a/vortex-compressor/src/scheme/mod.rs b/vortex-compressor/src/scheme/mod.rs index 341acc8acbf..de9e67690d4 100644 --- a/vortex-compressor/src/scheme/mod.rs +++ b/vortex-compressor/src/scheme/mod.rs @@ -23,6 +23,7 @@ pub use estimate::EstimateVerdict; pub use exclusion::AncestorExclusion; pub use exclusion::ChildSelection; pub use exclusion::DescendantExclusion; +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; @@ -123,6 +124,13 @@ pub trait Scheme: Debug + Send + Sync { /// Whether this scheme can compress the given canonical array. fn matches(&self, canonical: &Canonical) -> bool; + /// The array encodings this scheme itself may introduce into its compressed output. + /// + /// Cascaded children are compressed by other schemes, which declare their own encodings, + /// so only encodings constructed directly by [`compress`](Scheme::compress) belong here. + /// Canonical arrays the scheme merely rearranges do not need to be declared. + fn produced_encodings(&self) -> Vec; + /// Returns the stats generation options this scheme requires. The compressor merges all /// eligible schemes' options before generating stats so that a single stats pass satisfies /// every scheme. diff --git a/vortex-tensor/src/encodings/l2_denorm.rs b/vortex-tensor/src/encodings/l2_denorm.rs index d07734bc6ff..8bec6292ee0 100644 --- a/vortex-tensor/src/encodings/l2_denorm.rs +++ b/vortex-tensor/src/encodings/l2_denorm.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::scalar_fn::ScalarFnVTable; use vortex_compressor::CascadingCompressor; use vortex_compressor::scheme::CompressionEstimate; use vortex_compressor::scheme::CompressorContext; @@ -15,6 +17,7 @@ use vortex_compressor::stats::ArrayAndStats; use vortex_error::VortexResult; use crate::matcher::AnyTensor; +use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; #[derive(Debug)] @@ -32,6 +35,10 @@ impl Scheme for L2DenormScheme { ) } + fn produced_encodings(&self) -> Vec { + vec![L2Denorm.id()] + } + fn expected_compression_ratio( &self, _data: &ArrayAndStats, From 7de670ad460a2d4d7f507167a2657a6e3d593b20 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 21 Jul 2026 15:24:28 +0100 Subject: [PATCH 4/7] wip Signed-off-by: Joe Isaacs --- encodings/parquet-variant/src/vtable.rs | 7 +-- vortex-edition/src/session.rs | 84 ++++++++++++++++++++++++- vortex-file/src/writer.rs | 58 ++++++++++++++--- vortex-jni/src/writer.rs | 9 ++- vortex/src/editions/mod.rs | 63 +++++++++++++------ 5 files changed, 183 insertions(+), 38 deletions(-) diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 4fa3ea10690..2374e4a94f4 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -309,7 +309,6 @@ mod tests { use vortex_array::Canonical; use vortex_array::EqMode; use vortex_array::IntoArray; - use vortex_array::VTable; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; @@ -400,11 +399,7 @@ mod tests { #[fixture] fn write_strategy() -> Arc { - let mut allowed = vortex_file::ALLOWED_ENCODINGS.clone(); - allowed.insert(ParquetVariant.id()); - vortex_file::WriteStrategyBuilder::default() - .with_allow_encodings(allowed) - .build() + vortex_file::WriteStrategyBuilder::default().build() } #[test] diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index a2a9b50066a..e7b5d31eb58 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! The [`EditionSession`] session variable: the per-session registry of editions and -//! edition inclusions. +//! Session variables for registered and enabled editions. use std::any::Any; use std::collections::BTreeMap; @@ -41,6 +40,28 @@ struct Inner { inclusions: BTreeMap, } +/// The editions enabled for writing in a session. +/// +/// At most one edition is enabled per family. Enabling a newer or older edition from the +/// same family replaces the previous selection. This is separate from [`EditionSession`]: +/// registration describes what a session knows how to reason about, while enabling is the +/// explicit writer policy. +#[derive(Clone, Debug, Default)] +pub struct EnabledEditions { + inner: Arc>>, +} + +impl EnabledEditions { + /// Return the enabled editions, sorted by family. + pub fn editions(&self) -> Vec { + self.inner.read().values().copied().collect() + } + + fn enable(&self, edition: EditionId) { + self.inner.write().insert(edition.family, edition); + } +} + impl EditionSession { /// Create a session variable with no declarations. pub fn empty() -> Self { @@ -190,12 +211,71 @@ impl SessionVar for EditionSession { } } +impl SessionVar for EnabledEditions { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } +} + /// Session data for Vortex editions. pub trait EditionSessionExt: SessionExt { /// Returns the edition registry. fn editions(&self) -> SessionGuard<'_, EditionSession> { self.get::() } + + /// Returns the editions enabled for writing. + /// + /// Accessing this method installs the enabled-editions session variable if it is absent, + /// opting the session into edition-gated writing with an initially empty selection. + fn enabled_editions(&self) -> SessionGuard<'_, EnabledEditions> { + self.get::() + } + + /// Register an edition declaration with this session. + fn register_edition(&self, declaration: &EditionDeclaration) -> Result<(), EditionError> { + self.editions().declare(declaration) + } + + /// Enable a registered edition for writing. + /// + /// Enabling an edition replaces the enabled edition from the same family. An edition + /// must be registered first so a typo or unavailable third-party declaration cannot + /// silently produce an empty writable set. + fn enable_edition(&self, edition: EditionId) -> Result<(), EditionError> { + if self.editions().find(&edition).is_none() { + return Err(EditionError::new(format!( + "cannot enable unregistered edition {edition}" + ))); + } + self.enabled_editions().enable(edition); + Ok(()) + } + + /// Resolve the encodings in all enabled editions. + /// + /// Returns `None` when the enabled-editions variable is absent, which is the explicit + /// opt-out used by lower-level sessions. Once the variable is present, an empty enabled + /// set resolves to `Some(Vec::new())` and therefore permits no edition encodings. + fn enabled_encoding_ids(&self) -> Vec { + let Some(enabled) = self.get_opt::() else { + vec![] + }; + let editions = self.editions(); + let mut ids: Vec = enabled + .editions() + .iter() + .flat_map(|edition| editions.encodings_in(edition)) + .map(|inclusion| inclusion.encoding_id) + .collect(); + ids.sort_unstable(); + ids.dedup(); + ids + } } impl EditionSessionExt for S {} diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 80da9f4fb89..c6954fd4917 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -17,12 +17,14 @@ use futures::pin_mut; use futures::select; use itertools::Itertools; use vortex_array::ArrayContext; +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; use vortex_array::expr::stats::Stat; use vortex_array::iter::ArrayIterator; use vortex_array::iter::ArrayIteratorExt; +use vortex_array::session::ArrayRegistry; use vortex_array::session::ArraySessionExt; use vortex_array::stats::PRUNING_STATS; use vortex_array::stream::ArrayStream; @@ -30,6 +32,7 @@ use vortex_array::stream::ArrayStreamAdapter; use vortex_array::stream::ArrayStreamExt; use vortex_array::stream::SendableArrayStream; use vortex_buffer::ByteBuffer; +use vortex_edition::EditionSessionExt; use vortex_error::VortexError; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -49,7 +52,6 @@ use vortex_session::SessionExt; use vortex_session::VortexSession; use vortex_session::registry::ReadContext; -use crate::ALLOWED_ENCODINGS; use crate::Footer; use crate::MAGIC_BYTES; use crate::WriteStrategyBuilder; @@ -158,14 +160,14 @@ impl VortexWriteOptions { mut write: W, stream: SendableArrayStream, ) -> VortexResult { - // NOTE(os): Setup an array context that already has all known encodings pre-populated. - // This is preferred for now over having an empty context here, because only the - // serialised array order is deterministic. The serialisation of arrays are done - // parallel and with an empty context they can register their encodings to the context - // in different order, changing the written bytes from run to run. - let ctx = ArrayContext::new(ALLOWED_ENCODINGS.iter().cloned().sorted().collect()) - // Configure a registry just to ensure only known encodings are interned. - .with_registry(self.session.arrays().registry().clone()); + // Pre-populate the array context with the registered encodings selected by the enabled + // editions. This keeps the serialized ordering deterministic without advertising every + // encoding the session happens to know how to read. + let array_registry = self.session.arrays().registry().clone(); + let ctx = ArrayContext::new(self.session.enabled_encoding_ids()) + // The registry supplies serialization implementations; the layout strategy applies + // the enabled-edition write policy before arrays reach serialization. + .with_registry(array_registry); let dtype = stream.dtype().clone(); let (mut ptr, eof) = SequenceId::root().split(); @@ -532,3 +534,41 @@ impl WriteSummary { .collect()) } } + +#[cfg(test)] +mod tests { + use vortex_array::VTable; + use vortex_array::array_session; + use vortex_array::arrays::Bool; + use vortex_array::arrays::Primitive; + use vortex_array::session::ArraySessionExt; + use vortex_edition::Edition; + use vortex_edition::EditionDeclaration; + use vortex_edition::EditionId; + use vortex_edition::EditionSession; + use vortex_edition::EditionSessionExt; + + use super::initial_array_ids; + + #[test] + fn initial_array_ids_are_registered_and_enabled() -> Result<(), vortex_edition::EditionError> { + const EDITION: EditionId = EditionId::new("test", 2026, 7, 0); + static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: EDITION, + min_vortex_version: None, + }, + added: &[&"vortex.primitive", &"test.not_registered"], + }; + + let session = array_session().with::(); + session.register_edition(&DECLARATION)?; + session.enable_edition(EDITION)?; + + let registry = session.arrays().registry().clone(); + let ids = initial_array_ids(&session, ®istry); + assert_eq!(ids, [Primitive.id()]); + assert!(!ids.contains(&Bool.id())); + Ok(()) + } +} diff --git a/vortex-jni/src/writer.rs b/vortex-jni/src/writer.rs index 53649eef4ab..28e82158aa2 100644 --- a/vortex-jni/src/writer.rs +++ b/vortex-jni/src/writer.rs @@ -32,6 +32,7 @@ use jni::sys::jlong; use jni::sys::jobject; use object_store::ObjectStore; use object_store::path::Path as ObjectStorePath; +use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::VTable; use vortex::array::scalar::PValue; @@ -42,6 +43,7 @@ use vortex::array::stream::ArrayStreamAdapter; use vortex::dtype::DType; use vortex::dtype::Field as DTypeField; use vortex::dtype::FieldPath; +use vortex::editions::EditionSessionExt; use vortex::error::VortexError; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -60,6 +62,7 @@ use vortex::io::runtime::Task; use vortex::io::session::RuntimeSessionExt; use vortex::session::VortexSession; use vortex::utils::aliases::hash_map::HashMap; +use vortex::utils::aliases::hash_set::HashSet; use vortex_arrow::ArrowSessionExt; use vortex_parquet_variant::ParquetVariant; @@ -108,7 +111,11 @@ fn write_options_for_schema( return session.write_options(); } - let mut allowed = vortex::file::ALLOWED_ENCODINGS.clone(); + let mut allowed: HashSet = session + .enabled_encoding_ids() + .unwrap_or_default() + .into_iter() + .collect(); allowed.insert(ParquetVariant.id()); let strategy = WriteStrategyBuilder::default().with_allow_encodings(allowed); diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 1c3323d077f..6d60fa2b0b7 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -3,28 +3,27 @@ //! The Vortex edition declarations. //! -//! [`vortex_edition`] provides the types, the [`crate::editions::EditionSession`] session -//! variable, and the test harness; the actual declarations live here, one module per edition -//! (`editions::::`), and are seeded into the default session by -//! [`crate::editions::register_default_editions`]. +//! [`vortex_edition`] provides the types, session variables, and test harness. The actual +//! first-party declarations live here, one module per edition. The default session first +//! registers them with [`register_default_editions`] and then selects its write policy with +//! [`enable_default_editions`]. //! -//! Each edition module declares the edition together with the encodings that join the -//! family at it; members of earlier editions are inherited and never restated. Correctness -//! is enforced by unit tests: every edition module calls -//! [`vortex_edition::test_harness::validate_edition`] once from its `#[cfg(test)]` module, -//! and the computed set of a frozen edition is pinned by a golden test, so any change to -//! these declarations that alters a frozen set fails CI. New encodings are staged into the -//! newest draft edition. -//! -//! Note this is currently unused but a future PR will make this public and gate the writer behind -//! editions. +//! The default file writer resolves the session's enabled editions at write time. The +//! facade enables the newest frozen `core` edition, [`CORE_2026_07_0`], and additionally +//! enables the latest unstable edition when the `unstable_encodings` feature is selected. pub mod core; +#[cfg(test)] +mod tests; pub mod unstable; -use vortex_edition::EditionDeclaration; +pub use vortex_edition::Edition; +pub use vortex_edition::EditionDeclaration; +pub use vortex_edition::EditionId; +pub use vortex_edition::EditionInclusion; pub use vortex_edition::EditionSession; -use vortex_edition::EditionSessionExt; +pub use vortex_edition::EditionSessionExt; +pub use vortex_edition::EnabledEditions; use vortex_error::VortexExpect; use vortex_error::vortex_err; use vortex_session::VortexSession; @@ -38,7 +37,14 @@ pub use self::unstable::UNSTABLE_2026_02_0; pub use self::unstable::UNSTABLE_2026_04_0; pub use self::unstable::UNSTABLE_2026_06_0; -/// The Vortex editions, each declared together with the encodings it adds. +/// The `core` edition enabled for writing by the default Vortex session. +pub const DEFAULT_CORE_EDITION: EditionId = CORE_2026_07_0; + +/// The `unstable` edition enabled for writing by the default Vortex session when the +/// `unstable_encodings` feature is selected. +pub const DEFAULT_UNSTABLE_EDITION: EditionId = UNSTABLE_2026_06_0; + +/// The first-party Vortex edition declarations. pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ &core::v2025_05::DECLARATION, &core::v2025_06::DECLARATION, @@ -52,11 +58,28 @@ pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ /// Register the Vortex edition declarations with the session's [`EditionSession`]. pub fn register_default_editions(session: &VortexSession) { - let editions = session.editions(); for declaration in EDITION_DECLARATIONS { - editions - .declare(declaration) + session + .register_edition(declaration) .map_err(|e| vortex_err!("{e}")) .vortex_expect("edition declarations are valid"); } } + +/// Enable the default Vortex editions for writing. +/// +/// This selects the newest frozen `core` edition and, when configured, the newest unstable +/// edition. All declarations must have been registered first with +/// [`register_default_editions`]. +pub fn enable_default_editions(session: &VortexSession) { + session + .enable_edition(DEFAULT_CORE_EDITION) + .map_err(|e| vortex_err!("{e}")) + .vortex_expect("default core edition is registered"); + + #[cfg(feature = "unstable_encodings")] + session + .enable_edition(DEFAULT_UNSTABLE_EDITION) + .map_err(|e| vortex_err!("{e}")) + .vortex_expect("default unstable edition is registered"); +} From 55ade84f12b835c1e6cd9a4d346a17af275bbd1a Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 21 Jul 2026 16:42:43 +0100 Subject: [PATCH 5/7] wip Signed-off-by: Joe Isaacs --- vortex-edition/src/session.rs | 11 +- vortex-edition/src/tests.rs | 63 +++++ vortex-file/src/footer/field_sizes.rs | 1 + vortex-file/src/lib.rs | 37 ++- vortex-file/src/open.rs | 3 + vortex-file/src/strategy.rs | 117 ++------ vortex-file/src/tests.rs | 1 + vortex-file/src/writer.rs | 27 +- .../tests/issue_8819_footer_segment_oob.rs | 8 + vortex-file/tests/test_write_table.rs | 18 +- vortex-jni/src/writer.rs | 6 +- vortex/src/editions/tests.rs | 253 ++++++++++++++++++ 12 files changed, 417 insertions(+), 128 deletions(-) create mode 100644 vortex/src/editions/tests.rs diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs index e7b5d31eb58..a50d17b6b53 100644 --- a/vortex-edition/src/session.rs +++ b/vortex-edition/src/session.rs @@ -230,8 +230,8 @@ pub trait EditionSessionExt: SessionExt { /// Returns the editions enabled for writing. /// - /// Accessing this method installs the enabled-editions session variable if it is absent, - /// opting the session into edition-gated writing with an initially empty selection. + /// Accessing this method installs the enabled-editions session variable if it is absent, with + /// an initially empty selection. fn enabled_editions(&self) -> SessionGuard<'_, EnabledEditions> { self.get::() } @@ -258,12 +258,11 @@ pub trait EditionSessionExt: SessionExt { /// Resolve the encodings in all enabled editions. /// - /// Returns `None` when the enabled-editions variable is absent, which is the explicit - /// opt-out used by lower-level sessions. Once the variable is present, an empty enabled - /// set resolves to `Some(Vec::new())` and therefore permits no edition encodings. + /// When the enabled-editions variable is absent or no editions are enabled, this returns an + /// empty vector and therefore permits no edition encodings. fn enabled_encoding_ids(&self) -> Vec { let Some(enabled) = self.get_opt::() else { - vec![] + return vec![]; }; let editions = self.editions(); let mut ids: Vec = enabled diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs index 3791173fad8..3d6550982db 100644 --- a/vortex-edition/src/tests.rs +++ b/vortex-edition/src/tests.rs @@ -9,6 +9,7 @@ use crate::EditionId; use crate::EditionInclusion; use crate::EditionSession; use crate::EditionSessionExt; +use crate::EnabledEditions; const FIRST: EditionId = EditionId::new("test", 2026, 1, 0); const SECOND: EditionId = EditionId::new("test", 2026, 7, 0); @@ -126,6 +127,68 @@ fn session_exposes_edition_registry() { assert!(session.editions().find(&FIRST).is_some()); } +#[test] +fn registered_and_enabled_editions_are_separate() -> Result<(), crate::EditionError> { + let session = VortexSession::empty().with::(); + for declaration in DECLARATIONS { + session.register_edition(declaration)?; + } + + assert!(session.enabled_encoding_ids().is_empty()); + session.enable_edition(FIRST)?; + assert_eq!(session.enabled_editions().editions(), [FIRST]); + assert_eq!( + session + .enabled_encoding_ids() + .iter() + .map(|id| id.as_str()) + .collect::>(), + ["test.alpha", "test.beta"] + ); + + session.enable_edition(SECOND)?; + assert_eq!(session.enabled_editions().editions(), [SECOND]); + assert_eq!(session.enabled_encoding_ids().len(), 3); + + // Selecting an older edition in the same family replaces the newer one and removes + // encodings that joined after it. + session.enable_edition(FIRST)?; + assert_eq!(session.enabled_editions().editions(), [FIRST]); + let enabled = session.enabled_encoding_ids(); + assert_eq!(enabled.len(), 2); + assert!(enabled.iter().all(|id| id.as_str() != "test.gamma")); + Ok(()) +} + +#[test] +fn enabling_requires_registration() { + let session = VortexSession::empty().with::(); + assert!(session.enable_edition(FIRST).is_err()); + assert!(session.enabled_editions().editions().is_empty()); +} + +#[test] +fn enabled_editions_are_independent_across_families() -> Result<(), crate::EditionError> { + const OTHER: EditionId = EditionId::new("other", 2026, 4, 0); + static OTHER_DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: OTHER, + min_vortex_version: None, + }, + added: &[&"other.delta"], + }; + + let session = VortexSession::empty().with::(); + session.register_edition(&DECLARATIONS[0])?; + session.register_edition(&OTHER_DECLARATION)?; + session.enable_edition(FIRST)?; + session.enable_edition(OTHER)?; + + assert_eq!(session.enabled_editions().editions(), [OTHER, FIRST]); + assert_eq!(session.enabled_encoding_ids().len(), 3); + Ok(()) +} + #[test] fn duplicate_declarations_error() { let editions = session(); diff --git a/vortex-file/src/footer/field_sizes.rs b/vortex-file/src/footer/field_sizes.rs index 5d7d2ce7c02..91e794f3729 100644 --- a/vortex-file/src/footer/field_sizes.rs +++ b/vortex-file/src/footer/field_sizes.rs @@ -141,6 +141,7 @@ mod tests { .with::() .with::(); crate::register_default_encodings(&session); + crate::enable_all_registered_array_encodings(&session); session }); diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 73c6c7d9c7d..eba26834588 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -192,6 +192,37 @@ pub fn register_default_encodings(session: &VortexSession) { vortex_tensor::initialize(session); } +#[cfg(test)] +pub(crate) fn enable_all_registered_array_encodings(session: &VortexSession) { + use vortex_edition::Edition; + use vortex_edition::EditionId; + use vortex_edition::EditionInclusion; + use vortex_edition::EditionSessionExt; + use vortex_error::VortexExpect; + use vortex_error::vortex_err; + + const TEST_EDITION: EditionId = EditionId::new("test", 2026, 7, 0); + + let editions = session.editions(); + editions + .declare_edition(Edition { + id: TEST_EDITION, + min_vortex_version: None, + }) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("test edition is valid"); + for id in session.arrays().registry().ids() { + editions + .declare_inclusion(EditionInclusion::new(&id, TEST_EDITION)) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("registered array encoding has one test-edition inclusion"); + } + session + .enable_edition(TEST_EDITION) + .map_err(|error| vortex_err!("{error}")) + .vortex_expect("test edition is registered"); +} + #[cfg(test)] mod default_encoding_tests { use vortex_array::VTable as _; @@ -222,10 +253,4 @@ mod default_encoding_tests { .has_execute_parent(Filter.id(), OnPair.id()) ); } - - #[cfg(not(feature = "unstable_encodings"))] - #[test] - fn default_writer_does_not_allow_onpair() { - assert!(!crate::ALLOWED_ENCODINGS.contains(&OnPair.id())); - } } diff --git a/vortex-file/src/open.rs b/vortex-file/src/open.rs index fbc84888644..e2d308822f9 100644 --- a/vortex-file/src/open.rs +++ b/vortex-file/src/open.rs @@ -461,6 +461,7 @@ mod tests { .with::() .with::(); crate::register_default_encodings(&session); + crate::enable_all_registered_array_encodings(&session); session } @@ -517,6 +518,7 @@ mod tests { .with::(); crate::register_default_encodings(&session); + crate::enable_all_registered_array_encodings(&session); // Create a large file (> 1MB) let mut buf = ByteBufferMut::empty(); @@ -578,6 +580,7 @@ mod tests { .with::(); crate::register_default_encodings(&session); + crate::enable_all_registered_array_encodings(&session); let mut buf = ByteBufferMut::empty(); let array = Buffer::from((0i32..16_384).collect::>()).into_array(); diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index 763db758cb7..99682f1b37d 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -5,43 +5,13 @@ use std::num::NonZeroUsize; use std::sync::Arc; -use std::sync::LazyLock; -use vortex_alp::ALP; -use vortex_alp::ALPRD; use vortex_array::ArrayId; -use vortex_array::VTable; -use vortex_array::arrays::Bool; -use vortex_array::arrays::Chunked; -use vortex_array::arrays::Constant; -use vortex_array::arrays::Decimal; -use vortex_array::arrays::Dict; -use vortex_array::arrays::Extension; -use vortex_array::arrays::FixedSizeList; -use vortex_array::arrays::List; -use vortex_array::arrays::ListView; -use vortex_array::arrays::Masked; -use vortex_array::arrays::Null; -use vortex_array::arrays::Patched; -use vortex_array::arrays::Primitive; -use vortex_array::arrays::Struct; -use vortex_array::arrays::VarBin; -use vortex_array::arrays::VarBinView; -use vortex_array::arrays::Variant; -use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::dtype::FieldPath; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::schemes::integer::IntDictScheme; -use vortex_bytebool::ByteBool; -use vortex_datetime_parts::DateTimeParts; -use vortex_decimal_byte_parts::DecimalByteParts; use vortex_error::VortexExpect; -use vortex_fastlanes::BitPacked; -use vortex_fastlanes::Delta; -use vortex_fastlanes::FoR; -use vortex_fastlanes::RLE; -use vortex_fsst::FSST; use vortex_layout::LayoutStrategy; use vortex_layout::LayoutStrategyEncodingValidator; use vortex_layout::layouts::buffered::BufferedStrategy; @@ -58,80 +28,11 @@ use vortex_layout::layouts::table::TableStrategy; use vortex_layout::layouts::table::use_experimental_list_layout; use vortex_layout::layouts::zoned::writer::ZonedLayoutOptions; use vortex_layout::layouts::zoned::writer::ZonedStrategy; -#[cfg(feature = "unstable_encodings")] -use vortex_onpair::OnPair; -use vortex_pco::Pco; -use vortex_runend::RunEnd; -use vortex_sequence::Sequence; -use vortex_sparse::Sparse; use vortex_utils::aliases::hash_map::HashMap; use vortex_utils::aliases::hash_set::HashSet; -use vortex_zigzag::ZigZag; -#[cfg(feature = "zstd")] -use vortex_zstd::Zstd; -#[cfg(all(feature = "zstd", feature = "unstable_encodings"))] -use vortex_zstd::ZstdBuffers; const ONE_MEG: u64 = 1 << 20; -/// Static registry of all allowed array encodings for file writing. -/// -/// This includes all canonical encodings from vortex-array plus all compressed -/// encodings from the various encoding crates. -pub static ALLOWED_ENCODINGS: LazyLock> = LazyLock::new(|| { - let mut allowed = HashSet::new(); - - // Canonical encodings from vortex-array - allowed.insert(Null.id()); - allowed.insert(Bool.id()); - allowed.insert(Primitive.id()); - allowed.insert(Decimal.id()); - allowed.insert(VarBin.id()); - allowed.insert(VarBinView.id()); - allowed.insert(List.id()); - allowed.insert(ListView.id()); - allowed.insert(FixedSizeList.id()); - allowed.insert(Struct.id()); - allowed.insert(Extension.id()); - allowed.insert(Chunked.id()); - allowed.insert(Constant.id()); - allowed.insert(Masked.id()); - allowed.insert(Dict.id()); - allowed.insert(Variant.id()); - - // Compressed encodings from encoding crates - allowed.insert(ALP.id()); - allowed.insert(ALPRD.id()); - allowed.insert(BitPacked.id()); - allowed.insert(ByteBool.id()); - allowed.insert(DateTimeParts.id()); - allowed.insert(DecimalByteParts.id()); - allowed.insert(Delta.id()); - allowed.insert(FoR.id()); - allowed.insert(FSST.id()); - #[cfg(feature = "unstable_encodings")] - allowed.insert(OnPair.id()); - allowed.insert(Pco.id()); - allowed.insert(RLE.id()); - allowed.insert(RunEnd.id()); - allowed.insert(Sequence.id()); - allowed.insert(Sparse.id()); - allowed.insert(ZigZag.id()); - - // Experimental encodings - - if use_experimental_patches() { - allowed.insert(Patched.id()); - } - - #[cfg(feature = "zstd")] - allowed.insert(Zstd.id()); - #[cfg(all(feature = "zstd", feature = "unstable_encodings"))] - allowed.insert(ZstdBuffers.id()); - - allowed -}); - /// How the compressor was configured on [`WriteStrategyBuilder`]. enum CompressorConfig { /// A [`BtrBlocksCompressorBuilder`] that [`WriteStrategyBuilder::build`] will finalize. @@ -174,7 +75,7 @@ impl Default for WriteStrategyBuilder { compressor: CompressorConfig::BtrBlocks(BtrBlocksCompressorBuilder::default()), row_block_size: 8192, field_writers: HashMap::new(), - allow_encodings: Some(ALLOWED_ENCODINGS.clone()), + allow_encodings: None, flat_strategy: None, probe_compressor: None, use_list_layout: use_experimental_list_layout(), @@ -216,11 +117,18 @@ impl WriteStrategyBuilder { self } - /// Override the allowed array encodings for normalization. + /// Override the allowed array encodings for file writing. /// /// The configured flat leaf strategy is wrapped in a [`LayoutStrategyEncodingValidator`] - /// that recursively checks every chunk before passing it to the leaf writer. + /// that recursively checks every chunk before passing it to the leaf writer. A configured + /// BtrBlocks builder is restricted to schemes whose output encodings are all in this set. pub fn with_allow_encodings(mut self, allow_encodings: HashSet) -> Self { + self.compressor = match self.compressor { + CompressorConfig::BtrBlocks(builder) => { + CompressorConfig::BtrBlocks(builder.retain_allowed_encodings(&allow_encodings)) + } + compressor => compressor, + }; self.allow_encodings = Some(allow_encodings); self } @@ -239,6 +147,11 @@ impl WriteStrategyBuilder { /// The builder is finalized during [`build`](Self::build), producing two compressors: one for /// data (with `IntDictScheme` excluded) and one for stats. pub fn with_btrblocks_builder(mut self, builder: BtrBlocksCompressorBuilder) -> Self { + let builder = if let Some(allow_encodings) = &self.allow_encodings { + builder.retain_allowed_encodings(allow_encodings) + } else { + builder + }; self.compressor = CompressorConfig::BtrBlocks(builder); self } diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index e665ab18d50..26abb5e835d 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -88,6 +88,7 @@ static SESSION: LazyLock = LazyLock::new(|| { .with::(); crate::register_default_encodings(&session); + crate::enable_all_registered_array_encodings(&session); session }); diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index c6954fd4917..3a8e120df37 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -62,8 +62,8 @@ use crate::segments::writer::BufferedSegmentSink; /// Configure a new writer, which can eventually be used to write an [`ArrayStream`] into a sink /// that implements [`VortexWrite`]. /// -/// Unless overridden, the default [write strategy][crate::WriteStrategyBuilder] will be used with no -/// additional configuration. +/// Unless overridden, the default [write strategy][crate::WriteStrategyBuilder] will be restricted +/// to the encodings in the session's enabled editions. /// /// Construct with [`WriteOptionsSessionExt::write_options`] for normal use so the writer inherits /// the session's runtime, array registry, and memory configuration. @@ -80,8 +80,9 @@ pub trait WriteOptionsSessionExt: SessionExt { /// Create [`VortexWriteOptions`] for writing to a Vortex file. fn write_options(&self) -> VortexWriteOptions { let session = self.session(); + let strategy = default_write_strategy(&session); VortexWriteOptions { - strategy: WriteStrategyBuilder::default().build(), + strategy, session, exclude_dtype: false, file_statistics: PRUNING_STATS.to_vec(), @@ -91,11 +92,27 @@ pub trait WriteOptionsSessionExt: SessionExt { } impl WriteOptionsSessionExt for S {} +fn default_write_strategy(session: &VortexSession) -> Arc { + WriteStrategyBuilder::default() + .with_allow_encodings(session.enabled_encoding_ids().into_iter().collect()) + .build() +} + +fn initial_array_ids(session: &VortexSession, registry: &ArrayRegistry) -> Vec { + session + .enabled_encoding_ids() + .into_iter() + .filter(|id| registry.find(id).is_some()) + .sorted() + .collect() +} + impl VortexWriteOptions { /// Create a new [`VortexWriteOptions`] with the given session. pub fn new(session: VortexSession) -> Self { + let strategy = default_write_strategy(&session); VortexWriteOptions { - strategy: WriteStrategyBuilder::default().build(), + strategy, session, exclude_dtype: false, file_statistics: PRUNING_STATS.to_vec(), @@ -164,7 +181,7 @@ impl VortexWriteOptions { // editions. This keeps the serialized ordering deterministic without advertising every // encoding the session happens to know how to read. let array_registry = self.session.arrays().registry().clone(); - let ctx = ArrayContext::new(self.session.enabled_encoding_ids()) + let ctx = ArrayContext::new(initial_array_ids(&self.session, &array_registry)) // The registry supplies serialization implementations; the layout strategy applies // the enabled-edition write policy before arrays reach serialization. .with_registry(array_registry); diff --git a/vortex-file/tests/issue_8819_footer_segment_oob.rs b/vortex-file/tests/issue_8819_footer_segment_oob.rs index 7b09d7a9c35..96b838f8459 100644 --- a/vortex-file/tests/issue_8819_footer_segment_oob.rs +++ b/vortex-file/tests/issue_8819_footer_segment_oob.rs @@ -14,11 +14,13 @@ use std::mem::size_of; use std::sync::LazyLock; use vortex_array::IntoArray; +use vortex_array::session::ArraySessionExt; use vortex_buffer::Buffer; use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; use vortex_file::OpenOptionsSessionExt; use vortex_file::WriteOptionsSessionExt; +use vortex_file::WriteStrategyBuilder; use vortex_io::session::RuntimeSession; use vortex_layout::session::LayoutSession; use vortex_session::VortexSession; @@ -38,8 +40,14 @@ async fn open_buffer_rejects_out_of_bounds_footer_segment() { // Write a valid file to obtain a well-formed footer. let mut buf = ByteBufferMut::empty(); let array = Buffer::from((0i32..256).collect::>()).into_array(); + let allowed = SESSION.arrays().registry().ids().collect(); SESSION .write_options() + .with_strategy( + WriteStrategyBuilder::default() + .with_allow_encodings(allowed) + .build(), + ) .write(&mut buf, array.to_array_stream()) .await .expect("write"); diff --git a/vortex-file/tests/test_write_table.rs b/vortex-file/tests/test_write_table.rs index bd664cc24ed..7f3da8c4238 100644 --- a/vortex-file/tests/test_write_table.rs +++ b/vortex-file/tests/test_write_table.rs @@ -18,11 +18,14 @@ use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::FieldNames; use vortex_array::field_path; +use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_btrblocks::BtrBlocksCompressor; use vortex_buffer::ByteBuffer; use vortex_file::OpenOptionsSessionExt; +use vortex_file::VortexWriteOptions; use vortex_file::WriteOptionsSessionExt; +use vortex_file::WriteStrategyBuilder; use vortex_io::session::RuntimeSession; use vortex_layout::layouts::compressed::CompressingStrategy; use vortex_layout::layouts::flat::writer::FlatLayoutStrategy; @@ -39,6 +42,15 @@ static SESSION: LazyLock = LazyLock::new(|| { session }); +fn all_registered_write_options() -> VortexWriteOptions { + let allowed = SESSION.arrays().registry().ids().collect(); + SESSION.write_options().with_strategy( + WriteStrategyBuilder::default() + .with_allow_encodings(allowed) + .build(), + ) +} + #[tokio::test] async fn test_file_roundtrip() { let mut ctx = SESSION.create_execution_ctx(); @@ -142,8 +154,7 @@ async fn test_dict_listview_validity_roundtrip() { .into_array(); let mut bytes = Vec::new(); - SESSION - .write_options() + all_registered_write_options() .write(&mut bytes, data.to_array_stream()) .await .expect("write should not fail with fill_null serialization error"); @@ -188,8 +199,7 @@ async fn test_write_empty_nullable_struct_column() { .into_array(); let mut bytes = Vec::new(); - SESSION - .write_options() + all_registered_write_options() .write(&mut bytes, data.to_array_stream()) .await .expect("writing an empty nullable struct column should not panic"); diff --git a/vortex-jni/src/writer.rs b/vortex-jni/src/writer.rs index 28e82158aa2..0bcdad03894 100644 --- a/vortex-jni/src/writer.rs +++ b/vortex-jni/src/writer.rs @@ -111,11 +111,7 @@ fn write_options_for_schema( return session.write_options(); } - let mut allowed: HashSet = session - .enabled_encoding_ids() - .unwrap_or_default() - .into_iter() - .collect(); + let mut allowed: HashSet = session.enabled_encoding_ids().into_iter().collect(); allowed.insert(ParquetVariant.id()); let strategy = WriteStrategyBuilder::default().with_allow_encodings(allowed); diff --git a/vortex/src/editions/tests.rs b/vortex/src/editions/tests.rs new file mode 100644 index 00000000000..52dc3009a46 --- /dev/null +++ b/vortex/src/editions/tests.rs @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::IntoArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_buffer::ByteBufferMut; +use vortex_edition::EditionError; +use vortex_edition::EditionSession; +use vortex_edition::EditionSessionExt; +use vortex_edition::test_harness::validate_edition; +use vortex_error::VortexResult; +use vortex_error::vortex_err; +use vortex_file::WriteOptionsSessionExt; +use vortex_file::WriteStrategyBuilder; +use vortex_session::VortexSession; +use vortex_utils::aliases::hash_set::HashSet; + +use super::CORE_2025_05_0; +use super::CORE_2026_07_0; +use super::DEFAULT_CORE_EDITION; +use super::DEFAULT_UNSTABLE_EDITION; +use super::EDITION_DECLARATIONS; +use super::UNSTABLE_2026_06_0; + +fn session() -> Result { + let session = EditionSession::empty(); + for declaration in EDITION_DECLARATIONS { + session.declare(declaration)?; + } + Ok(session) +} + +#[test] +fn every_declared_edition_validates() -> Result<(), EditionError> { + let session = session()?; + for declaration in EDITION_DECLARATIONS { + validate_edition(&session, &declaration.edition.id)?; + } + Ok(()) +} + +/// The full encoding set of the newest frozen `core` edition. This set is frozen: the only +/// way it may change is by declaring a *new* edition, so a failure here means a frozen +/// declaration was edited. +#[test] +fn core_2026_07_encoding_set_is_pinned() { + let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); + let encodings = session.encodings_in(&CORE_2026_07_0); + let ids: Vec<&str> = encodings + .iter() + .map(|inclusion| inclusion.encoding_id.as_str()) + .collect(); + assert_eq!( + ids, + [ + "fastlanes.bitpacked", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", + ] + ); +} + +#[test] +fn encodings_in_editions_unions_families() { + let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); + let core_only: Vec<_> = session + .encodings_in(&CORE_2026_07_0) + .into_iter() + .map(|inclusion| inclusion.encoding_id) + .collect(); + let mut both = core_only.clone(); + both.extend( + session + .encodings_in(&UNSTABLE_2026_06_0) + .into_iter() + .map(|inclusion| inclusion.encoding_id), + ); + both.sort_unstable(); + both.dedup(); + + assert!(both.len() > core_only.len()); + assert!(both.iter().any(|id| id.as_str() == "fastlanes.delta")); + assert!(both.iter().any(|id| id.as_str() == "vortex.onpair")); + assert!(core_only.iter().all(|id| both.contains(id))); +} + +#[test] +fn earlier_editions_are_subsets() { + let session = session().unwrap_or_else(|e| panic!("registering editions: {e}")); + let first = session.encodings_in(&CORE_2025_05_0); + let latest = session.encodings_in(&CORE_2026_07_0); + assert!(first.iter().all(|inclusion| { + latest + .iter() + .any(|latest| latest.encoding_id == inclusion.encoding_id) + })); + assert!(first.len() < latest.len()); +} + +#[test] +fn default_session_enables_the_write_editions() { + use crate::VortexSessionDefault; + + let session = VortexSession::default(); + let enabled = session.enabled_editions().editions(); + assert!(enabled.contains(&DEFAULT_CORE_EDITION)); + + #[cfg(feature = "unstable_encodings")] + assert!(enabled.contains(&DEFAULT_UNSTABLE_EDITION)); + #[cfg(not(feature = "unstable_encodings"))] + assert!(!enabled.contains(&DEFAULT_UNSTABLE_EDITION)); +} + +#[test] +fn core_edition_ids_are_registered_array_encodings() { + use vortex_array::session::ArraySessionExt; + + use crate::VortexSessionDefault; + + let session = VortexSession::default(); + let registry = session.arrays().registry().clone(); + for inclusion in session.editions().encodings_in(&CORE_2026_07_0) { + assert!( + registry.find(&inclusion.encoding_id).is_some(), + "{} is declared in core but not registered as an array encoding", + inclusion.encoding_id + ); + } +} + +fn baseline_core_session() -> VortexResult { + use crate::VortexSessionDefault; + + let session = VortexSession::default(); + session + .enable_edition(CORE_2025_05_0) + .map_err(|error| vortex_err!("{error}"))?; + Ok(session) +} + +fn sequential_integers() -> PrimitiveArray { + PrimitiveArray::from_iter(0..65_536i32) +} + +#[tokio::test] +async fn default_writer_filters_compressor_to_enabled_editions() -> VortexResult<()> { + let session = baseline_core_session()?; + let mut buffer = ByteBufferMut::empty(); + + session + .write_options() + .write( + &mut buffer, + sequential_integers().into_array().to_array_stream(), + ) + .await?; + + Ok(()) +} + +#[tokio::test] +async fn configured_btrblocks_builder_uses_enabled_editions_in_either_order() -> VortexResult<()> { + let session = baseline_core_session()?; + let allowed: HashSet<_> = session.enabled_encoding_ids().into_iter().collect(); + let strategies = [ + WriteStrategyBuilder::default() + .with_btrblocks_builder(BtrBlocksCompressorBuilder::default()) + .with_allow_encodings(allowed.clone()) + .build(), + WriteStrategyBuilder::default() + .with_allow_encodings(allowed) + .with_btrblocks_builder(BtrBlocksCompressorBuilder::default()) + .build(), + ]; + + for strategy in strategies { + let mut buffer = ByteBufferMut::empty(); + session + .write_options() + .with_strategy(strategy) + .write( + &mut buffer, + sequential_integers().into_array().to_array_stream(), + ) + .await?; + } + + Ok(()) +} + +#[tokio::test] +async fn opaque_compressor_cannot_write_outside_enabled_editions() -> VortexResult<()> { + let session = baseline_core_session()?; + let allowed = session.enabled_encoding_ids().into_iter().collect(); + let strategy = WriteStrategyBuilder::default() + .with_compressor(BtrBlocksCompressorBuilder::default().build()) + .with_allow_encodings(allowed) + .build(); + let mut buffer = ByteBufferMut::empty(); + + let result = session + .write_options() + .with_strategy(strategy) + .write( + &mut buffer, + sequential_integers().into_array().to_array_stream(), + ) + .await; + let error = match result { + Ok(_) => { + return Err(vortex_err!( + "the unrestricted opaque compressor wrote an encoding outside core@2025.05" + )); + } + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("normalize forbids encoding (vortex.sequence)"), + "unexpected error: {message}" + ); + + Ok(()) +} From 58776e35be965dfd283245d02b64fab32407c1ef Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Wed, 22 Jul 2026 11:16:32 +0100 Subject: [PATCH 6/7] fix Signed-off-by: Joe Isaacs --- vortex-file/src/writer.rs | 39 +++++++-------------------------------- 1 file changed, 7 insertions(+), 32 deletions(-) diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 3a8e120df37..947907977f7 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -17,14 +17,12 @@ use futures::pin_mut; use futures::select; use itertools::Itertools; use vortex_array::ArrayContext; -use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; use vortex_array::expr::stats::Stat; use vortex_array::iter::ArrayIterator; use vortex_array::iter::ArrayIteratorExt; -use vortex_array::session::ArrayRegistry; use vortex_array::session::ArraySessionExt; use vortex_array::stats::PRUNING_STATS; use vortex_array::stream::ArrayStream; @@ -80,37 +78,17 @@ pub trait WriteOptionsSessionExt: SessionExt { /// Create [`VortexWriteOptions`] for writing to a Vortex file. fn write_options(&self) -> VortexWriteOptions { let session = self.session(); - let strategy = default_write_strategy(&session); - VortexWriteOptions { - strategy, - session, - exclude_dtype: false, - file_statistics: PRUNING_STATS.to_vec(), - max_variable_length_statistics_size: 64, - } + VortexWriteOptions::new(session) } } impl WriteOptionsSessionExt for S {} -fn default_write_strategy(session: &VortexSession) -> Arc { - WriteStrategyBuilder::default() - .with_allow_encodings(session.enabled_encoding_ids().into_iter().collect()) - .build() -} - -fn initial_array_ids(session: &VortexSession, registry: &ArrayRegistry) -> Vec { - session - .enabled_encoding_ids() - .into_iter() - .filter(|id| registry.find(id).is_some()) - .sorted() - .collect() -} - impl VortexWriteOptions { /// Create a new [`VortexWriteOptions`] with the given session. pub fn new(session: VortexSession) -> Self { - let strategy = default_write_strategy(&session); + let strategy = WriteStrategyBuilder::default() + .with_allow_encodings(session.enabled_encoding_ids().into_iter().collect()) + .build(); VortexWriteOptions { strategy, session, @@ -180,11 +158,10 @@ impl VortexWriteOptions { // Pre-populate the array context with the registered encodings selected by the enabled // editions. This keeps the serialized ordering deterministic without advertising every // encoding the session happens to know how to read. - let array_registry = self.session.arrays().registry().clone(); - let ctx = ArrayContext::new(initial_array_ids(&self.session, &array_registry)) + let ctx = ArrayContext::new(self.session.enabled_encoding_ids()) // The registry supplies serialization implementations; the layout strategy applies // the enabled-edition write policy before arrays reach serialization. - .with_registry(array_registry); + .with_registry(self.session.arrays().registry().clone()); let dtype = stream.dtype().clone(); let (mut ptr, eof) = SequenceId::root().split(); @@ -565,8 +542,6 @@ mod tests { use vortex_edition::EditionSession; use vortex_edition::EditionSessionExt; - use super::initial_array_ids; - #[test] fn initial_array_ids_are_registered_and_enabled() -> Result<(), vortex_edition::EditionError> { const EDITION: EditionId = EditionId::new("test", 2026, 7, 0); @@ -583,7 +558,7 @@ mod tests { session.enable_edition(EDITION)?; let registry = session.arrays().registry().clone(); - let ids = initial_array_ids(&session, ®istry); + let ids = session.enabled_encoding_ids(); assert_eq!(ids, [Primitive.id()]); assert!(!ids.contains(&Bool.id())); Ok(()) From c85b1c3b62ed0ec7a6eca2e01a408d14130ae12c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 22 Jul 2026 10:48:35 +0000 Subject: [PATCH 7/7] Fix CI: intersect initial array ids with the registry, docs links, doctest sizes - Implement the intended semantics of the writer's pre-populated array context: `initial_array_ids` is the intersection of the session's enabled-edition encodings with the registered encodings, since an enabled edition may include encodings the session cannot serialize (e.g. an optional crate that is not compiled in). This completes the `initial_array_ids_are_registered_and_enabled` test and removes the unused `registry` binding and `Itertools` import that failed the lint jobs. - Qualify the intra-doc links in the editions module docs: links in the module's inner docs resolve at the facade's `pub mod` site, so unqualified links break rustdoc under -D warnings. - Restore the updated file-size golden values in the Python write-options doctests: the footer's encoding list changed with the edition-derived array context, shrinking the written files by 40 bytes. Signed-off-by: Claude Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CK2nCuXnyd2g3mNHQC8Lz2 --- vortex-file/src/writer.rs | 28 ++++++++++++++++++++++------ vortex-python/src/io.rs | 4 ++-- vortex/src/editions/mod.rs | 9 +++++---- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 947907977f7..58aab3732c8 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -15,8 +15,8 @@ use futures::future::LocalBoxFuture; use futures::future::ready; use futures::pin_mut; use futures::select; -use itertools::Itertools; use vortex_array::ArrayContext; +use vortex_array::ArrayId; use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; @@ -157,8 +157,9 @@ impl VortexWriteOptions { ) -> VortexResult { // Pre-populate the array context with the registered encodings selected by the enabled // editions. This keeps the serialized ordering deterministic without advertising every - // encoding the session happens to know how to read. - let ctx = ArrayContext::new(self.session.enabled_encoding_ids()) + // encoding the session happens to read or every enabled encoding the session cannot + // serialize. + let ctx = ArrayContext::new(initial_array_ids(&self.session)) // The registry supplies serialization implementations; the layout strategy applies // the enabled-edition write policy before arrays reach serialization. .with_registry(self.session.arrays().registry().clone()); @@ -529,19 +530,35 @@ impl WriteSummary { } } +/// The encoding ids used to pre-populate the writer's array context: every encoding of the +/// session's enabled editions that is also registered with the session, in sorted order. +/// +/// An enabled edition may include encodings the session has no registration for (e.g. an +/// optional encoding crate that is not compiled in); those cannot be serialized, so they are +/// not advertised in the context. +fn initial_array_ids(session: &VortexSession) -> Vec { + let registry = session.arrays().registry().clone(); + session + .enabled_encoding_ids() + .into_iter() + .filter(|id| registry.find(id).is_some()) + .collect() +} + #[cfg(test)] mod tests { use vortex_array::VTable; use vortex_array::array_session; use vortex_array::arrays::Bool; use vortex_array::arrays::Primitive; - use vortex_array::session::ArraySessionExt; use vortex_edition::Edition; use vortex_edition::EditionDeclaration; use vortex_edition::EditionId; use vortex_edition::EditionSession; use vortex_edition::EditionSessionExt; + use super::initial_array_ids; + #[test] fn initial_array_ids_are_registered_and_enabled() -> Result<(), vortex_edition::EditionError> { const EDITION: EditionId = EditionId::new("test", 2026, 7, 0); @@ -557,8 +574,7 @@ mod tests { session.register_edition(&DECLARATION)?; session.enable_edition(EDITION)?; - let registry = session.arrays().registry().clone(); - let ids = session.enabled_encoding_ids(); + let ids = initial_array_ids(&session); assert_eq!(ids, [Primitive.id()]); assert!(!ids.contains(&Bool.id())); Ok(()) diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs index 182ab7fbe74..c411b4035e9 100644 --- a/vortex-python/src/io.rs +++ b/vortex-python/src/io.rs @@ -258,7 +258,7 @@ impl PyVortexWriteOptions { /// >>> vx.io.VortexWriteOptions.default().write(sprl, "chonky.vortex") /// >>> import os /// >>> os.path.getsize('chonky.vortex') - /// 215940 + /// 215900 /// /// Wow, Vortex manages to use about two bytes per integer! So advanced. So tiny. /// @@ -268,7 +268,7 @@ impl PyVortexWriteOptions { /// /// >>> vx.io.VortexWriteOptions.compact().write(sprl, "tiny.vortex") /// >>> os.path.getsize('tiny.vortex') - /// 55068 + /// 55028 /// /// Random numbers are not (usually) composed of random bytes! #[staticmethod] diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs index 6d60fa2b0b7..0a425b27096 100644 --- a/vortex/src/editions/mod.rs +++ b/vortex/src/editions/mod.rs @@ -5,12 +5,13 @@ //! //! [`vortex_edition`] provides the types, session variables, and test harness. The actual //! first-party declarations live here, one module per edition. The default session first -//! registers them with [`register_default_editions`] and then selects its write policy with -//! [`enable_default_editions`]. +//! registers them with [`crate::editions::register_default_editions`] and then selects its +//! write policy with [`crate::editions::enable_default_editions`]. //! //! The default file writer resolves the session's enabled editions at write time. The -//! facade enables the newest frozen `core` edition, [`CORE_2026_07_0`], and additionally -//! enables the latest unstable edition when the `unstable_encodings` feature is selected. +//! facade enables the newest frozen `core` edition, [`crate::editions::CORE_2026_07_0`], +//! and additionally enables the latest unstable edition when the `unstable_encodings` +//! feature is selected. pub mod core; #[cfg(test)]