Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions vortex-arrow/src/executor/bool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ use vortex_array::ArrayRef;
use vortex_array::ExecutionCtx;
use vortex_array::arrays::BoolArray;
use vortex_array::arrays::bool::BoolArrayExt;
use vortex_array::dtype::DType;
use vortex_error::VortexResult;
use vortex_error::vortex_ensure;

use crate::null_buffer::to_null_buffer;

Expand All @@ -33,6 +35,11 @@ pub(super) fn to_arrow_bool(
array: ArrayRef,
ctx: &mut ExecutionCtx,
) -> VortexResult<ArrowArrayRef> {
vortex_ensure!(
matches!(array.dtype(), DType::Bool(_)),
"Cannot convert Vortex array with dtype {} to an Arrow Boolean array",
array.dtype()
);
let bool_array = array.execute::<BoolArray>(ctx)?;
canonical_bool_to_arrow(&bool_array, ctx)
}
9 changes: 9 additions & 0 deletions vortex-arrow/src/executor/decimal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,11 @@ use num_traits::ToPrimitive;
use vortex_array::ArrayRef;
use vortex_array::ExecutionCtx;
use vortex_array::arrays::DecimalArray;
use vortex_array::dtype::DType;
use vortex_array::dtype::DecimalType;
use vortex_buffer::Buffer;
use vortex_error::VortexResult;
use vortex_error::vortex_ensure;
use vortex_error::vortex_err;

use crate::null_buffer::to_null_buffer;
Expand All @@ -28,6 +30,13 @@ pub(super) fn to_arrow_decimal(
data_type: &DataType,
ctx: &mut ExecutionCtx,
) -> VortexResult<ArrowArrayRef> {
vortex_ensure!(
matches!(array.dtype(), DType::Decimal(..)),
"Cannot convert Vortex array with dtype {} to an Arrow {} array",
array.dtype(),
data_type
);

// Execute the array as a DecimalArray.
let decimal_array = array.execute::<DecimalArray>(ctx)?;

Expand Down
7 changes: 7 additions & 0 deletions vortex-arrow/src/executor/fixed_size_list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use vortex_array::arrays::FixedSizeList;
use vortex_array::arrays::FixedSizeListArray;
use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
use vortex_array::arrays::fixed_size_list::FixedSizeListArraySlotsExt;
use vortex_array::dtype::DType;
use vortex_error::VortexResult;
use vortex_error::vortex_ensure;

Expand All @@ -22,6 +23,12 @@ pub(super) fn to_arrow_fixed_list(
elements_field: &FieldRef,
ctx: &mut ExecutionCtx,
) -> VortexResult<arrow_array::ArrayRef> {
vortex_ensure!(
matches!(array.dtype(), DType::FixedSizeList(..)),
"Cannot convert Vortex array with dtype {} to an Arrow FixedSizeList array",
array.dtype()
);

// Check for Vortex FixedSizeListArray and convert directly.
if let Some(array) = array.as_opt::<FixedSizeList>() {
return list_to_list(&array.into_owned(), elements_field, list_size, ctx);
Expand Down
6 changes: 6 additions & 0 deletions vortex-arrow/src/executor/list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ pub(super) fn to_arrow_list<O: OffsetSizeTrait + NativePType>(
elements_field: &FieldRef,
ctx: &mut ExecutionCtx,
) -> VortexResult<ArrowArrayRef> {
vortex_ensure!(
matches!(array.dtype(), DType::List(..)),
"Cannot convert Vortex array with dtype {} to an Arrow list array",
array.dtype()
);

let array = array.execute_until::<ArrowListExportable>(ctx)?;

// If the Vortex array is already in List format, we can directly convert it.
Expand Down
6 changes: 6 additions & 0 deletions vortex-arrow/src/executor/list_view.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ pub(super) fn to_arrow_list_view<O: OffsetSizeTrait + IntegerPType>(
elements_field: &FieldRef,
ctx: &mut ExecutionCtx,
) -> VortexResult<arrow_array::ArrayRef> {
vortex_ensure!(
matches!(array.dtype(), DType::List(..)),
"Cannot convert Vortex array with dtype {} to an Arrow list-view array",
array.dtype()
);

let array = array.execute::<ListViewArray>(ctx)?;

// Reclaim unreferenced elements before handing the array to Arrow. Otherwise downstream
Expand Down
91 changes: 91 additions & 0 deletions vortex-arrow/src/executor/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -251,3 +251,94 @@ fn infer_nearest_arrow_type(array: &ArrayRef) -> VortexResult<DataType> {
// Everything else: use canonical dtype conversion
to_data_type_naive(array.dtype())
}

#[cfg(test)]
mod tests {
use std::sync::Arc;

use arrow_schema::DataType;
use arrow_schema::Field;
use arrow_schema::TimeUnit;
use rstest::rstest;
use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::VortexSessionExecute;
use vortex_array::array_session;
use vortex_array::arrays::BoolArray;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::arrays::VarBinViewArray;

use crate::ArrowSessionExt;

fn utf8() -> ArrayRef {
VarBinViewArray::from_iter_str(["a", "bb"]).into_array()
}

fn primitive() -> ArrayRef {
PrimitiveArray::from_iter([1i32, 2]).into_array()
}

fn boolean() -> ArrayRef {
BoolArray::from_iter([true, false]).into_array()
}

fn list_target() -> DataType {
DataType::List(Arc::new(Field::new("item", DataType::Int32, true)))
}

/// Must error rather than panic inside `execute::<T>`.
#[rstest]
#[case::bool_from_utf8(utf8(), DataType::Boolean)]
#[case::bool_from_primitive(primitive(), DataType::Boolean)]
#[case::null_from_utf8(utf8(), DataType::Null)]
#[case::null_from_primitive(primitive(), DataType::Null)]
#[case::decimal_from_utf8(utf8(), DataType::Decimal128(10, 2))]
#[case::decimal_from_primitive(primitive(), DataType::Decimal128(10, 2))]
#[case::list_from_utf8(utf8(), list_target())]
#[case::list_from_primitive(primitive(), list_target())]
#[case::list_view_from_primitive(
primitive(),
DataType::ListView(Arc::new(Field::new("item", DataType::Int32, true)))
)]
#[case::fixed_size_list_from_utf8(
utf8(),
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int32, true)), 2)
)]
#[case::byte_from_primitive(primitive(), DataType::Utf8)]
#[case::byte_from_bool(boolean(), DataType::Binary)]
fn incompatible_target_returns_error(#[case] array: ArrayRef, #[case] target: DataType) {
let session = array_session();
let mut ctx = session.create_execution_ctx();
let field = Field::new("f", target.clone(), array.dtype().is_nullable());

let result = session.arrow().execute_arrow(array, Some(&field), &mut ctx);

assert!(
result.is_err(),
"expected an error exporting to {target:?}, got Ok"
);
}

/// Cross-class conversions that are genuinely supported must keep working.
#[rstest]
#[case::bool_to_int32(boolean(), DataType::Int32)]
#[case::bool_to_float64(boolean(), DataType::Float64)]
#[case::primitive_to_int64(primitive(), DataType::Int64)]
#[case::primitive_to_date32(primitive(), DataType::Date32)]
#[case::primitive_to_timestamp(primitive(), DataType::Timestamp(TimeUnit::Microsecond, None))]
#[case::utf8_to_binary(utf8(), DataType::Binary)]
#[case::utf8_to_large_utf8(utf8(), DataType::LargeUtf8)]
fn supported_cross_class_target_still_works(#[case] array: ArrayRef, #[case] target: DataType) {
let session = array_session();
let mut ctx = session.create_execution_ctx();
let field = Field::new("f", target.clone(), array.dtype().is_nullable());

let result = session.arrow().execute_arrow(array, Some(&field), &mut ctx);

assert!(
result.is_ok(),
"expected {target:?} export to succeed, got {:?}",
result.err()
);
}
}
7 changes: 7 additions & 0 deletions vortex-arrow/src/executor/null.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ use arrow_array::NullArray as ArrowNullArray;
use vortex_array::ArrayRef;
use vortex_array::ExecutionCtx;
use vortex_array::arrays::NullArray;
use vortex_array::dtype::DType;
use vortex_error::VortexResult;
use vortex_error::vortex_ensure;

/// Convert a canonical NullArray directly to Arrow.
pub fn canonical_null_to_arrow(array: &NullArray) -> ArrowArrayRef {
Expand All @@ -19,6 +21,11 @@ pub(super) fn to_arrow_null(
array: ArrayRef,
ctx: &mut ExecutionCtx,
) -> VortexResult<ArrowArrayRef> {
vortex_ensure!(
matches!(array.dtype(), DType::Null),
"Cannot convert Vortex array with dtype {} to an Arrow Null array",
array.dtype()
);
let null_array = array.execute::<NullArray>(ctx)?;
Ok(canonical_null_to_arrow(&null_array))
}
Loading