From 929fd2e0d7a4cefd533dbd11c2da5e3bfc40637d Mon Sep 17 00:00:00 2001 From: vishnu prakash Date: Wed, 22 Jul 2026 23:12:57 +0530 Subject: [PATCH] Parquet: Fix variant BINARY upper bound to truncate up (#16880) * Parquet: Fix variant BINARY upper bound to truncate up * Parquet: Add test for variant BINARY upper bound overflow (cherry picked from commit 470f4642bdda336bac5372c79c762a2654dfe3f3) --- .../iceberg/parquet/ParquetVariantUtil.java | 2 +- .../iceberg/parquet/TestVariantMetrics.java | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetVariantUtil.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetVariantUtil.java index 22ea1de147cd..f8bc2a8b035d 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetVariantUtil.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetVariantUtil.java @@ -333,7 +333,7 @@ private static VariantValue truncateUpperBound(VariantValue value) { return truncatedString != null ? Variants.of(PhysicalType.STRING, truncatedString) : null; case BINARY: ByteBuffer truncatedBuffer = - BinaryUtil.truncateBinaryMin((ByteBuffer) value.asPrimitive().get(), 16); + BinaryUtil.truncateBinaryMax((ByteBuffer) value.asPrimitive().get(), 16); return truncatedBuffer != null ? Variants.of(PhysicalType.BINARY, truncatedBuffer) : null; default: return value; diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java index aa1c0e942a22..5057290e77db 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java @@ -199,6 +199,56 @@ public void testShreddedPrimitiveTypeMismatch(VariantValue value) throws IOExcep .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 1L))); } + @Test + public void testShreddedBinaryBoundsTruncation() throws IOException { + // binary longer than the 16-byte truncation length so the bounds are truncated + byte[] bytes = new byte[20]; + for (int i = 0; i < bytes.length; i += 1) { + bytes[i] = (byte) (i + 1); + } + VariantValue value = Variants.of(ByteBuffer.wrap(bytes)); + + Metrics metrics = + writeParquet( + (id, name) -> ParquetVariantUtil.toParquetSchema(value), + Variant.of(EMPTY, value), + Variant.of(EMPTY, Variants.ofNull()), + null); + + assertThat(metrics.lowerBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(BinaryUtil.truncateBinaryMin(ByteBuffer.wrap(bytes), 16))); + + assertThat(metrics.upperBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(BinaryUtil.truncateBinaryMax(ByteBuffer.wrap(bytes), 16))); + } + + @Test + public void testShreddedBinaryUpperBoundOverflow() throws IOException { + // an all-0xFF binary cannot be truncated up so the upper bound is omitted + byte[] bytes = new byte[20]; + for (int i = 0; i < bytes.length; i += 1) { + bytes[i] = (byte) 0xFF; + } + VariantValue value = Variants.of(ByteBuffer.wrap(bytes)); + + Metrics metrics = + writeParquet( + (id, name) -> ParquetVariantUtil.toParquetSchema(value), + Variant.of(EMPTY, value), + Variant.of(EMPTY, Variants.ofNull()), + null); + + assertThat(metrics.lowerBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(BinaryUtil.truncateBinaryMin(ByteBuffer.wrap(bytes), 16))); + + assertThat(metrics.upperBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isNull(); + } + @Test public void testVariantFloatNaN() throws IOException { // NaN values are not counted because there is no ID for FieldMetrics