From a0c81f8549c9a8c5c6c2b8c4e8bded2adeefe3b0 Mon Sep 17 00:00:00 2001 From: vishnuprakaz Date: Thu, 30 Jul 2026 15:31:09 +0530 Subject: [PATCH] Parquet: Guard ordered predicates on variant/nested types in metrics row group filter --- .../data/TestMetricsRowGroupFilter.java | 56 +++++++++++++++++++ .../parquet/ParquetMetricsRowGroupFilter.java | 28 ++++++++++ 2 files changed, 84 insertions(+) diff --git a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java index edb01d7e77ef..a8e19f32492e 100644 --- a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java +++ b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java @@ -1259,6 +1259,62 @@ public void testVariantFieldIn() throws IOException { .isTrue(); } + @TestTemplate + public void testVariantFieldLt() throws IOException { + assumeThat(format).isEqualTo(FileFormat.PARQUET); + + VariantMetadata md = Variants.metadata("k"); + Variant v0 = createVariantWithKey(md, "v0"); + List records = createVariantRecords(v0); + + boolean shouldRead = shouldReadVariant(lessThan("variant_field", v0), records); + assertThat(shouldRead) + .as("Should read: variant lt filters must be evaluated post scan") + .isTrue(); + } + + @TestTemplate + public void testVariantFieldLtEq() throws IOException { + assumeThat(format).isEqualTo(FileFormat.PARQUET); + + VariantMetadata md = Variants.metadata("k"); + Variant v0 = createVariantWithKey(md, "v0"); + List records = createVariantRecords(v0); + + boolean shouldRead = shouldReadVariant(lessThanOrEqual("variant_field", v0), records); + assertThat(shouldRead) + .as("Should read: variant ltEq filters must be evaluated post scan") + .isTrue(); + } + + @TestTemplate + public void testVariantFieldGt() throws IOException { + assumeThat(format).isEqualTo(FileFormat.PARQUET); + + VariantMetadata md = Variants.metadata("k"); + Variant v0 = createVariantWithKey(md, "v0"); + List records = createVariantRecords(v0); + + boolean shouldRead = shouldReadVariant(greaterThan("variant_field", v0), records); + assertThat(shouldRead) + .as("Should read: variant gt filters must be evaluated post scan") + .isTrue(); + } + + @TestTemplate + public void testVariantFieldGtEq() throws IOException { + assumeThat(format).isEqualTo(FileFormat.PARQUET); + + VariantMetadata md = Variants.metadata("k"); + Variant v0 = createVariantWithKey(md, "v0"); + List records = createVariantRecords(v0); + + boolean shouldRead = shouldReadVariant(greaterThanOrEqual("variant_field", v0), records); + assertThat(shouldRead) + .as("Should read: variant gtEq filters must be evaluated post scan") + .isTrue(); + } + @TestTemplate public void testUUID() { assumeThat(format).as("Only valid for Parquet").isEqualTo(FileFormat.PARQUET); diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetricsRowGroupFilter.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetricsRowGroupFilter.java index 0cf4aa2a3633..42b06f7c1378 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetricsRowGroupFilter.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetricsRowGroupFilter.java @@ -237,6 +237,13 @@ public Boolean notNaN(BoundReference ref) { public Boolean lt(BoundReference ref, Literal lit) { int id = ref.fieldId(); + // Leave all nested column type and variant type filters to be + // evaluated post scan. + Type type = schema.findType(id); + if (type instanceof Type.NestedType || type.isVariantType()) { + return ROWS_MIGHT_MATCH; + } + Long valueCount = valueCounts.get(id); if (valueCount == null) { // the column is not present and is all nulls @@ -267,6 +274,13 @@ public Boolean lt(BoundReference ref, Literal lit) { public Boolean ltEq(BoundReference ref, Literal lit) { int id = ref.fieldId(); + // Leave all nested column type and variant type filters to be + // evaluated post scan. + Type type = schema.findType(id); + if (type instanceof Type.NestedType || type.isVariantType()) { + return ROWS_MIGHT_MATCH; + } + Long valueCount = valueCounts.get(id); if (valueCount == null) { // the column is not present and is all nulls @@ -297,6 +311,13 @@ public Boolean ltEq(BoundReference ref, Literal lit) { public Boolean gt(BoundReference ref, Literal lit) { int id = ref.fieldId(); + // Leave all nested column type and variant type filters to be + // evaluated post scan. + Type type = schema.findType(id); + if (type instanceof Type.NestedType || type.isVariantType()) { + return ROWS_MIGHT_MATCH; + } + Long valueCount = valueCounts.get(id); if (valueCount == null) { // the column is not present and is all nulls @@ -327,6 +348,13 @@ public Boolean gt(BoundReference ref, Literal lit) { public Boolean gtEq(BoundReference ref, Literal lit) { int id = ref.fieldId(); + // Leave all nested column type and variant type filters to be + // evaluated post scan. + Type type = schema.findType(id); + if (type instanceof Type.NestedType || type.isVariantType()) { + return ROWS_MIGHT_MATCH; + } + Long valueCount = valueCounts.get(id); if (valueCount == null) { // the column is not present and is all nulls