From 2a114a066e20087fb76894214b1d7bc1fa83a966 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 16:34:30 +0200 Subject: [PATCH 01/12] API: Fix null counts on a required column in evaluator test The single_value_nulls fixtures in testNotEqWithSingleValue and testNotInWithSingleValue reported 2 nulls for field 3, the "required" column. A required column cannot contain nulls, so no writer produces those metrics. Move both to the optional "some_empty" column, which keeps the case the tests cover: a file whose bounds hold a single value still has to be read when the column also contains nulls, because those nulls match != and NOT IN. The NOT IN fixture also reported a NaN count for a string column, which only float and double columns track. --- .../TestInclusiveMetricsEvaluator.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index 8d652df0df03..6166a96380e6 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -1060,14 +1060,14 @@ public void testNotEqWithSingleValue() { "single_value_nulls.avro", Row.of(), 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 2L), + ImmutableMap.of(14, 10L), + ImmutableMap.of(14, 2L), null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc"))); shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")) + new InclusiveMetricsEvaluator(SCHEMA, notEqual("some_empty", "abc")) .eval(singleValueWithNulls); assertThat(shouldRead).as("Should read: file has nulls which match != predicate").isTrue(); @@ -1151,14 +1151,14 @@ public void testNotInWithSingleValue() { "single_value_nulls.avro", Row.of(), 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 2L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); + ImmutableMap.of(14, 10L), + ImmutableMap.of(14, 2L), + null, + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc"))); shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def")) + new InclusiveMetricsEvaluator(SCHEMA, notIn("some_empty", "abc", "def")) .eval(singleValueWithNulls); assertThat(shouldRead).as("Should read: file has nulls which match NOT IN predicate").isTrue(); From 1cab2903580bcfcc1076bee2b30253b2b2c7961d Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 16:51:28 +0200 Subject: [PATCH 02/12] API: Remove NaN counts for a string column in evaluator test The range_of_values and single_value fixtures reported a NaN count for field 3, a string column. Only float and double columns track NaN counts, which is why the single_value fixture in testNotEqWithSingleValue already passed null for them. Generated-by: Cursor --- .../iceberg/expressions/TestInclusiveMetricsEvaluator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index 6166a96380e6..eb00a1b6fed0 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -1022,7 +1022,7 @@ public void testNotEqWithSingleValue() { 10, ImmutableMap.of(3, 10L), ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), + null, ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); @@ -1111,7 +1111,7 @@ public void testNotInWithSingleValue() { 10, ImmutableMap.of(3, 10L), ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), + null, ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); @@ -1128,7 +1128,7 @@ public void testNotInWithSingleValue() { 10, ImmutableMap.of(3, 10L), ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), + null, ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); From e191ca7beafc1e17819dab71c3ef624cd9f8181d Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 14:12:39 +0200 Subject: [PATCH 03/12] API: Fix struct metrics in nested struct evaluator test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `FILE_6` in `TestInclusiveMetricsEvaluator` carried value and null counts for the `required_address` and `optional_address` structs. Parquet produces metrics for leaf columns only in `ParquetMetrics.MetricsVisitor.struct` and returns just the concatenation of its children and never emits a `FieldMetrics` for the struct — so no Parquet file yields those entries. They made `notNull("optional_address")` appear prunable when in practice it is not --- .../expressions/TestInclusiveMetricsEvaluator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index eb00a1b6fed0..e21e69fa59a0 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -210,9 +210,9 @@ public class TestInclusiveMetricsEvaluator { Row.of(), 10, // any value counts, including nulls - ImmutableMap.of(100, 5L, 101, 5L, 102, 5L, 103, 5L, 104, 5L, 105, 5L), + ImmutableMap.of(102, 5L, 103, 5L, 104, 5L, 105, 5L), // null value counts - ImmutableMap.of(100, 0L, 101, 5L, 103, 5L, 104, 5L, 105, 5L), + ImmutableMap.of(103, 5L, 104, 5L, 105, 5L), // nan value counts null, // lower bounds @@ -921,7 +921,7 @@ public void testIsNullInNestedStruct() { // read optional_address and its nested fields shouldRead = new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address")).eval(FILE_6); - assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); + assertThat(shouldRead).as("Should read: metrics are not tracked for structs").isTrue(); shouldRead = new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.required_street2")) @@ -958,7 +958,7 @@ public void testNotNullInNestedStruct() { // read optional_address and its nested fields shouldRead = new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address")).eval(FILE_6); - assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); + assertThat(shouldRead).as("Should read: metrics are not tracked for structs").isTrue(); shouldRead = new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.required_street2")) From 60d7aaa744dbeb10830b1569dbf6fa89a045386a Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Thu, 30 Jul 2026 11:12:29 +0200 Subject: [PATCH 04/12] API, Core: Make TestInclusiveMetricsEvaluator/TestInclusiveMetricsEvaluatorWithExtract/TestInclusiveMetricsEvaluatorWithTransforms extendable --- .../TestInclusiveMetricsEvaluator.java | 793 ++++++++---------- ...tInclusiveMetricsEvaluatorWithExtract.java | 91 +- ...clusiveMetricsEvaluatorWithTransforms.java | 37 +- 3 files changed, 405 insertions(+), 516 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index e21e69fa59a0..310e90c9661c 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -57,8 +57,8 @@ import org.apache.iceberg.util.UnicodeUtil; import org.junit.jupiter.api.Test; -public class TestInclusiveMetricsEvaluator { - private static final Schema SCHEMA = +public class TestInclusiveMetricsEvaluator { + protected static final Schema SCHEMA = new Schema( required(1, "id", IntegerType.get()), optional(2, "no_stats", Types.IntegerType.get()), @@ -75,7 +75,7 @@ public class TestInclusiveMetricsEvaluator { optional(13, "no_nan_stats", Types.DoubleType.get()), optional(14, "some_empty", Types.StringType.get())); - private static final Schema NESTED_SCHEMA = + protected static final Schema NESTED_SCHEMA = new Schema( required( 100, @@ -90,8 +90,10 @@ public class TestInclusiveMetricsEvaluator { required(104, "required_street2", Types.StringType.get()), optional(105, "optional_street2", Types.StringType.get())))); - private static final int INT_MIN_VALUE = 30; - private static final int INT_MAX_VALUE = 79; + protected static final Schema FLOAT_SCHEMA = new Schema(required(1, "f", Types.FloatType.get())); + + protected static final int INT_MIN_VALUE = 30; + protected static final int INT_MAX_VALUE = 79; private static final DataFile FILE = new TestDataFile( @@ -220,78 +222,234 @@ public class TestInclusiveMetricsEvaluator { // upper bounds null); + private static final DataFile MISSING_STATS = new TestDataFile("file.parquet", Row.of(), 50); + + private static final DataFile EMPTY_FILE = new TestDataFile("file.parquet", Row.of(), 0); + + private static final DataFile RANGE_OF_VALUES = + new TestDataFile( + "range_of_values.avro", + Row.of(), + 10, + ImmutableMap.of(3, 10L), + ImmutableMap.of(3, 0L), + null, + ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); + + private static final DataFile SINGLE_VALUE_FILE = + new TestDataFile( + "single_value.avro", + Row.of(), + 10, + ImmutableMap.of(3, 10L), + ImmutableMap.of(3, 0L), + null, + ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); + + // some_empty is optional because a required column cannot contain nulls + private static final DataFile SINGLE_VALUE_WITH_NULLS = + new TestDataFile( + "single_value_nulls.avro", + Row.of(), + 10, + ImmutableMap.of(14, 10L), + ImmutableMap.of(14, 2L), + null, + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc"))); + + private static final DataFile SINGLE_VALUE_WITH_NAN = + new TestDataFile( + "single_value_nan.avro", + Row.of(), + 10, + ImmutableMap.of(9, 10L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, 2L), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); + + private static final DataFile SINGLE_VALUE_NAN_BOUNDS = + new TestDataFile( + "single_value_nan_bounds.avro", + Row.of(), + 10, + ImmutableMap.of(9, 10L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN))); + + private static final Map FLOAT_BOUND = + ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); + + private static final DataFile SINGLE_FLOAT_VALUE_FILE = + new TestDataFile( + "single_value_file.avro", + Row.of(), + 10, + ImmutableMap.of(1, 10L), + ImmutableMap.of(1, 0L), + ImmutableMap.of(1, 0L), + FLOAT_BOUND, + FLOAT_BOUND); + + private static final DataFile SINGLE_FLOAT_VALUE_FILE_WITH_NAN = + new TestDataFile( + "single_value_file.avro", + Row.of(), + 10, + ImmutableMap.of(1, 10L), + ImmutableMap.of(1, 0L), + ImmutableMap.of(1, 1L), // contains a NaN value + FLOAT_BOUND, + FLOAT_BOUND); + + protected boolean shouldRead(Schema schema, Expression expr, boolean caseSensitive, F testFile) { + return new InclusiveMetricsEvaluator(schema, expr, caseSensitive).eval((DataFile) testFile); + } + + protected boolean shouldRead(Schema schema, Expression expr, F testFile) { + return shouldRead(schema, expr, true, testFile); + } + + protected F file() { + return asFile(FILE); + } + + protected F file2() { + return asFile(FILE_2); + } + + protected F file3() { + return asFile(FILE_3); + } + + protected F file4() { + return asFile(FILE_4); + } + + protected F file5() { + return asFile(FILE_5); + } + + protected F file6() { + return asFile(FILE_6); + } + + protected F missingStats() { + return asFile(MISSING_STATS); + } + + protected F emptyFile() { + return asFile(EMPTY_FILE); + } + + protected F rangeOfValues() { + return asFile(RANGE_OF_VALUES); + } + + protected F singleValueFile() { + return asFile(SINGLE_VALUE_FILE); + } + + protected F singleValueWithNulls() { + return asFile(SINGLE_VALUE_WITH_NULLS); + } + + protected F singleValueWithNaN() { + return asFile(SINGLE_VALUE_WITH_NAN); + } + + protected F singleValueNaNBounds() { + return asFile(SINGLE_VALUE_NAN_BOUNDS); + } + + protected F singleFloatValueFile() { + return asFile(SINGLE_FLOAT_VALUE_FILE); + } + + protected F singleFloatValueFileWithNaN() { + return asFile(SINGLE_FLOAT_VALUE_FILE_WITH_NAN); + } + + @SuppressWarnings("unchecked") + private F asFile(DataFile dataFile) { + return (F) dataFile; + } + @Test public void testAllNulls() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("all_nulls")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNull("all_nulls"), file()); assertThat(shouldRead).as("Should skip: no non-null value in all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: lessThan on all null column").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: lessThanOrEqual on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: greaterThan on all null column").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: greaterThanOrEqual on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: equal on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: startsWith on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notStartsWith("all_nulls", "a"), file()); assertThat(shouldRead).as("Should read: notStartsWith on all null column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("some_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNull("some_nulls"), file()); assertThat(shouldRead) .as("Should read: column with some nulls contains a non-null value") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("no_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNull("no_nulls"), file()); assertThat(shouldRead).as("Should read: non-null column contains a non-null value").isTrue(); } @Test public void testNoNulls() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("all_nulls")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, isNull("all_nulls"), file()); assertThat(shouldRead).as("Should read: at least one null value in all null column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("some_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("some_nulls"), file()); assertThat(shouldRead).as("Should read: column with some nulls contains a null value").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("no_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("no_nulls"), file()); assertThat(shouldRead).as("Should skip: non-null column contains no null values").isFalse(); } @Test public void testIsNaN() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nans")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, isNaN("all_nans"), file()); assertThat(shouldRead).as("Should read: at least one nan value in all nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("some_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("some_nans"), file()); assertThat(shouldRead).as("Should read: at least one nan value in some nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("no_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("no_nans"), file()); assertThat(shouldRead).as("Should skip: no-nans column contains no nan values").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nulls_double")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("all_nulls_double"), file()); assertThat(shouldRead).as("Should skip: all-null column doesn't contain nan value").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("no_nan_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("no_nan_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nans_v1_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("all_nans_v1_stats"), file()); assertThat(shouldRead).as("Should read: at least one nan value in all nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("nan_and_null_only")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("nan_and_null_only"), file()); assertThat(shouldRead) .as("Should read: at least one nan value in nan and nulls only column") .isTrue(); @@ -299,35 +457,35 @@ public void testIsNaN() { @Test public void testNotNaN() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nans")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNaN("all_nans"), file()); assertThat(shouldRead) .as("Should skip: column with all nans will not contain non-nan") .isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("some_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("some_nans"), file()); assertThat(shouldRead) .as("Should read: at least one non-nan value in some nan column") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("no_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("no_nans"), file()); assertThat(shouldRead).as("Should read: at least one non-nan value in no nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nulls_double")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("all_nulls_double"), file()); assertThat(shouldRead) .as("Should read: at least one non-nan value in all null column") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("no_nan_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("no_nan_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nans_v1_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("all_nans_v1_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("nan_and_null_only")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("nan_and_null_only"), file()); assertThat(shouldRead) .as("Should read: at least one null value in nan and nulls only column") .isTrue(); @@ -335,25 +493,22 @@ public void testNotNaN() { @Test public void testRequiredColumn() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("required")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNull("required"), file()); assertThat(shouldRead).as("Should read: required columns are always non-null").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("required")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("required"), file()); assertThat(shouldRead).as("Should skip: required columns are always non-null").isFalse(); } @Test public void testMissingColumn() { - assertThatThrownBy( - () -> new InclusiveMetricsEvaluator(SCHEMA, lessThan("missing", 5)).eval(FILE)) + assertThatThrownBy(() -> shouldRead(SCHEMA, lessThan("missing", 5), file())) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot find field 'missing'"); } @Test public void testMissingStats() { - DataFile missingStats = new TestDataFile("file.parquet", Row.of(), 50); - Expression[] exprs = new Expression[] { lessThan("no_stats", 5), @@ -369,15 +524,13 @@ public void testMissingStats() { }; for (Expression expr : exprs) { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, expr).eval(missingStats); + boolean shouldRead = shouldRead(SCHEMA, expr, missingStats()); assertThat(shouldRead).as("Should read when missing stats for expr: " + expr).isTrue(); } } @Test public void testZeroRecordFile() { - DataFile empty = new TestDataFile("file.parquet", Row.of(), 0); - Expression[] exprs = new Expression[] { lessThan("id", 5), @@ -393,7 +546,7 @@ public void testZeroRecordFile() { }; for (Expression expr : exprs) { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, expr).eval(empty); + boolean shouldRead = shouldRead(SCHEMA, expr, emptyFile()); assertThat(shouldRead).as("Should never read 0-record file: " + expr).isFalse(); } } @@ -401,13 +554,10 @@ public void testZeroRecordFile() { @Test public void testNot() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(lessThan("id", INT_MIN_VALUE - 25))).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(lessThan("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should read: not(false)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(greaterThan("id", INT_MIN_VALUE - 25))) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(greaterThan("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should skip: not(true)").isFalse(); } @@ -415,28 +565,24 @@ public void testNot() { public void testAnd() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MIN_VALUE - 30))) - .eval(FILE); + shouldRead( + SCHEMA, + and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MIN_VALUE - 30)), + file()); assertThat(shouldRead).as("Should skip: and(false, true)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MAX_VALUE + 1))) - .eval(FILE); + shouldRead( + SCHEMA, + and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1)), + file()); assertThat(shouldRead).as("Should skip: and(false, false)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id", INT_MIN_VALUE))) - .eval(FILE); + shouldRead( + SCHEMA, + and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id", INT_MIN_VALUE)), + file()); assertThat(shouldRead).as("Should read: and(true, true)").isTrue(); } @@ -444,398 +590,312 @@ public void testAnd() { public void testOr() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1))) - .eval(FILE); + shouldRead( + SCHEMA, + or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1)), + file()); assertThat(shouldRead).as("Should skip: or(false, false)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - or( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MAX_VALUE - 19))) - .eval(FILE); + shouldRead( + SCHEMA, + or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE - 19)), + file()); assertThat(shouldRead).as("Should read: or(false, true)").isTrue(); } @Test public void testIntegerLt() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (5 < 30)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file()); assertThat(shouldRead) .as("Should not read: id range below lower bound (30 is not < 30)") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE + 1), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerLtEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (5 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (29 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerGt() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE + 6)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (85 < 79)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE), file()); assertThat(shouldRead) .as("Should not read: id range above upper bound (79 is not > 79)") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerGtEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 6)) - .eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (85 < 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (80 > 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id below lower bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id below lower bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should not read: id above upper bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE + 6)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id above upper bound").isFalse(); } @Test public void testIntegerNotEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE + 6)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testIntegerNotEqRewritten() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE - 25))).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE - 1))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 1)), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE)), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE - 4))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE - 4)), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE)), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE + 1))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 1)), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE + 6))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 6)), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testCaseInsensitiveIntegerNotEqRewritten() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE - 25)), false) - .eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 25)), false, file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)), false, file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false, file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)), false, file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false, file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)), false, file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)), false, file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testCaseSensitiveIntegerNotEqRewritten() { - assertThatThrownBy( - () -> new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", 5)), true).eval(FILE)) + assertThatThrownBy(() -> shouldRead(SCHEMA, not(equal("ID", 5)), true, file())) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot find field 'ID'"); } @Test public void testStringStartsWith() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"), true).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true, file()); assertThat(shouldRead).as("Should read: no stats").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aaa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aaa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1s"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "1s"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1str1x"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "1str1x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "ff"), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, startsWith("required", "ff"), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aB"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aB"), true, file2()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "dWX"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "dWX"), true, file2()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "5"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "5"), true, file3()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "3str3x"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "3str3x"), true, file3()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("some_empty", "房东整租霍"), true).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("some_empty", "房东整租霍"), true, file()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", ""), true).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", ""), true, file()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", aboveMax), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, startsWith("required", aboveMax), true, file4()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); } @Test public void testStringNotStartsWith() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"), true).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true, file()); assertThat(shouldRead).as("Should read: no stats").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aaa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aaa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1s"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1s"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1str1x"), true) - .eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1str1x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "ff"), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "ff"), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aB"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aB"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "dWX"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "dWX"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "5"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "5"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "3str3x"), true) - .eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "3str3x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", aboveMax), true) - .eval(FILE_4); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", aboveMax), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "abc"), true).eval(FILE_5); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abc"), true, file5()); assertThat(shouldRead).as("Should not read: all strings start with prefix").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "abcd"), true).eval(FILE_5); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abcd"), true, file5()); assertThat(shouldRead).as("Should not read: lower shorter than prefix, cannot match").isTrue(); } @Test public void testIntegerIn() { boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)) - .eval(FILE); + shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24), file()); assertThat(shouldRead).as("Should not read: id below lower bound (5 < 30, 6 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id below lower bound (28 < 30, 29 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 1, INT_MIN_VALUE)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 1, INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound (30 == 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3), file()); assertThat(shouldRead) .as("Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id equal to upper bound (79 == 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2), file()); assertThat(shouldRead).as("Should not read: id above upper bound (80 > 79, 81 > 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7), file()); assertThat(shouldRead).as("Should not read: id above upper bound (85 > 79, 86 > 79)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("all_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("all_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should skip: in on all nulls column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("some_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("some_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: in on some nulls column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("no_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("no_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: in on no nulls column").isTrue(); // should read as the number of elements in the in expression is too big @@ -843,131 +903,97 @@ public void testIntegerIn() { for (int id = -400; id <= 0; id++) { ids.add(id); } - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("id", ids)).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", ids), file()); assertThat(shouldRead).as("Should read: large in expression").isTrue(); } @Test public void testIntegerNotIn() { boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)) - .eval(FILE); + shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24), file()); assertThat(shouldRead).as("Should read: id below lower bound (5 < 30, 6 < 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should read: id below lower bound (28 < 30, 29 < 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 1, INT_MIN_VALUE)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 1, INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound (30 == 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3), file()); assertThat(shouldRead) .as("Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id equal to upper bound (79 == 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2), file()); assertThat(shouldRead).as("Should read: id above upper bound (80 > 79, 81 > 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7), file()); assertThat(shouldRead).as("Should read: id above upper bound (85 > 79, 86 > 79)").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("all_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("all_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on all nulls column").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("some_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("some_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on some nulls column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("no_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on no nulls column").isTrue(); } @Test public void testIsNullInNestedStruct() { // read required_address and its nested fields - boolean shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address")).eval(FILE_6); + boolean shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address"), file6()); assertThat(shouldRead).as("Should not read: required_address is required").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address.required_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address.required_street1"), file6()); assertThat(shouldRead) .as("Should not read: required_address.required_street1 is required") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address.optional_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address.optional_street1"), file6()); assertThat(shouldRead) .as("Should read: required_address.optional_street1 is optional") .isTrue(); // read optional_address and its nested fields - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address")).eval(FILE_6); - assertThat(shouldRead).as("Should read: metrics are not tracked for structs").isTrue(); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address"), file6()); + assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.required_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address.required_street2"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.optional_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address.optional_street2"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); } @Test public void testNotNullInNestedStruct() { // read required_address and its nested fields - boolean shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address")).eval(FILE_6); + boolean shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address"), file6()); assertThat(shouldRead).as("Should read: required_address is required").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address.required_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address.required_street1"), file6()); assertThat(shouldRead) .as("Should read: required_address.required_street1 is required") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address.optional_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address.optional_street1"), file6()); assertThat(shouldRead) .as("Should not read: required_address.optional_street1 is optional") .isFalse(); // read optional_address and its nested fields - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address")).eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address"), file6()); assertThat(shouldRead).as("Should read: metrics are not tracked for structs").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.required_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.optional_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.optional_street2"), file6()); assertThat(shouldRead) .as("Should not read: optional_address.optional_street2 is optional") .isFalse(); @@ -975,207 +1001,66 @@ public void testNotNullInNestedStruct() { @Test public void testNotEqSingleValueWithoutNaN() { - Schema schema = new Schema(required(1, "f", Types.FloatType.get())); - Map bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); - DataFile singleValueFile = - new TestDataFile( - "single_value_file.avro", - Row.of(), - 10, - ImmutableMap.of(1, 10L), - ImmutableMap.of(1, 0L), - ImmutableMap.of(1, 0L), - bound, - bound); - - assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile)) + assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFile())) .as("Should skip: file contains no values not equal to 1.0") .isFalse(); } @Test public void testNotEqSingleValueWithNaN() { - Schema schema = new Schema(required(1, "f", Types.FloatType.get())); - Map bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); - DataFile singleValueFile = - new TestDataFile( - "single_value_file.avro", - Row.of(), - 10, - ImmutableMap.of(1, 10L), - ImmutableMap.of(1, 0L), - ImmutableMap.of(1, 1L), // contains a NaN value - bound, - bound); - - assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile)) + assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFileWithNaN())) .as("Should read: file contains a NaN value not equal to 1.0") .isTrue(); } @Test public void testNotEqWithSingleValue() { - DataFile rangeOfValues = - new TestDataFile( - "range_of_values.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); - - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "aaa")).eval(rangeOfValues); + boolean shouldRead = shouldRead(SCHEMA, notEqual("required", "aaa"), rangeOfValues()); assertThat(shouldRead) .as("Should read: file has range of values, cannot exclude based on literal") .isTrue(); - DataFile singleValueFile = - new TestDataFile( - "single_value.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")).eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notEqual("required", "abc"), singleValueFile()); assertThat(shouldRead) .as("Should not read: file contains single value equal to literal") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "def")).eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notEqual("required", "def"), singleValueFile()); assertThat(shouldRead) .as("Should read: file contains single value not equal to literal") .isTrue(); - DataFile singleValueWithNulls = - new TestDataFile( - "single_value_nulls.avro", - Row.of(), - 10, - ImmutableMap.of(14, 10L), - ImmutableMap.of(14, 2L), - null, - ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("some_empty", "abc")) - .eval(singleValueWithNulls); + shouldRead = shouldRead(SCHEMA, notEqual("some_empty", "abc"), singleValueWithNulls()); assertThat(shouldRead).as("Should read: file has nulls which match != predicate").isTrue(); - DataFile singleValueWithNaN = - new TestDataFile( - "single_value_nan.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 2L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueWithNaN); + shouldRead = shouldRead(SCHEMA, notEqual("no_nans", 5.0F), singleValueWithNaN()); assertThat(shouldRead).as("Should read: file has NaN values which match != predicate").isTrue(); - DataFile singleValueNaNBounds = - new TestDataFile( - "single_value_nan_bounds.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueNaNBounds); + shouldRead = shouldRead(SCHEMA, notEqual("no_nans", 5.0F), singleValueNaNBounds()); assertThat(shouldRead).as("Should read: bounds are NaN").isTrue(); } @Test public void testNotInWithSingleValue() { - DataFile rangeOfValues = - new TestDataFile( - "range_of_values.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); - - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "aaa", "bbb")).eval(rangeOfValues); + boolean shouldRead = shouldRead(SCHEMA, notIn("required", "aaa", "bbb"), rangeOfValues()); assertThat(shouldRead) .as("Should read: file has range of values, cannot exclude based on literal") .isTrue(); - DataFile singleValueFile = - new TestDataFile( - "single_value.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def")) - .eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notIn("required", "abc", "def"), singleValueFile()); assertThat(shouldRead) .as("Should not read: file contains single value in exclusion list") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "def", "ghi")) - .eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notIn("required", "def", "ghi"), singleValueFile()); assertThat(shouldRead) .as("Should read: file contains single value not in exclusion list") .isTrue(); - DataFile singleValueWithNulls = - new TestDataFile( - "single_value_nulls.avro", - Row.of(), - 10, - ImmutableMap.of(14, 10L), - ImmutableMap.of(14, 2L), - null, - ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("some_empty", "abc", "def")) - .eval(singleValueWithNulls); + shouldRead = shouldRead(SCHEMA, notIn("some_empty", "abc", "def"), singleValueWithNulls()); assertThat(shouldRead).as("Should read: file has nulls which match NOT IN predicate").isTrue(); - DataFile singleValueWithNaN = - new TestDataFile( - "single_value_nan.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 2L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nans", 5.0F, 10.0F)) - .eval(singleValueWithNaN); + shouldRead = shouldRead(SCHEMA, notIn("no_nans", 5.0F, 10.0F), singleValueWithNaN()); assertThat(shouldRead) .as("Should read: file has NaN values which match NOT IN predicate") .isTrue(); diff --git a/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithExtract.java b/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithExtract.java index 03c1c12f4fba..01944969a9cc 100644 --- a/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithExtract.java +++ b/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithExtract.java @@ -63,15 +63,15 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.FieldSource; -public class TestInclusiveMetricsEvaluatorWithExtract { - private static final Schema SCHEMA = +public class TestInclusiveMetricsEvaluatorWithExtract { + protected static final Schema SCHEMA = new Schema( required(1, "id", IntegerType.get()), required(2, "variant", Types.VariantType.get()), optional(3, "all_nulls", Types.VariantType.get())); - private static final int INT_MIN_VALUE = 30; - private static final int INT_MAX_VALUE = 79; + protected static final int INT_MIN_VALUE = 30; + protected static final int INT_MAX_VALUE = 79; private static final DataFile FILE = new TestDataFile( @@ -108,19 +108,43 @@ public class TestInclusiveMetricsEvaluatorWithExtract { Variants.of("abe"))))); private boolean shouldRead(Expression expr) { - return shouldRead(expr, FILE); + return shouldRead(expr, file()); } - private boolean shouldRead(Expression expr, DataFile file) { + private boolean shouldRead(Expression expr, F file) { return shouldRead(expr, file, true); } private boolean shouldReadCaseInsensitive(Expression expr) { - return shouldRead(expr, FILE, false); + return shouldRead(expr, file(), false); } - private boolean shouldRead(Expression expr, DataFile file, boolean caseSensitive) { - return new InclusiveMetricsEvaluator(SCHEMA, expr, caseSensitive).eval(file); + protected boolean shouldRead(Expression expr, F file, boolean caseSensitive) { + return new InclusiveMetricsEvaluator(SCHEMA, expr, caseSensitive).eval((DataFile) file); + } + + protected F file() { + return asFile(FILE); + } + + protected F emptyFile() { + return asFile(new TestDataFile("file.parquet", Row.of(), 0)); + } + + /** Returns a file with variant bounds for the given path in the {@code variant} column. */ + protected F fileWithVariantBounds(String path, VariantValue lower, VariantValue upper) { + Map lowerBounds = + ImmutableMap.of(2, VariantTestUtil.variantBuffer(Map.of(path, lower))); + Map upperBounds = + ImmutableMap.of(2, VariantTestUtil.variantBuffer(Map.of(path, upper))); + + return asFile( + new TestDataFile("file.parquet", Row.of(), 50, null, null, null, lowerBounds, upperBounds)); + } + + @SuppressWarnings("unchecked") + private F asFile(DataFile dataFile) { + return (F) dataFile; } @Test @@ -223,7 +247,7 @@ public void testMissingStats(Expression expr) { @ParameterizedTest @FieldSource("MISSING_STATS_EXPRESSIONS") public void testZeroRecordFile(Expression expr) { - DataFile empty = new TestDataFile("file.parquet", Row.of(), 0); + F empty = emptyFile(); assertThat(shouldRead(expr, empty)).as("Should never read 0-record file: " + expr).isFalse(); } @@ -851,21 +875,9 @@ public void testIntegerNotIn() { @ParameterizedTest @FieldSource("DATEANDTIMESTAMPTYPESEQPARAMETERS") public void testDateAndTimestampTypesEq(String variantType, Arguments args) { - // lower bounds - Map lowerBounds = - ImmutableMap.of( - 2, - VariantTestUtil.variantBuffer( - Map.of("$['event_timestamp']", (VariantValue) args.get()[1]))); - // upper bounds - Map upperBounds = - ImmutableMap.of( - 2, - VariantTestUtil.variantBuffer( - Map.of("$['event_timestamp']", (VariantValue) args.get()[2]))); - - DataFile file = - new TestDataFile("file.parquet", Row.of(), 50, null, null, null, lowerBounds, upperBounds); + F file = + fileWithVariantBounds( + "$['event_timestamp']", (VariantValue) args.get()[1], (VariantValue) args.get()[2]); Expression expr = equal(extract("variant", "$.event_timestamp", variantType), args.get()[0]); assertThat(shouldRead(expr, file)).isEqualTo(args.get()[3]); } @@ -998,21 +1010,9 @@ public void testDateAndTimestampTypesEq(String variantType, Arguments args) { @ParameterizedTest @FieldSource("DATEANDTIMESTAMPTYPESNOTEQPARAMETERS") public void testDateAndTimestampTypesNotEq(String variantType, Arguments args) { - // lower bounds - Map lowerBounds = - ImmutableMap.of( - 2, - VariantTestUtil.variantBuffer( - Map.of("$['event_timestamp']", (VariantValue) args.get()[1]))); - // upper bounds - Map upperBounds = - ImmutableMap.of( - 2, - VariantTestUtil.variantBuffer( - Map.of("$['event_timestamp']", (VariantValue) args.get()[2]))); - - DataFile file = - new TestDataFile("file.parquet", Row.of(), 50, null, null, null, lowerBounds, upperBounds); + F file = + fileWithVariantBounds( + "$['event_timestamp']", (VariantValue) args.get()[1], (VariantValue) args.get()[2]); Expression expr = notEqual(extract("variant", "$.event_timestamp", variantType), args.get()[0]); assertThat(shouldRead(expr, file)).as("Should read: many possible timestamps" + expr).isTrue(); } @@ -1020,16 +1020,7 @@ public void testDateAndTimestampTypesNotEq(String variantType, Arguments args) { @Test public void testUUIDEq() { UUID uuid = UUID.randomUUID(); - // lower bounds - Map lowerBounds = - ImmutableMap.of( - 2, VariantTestUtil.variantBuffer(Map.of("$['event_uuid']", Variants.ofUUID(uuid)))); - // upper bounds - Map upperBounds = - ImmutableMap.of( - 2, VariantTestUtil.variantBuffer(Map.of("$['event_uuid']", Variants.ofUUID(uuid)))); - DataFile file = - new TestDataFile("file.parquet", Row.of(), 50, null, null, null, lowerBounds, upperBounds); + F file = fileWithVariantBounds("$['event_uuid']", Variants.ofUUID(uuid), Variants.ofUUID(uuid)); Expression expr = equal(extract("variant", "$.event_uuid", PhysicalType.UUID.name()), uuid); assertThat(shouldRead(expr, file)).as("Should read: many possible UUIDs" + expr).isTrue(); } diff --git a/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithTransforms.java b/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithTransforms.java index 41a14667eae1..69db255018de 100644 --- a/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithTransforms.java +++ b/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithTransforms.java @@ -58,8 +58,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.FieldSource; -public class TestInclusiveMetricsEvaluatorWithTransforms { - private static final Schema SCHEMA = +public class TestInclusiveMetricsEvaluatorWithTransforms { + protected static final Schema SCHEMA = new Schema( required(1, "id", IntegerType.get()), required(2, "ts", Types.TimestampType.withZone()), @@ -68,13 +68,13 @@ public class TestInclusiveMetricsEvaluatorWithTransforms { optional(5, "no_stats", IntegerType.get()), optional(6, "str", Types.StringType.get())); - private static final int INT_MIN_VALUE = 30; - private static final int INT_MAX_VALUE = 79; + protected static final int INT_MIN_VALUE = 30; + protected static final int INT_MAX_VALUE = 79; - private static final long TS_MIN_VALUE = + protected static final long TS_MIN_VALUE = DateTimeUtil.microsFromTimestamptz( DateTimeUtil.dateFromDays(30).atStartOfDay().atOffset(ZoneOffset.UTC)); - private static final long TS_MAX_VALUE = + protected static final long TS_MAX_VALUE = DateTimeUtil.microsFromTimestamptz( DateTimeUtil.dateFromDays(79).atStartOfDay().atOffset(ZoneOffset.UTC)); @@ -109,19 +109,32 @@ public class TestInclusiveMetricsEvaluatorWithTransforms { 6, Conversions.toByteBuffer(Types.StringType.get(), "abe"))); private boolean shouldRead(Expression expr) { - return shouldRead(expr, FILE); + return shouldRead(expr, file()); } - private boolean shouldRead(Expression expr, DataFile file) { + private boolean shouldRead(Expression expr, F file) { return shouldRead(expr, file, true); } private boolean shouldReadCaseInsensitive(Expression expr) { - return shouldRead(expr, FILE, false); + return shouldRead(expr, file(), false); } - private boolean shouldRead(Expression expr, DataFile file, boolean caseSensitive) { - return new InclusiveMetricsEvaluator(SCHEMA, expr, caseSensitive).eval(file); + protected boolean shouldRead(Expression expr, F file, boolean caseSensitive) { + return new InclusiveMetricsEvaluator(SCHEMA, expr, caseSensitive).eval((DataFile) file); + } + + protected F file() { + return asFile(FILE); + } + + protected F emptyFile() { + return asFile(new TestDataFile("file.parquet", Row.of(), 0)); + } + + @SuppressWarnings("unchecked") + private F asFile(DataFile dataFile) { + return (F) dataFile; } @Test @@ -289,7 +302,7 @@ public void testMissingStats(Expression expr) { @ParameterizedTest @FieldSource("MISSING_STATS_EXPRESSIONS") public void testZeroRecordFile(Expression expr) { - DataFile empty = new TestDataFile("file.parquet", Row.of(), 0); + F empty = emptyFile(); assertThat(shouldRead(expr, empty)).as("Should never read 0-record file: " + expr).isFalse(); } From e091081e6326f5ed80a0d8a0394c4008401ceb05 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Tue, 28 Jul 2026 18:04:29 +0200 Subject: [PATCH 05/12] Core: Add content stats evaluator --- .../java/org/apache/iceberg/ContentStats.java | 2 +- .../java/org/apache/iceberg/FieldStats.java | 2 +- .../java/org/apache/iceberg/TrackedFile.java | 2 +- .../expressions/InclusiveStatsEvaluator.java | 159 ++++++++++++++ .../org/apache/iceberg/StatsTestUtil.java | 68 ++++++ .../iceberg/TestInclusiveStatsEvaluator.java | 196 ++++++++++++++++++ ...estInclusiveStatsEvaluatorWithExtract.java | 87 ++++++++ ...InclusiveStatsEvaluatorWithTransforms.java | 59 ++++++ 8 files changed, 572 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java create mode 100644 core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java create mode 100644 core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java create mode 100644 core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java diff --git a/core/src/main/java/org/apache/iceberg/ContentStats.java b/core/src/main/java/org/apache/iceberg/ContentStats.java index aa7a65ac934d..51bfa52a67ad 100644 --- a/core/src/main/java/org/apache/iceberg/ContentStats.java +++ b/core/src/main/java/org/apache/iceberg/ContentStats.java @@ -21,7 +21,7 @@ import java.util.Set; import org.apache.iceberg.types.Types; -interface ContentStats { +public interface ContentStats { /** Returns an iterable of the {@link FieldStats} held by this container struct */ Iterable> fieldStats(); diff --git a/core/src/main/java/org/apache/iceberg/FieldStats.java b/core/src/main/java/org/apache/iceberg/FieldStats.java index 495dddba913d..693765a8977b 100644 --- a/core/src/main/java/org/apache/iceberg/FieldStats.java +++ b/core/src/main/java/org/apache/iceberg/FieldStats.java @@ -20,7 +20,7 @@ import org.apache.iceberg.types.Types; -interface FieldStats { +public interface FieldStats { /** The field ID of the statistic */ int fieldId(); diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 9aaeae48d688..374808b7cb53 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -25,7 +25,7 @@ import org.apache.iceberg.types.Types; /** A file tracked by a manifest. */ -interface TrackedFile { +public interface TrackedFile { Types.NestedField TRACKING = Types.NestedField.required( 147, "tracking", Tracking.schema(), "Tracking information for this entry"); diff --git a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java new file mode 100644 index 000000000000..5e88e4c5fd9e --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.expressions; + +import static org.apache.iceberg.expressions.Expressions.rewriteNot; + +import org.apache.iceberg.ContentStats; +import org.apache.iceberg.FieldStats; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TrackedFile; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.variants.Variant; +import org.apache.iceberg.variants.VariantObject; + +/** + * Evaluates an {@link Expression} on a {@link TrackedFile} to test whether rows in the file may + * match. + * + *

This evaluation is inclusive: it returns true if a file may match and false if it cannot + * match. + * + *

Files are passed to {@link #eval(TrackedFile)}, which returns true if the file may contain + * matching rows and false if the file cannot contain matching rows. Files may be skipped if and + * only if the return value of {@code eval} is false. + * + *

Due to the comparison implementation of ORC stats, for float/double columns in ORC files, if + * the first value in a file is NaN, metrics of this file will report NaN for both upper and lower + * bound despite that the column could contain non-NaN data. Thus, in some scenarios explicitly + * checks for NaN is necessary in order to not skip files that may contain matching data. + */ +public class InclusiveStatsEvaluator { + private final Expression expr; + + public InclusiveStatsEvaluator(Schema schema, Expression unbound) { + this(schema, unbound, true); + } + + public InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { + Types.StructType struct = schema.asStruct(); + this.expr = Binder.bind(struct, rewriteNot(unbound), caseSensitive); + } + + /** + * Test whether the file may contain records that match the expression. + * + * @param file a tracked file + * @return false if the file cannot contain rows that match the expression, true otherwise. + */ + public boolean eval(TrackedFile file) { + return new StatsEvalVisitor().eval(file); + } + + private class StatsEvalVisitor extends InclusiveEvalVisitor { + private ContentStats stats = null; + + private boolean eval(TrackedFile file) { + if (file.recordCount() == 0) { + return ROWS_CANNOT_MATCH; + } + + if (file.recordCount() < 0) { + // imported Avro files may have an incorrect -1 row count + return ROWS_MIGHT_MATCH; + } + + if (null == file.contentStats()) { + return ROWS_MIGHT_MATCH; + } + + this.stats = file.contentStats(); + + return ExpressionVisitors.visitEvaluator(expr, this); + } + + @Override + protected boolean mayContainNull(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats == null + || !fieldStats.hasNullValueCount() + || fieldStats.nullValueCount() != 0; + } + + @Override + protected boolean containsNullsOnly(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats != null + && fieldStats.hasValueCount() + && fieldStats.hasNullValueCount() + && fieldStats.valueCount() - fieldStats.nullValueCount() == 0; + } + + @Override + protected boolean mayContainNaN(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats == null + || !fieldStats.hasNanValueCount() + || fieldStats.nanValueCount() != 0; + } + + @Override + protected boolean containsNaNsOnly(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats != null + && fieldStats.hasValueCount() + && fieldStats.hasNanValueCount() + && fieldStats.valueCount() - fieldStats.nanValueCount() == 0; + } + + @Override + protected T lowerBound(BoundReference ref) { + FieldStats fieldStats = stats.statsFor(ref.fieldId()); + return fieldStats != null ? fieldStats.lowerBound() : null; + } + + @Override + protected T upperBound(BoundReference ref) { + FieldStats fieldStats = stats.statsFor(ref.fieldId()); + return fieldStats != null ? fieldStats.upperBound() : null; + } + + @Override + protected T extractLowerBound(BoundExtract bound) { + FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); + if (fieldStats != null && fieldStats.lowerBound() != null) { + VariantObject fieldLowerBounds = fieldStats.lowerBound().value().asObject(); + return VariantExpressionUtil.castTo(fieldLowerBounds.get(bound.path()), bound.type()); + } + + return null; + } + + @Override + protected T extractUpperBound(BoundExtract bound) { + FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); + if (fieldStats != null && fieldStats.upperBound() != null) { + VariantObject fieldUpperBounds = fieldStats.upperBound().value().asObject(); + return VariantExpressionUtil.castTo(fieldUpperBounds.get(bound.path()), bound.type()); + } + + return null; + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java index 1a6ee1923292..32b4c249c2f6 100644 --- a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java +++ b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java @@ -18,12 +18,80 @@ */ package org.apache.iceberg; +import java.util.List; import org.apache.iceberg.types.Types; import org.mockito.Mockito; class StatsTestUtil { + private static final int FORMAT_VERSION_V4 = 4; + private StatsTestUtil() {} + static TrackedFile trackedFile(String location, long recordCount, ContentStats stats) { + return new TrackedFileStruct( + null, + FileContent.DATA, + FORMAT_VERSION_V4, + location, + FileFormat.fromFileName(location), + null, + recordCount, + 1024L, + null, + stats, + null, + null, + null, + null, + null, + null); + } + + static ContentStats contentStats(Types.StructType statsType, FieldStats... fieldStats) { + ContentStatsStruct stats = new ContentStatsStruct(statsType); + for (FieldStats field : fieldStats) { + stats.setStats(field.fieldId(), field); + } + + return stats; + } + + /** Returns the stats for a field, where a null metric is one that the column does not track. */ + static FieldStats fieldStats( + Types.StructType statsType, + int fieldId, + Object lower, + Object upper, + Long valueCount, + Long nullCount, + Long nanCount) { + Types.StructType type = statsType.field(StatsUtil.toBaseId(fieldId)).type().asStructType(); + FieldStatsStruct stats = new FieldStatsStruct<>(type); + set(stats, StatsUtil.LOWER_BOUND_NAME, lower); + set(stats, StatsUtil.UPPER_BOUND_NAME, upper); + set(stats, "value_count", valueCount); + set(stats, "null_value_count", nullCount); + set(stats, "nan_value_count", nanCount); + + return stats; + } + + private static void set(FieldStatsStruct stats, String metric, Object value) { + if (value == null) { + return; + } + + List fields = stats.type().fields(); + for (int pos = 0; pos < fields.size(); pos += 1) { + if (fields.get(pos).name().equals(metric)) { + stats.set(pos, value); + return; + } + } + + throw new IllegalArgumentException("Cannot set " + metric + ": not tracked by " + stats.type()); + } + /** * Mocks a {@link FieldStats} for a stats struct type, stubbing the bounds and only the counts * that are present. A null count leaves the matching {@code has*Count()} reporting false, so a diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java new file mode 100644 index 000000000000..33767d81de41 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.StatsTestUtil.contentStats; +import static org.apache.iceberg.StatsTestUtil.fieldStats; +import static org.apache.iceberg.StatsTestUtil.trackedFile; +import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.InclusiveStatsEvaluator; +import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +class TestInclusiveStatsEvaluator extends TestInclusiveMetricsEvaluator { + private static final Types.StructType STATS_TYPE = + StatsUtil.statsReadSchema( + SCHEMA, ImmutableList.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)); + + // stats are tracked for the leaf fields, not for the address structs themselves + private static final Types.StructType NESTED_STATS_TYPE = + StatsUtil.statsReadSchema(NESTED_SCHEMA, ImmutableList.of(102, 103, 104, 105)); + + private static final Types.StructType FLOAT_STATS_TYPE = + StatsUtil.statsReadSchema(FLOAT_SCHEMA, ImmutableList.of(1)); + + @Override + protected boolean shouldRead( + Schema schema, Expression expr, boolean caseSensitive, TrackedFile testFile) { + return new InclusiveStatsEvaluator(schema, expr, caseSensitive).eval(testFile); + } + + @Override + protected TrackedFile file() { + return trackedFile( + "file.avro", + 50, + contentStats( + STATS_TYPE, + stats(1, INT_MIN_VALUE, INT_MAX_VALUE, null, null, null), + stats(4, null, null, 50L, 50L, null), + stats(5, null, null, 50L, 10L, null), + stats(6, null, null, 50L, 0L, null), + stats(7, null, null, 50L, null, 50L), + stats(8, null, null, 50L, null, 10L), + stats(9, null, null, 50L, null, 0L), + stats(10, null, null, 50L, 50L, null), + stats(11, Float.NaN, Float.NaN, 50L, 0L, null), + stats(12, Double.NaN, Double.NaN, 50L, 1L, null), + stats(13, null, null, 50L, null, null), + stats(14, "", "房东整租霍营小区二层两居室", 50L, 0L, null))); + } + + @Override + protected TrackedFile file2() { + return trackedFile( + "file_2.avro", 50, contentStats(STATS_TYPE, stats(3, "aa", "dC", 50L, null, null))); + } + + @Override + protected TrackedFile file3() { + return trackedFile( + "file_3.avro", 50, contentStats(STATS_TYPE, stats(3, "1str1", "3str3", 50L, null, null))); + } + + @Override + protected TrackedFile file4() { + return trackedFile( + "file_4.avro", 50, contentStats(STATS_TYPE, stats(3, "abc", "イロハニホヘト", 50L, null, null))); + } + + @Override + protected TrackedFile file5() { + return trackedFile( + "file_5.avro", 50, contentStats(STATS_TYPE, stats(3, "abc", "abcdefghi", 50L, null, null))); + } + + @Override + protected TrackedFile file6() { + return trackedFile( + "file_6.avro", + 10, + contentStats( + NESTED_STATS_TYPE, + stats(NESTED_STATS_TYPE, 102, null, null, 5L, null, null), + stats(NESTED_STATS_TYPE, 103, null, null, 5L, 5L, null), + stats(NESTED_STATS_TYPE, 104, null, null, 5L, 5L, null), + stats(NESTED_STATS_TYPE, 105, null, null, 5L, 5L, null))); + } + + @Override + protected TrackedFile missingStats() { + return trackedFile("file.parquet", 50, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile emptyFile() { + return trackedFile("file.parquet", 0, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile rangeOfValues() { + return trackedFile( + "range_of_values.avro", + 10, + contentStats(STATS_TYPE, stats(3, "aaa", "zzz", 10L, null, null))); + } + + @Override + protected TrackedFile singleValueFile() { + return trackedFile( + "single_value.avro", 10, contentStats(STATS_TYPE, stats(3, "abc", "abc", 10L, null, null))); + } + + @Override + protected TrackedFile singleValueWithNulls() { + return trackedFile( + "single_value_nulls.avro", + 10, + contentStats(STATS_TYPE, stats(14, "abc", "abc", 10L, 2L, null))); + } + + @Override + protected TrackedFile singleValueWithNaN() { + return trackedFile( + "single_value_nan.avro", 10, contentStats(STATS_TYPE, stats(9, 5.0f, 5.0f, 10L, 0L, 2L))); + } + + @Override + protected TrackedFile singleValueNaNBounds() { + return trackedFile( + "single_value_nan_bounds.avro", + 10, + contentStats(STATS_TYPE, stats(9, Float.NaN, Float.NaN, 10L, 0L, 0L))); + } + + @Override + protected TrackedFile singleFloatValueFile() { + return trackedFile( + "single_value_file.avro", + 10, + contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 0L))); + } + + @Override + protected TrackedFile singleFloatValueFileWithNaN() { + return trackedFile( + "single_value_file.avro", + 10, + contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 1L))); + } + + @Test + void fileWithoutContentStats() { + TrackedFile file = trackedFile("file.avro", 50, null); + + assertThat(shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file)) + .as("Should read: file does not track content stats") + .isTrue(); + } + + private static FieldStats stats( + int fieldId, Object lower, Object upper, Long valueCount, Long nullCount, Long nanCount) { + return stats(STATS_TYPE, fieldId, lower, upper, valueCount, nullCount, nanCount); + } + + private static FieldStats stats( + Types.StructType statsType, + int fieldId, + Object lower, + Object upper, + Long valueCount, + Long nullCount, + Long nanCount) { + return fieldStats(statsType, fieldId, lower, upper, valueCount, nullCount, nanCount); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java new file mode 100644 index 000000000000..317c56112827 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.StatsTestUtil.contentStats; +import static org.apache.iceberg.StatsTestUtil.fieldStats; +import static org.apache.iceberg.StatsTestUtil.trackedFile; + +import java.util.Map; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.InclusiveStatsEvaluator; +import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluatorWithExtract; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.variants.Variant; +import org.apache.iceberg.variants.VariantTestUtil; +import org.apache.iceberg.variants.VariantValue; +import org.apache.iceberg.variants.Variants; + +class TestInclusiveStatsEvaluatorWithExtract + extends TestInclusiveMetricsEvaluatorWithExtract { + private static final Types.StructType STATS_TYPE = + StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3)); + + private static final Variant LOWER_BOUND = + VariantTestUtil.variant( + Map.of("$['event_id']", Variants.of(INT_MIN_VALUE), "$['str']", Variants.of("abc"))); + + private static final Variant UPPER_BOUND = + VariantTestUtil.variant( + Map.of("$['event_id']", Variants.of(INT_MAX_VALUE), "$['str']", Variants.of("abe"))); + + @Override + protected boolean shouldRead(Expression expr, TrackedFile testFile, boolean caseSensitive) { + return new InclusiveStatsEvaluator(SCHEMA, expr, caseSensitive).eval(testFile); + } + + @Override + protected TrackedFile file() { + return trackedFile( + "file.avro", + 50, + contentStats( + STATS_TYPE, + fieldStats(STATS_TYPE, 1, null, null, 50L, null, null), + fieldStats(STATS_TYPE, 2, LOWER_BOUND, UPPER_BOUND, 50L, null, null), + fieldStats(STATS_TYPE, 3, null, null, 50L, 50L, null))); + } + + @Override + protected TrackedFile emptyFile() { + return trackedFile("file.parquet", 0, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile fileWithVariantBounds(String path, VariantValue lower, VariantValue upper) { + return trackedFile( + "file.parquet", + 50, + contentStats( + STATS_TYPE, + fieldStats( + STATS_TYPE, + 2, + VariantTestUtil.variant(Map.of(path, lower)), + VariantTestUtil.variant(Map.of(path, upper)), + null, + null, + null))); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java new file mode 100644 index 000000000000..613e6504dc5a --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.StatsTestUtil.contentStats; +import static org.apache.iceberg.StatsTestUtil.fieldStats; +import static org.apache.iceberg.StatsTestUtil.trackedFile; + +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.InclusiveStatsEvaluator; +import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluatorWithTransforms; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; + +class TestInclusiveStatsEvaluatorWithTransforms + extends TestInclusiveMetricsEvaluatorWithTransforms { + private static final Types.StructType STATS_TYPE = + StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4, 5, 6)); + + @Override + protected boolean shouldRead(Expression expr, TrackedFile testFile, boolean caseSensitive) { + return new InclusiveStatsEvaluator(SCHEMA, expr, caseSensitive).eval(testFile); + } + + @Override + protected TrackedFile file() { + return trackedFile( + "file.avro", + 50, + contentStats( + STATS_TYPE, + fieldStats(STATS_TYPE, 1, null, null, 50L, null, null), + fieldStats(STATS_TYPE, 2, TS_MIN_VALUE, TS_MAX_VALUE, 50L, null, null), + fieldStats(STATS_TYPE, 3, null, null, 50L, 50L, null), + fieldStats(STATS_TYPE, 4, null, null, 50L, 50L, null), + fieldStats(STATS_TYPE, 6, "abc", "abe", null, null, null))); + } + + @Override + protected TrackedFile emptyFile() { + return trackedFile("file.parquet", 0, contentStats(STATS_TYPE)); + } +} From 5a1c9123438a931ce9d3bf28afdf7d9268ab2ced Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 14:25:15 +0200 Subject: [PATCH 06/12] Core: Treat missing null count as zero for required fields Content stats omit null_value_count for a field that cannot be null, but InclusiveStatsEvaluator read the missing count as unknown and stopped pruning files for notEqual, notIn, and notStartsWith. The v3 metrics maps carry an explicit zero from Parquet, so the same filters pruned there. Detect the fields that cannot be null the same way expression binding does, by requiring the field and all of its ancestors to be required, and report no nulls for them. --- .../expressions/InclusiveStatsEvaluator.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java index 5e88e4c5fd9e..3de33f5d01de 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java +++ b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java @@ -20,10 +20,14 @@ import static org.apache.iceberg.expressions.Expressions.rewriteNot; +import java.util.Collections; +import java.util.Set; import org.apache.iceberg.ContentStats; import org.apache.iceberg.FieldStats; import org.apache.iceberg.Schema; import org.apache.iceberg.TrackedFile; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.variants.Variant; import org.apache.iceberg.variants.VariantObject; @@ -46,6 +50,7 @@ */ public class InclusiveStatsEvaluator { private final Expression expr; + private final Set neverNullIds; public InclusiveStatsEvaluator(Schema schema, Expression unbound) { this(schema, unbound, true); @@ -54,6 +59,32 @@ public InclusiveStatsEvaluator(Schema schema, Expression unbound) { public InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { Types.StructType struct = schema.asStruct(); this.expr = Binder.bind(struct, rewriteNot(unbound), caseSensitive); + this.neverNullIds = + neverNullIds( + schema, Binder.boundReferences(struct, Collections.singletonList(expr), caseSensitive)); + } + + /** + * Returns the IDs of the referenced fields that cannot contain null values. + * + *

Stats omit the null count for such fields, which must not be confused with an unknown count. + */ + private static Set neverNullIds(Schema schema, Set referencedIds) { + ImmutableSet.Builder neverNull = ImmutableSet.builder(); + + for (int id : referencedIds) { + Types.NestedField field = schema.findField(id); + if (field != null && field.isRequired() && allAncestorFieldsAreRequired(schema, id)) { + neverNull.add(id); + } + } + + return neverNull.build(); + } + + private static boolean allAncestorFieldsAreRequired(Schema schema, int fieldId) { + return TypeUtil.ancestorFields(schema, fieldId).stream() + .allMatch(Types.NestedField::isRequired); } /** @@ -90,6 +121,10 @@ private boolean eval(TrackedFile file) { @Override protected boolean mayContainNull(int id) { + if (neverNullIds.contains(id)) { + return false; + } + FieldStats fieldStats = stats.statsFor(id); return fieldStats == null || !fieldStats.hasNullValueCount() From b656ad167e437f3f9fd6b1f2921f080df97a26be Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 14:57:19 +0200 Subject: [PATCH 07/12] accept revapi failures --- .palantir/revapi.yml | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 478b78dea3e3..e726f0d2d5ca 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -564,6 +564,56 @@ acceptedBreaks: - code: "java.field.removedWithConstant" old: "field org.apache.iceberg.PartitionStatsHandler.PARTITION_FIELD_NAME" justification: "Removed deprecated functionality for partition stats" + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasNanValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasNullValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::hasValueCount()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method boolean org.apache.iceberg.FieldStats::tightBounds()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method int org.apache.iceberg.TrackedFile::formatVersion()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method java.lang.Integer org.apache.iceberg.FieldStats::avgValueSizeInBytes()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.ContentStats org.apache.iceberg.ContentStats::copy()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.ContentStats org.apache.iceberg.ContentStats::copy(java.util.Set)" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.FieldStats org.apache.iceberg.FieldStats::copy()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." + - code: "java.method.addedToInterface" + new: "method org.apache.iceberg.types.Types.StructType org.apache.iceberg.ContentStats::type()" + justification: "These interfaces were package-private in 1.11.0 and could not\ + \ be implemented outside the project. Promoting them to public reads as an\ + \ existing type gaining methods, not as a new type appearing." - code: "java.method.removed" old: "method org.apache.iceberg.MetricsConfig org.apache.iceberg.MetricsConfig::forPositionDelete(org.apache.iceberg.Table)" justification: "Deprecated for removal in 1.12.0" From 296e068bbfa4f82a6b24863ec417cf8ae4d09bfc Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 12:42:36 +0200 Subject: [PATCH 08/12] Spec, Core: Track null counts for required fields in optional structs The content stats schema omitted null_value_count whenever a field was required, but a required field nested in an optional struct is null on every row where its parent is null, and Parquet reports that count from the leaf column's definition levels. Include the count for any field that may be null, either because it or one of its ancestors is optional. --- .../java/org/apache/iceberg/StatsUtil.java | 29 ++++++++++++++++--- .../org/apache/iceberg/TestStatsUtil.java | 4 +++ format/spec.md | 8 +++-- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java index 545275002c9a..58a866b205ec 100644 --- a/core/src/main/java/org/apache/iceberg/StatsUtil.java +++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java @@ -147,7 +147,10 @@ public static Types.StructType statsWriteSchema(Schema tableSchema, MetricsConfi int baseId = toBaseId(id); Types.StructType fieldStruct = fieldStatsStruct( - field.isOptional(), field.type(), baseId, metricsConfig.columnMode(id)); + isNullable(tableSchema, parentIndex, id), + field.type(), + baseId, + metricsConfig.columnMode(id)); if (fieldStruct != null) { fieldStructs.add(optional(baseId, fieldName, fieldStruct)); @@ -178,7 +181,11 @@ public static Types.StructType statsReadSchema(Schema tableSchema, Iterable parentIndex, int id) { + Integer currentId = id; + while (currentId != null) { + if (schema.findField(currentId).isOptional()) { + return true; + } + + currentId = parentIndex.get(currentId); + } + + return false; + } + /** Return whether a field has one value or may be repeated in map or list. */ private static boolean isScalar(Schema schema, Map parentIndex, int id) { Integer currentId = id; diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java index e145dd61e14c..f8fa596fcc61 100644 --- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java @@ -395,12 +395,14 @@ public void testContentStatsReadSchemaNestedStruct() { Types.NestedField.required(3, "lat", Types.DoubleType.get()), Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); + // lat is required, but tracks a null count because it is null when location is null Types.StructType latStats = Types.StructType.of( Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); Types.StructType lonStats = Types.StructType.of( @@ -518,12 +520,14 @@ public void testContentStatsWriteSchemaNestedStruct() { Types.NestedField.required(3, "lat", Types.DoubleType.get()), Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); + // lat is required, but tracks a null count because it is null when location is null Types.StructType latStats = Types.StructType.of( Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); Types.StructType lonStats = Types.StructType.of( diff --git a/format/spec.md b/format/spec.md index ebb012598924..fdba1dfa75ad 100644 --- a/format/spec.md +++ b/format/spec.md @@ -799,6 +799,8 @@ In Iceberg v4, statistics are stored in typed fields grouped in a struct that co Field-level structs in `content_stats` are based on the corresponding table field's type, requirement, and ID (`field-id`). +Stats are tracked for primitive and `variant` fields. Stats are not tracked for `struct`, `list`, and `map` fields, or for any field contained in a `list` or `map` because it may be repeated. + Field stats structs are assigned a range of 200 IDs, starting at `10_000 + 200 * field-id`. The first ID in the range (`base-id`) is the ID of the struct field in `content_stats`. Fields within the stats struct are assigned IDs from the range by adding an offset to the `base-id`. For example, the stats struct for table field 2 uses IDs in the range `[10_400, 10_599]`, the field within `content_stats` uses the `base-id`, ID `10_400`, and its `lower_bound` field (offset 1) uses ID `10_401`. Content stats must be resolved by ID; field names used for stats structs are informational. The recommended name for each field is the full name of the field in the table schema. @@ -820,10 +822,12 @@ Each stats struct holds statistics for one table field. It may contain the follo | _optional_ | 2 | `upper_bound` | Field type or `geo_upper` | all primitives or `variant` | Upper bound stored as the field's type, or `geo_upper` for geo types | | _optional_ | 3 | `tight_bounds` | `boolean` | all except `geometry`, `geography`, `variant` | When true, `lower_bound` and `upper_bound` must be equal to the min and max values | | _optional_ | 4 | `value_count` | `long` | all | Number of values in the column (including null and NaN values) | -| _optional_ | 5 | `null_value_count` | `long` | optional fields | Number of null values in the column | +| _optional_ | 5 | `null_value_count` | `long` | fields that may be null | Number of null values in the column | | _optional_ | 6 | `nan_value_count` | `long` | `float`, `double` | Number of NaN values in the column | | _optional_ | 7 | `avg_value_size_in_bytes` | `int` | `string`, `binary`, `variant`, `geometry`, `geography` | Avg value size in memory (uncompressed) in bytes over non-null values to estimate memory consumption | +A field may be null if it is optional or if any field that contains it is optional. For example, a `required` field nested in an `optional` struct is null whenever the struct is null. + For example, stats for a `required` `int` field named `id` with field-id `2` are stored using: ``` @@ -833,7 +837,7 @@ For example, stats for a `required` `int` field named `id` with field-id `2` are 10_403: optional boolean tight_bounds; 10_404: optional long value_count; - // null_value_count is only used for optional fields + // null_value_count is only used for fields that may be null // nan_value_count is only used for float and double // avg_value_size_in_bytes is only used for variable length types } From 9142cf1002812be03aaff4ae04de228eb793b104 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Wed, 29 Jul 2026 16:15:01 +0200 Subject: [PATCH 09/12] API, Core: Unify isNullable() into TypeUtil --- .../iceberg/expressions/UnboundPredicate.java | 9 +- .../org/apache/iceberg/types/TypeUtil.java | 19 ++++ .../apache/iceberg/types/TestTypeUtil.java | 99 +++++++++++++++++++ .../java/org/apache/iceberg/StatsUtil.java | 18 +--- .../expressions/InclusiveStatsEvaluator.java | 8 +- 5 files changed, 123 insertions(+), 30 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java index 75ca9d5835bc..17f3873d70af 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java +++ b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java @@ -127,7 +127,7 @@ private Expression bindUnaryOperation(StructType struct, BoundTerm boundTerm) switch (op()) { case IS_NULL: if (!boundTerm.producesNull() - && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { + && !TypeUtil.isNullable(struct.asSchema(), boundTerm.ref().fieldId())) { return Expressions.alwaysFalse(); } else if (boundTerm.type().equals(Types.UnknownType.get())) { return Expressions.alwaysTrue(); @@ -135,7 +135,7 @@ && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { return new BoundUnaryPredicate<>(Operation.IS_NULL, boundTerm); case NOT_NULL: if (!boundTerm.producesNull() - && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { + && !TypeUtil.isNullable(struct.asSchema(), boundTerm.ref().fieldId())) { return Expressions.alwaysTrue(); } else if (boundTerm.type().equals(Types.UnknownType.get())) { return Expressions.alwaysFalse(); @@ -158,11 +158,6 @@ && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { } } - private boolean allAncestorFieldsAreRequired(StructType struct, int fieldId) { - return TypeUtil.ancestorFields(struct.asSchema(), fieldId).stream() - .allMatch(Types.NestedField::isRequired); - } - private boolean floatingType(Type.TypeID typeID) { return Type.TypeID.DOUBLE.equals(typeID) || Type.TypeID.FLOAT.equals(typeID); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index a3bee3e3d860..5f7ad3310c55 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -273,6 +273,25 @@ public static List ancestorFields(Schema schema, int fieldId) return parents; } + /** + * Returns whether a field may contain null values. + * + *

A field may be null if it is optional or if any field that contains it is optional. A + * required field nested in an optional struct is null whenever that struct is null. + * + * @param schema The schema that contains the field ID + * @param fieldId The field ID to check + * @return true if the field may be null, false if it cannot be null + * @throws IllegalArgumentException if the field ID is not present in the schema + */ + public static boolean isNullable(Schema schema, int fieldId) { + Types.NestedField field = schema.findField(fieldId); + Preconditions.checkArgument(field != null, "Cannot find field with ID: %s", fieldId); + + return field.isOptional() + || ancestorFields(schema, fieldId).stream().anyMatch(Types.NestedField::isOptional); + } + /** * Assigns fresh ids from the {@link NextID nextId function} for all fields in a type. * diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index b7da4b3108e6..872bccb987c4 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -971,6 +971,105 @@ public void ancestorFieldsInNestedSchema() { assertThat(TypeUtil.ancestorFields(schema, 17)).containsExactly(pointsElement, points); } + @Test + public void isNullableWithUnknownFieldId() { + Schema schema = new Schema(required(1, "id", IntegerType.get())); + + assertThatThrownBy(() -> TypeUtil.isNullable(schema, 2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot find field with ID: 2"); + } + + @Test + public void isNullableWithTopLevelFields() { + Schema schema = + new Schema( + required(1, "id", IntegerType.get()), optional(2, "data", Types.StringType.get())); + + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + } + + @Test + public void isNullableWithNestedStructs() { + Schema schema = + new Schema( + required( + 1, + "required_location", + Types.StructType.of( + required(3, "required_lat", Types.DoubleType.get()), + optional(4, "optional_lon", Types.DoubleType.get()), + required( + 5, + "required_inner", + Types.StructType.of(required(6, "required_zip", IntegerType.get()))))), + optional( + 2, + "optional_location", + Types.StructType.of( + required(7, "required_lat", Types.DoubleType.get()), + required( + 8, + "required_inner", + Types.StructType.of(required(9, "required_zip", IntegerType.get())))))); + + // a required field is not null when every field that contains it is required + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 3)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 5)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 6)).isFalse(); + + // an optional field is null regardless of the fields that contain it + assertThat(TypeUtil.isNullable(schema, 4)).isTrue(); + + // a required field nested in an optional struct is null when the struct is null + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 7)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 8)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 9)).isTrue(); + } + + @Test + public void isNullableWithListsAndMaps() { + Schema schema = + new Schema( + required( + 1, + "required_points", + Types.ListType.ofRequired( + 4, Types.StructType.of(required(5, "required_x", Types.LongType.get())))), + optional( + 2, + "optional_points", + Types.ListType.ofOptional( + 6, Types.StructType.of(required(7, "required_x", Types.LongType.get())))), + required( + 3, + "locations", + Types.MapType.ofRequired( + 8, + 9, + Types.StringType.get(), + Types.StructType.of(required(10, "required_lat", Types.DoubleType.get()))))); + + // a required element of a required list is not null, nor is anything it contains + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 4)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 5)).isFalse(); + + // an optional element is null, as is anything it contains + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 6)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 7)).isTrue(); + + // required map keys and values are not null + assertThat(TypeUtil.isNullable(schema, 3)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 8)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 9)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 10)).isFalse(); + } + @Test public void testIndexStatsNames() { Schema schema = diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java index 58a866b205ec..47decef0ce90 100644 --- a/core/src/main/java/org/apache/iceberg/StatsUtil.java +++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java @@ -147,7 +147,7 @@ public static Types.StructType statsWriteSchema(Schema tableSchema, MetricsConfi int baseId = toBaseId(id); Types.StructType fieldStruct = fieldStatsStruct( - isNullable(tableSchema, parentIndex, id), + TypeUtil.isNullable(tableSchema, id), field.type(), baseId, metricsConfig.columnMode(id)); @@ -182,7 +182,7 @@ public static Types.StructType statsReadSchema(Schema tableSchema, Iterable parentIndex, int id) { - Integer currentId = id; - while (currentId != null) { - if (schema.findField(currentId).isOptional()) { - return true; - } - - currentId = parentIndex.get(currentId); - } - - return false; - } - /** Return whether a field has one value or may be repeated in map or list. */ private static boolean isScalar(Schema schema, Map parentIndex, int id) { Integer currentId = id; diff --git a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java index 3de33f5d01de..97e94eeba927 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java +++ b/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java @@ -73,8 +73,7 @@ private static Set neverNullIds(Schema schema, Set referencedI ImmutableSet.Builder neverNull = ImmutableSet.builder(); for (int id : referencedIds) { - Types.NestedField field = schema.findField(id); - if (field != null && field.isRequired() && allAncestorFieldsAreRequired(schema, id)) { + if (!TypeUtil.isNullable(schema, id)) { neverNull.add(id); } } @@ -82,11 +81,6 @@ private static Set neverNullIds(Schema schema, Set referencedI return neverNull.build(); } - private static boolean allAncestorFieldsAreRequired(Schema schema, int fieldId) { - return TypeUtil.ancestorFields(schema, fieldId).stream() - .allMatch(Types.NestedField::isRequired); - } - /** * Test whether the file may contain records that match the expression. * From eec73590bcf18b66c32e6fdd9c8bbc630f8d66e8 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Thu, 30 Jul 2026 14:57:28 +0200 Subject: [PATCH 10/12] mostly revert tracking null counts for required fields inside optional structs --- .../TestInclusiveMetricsEvaluator.java | 10 +++++++--- .../main/java/org/apache/iceberg/StatsUtil.java | 15 ++++----------- .../expressions/InclusiveStatsEvaluator.java | 4 +++- .../iceberg/TestInclusiveStatsEvaluator.java | 16 +++++++++++++++- .../java/org/apache/iceberg/TestStatsUtil.java | 4 ---- format/spec.md | 8 ++------ 6 files changed, 31 insertions(+), 26 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index 310e90c9661c..1767bfac5967 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -990,15 +990,19 @@ public void testNotNullInNestedStruct() { shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address"), file6()); assertThat(shouldRead).as("Should read: metrics are not tracked for structs").isTrue(); - shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); - assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); - shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.optional_street2"), file6()); assertThat(shouldRead) .as("Should not read: optional_address.optional_street2 is optional") .isFalse(); } + @Test + public void notNullForRequiredFieldInOptionalStruct() { + boolean shouldRead = + shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); + assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); + } + @Test public void testNotEqSingleValueWithoutNaN() { assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFile())) diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java index 47decef0ce90..545275002c9a 100644 --- a/core/src/main/java/org/apache/iceberg/StatsUtil.java +++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java @@ -147,10 +147,7 @@ public static Types.StructType statsWriteSchema(Schema tableSchema, MetricsConfi int baseId = toBaseId(id); Types.StructType fieldStruct = fieldStatsStruct( - TypeUtil.isNullable(tableSchema, id), - field.type(), - baseId, - metricsConfig.columnMode(id)); + field.isOptional(), field.type(), baseId, metricsConfig.columnMode(id)); if (fieldStruct != null) { fieldStructs.add(optional(baseId, fieldName, fieldStruct)); @@ -181,11 +178,7 @@ public static Types.StructType statsReadSchema(Schema tableSchema, IterableStats omit the null count for such fields, which must not be confused with an unknown count. + *

Stats omit the null count for every required field. The count is zero for a field that + * cannot be null, but it is unknown for a required field that an optional struct contains because + * that field is null whenever the struct is null. */ private static Set neverNullIds(Schema schema, Set referencedIds) { ImmutableSet.Builder neverNull = ImmutableSet.builder(); diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java index 33767d81de41..602246616068 100644 --- a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java @@ -22,6 +22,7 @@ import static org.apache.iceberg.StatsTestUtil.fieldStats; import static org.apache.iceberg.StatsTestUtil.trackedFile; import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.apache.iceberg.expressions.Expressions.notNull; import static org.assertj.core.api.Assertions.assertThat; import org.apache.iceberg.expressions.Expression; @@ -103,7 +104,8 @@ protected TrackedFile file6() { NESTED_STATS_TYPE, stats(NESTED_STATS_TYPE, 102, null, null, 5L, null, null), stats(NESTED_STATS_TYPE, 103, null, null, 5L, 5L, null), - stats(NESTED_STATS_TYPE, 104, null, null, 5L, 5L, null), + // required_street2 is required, so its stats do not track a null count + stats(NESTED_STATS_TYPE, 104, null, null, 5L, null, null), stats(NESTED_STATS_TYPE, 105, null, null, 5L, 5L, null))); } @@ -169,6 +171,18 @@ protected TrackedFile singleFloatValueFileWithNaN() { contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 1L))); } + /** + * Content stats omit the null count for a required field, even when an optional struct contains + * it, so a file where every value is null cannot be pruned. + */ + @Override + @Test + public void notNullForRequiredFieldInOptionalStruct() { + boolean shouldRead = + shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); + assertThat(shouldRead).as("Should read: the null count is not tracked").isTrue(); + } + @Test void fileWithoutContentStats() { TrackedFile file = trackedFile("file.avro", 50, null); diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java index f8fa596fcc61..e145dd61e14c 100644 --- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java @@ -395,14 +395,12 @@ public void testContentStatsReadSchemaNestedStruct() { Types.NestedField.required(3, "lat", Types.DoubleType.get()), Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); - // lat is required, but tracks a null count because it is null when location is null Types.StructType latStats = Types.StructType.of( Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), - Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); Types.StructType lonStats = Types.StructType.of( @@ -520,14 +518,12 @@ public void testContentStatsWriteSchemaNestedStruct() { Types.NestedField.required(3, "lat", Types.DoubleType.get()), Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); - // lat is required, but tracks a null count because it is null when location is null Types.StructType latStats = Types.StructType.of( Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), - Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); Types.StructType lonStats = Types.StructType.of( diff --git a/format/spec.md b/format/spec.md index fdba1dfa75ad..ebb012598924 100644 --- a/format/spec.md +++ b/format/spec.md @@ -799,8 +799,6 @@ In Iceberg v4, statistics are stored in typed fields grouped in a struct that co Field-level structs in `content_stats` are based on the corresponding table field's type, requirement, and ID (`field-id`). -Stats are tracked for primitive and `variant` fields. Stats are not tracked for `struct`, `list`, and `map` fields, or for any field contained in a `list` or `map` because it may be repeated. - Field stats structs are assigned a range of 200 IDs, starting at `10_000 + 200 * field-id`. The first ID in the range (`base-id`) is the ID of the struct field in `content_stats`. Fields within the stats struct are assigned IDs from the range by adding an offset to the `base-id`. For example, the stats struct for table field 2 uses IDs in the range `[10_400, 10_599]`, the field within `content_stats` uses the `base-id`, ID `10_400`, and its `lower_bound` field (offset 1) uses ID `10_401`. Content stats must be resolved by ID; field names used for stats structs are informational. The recommended name for each field is the full name of the field in the table schema. @@ -822,12 +820,10 @@ Each stats struct holds statistics for one table field. It may contain the follo | _optional_ | 2 | `upper_bound` | Field type or `geo_upper` | all primitives or `variant` | Upper bound stored as the field's type, or `geo_upper` for geo types | | _optional_ | 3 | `tight_bounds` | `boolean` | all except `geometry`, `geography`, `variant` | When true, `lower_bound` and `upper_bound` must be equal to the min and max values | | _optional_ | 4 | `value_count` | `long` | all | Number of values in the column (including null and NaN values) | -| _optional_ | 5 | `null_value_count` | `long` | fields that may be null | Number of null values in the column | +| _optional_ | 5 | `null_value_count` | `long` | optional fields | Number of null values in the column | | _optional_ | 6 | `nan_value_count` | `long` | `float`, `double` | Number of NaN values in the column | | _optional_ | 7 | `avg_value_size_in_bytes` | `int` | `string`, `binary`, `variant`, `geometry`, `geography` | Avg value size in memory (uncompressed) in bytes over non-null values to estimate memory consumption | -A field may be null if it is optional or if any field that contains it is optional. For example, a `required` field nested in an `optional` struct is null whenever the struct is null. - For example, stats for a `required` `int` field named `id` with field-id `2` are stored using: ``` @@ -837,7 +833,7 @@ For example, stats for a `required` `int` field named `id` with field-id `2` are 10_403: optional boolean tight_bounds; 10_404: optional long value_count; - // null_value_count is only used for fields that may be null + // null_value_count is only used for optional fields // nan_value_count is only used for float and double // avg_value_size_in_bytes is only used for variable length types } From 1f62852b921bd14e1aa7f3b64a4467fce0330336 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Thu, 30 Jul 2026 15:15:58 +0200 Subject: [PATCH 11/12] revert revapi failures and move classes around --- .palantir/revapi.yml | 50 ------------------- .../expressions/InclusiveEvalVisitor.java | 9 +++- .../java/org/apache/iceberg/ContentStats.java | 2 +- .../java/org/apache/iceberg/FieldStats.java | 2 +- .../InclusiveStatsEvaluator.java | 24 +++++---- .../java/org/apache/iceberg/TrackedFile.java | 2 +- .../iceberg/TestInclusiveStatsEvaluator.java | 1 - ...estInclusiveStatsEvaluatorWithExtract.java | 1 - ...InclusiveStatsEvaluatorWithTransforms.java | 1 - 9 files changed, 24 insertions(+), 68 deletions(-) rename core/src/main/java/org/apache/iceberg/{expressions => }/InclusiveStatsEvaluator.java (89%) diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index e726f0d2d5ca..478b78dea3e3 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -564,56 +564,6 @@ acceptedBreaks: - code: "java.field.removedWithConstant" old: "field org.apache.iceberg.PartitionStatsHandler.PARTITION_FIELD_NAME" justification: "Removed deprecated functionality for partition stats" - - code: "java.method.addedToInterface" - new: "method boolean org.apache.iceberg.FieldStats::hasNanValueCount()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method boolean org.apache.iceberg.FieldStats::hasNullValueCount()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method boolean org.apache.iceberg.FieldStats::hasValueCount()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method boolean org.apache.iceberg.FieldStats::tightBounds()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method int org.apache.iceberg.TrackedFile::formatVersion()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method java.lang.Integer org.apache.iceberg.FieldStats::avgValueSizeInBytes()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method org.apache.iceberg.ContentStats org.apache.iceberg.ContentStats::copy()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method org.apache.iceberg.ContentStats org.apache.iceberg.ContentStats::copy(java.util.Set)" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method org.apache.iceberg.FieldStats org.apache.iceberg.FieldStats::copy()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - - code: "java.method.addedToInterface" - new: "method org.apache.iceberg.types.Types.StructType org.apache.iceberg.ContentStats::type()" - justification: "These interfaces were package-private in 1.11.0 and could not\ - \ be implemented outside the project. Promoting them to public reads as an\ - \ existing type gaining methods, not as a new type appearing." - code: "java.method.removed" old: "method org.apache.iceberg.MetricsConfig org.apache.iceberg.MetricsConfig::forPositionDelete(org.apache.iceberg.Table)" justification: "Deprecated for removal in 1.12.0" diff --git a/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java b/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java index 02ee2c7ffc9f..3476abee3a2e 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java +++ b/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java @@ -24,9 +24,11 @@ import java.util.stream.Collectors; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.types.Comparators; +import org.apache.iceberg.types.Type; import org.apache.iceberg.util.NaNUtil; +import org.apache.iceberg.variants.VariantValue; -abstract class InclusiveEvalVisitor extends ExpressionVisitors.BoundVisitor { +public abstract class InclusiveEvalVisitor extends ExpressionVisitors.BoundVisitor { private static final int IN_PREDICATE_LIMIT = 200; protected static final boolean ROWS_MIGHT_MATCH = true; @@ -64,6 +66,11 @@ protected T extractUpperBound(BoundExtract bound) { return null; } + /** Return a variant value as the given type, or null if it cannot be represented as that type. */ + protected static T castTo(VariantValue value, Type type) { + return VariantExpressionUtil.castTo(value, type); + } + @Override public Boolean alwaysTrue() { return ROWS_MIGHT_MATCH; // all rows match diff --git a/core/src/main/java/org/apache/iceberg/ContentStats.java b/core/src/main/java/org/apache/iceberg/ContentStats.java index 51bfa52a67ad..aa7a65ac934d 100644 --- a/core/src/main/java/org/apache/iceberg/ContentStats.java +++ b/core/src/main/java/org/apache/iceberg/ContentStats.java @@ -21,7 +21,7 @@ import java.util.Set; import org.apache.iceberg.types.Types; -public interface ContentStats { +interface ContentStats { /** Returns an iterable of the {@link FieldStats} held by this container struct */ Iterable> fieldStats(); diff --git a/core/src/main/java/org/apache/iceberg/FieldStats.java b/core/src/main/java/org/apache/iceberg/FieldStats.java index 693765a8977b..495dddba913d 100644 --- a/core/src/main/java/org/apache/iceberg/FieldStats.java +++ b/core/src/main/java/org/apache/iceberg/FieldStats.java @@ -20,7 +20,7 @@ import org.apache.iceberg.types.Types; -public interface FieldStats { +interface FieldStats { /** The field ID of the statistic */ int fieldId(); diff --git a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/InclusiveStatsEvaluator.java similarity index 89% rename from core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java rename to core/src/main/java/org/apache/iceberg/InclusiveStatsEvaluator.java index 6cd8bf21f5a6..2545c63b3ec9 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/InclusiveStatsEvaluator.java +++ b/core/src/main/java/org/apache/iceberg/InclusiveStatsEvaluator.java @@ -16,16 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iceberg.expressions; +package org.apache.iceberg; import static org.apache.iceberg.expressions.Expressions.rewriteNot; import java.util.Collections; import java.util.Set; -import org.apache.iceberg.ContentStats; -import org.apache.iceberg.FieldStats; -import org.apache.iceberg.Schema; -import org.apache.iceberg.TrackedFile; +import org.apache.iceberg.expressions.Binder; +import org.apache.iceberg.expressions.BoundExtract; +import org.apache.iceberg.expressions.BoundReference; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.ExpressionVisitors; +import org.apache.iceberg.expressions.InclusiveEvalVisitor; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -48,15 +50,15 @@ * bound despite that the column could contain non-NaN data. Thus, in some scenarios explicitly * checks for NaN is necessary in order to not skip files that may contain matching data. */ -public class InclusiveStatsEvaluator { +class InclusiveStatsEvaluator { private final Expression expr; private final Set neverNullIds; - public InclusiveStatsEvaluator(Schema schema, Expression unbound) { + InclusiveStatsEvaluator(Schema schema, Expression unbound) { this(schema, unbound, true); } - public InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { + InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { Types.StructType struct = schema.asStruct(); this.expr = Binder.bind(struct, rewriteNot(unbound), caseSensitive); this.neverNullIds = @@ -89,7 +91,7 @@ private static Set neverNullIds(Schema schema, Set referencedI * @param file a tracked file * @return false if the file cannot contain rows that match the expression, true otherwise. */ - public boolean eval(TrackedFile file) { + boolean eval(TrackedFile file) { return new StatsEvalVisitor().eval(file); } @@ -170,7 +172,7 @@ protected T extractLowerBound(BoundExtract bound) { FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); if (fieldStats != null && fieldStats.lowerBound() != null) { VariantObject fieldLowerBounds = fieldStats.lowerBound().value().asObject(); - return VariantExpressionUtil.castTo(fieldLowerBounds.get(bound.path()), bound.type()); + return castTo(fieldLowerBounds.get(bound.path()), bound.type()); } return null; @@ -181,7 +183,7 @@ protected T extractUpperBound(BoundExtract bound) { FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); if (fieldStats != null && fieldStats.upperBound() != null) { VariantObject fieldUpperBounds = fieldStats.upperBound().value().asObject(); - return VariantExpressionUtil.castTo(fieldUpperBounds.get(bound.path()), bound.type()); + return castTo(fieldUpperBounds.get(bound.path()), bound.type()); } return null; diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 374808b7cb53..9aaeae48d688 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -25,7 +25,7 @@ import org.apache.iceberg.types.Types; /** A file tracked by a manifest. */ -public interface TrackedFile { +interface TrackedFile { Types.NestedField TRACKING = Types.NestedField.required( 147, "tracking", Tracking.schema(), "Tracking information for this entry"); diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java index 602246616068..1cf8af4f0093 100644 --- a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java @@ -26,7 +26,6 @@ import static org.assertj.core.api.Assertions.assertThat; import org.apache.iceberg.expressions.Expression; -import org.apache.iceberg.expressions.InclusiveStatsEvaluator; import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluator; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.types.Types; diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java index 317c56112827..7e82c0af40c1 100644 --- a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java @@ -24,7 +24,6 @@ import java.util.Map; import org.apache.iceberg.expressions.Expression; -import org.apache.iceberg.expressions.InclusiveStatsEvaluator; import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluatorWithExtract; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.types.Types; diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java index 613e6504dc5a..a034d9a58ee2 100644 --- a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java @@ -23,7 +23,6 @@ import static org.apache.iceberg.StatsTestUtil.trackedFile; import org.apache.iceberg.expressions.Expression; -import org.apache.iceberg.expressions.InclusiveStatsEvaluator; import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluatorWithTransforms; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.types.Types; From 4f232b9544959d946653832e87b5daaeea19c933 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Thu, 30 Jul 2026 15:31:39 +0200 Subject: [PATCH 12/12] fixup! API, Core: Unify isNullable() into TypeUtil --- api/src/main/java/org/apache/iceberg/types/TypeUtil.java | 8 ++++---- .../test/java/org/apache/iceberg/types/TestTypeUtil.java | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index 5f7ad3310c55..8c4a6b3351da 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -277,18 +277,18 @@ public static List ancestorFields(Schema schema, int fieldId) * Returns whether a field may contain null values. * *

A field may be null if it is optional or if any field that contains it is optional. A - * required field nested in an optional struct is null whenever that struct is null. + * required field nested in an optional struct is null whenever that struct is null. A field that + * is not present in the schema may be null because its requirement is unknown. * * @param schema The schema that contains the field ID * @param fieldId The field ID to check * @return true if the field may be null, false if it cannot be null - * @throws IllegalArgumentException if the field ID is not present in the schema */ public static boolean isNullable(Schema schema, int fieldId) { Types.NestedField field = schema.findField(fieldId); - Preconditions.checkArgument(field != null, "Cannot find field with ID: %s", fieldId); - return field.isOptional() + return field == null + || field.isOptional() || ancestorFields(schema, fieldId).stream().anyMatch(Types.NestedField::isOptional); } diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index 872bccb987c4..28231fd4a171 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -975,9 +975,7 @@ public void ancestorFieldsInNestedSchema() { public void isNullableWithUnknownFieldId() { Schema schema = new Schema(required(1, "id", IntegerType.get())); - assertThatThrownBy(() -> TypeUtil.isNullable(schema, 2)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Cannot find field with ID: 2"); + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); } @Test