diff --git a/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java b/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java index 02ee2c7ffc9f..3476abee3a2e 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java +++ b/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java @@ -24,9 +24,11 @@ import java.util.stream.Collectors; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.types.Comparators; +import org.apache.iceberg.types.Type; import org.apache.iceberg.util.NaNUtil; +import org.apache.iceberg.variants.VariantValue; -abstract class InclusiveEvalVisitor extends ExpressionVisitors.BoundVisitor { +public abstract class InclusiveEvalVisitor extends ExpressionVisitors.BoundVisitor { private static final int IN_PREDICATE_LIMIT = 200; protected static final boolean ROWS_MIGHT_MATCH = true; @@ -64,6 +66,11 @@ protected T extractUpperBound(BoundExtract bound) { return null; } + /** Return a variant value as the given type, or null if it cannot be represented as that type. */ + protected static T castTo(VariantValue value, Type type) { + return VariantExpressionUtil.castTo(value, type); + } + @Override public Boolean alwaysTrue() { return ROWS_MIGHT_MATCH; // all rows match diff --git a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java index 75ca9d5835bc..17f3873d70af 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java +++ b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java @@ -127,7 +127,7 @@ private Expression bindUnaryOperation(StructType struct, BoundTerm boundTerm) switch (op()) { case IS_NULL: if (!boundTerm.producesNull() - && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { + && !TypeUtil.isNullable(struct.asSchema(), boundTerm.ref().fieldId())) { return Expressions.alwaysFalse(); } else if (boundTerm.type().equals(Types.UnknownType.get())) { return Expressions.alwaysTrue(); @@ -135,7 +135,7 @@ && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { return new BoundUnaryPredicate<>(Operation.IS_NULL, boundTerm); case NOT_NULL: if (!boundTerm.producesNull() - && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { + && !TypeUtil.isNullable(struct.asSchema(), boundTerm.ref().fieldId())) { return Expressions.alwaysTrue(); } else if (boundTerm.type().equals(Types.UnknownType.get())) { return Expressions.alwaysFalse(); @@ -158,11 +158,6 @@ && allAncestorFieldsAreRequired(struct, boundTerm.ref().fieldId())) { } } - private boolean allAncestorFieldsAreRequired(StructType struct, int fieldId) { - return TypeUtil.ancestorFields(struct.asSchema(), fieldId).stream() - .allMatch(Types.NestedField::isRequired); - } - private boolean floatingType(Type.TypeID typeID) { return Type.TypeID.DOUBLE.equals(typeID) || Type.TypeID.FLOAT.equals(typeID); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index a3bee3e3d860..8c4a6b3351da 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -273,6 +273,25 @@ public static List ancestorFields(Schema schema, int fieldId) return parents; } + /** + * Returns whether a field may contain null values. + * + *

A field may be null if it is optional or if any field that contains it is optional. A + * required field nested in an optional struct is null whenever that struct is null. A field that + * is not present in the schema may be null because its requirement is unknown. + * + * @param schema The schema that contains the field ID + * @param fieldId The field ID to check + * @return true if the field may be null, false if it cannot be null + */ + public static boolean isNullable(Schema schema, int fieldId) { + Types.NestedField field = schema.findField(fieldId); + + return field == null + || field.isOptional() + || ancestorFields(schema, fieldId).stream().anyMatch(Types.NestedField::isOptional); + } + /** * Assigns fresh ids from the {@link NextID nextId function} for all fields in a type. * diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java index 8d652df0df03..1767bfac5967 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java @@ -57,8 +57,8 @@ import org.apache.iceberg.util.UnicodeUtil; import org.junit.jupiter.api.Test; -public class TestInclusiveMetricsEvaluator { - private static final Schema SCHEMA = +public class TestInclusiveMetricsEvaluator { + protected static final Schema SCHEMA = new Schema( required(1, "id", IntegerType.get()), optional(2, "no_stats", Types.IntegerType.get()), @@ -75,7 +75,7 @@ public class TestInclusiveMetricsEvaluator { optional(13, "no_nan_stats", Types.DoubleType.get()), optional(14, "some_empty", Types.StringType.get())); - private static final Schema NESTED_SCHEMA = + protected static final Schema NESTED_SCHEMA = new Schema( required( 100, @@ -90,8 +90,10 @@ public class TestInclusiveMetricsEvaluator { required(104, "required_street2", Types.StringType.get()), optional(105, "optional_street2", Types.StringType.get())))); - private static final int INT_MIN_VALUE = 30; - private static final int INT_MAX_VALUE = 79; + protected static final Schema FLOAT_SCHEMA = new Schema(required(1, "f", Types.FloatType.get())); + + protected static final int INT_MIN_VALUE = 30; + protected static final int INT_MAX_VALUE = 79; private static final DataFile FILE = new TestDataFile( @@ -210,9 +212,9 @@ public class TestInclusiveMetricsEvaluator { Row.of(), 10, // any value counts, including nulls - ImmutableMap.of(100, 5L, 101, 5L, 102, 5L, 103, 5L, 104, 5L, 105, 5L), + ImmutableMap.of(102, 5L, 103, 5L, 104, 5L, 105, 5L), // null value counts - ImmutableMap.of(100, 0L, 101, 5L, 103, 5L, 104, 5L, 105, 5L), + ImmutableMap.of(103, 5L, 104, 5L, 105, 5L), // nan value counts null, // lower bounds @@ -220,78 +222,234 @@ public class TestInclusiveMetricsEvaluator { // upper bounds null); + private static final DataFile MISSING_STATS = new TestDataFile("file.parquet", Row.of(), 50); + + private static final DataFile EMPTY_FILE = new TestDataFile("file.parquet", Row.of(), 0); + + private static final DataFile RANGE_OF_VALUES = + new TestDataFile( + "range_of_values.avro", + Row.of(), + 10, + ImmutableMap.of(3, 10L), + ImmutableMap.of(3, 0L), + null, + ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); + + private static final DataFile SINGLE_VALUE_FILE = + new TestDataFile( + "single_value.avro", + Row.of(), + 10, + ImmutableMap.of(3, 10L), + ImmutableMap.of(3, 0L), + null, + ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); + + // some_empty is optional because a required column cannot contain nulls + private static final DataFile SINGLE_VALUE_WITH_NULLS = + new TestDataFile( + "single_value_nulls.avro", + Row.of(), + 10, + ImmutableMap.of(14, 10L), + ImmutableMap.of(14, 2L), + null, + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc")), + ImmutableMap.of(14, toByteBuffer(StringType.get(), "abc"))); + + private static final DataFile SINGLE_VALUE_WITH_NAN = + new TestDataFile( + "single_value_nan.avro", + Row.of(), + 10, + ImmutableMap.of(9, 10L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, 2L), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); + + private static final DataFile SINGLE_VALUE_NAN_BOUNDS = + new TestDataFile( + "single_value_nan_bounds.avro", + Row.of(), + 10, + ImmutableMap.of(9, 10L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, 0L), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)), + ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN))); + + private static final Map FLOAT_BOUND = + ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); + + private static final DataFile SINGLE_FLOAT_VALUE_FILE = + new TestDataFile( + "single_value_file.avro", + Row.of(), + 10, + ImmutableMap.of(1, 10L), + ImmutableMap.of(1, 0L), + ImmutableMap.of(1, 0L), + FLOAT_BOUND, + FLOAT_BOUND); + + private static final DataFile SINGLE_FLOAT_VALUE_FILE_WITH_NAN = + new TestDataFile( + "single_value_file.avro", + Row.of(), + 10, + ImmutableMap.of(1, 10L), + ImmutableMap.of(1, 0L), + ImmutableMap.of(1, 1L), // contains a NaN value + FLOAT_BOUND, + FLOAT_BOUND); + + protected boolean shouldRead(Schema schema, Expression expr, boolean caseSensitive, F testFile) { + return new InclusiveMetricsEvaluator(schema, expr, caseSensitive).eval((DataFile) testFile); + } + + protected boolean shouldRead(Schema schema, Expression expr, F testFile) { + return shouldRead(schema, expr, true, testFile); + } + + protected F file() { + return asFile(FILE); + } + + protected F file2() { + return asFile(FILE_2); + } + + protected F file3() { + return asFile(FILE_3); + } + + protected F file4() { + return asFile(FILE_4); + } + + protected F file5() { + return asFile(FILE_5); + } + + protected F file6() { + return asFile(FILE_6); + } + + protected F missingStats() { + return asFile(MISSING_STATS); + } + + protected F emptyFile() { + return asFile(EMPTY_FILE); + } + + protected F rangeOfValues() { + return asFile(RANGE_OF_VALUES); + } + + protected F singleValueFile() { + return asFile(SINGLE_VALUE_FILE); + } + + protected F singleValueWithNulls() { + return asFile(SINGLE_VALUE_WITH_NULLS); + } + + protected F singleValueWithNaN() { + return asFile(SINGLE_VALUE_WITH_NAN); + } + + protected F singleValueNaNBounds() { + return asFile(SINGLE_VALUE_NAN_BOUNDS); + } + + protected F singleFloatValueFile() { + return asFile(SINGLE_FLOAT_VALUE_FILE); + } + + protected F singleFloatValueFileWithNaN() { + return asFile(SINGLE_FLOAT_VALUE_FILE_WITH_NAN); + } + + @SuppressWarnings("unchecked") + private F asFile(DataFile dataFile) { + return (F) dataFile; + } + @Test public void testAllNulls() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("all_nulls")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNull("all_nulls"), file()); assertThat(shouldRead).as("Should skip: no non-null value in all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: lessThan on all null column").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: lessThanOrEqual on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: greaterThan on all null column").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: greaterThanOrEqual on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: equal on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", "a"), file()); assertThat(shouldRead).as("Should skip: startsWith on all null column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("all_nulls", "a")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notStartsWith("all_nulls", "a"), file()); assertThat(shouldRead).as("Should read: notStartsWith on all null column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("some_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNull("some_nulls"), file()); assertThat(shouldRead) .as("Should read: column with some nulls contains a non-null value") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("no_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNull("no_nulls"), file()); assertThat(shouldRead).as("Should read: non-null column contains a non-null value").isTrue(); } @Test public void testNoNulls() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("all_nulls")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, isNull("all_nulls"), file()); assertThat(shouldRead).as("Should read: at least one null value in all null column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("some_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("some_nulls"), file()); assertThat(shouldRead).as("Should read: column with some nulls contains a null value").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("no_nulls")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("no_nulls"), file()); assertThat(shouldRead).as("Should skip: non-null column contains no null values").isFalse(); } @Test public void testIsNaN() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nans")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, isNaN("all_nans"), file()); assertThat(shouldRead).as("Should read: at least one nan value in all nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("some_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("some_nans"), file()); assertThat(shouldRead).as("Should read: at least one nan value in some nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("no_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("no_nans"), file()); assertThat(shouldRead).as("Should skip: no-nans column contains no nan values").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nulls_double")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("all_nulls_double"), file()); assertThat(shouldRead).as("Should skip: all-null column doesn't contain nan value").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("no_nan_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("no_nan_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("all_nans_v1_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("all_nans_v1_stats"), file()); assertThat(shouldRead).as("Should read: at least one nan value in all nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNaN("nan_and_null_only")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNaN("nan_and_null_only"), file()); assertThat(shouldRead) .as("Should read: at least one nan value in nan and nulls only column") .isTrue(); @@ -299,35 +457,35 @@ public void testIsNaN() { @Test public void testNotNaN() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nans")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNaN("all_nans"), file()); assertThat(shouldRead) .as("Should skip: column with all nans will not contain non-nan") .isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("some_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("some_nans"), file()); assertThat(shouldRead) .as("Should read: at least one non-nan value in some nan column") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("no_nans")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("no_nans"), file()); assertThat(shouldRead).as("Should read: at least one non-nan value in no nan column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nulls_double")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("all_nulls_double"), file()); assertThat(shouldRead) .as("Should read: at least one non-nan value in all null column") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("no_nan_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("no_nan_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("all_nans_v1_stats")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("all_nans_v1_stats"), file()); assertThat(shouldRead) .as("Should read: no guarantee on if contains nan value without nan stats") .isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNaN("nan_and_null_only")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notNaN("nan_and_null_only"), file()); assertThat(shouldRead) .as("Should read: at least one null value in nan and nulls only column") .isTrue(); @@ -335,25 +493,22 @@ public void testNotNaN() { @Test public void testRequiredColumn() { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notNull("required")).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notNull("required"), file()); assertThat(shouldRead).as("Should read: required columns are always non-null").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, isNull("required")).eval(FILE); + shouldRead = shouldRead(SCHEMA, isNull("required"), file()); assertThat(shouldRead).as("Should skip: required columns are always non-null").isFalse(); } @Test public void testMissingColumn() { - assertThatThrownBy( - () -> new InclusiveMetricsEvaluator(SCHEMA, lessThan("missing", 5)).eval(FILE)) + assertThatThrownBy(() -> shouldRead(SCHEMA, lessThan("missing", 5), file())) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot find field 'missing'"); } @Test public void testMissingStats() { - DataFile missingStats = new TestDataFile("file.parquet", Row.of(), 50); - Expression[] exprs = new Expression[] { lessThan("no_stats", 5), @@ -369,15 +524,13 @@ public void testMissingStats() { }; for (Expression expr : exprs) { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, expr).eval(missingStats); + boolean shouldRead = shouldRead(SCHEMA, expr, missingStats()); assertThat(shouldRead).as("Should read when missing stats for expr: " + expr).isTrue(); } } @Test public void testZeroRecordFile() { - DataFile empty = new TestDataFile("file.parquet", Row.of(), 0); - Expression[] exprs = new Expression[] { lessThan("id", 5), @@ -393,7 +546,7 @@ public void testZeroRecordFile() { }; for (Expression expr : exprs) { - boolean shouldRead = new InclusiveMetricsEvaluator(SCHEMA, expr).eval(empty); + boolean shouldRead = shouldRead(SCHEMA, expr, emptyFile()); assertThat(shouldRead).as("Should never read 0-record file: " + expr).isFalse(); } } @@ -401,13 +554,10 @@ public void testZeroRecordFile() { @Test public void testNot() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(lessThan("id", INT_MIN_VALUE - 25))).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(lessThan("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should read: not(false)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(greaterThan("id", INT_MIN_VALUE - 25))) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(greaterThan("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should skip: not(true)").isFalse(); } @@ -415,28 +565,24 @@ public void testNot() { public void testAnd() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MIN_VALUE - 30))) - .eval(FILE); + shouldRead( + SCHEMA, + and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MIN_VALUE - 30)), + file()); assertThat(shouldRead).as("Should skip: and(false, true)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MAX_VALUE + 1))) - .eval(FILE); + shouldRead( + SCHEMA, + and(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1)), + file()); assertThat(shouldRead).as("Should skip: and(false, false)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id", INT_MIN_VALUE))) - .eval(FILE); + shouldRead( + SCHEMA, + and(greaterThan("id", INT_MIN_VALUE - 25), lessThanOrEqual("id", INT_MIN_VALUE)), + file()); assertThat(shouldRead).as("Should read: and(true, true)").isTrue(); } @@ -444,398 +590,312 @@ public void testAnd() { public void testOr() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1))) - .eval(FILE); + shouldRead( + SCHEMA, + or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE + 1)), + file()); assertThat(shouldRead).as("Should skip: or(false, false)").isFalse(); shouldRead = - new InclusiveMetricsEvaluator( - SCHEMA, - or( - lessThan("id", INT_MIN_VALUE - 25), - greaterThanOrEqual("id", INT_MAX_VALUE - 19))) - .eval(FILE); + shouldRead( + SCHEMA, + or(lessThan("id", INT_MIN_VALUE - 25), greaterThanOrEqual("id", INT_MAX_VALUE - 19)), + file()); assertThat(shouldRead).as("Should read: or(false, true)").isTrue(); } @Test public void testIntegerLt() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (5 < 30)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file()); assertThat(shouldRead) .as("Should not read: id range below lower bound (30 is not < 30)") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MIN_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE + 1), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThan("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerLtEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (5 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id range below lower bound (29 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, lessThanOrEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerGt() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE + 6)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (85 < 79)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE), file()); assertThat(shouldRead) .as("Should not read: id range above upper bound (79 is not > 79)") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 1), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerGtEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 6)) - .eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (85 < 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should not read: id range above upper bound (80 > 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: one possible id").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: many possible ids").isTrue(); } @Test public void testIntegerEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should not read: id below lower bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id below lower bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should not read: id above upper bound").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, equal("id", INT_MAX_VALUE + 6)).eval(FILE); + shouldRead = shouldRead(SCHEMA, equal("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should not read: id above upper bound").isFalse(); } @Test public void testIntegerNotEq() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE - 25)).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 25), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE - 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MIN_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE - 4)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE - 4), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE + 1)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("id", INT_MAX_VALUE + 6)).eval(FILE); + shouldRead = shouldRead(SCHEMA, notEqual("id", INT_MAX_VALUE + 6), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testIntegerNotEqRewritten() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE - 25))).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 25)), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE - 1))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE - 1)), file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MIN_VALUE))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MIN_VALUE)), file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE - 4))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE - 4)), file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE)), file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE + 1))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 1)), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("id", INT_MAX_VALUE + 6))).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("id", INT_MAX_VALUE + 6)), file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testCaseInsensitiveIntegerNotEqRewritten() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE - 25)), false) - .eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 25)), false, file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE - 1)), false, file()); assertThat(shouldRead).as("Should read: id below lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MIN_VALUE)), false, file()); assertThat(shouldRead).as("Should read: id equal to lower bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE - 4)), false, file()); assertThat(shouldRead).as("Should read: id between lower and upper bounds").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false).eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE)), false, file()); assertThat(shouldRead).as("Should read: id equal to upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 1)), false, file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)), false) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, not(equal("ID", INT_MAX_VALUE + 6)), false, file()); assertThat(shouldRead).as("Should read: id above upper bound").isTrue(); } @Test public void testCaseSensitiveIntegerNotEqRewritten() { - assertThatThrownBy( - () -> new InclusiveMetricsEvaluator(SCHEMA, not(equal("ID", 5)), true).eval(FILE)) + assertThatThrownBy(() -> shouldRead(SCHEMA, not(equal("ID", 5)), true, file())) .isInstanceOf(ValidationException.class) .hasMessageContaining("Cannot find field 'ID'"); } @Test public void testStringStartsWith() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"), true).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true, file()); assertThat(shouldRead).as("Should read: no stats").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "a"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "a"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aaa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aaa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1s"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "1s"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "1str1x"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "1str1x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "ff"), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, startsWith("required", "ff"), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "aB"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "aB"), true, file2()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "dWX"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, startsWith("required", "dWX"), true, file2()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "5"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "5"), true, file3()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", "3str3x"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, startsWith("required", "3str3x"), true, file3()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("some_empty", "房东整租霍"), true).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("some_empty", "房东整租霍"), true, file()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("all_nulls", ""), true).eval(FILE); + shouldRead = shouldRead(SCHEMA, startsWith("all_nulls", ""), true, file()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, startsWith("required", aboveMax), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, startsWith("required", aboveMax), true, file4()); assertThat(shouldRead).as("Should not read: range doesn't match").isFalse(); } @Test public void testStringNotStartsWith() { - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"), true).eval(FILE); + boolean shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true, file()); assertThat(shouldRead).as("Should read: no stats").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "a"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "a"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aaa"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aaa"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1s"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1s"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "1str1x"), true) - .eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "1str1x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "ff"), true).eval(FILE_4); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "ff"), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "aB"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "aB"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "dWX"), true).eval(FILE_2); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "dWX"), true, file2()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "5"), true).eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "5"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "3str3x"), true) - .eval(FILE_3); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "3str3x"), true, file3()); assertThat(shouldRead).as("Should read: range matches").isTrue(); String aboveMax = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", aboveMax), true) - .eval(FILE_4); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", aboveMax), true, file4()); assertThat(shouldRead).as("Should read: range matches").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "abc"), true).eval(FILE_5); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abc"), true, file5()); assertThat(shouldRead).as("Should not read: all strings start with prefix").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notStartsWith("required", "abcd"), true).eval(FILE_5); + shouldRead = shouldRead(SCHEMA, notStartsWith("required", "abcd"), true, file5()); assertThat(shouldRead).as("Should not read: lower shorter than prefix, cannot match").isTrue(); } @Test public void testIntegerIn() { boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)) - .eval(FILE); + shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24), file()); assertThat(shouldRead).as("Should not read: id below lower bound (5 < 30, 6 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should not read: id below lower bound (28 < 30, 29 < 30)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MIN_VALUE - 1, INT_MIN_VALUE)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MIN_VALUE - 1, INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound (30 == 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3), file()); assertThat(shouldRead) .as("Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE, INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id equal to upper bound (79 == 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2), file()); assertThat(shouldRead).as("Should not read: id above upper bound (80 > 79, 81 > 79)").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7), file()); assertThat(shouldRead).as("Should not read: id above upper bound (85 > 79, 86 > 79)").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("all_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("all_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should skip: in on all nulls column").isFalse(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("some_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("some_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: in on some nulls column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("no_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("no_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: in on no nulls column").isTrue(); // should read as the number of elements in the in expression is too big @@ -843,339 +903,168 @@ public void testIntegerIn() { for (int id = -400; id <= 0; id++) { ids.add(id); } - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, in("id", ids)).eval(FILE); + shouldRead = shouldRead(SCHEMA, in("id", ids), file()); assertThat(shouldRead).as("Should read: large in expression").isTrue(); } @Test public void testIntegerNotIn() { boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)) - .eval(FILE); + shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 25, INT_MIN_VALUE - 24), file()); assertThat(shouldRead).as("Should read: id below lower bound (5 < 30, 6 < 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 2, INT_MIN_VALUE - 1), file()); assertThat(shouldRead).as("Should read: id below lower bound (28 < 30, 29 < 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MIN_VALUE - 1, INT_MIN_VALUE)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MIN_VALUE - 1, INT_MIN_VALUE), file()); assertThat(shouldRead).as("Should read: id equal to lower bound (30 == 30)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE - 4, INT_MAX_VALUE - 3), file()); assertThat(shouldRead) .as("Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE + 1)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE, INT_MAX_VALUE + 1), file()); assertThat(shouldRead).as("Should read: id equal to upper bound (79 == 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 1, INT_MAX_VALUE + 2), file()); assertThat(shouldRead).as("Should read: id above upper bound (80 > 79, 81 > 79)").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)) - .eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("id", INT_MAX_VALUE + 6, INT_MAX_VALUE + 7), file()); assertThat(shouldRead).as("Should read: id above upper bound (85 > 79, 86 > 79)").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("all_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("all_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on all nulls column").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("some_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("some_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on some nulls column").isTrue(); - shouldRead = new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nulls", "abc", "def")).eval(FILE); + shouldRead = shouldRead(SCHEMA, notIn("no_nulls", "abc", "def"), file()); assertThat(shouldRead).as("Should read: notIn on no nulls column").isTrue(); } @Test public void testIsNullInNestedStruct() { // read required_address and its nested fields - boolean shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address")).eval(FILE_6); + boolean shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address"), file6()); assertThat(shouldRead).as("Should not read: required_address is required").isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address.required_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address.required_street1"), file6()); assertThat(shouldRead) .as("Should not read: required_address.required_street1 is required") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("required_address.optional_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("required_address.optional_street1"), file6()); assertThat(shouldRead) .as("Should read: required_address.optional_street1 is optional") .isTrue(); // read optional_address and its nested fields - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address")).eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.required_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address.required_street2"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, isNull("optional_address.optional_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, isNull("optional_address.optional_street2"), file6()); assertThat(shouldRead).as("Should read: optional_address is optional").isTrue(); } @Test public void testNotNullInNestedStruct() { // read required_address and its nested fields - boolean shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address")).eval(FILE_6); + boolean shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address"), file6()); assertThat(shouldRead).as("Should read: required_address is required").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address.required_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address.required_street1"), file6()); assertThat(shouldRead) .as("Should read: required_address.required_street1 is required") .isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("required_address.optional_street1")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("required_address.optional_street1"), file6()); assertThat(shouldRead) .as("Should not read: required_address.optional_street1 is optional") .isFalse(); // read optional_address and its nested fields - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address")).eval(FILE_6); - assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address"), file6()); + assertThat(shouldRead).as("Should read: metrics are not tracked for structs").isTrue(); - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.required_street2")) - .eval(FILE_6); - assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); - - shouldRead = - new InclusiveMetricsEvaluator(NESTED_SCHEMA, notNull("optional_address.optional_street2")) - .eval(FILE_6); + shouldRead = shouldRead(NESTED_SCHEMA, notNull("optional_address.optional_street2"), file6()); assertThat(shouldRead) .as("Should not read: optional_address.optional_street2 is optional") .isFalse(); } + @Test + public void notNullForRequiredFieldInOptionalStruct() { + boolean shouldRead = + shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); + assertThat(shouldRead).as("Should not read: optional_address is optional").isFalse(); + } + @Test public void testNotEqSingleValueWithoutNaN() { - Schema schema = new Schema(required(1, "f", Types.FloatType.get())); - Map bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); - DataFile singleValueFile = - new TestDataFile( - "single_value_file.avro", - Row.of(), - 10, - ImmutableMap.of(1, 10L), - ImmutableMap.of(1, 0L), - ImmutableMap.of(1, 0L), - bound, - bound); - - assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile)) + assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFile())) .as("Should skip: file contains no values not equal to 1.0") .isFalse(); } @Test public void testNotEqSingleValueWithNaN() { - Schema schema = new Schema(required(1, "f", Types.FloatType.get())); - Map bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f)); - DataFile singleValueFile = - new TestDataFile( - "single_value_file.avro", - Row.of(), - 10, - ImmutableMap.of(1, 10L), - ImmutableMap.of(1, 0L), - ImmutableMap.of(1, 1L), // contains a NaN value - bound, - bound); - - assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile)) + assertThat(shouldRead(FLOAT_SCHEMA, notEqual("f", 1.0f), singleFloatValueFileWithNaN())) .as("Should read: file contains a NaN value not equal to 1.0") .isTrue(); } @Test public void testNotEqWithSingleValue() { - DataFile rangeOfValues = - new TestDataFile( - "range_of_values.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); - - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "aaa")).eval(rangeOfValues); + boolean shouldRead = shouldRead(SCHEMA, notEqual("required", "aaa"), rangeOfValues()); assertThat(shouldRead) .as("Should read: file has range of values, cannot exclude based on literal") .isTrue(); - DataFile singleValueFile = - new TestDataFile( - "single_value.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")).eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notEqual("required", "abc"), singleValueFile()); assertThat(shouldRead) .as("Should not read: file contains single value equal to literal") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "def")).eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notEqual("required", "def"), singleValueFile()); assertThat(shouldRead) .as("Should read: file contains single value not equal to literal") .isTrue(); - DataFile singleValueWithNulls = - new TestDataFile( - "single_value_nulls.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 2L), - null, - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")) - .eval(singleValueWithNulls); + shouldRead = shouldRead(SCHEMA, notEqual("some_empty", "abc"), singleValueWithNulls()); assertThat(shouldRead).as("Should read: file has nulls which match != predicate").isTrue(); - DataFile singleValueWithNaN = - new TestDataFile( - "single_value_nan.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 2L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueWithNaN); + shouldRead = shouldRead(SCHEMA, notEqual("no_nans", 5.0F), singleValueWithNaN()); assertThat(shouldRead).as("Should read: file has NaN values which match != predicate").isTrue(); - DataFile singleValueNaNBounds = - new TestDataFile( - "single_value_nan_bounds.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueNaNBounds); + shouldRead = shouldRead(SCHEMA, notEqual("no_nans", 5.0F), singleValueNaNBounds()); assertThat(shouldRead).as("Should read: bounds are NaN").isTrue(); } @Test public void testNotInWithSingleValue() { - DataFile rangeOfValues = - new TestDataFile( - "range_of_values.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz"))); - - boolean shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "aaa", "bbb")).eval(rangeOfValues); + boolean shouldRead = shouldRead(SCHEMA, notIn("required", "aaa", "bbb"), rangeOfValues()); assertThat(shouldRead) .as("Should read: file has range of values, cannot exclude based on literal") .isTrue(); - DataFile singleValueFile = - new TestDataFile( - "single_value.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def")) - .eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notIn("required", "abc", "def"), singleValueFile()); assertThat(shouldRead) .as("Should not read: file contains single value in exclusion list") .isFalse(); - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "def", "ghi")) - .eval(singleValueFile); + shouldRead = shouldRead(SCHEMA, notIn("required", "def", "ghi"), singleValueFile()); assertThat(shouldRead) .as("Should read: file contains single value not in exclusion list") .isTrue(); - DataFile singleValueWithNulls = - new TestDataFile( - "single_value_nulls.avro", - Row.of(), - 10, - ImmutableMap.of(3, 10L), - ImmutableMap.of(3, 2L), - ImmutableMap.of(3, 0L), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")), - ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc"))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def")) - .eval(singleValueWithNulls); + shouldRead = shouldRead(SCHEMA, notIn("some_empty", "abc", "def"), singleValueWithNulls()); assertThat(shouldRead).as("Should read: file has nulls which match NOT IN predicate").isTrue(); - DataFile singleValueWithNaN = - new TestDataFile( - "single_value_nan.avro", - Row.of(), - 10, - ImmutableMap.of(9, 10L), - ImmutableMap.of(9, 0L), - ImmutableMap.of(9, 2L), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)), - ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F))); - - shouldRead = - new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nans", 5.0F, 10.0F)) - .eval(singleValueWithNaN); + shouldRead = shouldRead(SCHEMA, notIn("no_nans", 5.0F, 10.0F), singleValueWithNaN()); assertThat(shouldRead) .as("Should read: file has NaN values which match NOT IN predicate") .isTrue(); diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index b7da4b3108e6..28231fd4a171 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -971,6 +971,103 @@ public void ancestorFieldsInNestedSchema() { assertThat(TypeUtil.ancestorFields(schema, 17)).containsExactly(pointsElement, points); } + @Test + public void isNullableWithUnknownFieldId() { + Schema schema = new Schema(required(1, "id", IntegerType.get())); + + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + } + + @Test + public void isNullableWithTopLevelFields() { + Schema schema = + new Schema( + required(1, "id", IntegerType.get()), optional(2, "data", Types.StringType.get())); + + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + } + + @Test + public void isNullableWithNestedStructs() { + Schema schema = + new Schema( + required( + 1, + "required_location", + Types.StructType.of( + required(3, "required_lat", Types.DoubleType.get()), + optional(4, "optional_lon", Types.DoubleType.get()), + required( + 5, + "required_inner", + Types.StructType.of(required(6, "required_zip", IntegerType.get()))))), + optional( + 2, + "optional_location", + Types.StructType.of( + required(7, "required_lat", Types.DoubleType.get()), + required( + 8, + "required_inner", + Types.StructType.of(required(9, "required_zip", IntegerType.get())))))); + + // a required field is not null when every field that contains it is required + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 3)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 5)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 6)).isFalse(); + + // an optional field is null regardless of the fields that contain it + assertThat(TypeUtil.isNullable(schema, 4)).isTrue(); + + // a required field nested in an optional struct is null when the struct is null + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 7)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 8)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 9)).isTrue(); + } + + @Test + public void isNullableWithListsAndMaps() { + Schema schema = + new Schema( + required( + 1, + "required_points", + Types.ListType.ofRequired( + 4, Types.StructType.of(required(5, "required_x", Types.LongType.get())))), + optional( + 2, + "optional_points", + Types.ListType.ofOptional( + 6, Types.StructType.of(required(7, "required_x", Types.LongType.get())))), + required( + 3, + "locations", + Types.MapType.ofRequired( + 8, + 9, + Types.StringType.get(), + Types.StructType.of(required(10, "required_lat", Types.DoubleType.get()))))); + + // a required element of a required list is not null, nor is anything it contains + assertThat(TypeUtil.isNullable(schema, 1)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 4)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 5)).isFalse(); + + // an optional element is null, as is anything it contains + assertThat(TypeUtil.isNullable(schema, 2)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 6)).isTrue(); + assertThat(TypeUtil.isNullable(schema, 7)).isTrue(); + + // required map keys and values are not null + assertThat(TypeUtil.isNullable(schema, 3)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 8)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 9)).isFalse(); + assertThat(TypeUtil.isNullable(schema, 10)).isFalse(); + } + @Test public void testIndexStatsNames() { Schema schema = diff --git a/core/src/main/java/org/apache/iceberg/InclusiveStatsEvaluator.java b/core/src/main/java/org/apache/iceberg/InclusiveStatsEvaluator.java new file mode 100644 index 000000000000..2545c63b3ec9 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/InclusiveStatsEvaluator.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.expressions.Expressions.rewriteNot; + +import java.util.Collections; +import java.util.Set; +import org.apache.iceberg.expressions.Binder; +import org.apache.iceberg.expressions.BoundExtract; +import org.apache.iceberg.expressions.BoundReference; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.ExpressionVisitors; +import org.apache.iceberg.expressions.InclusiveEvalVisitor; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.variants.Variant; +import org.apache.iceberg.variants.VariantObject; + +/** + * Evaluates an {@link Expression} on a {@link TrackedFile} to test whether rows in the file may + * match. + * + *

This evaluation is inclusive: it returns true if a file may match and false if it cannot + * match. + * + *

Files are passed to {@link #eval(TrackedFile)}, which returns true if the file may contain + * matching rows and false if the file cannot contain matching rows. Files may be skipped if and + * only if the return value of {@code eval} is false. + * + *

Due to the comparison implementation of ORC stats, for float/double columns in ORC files, if + * the first value in a file is NaN, metrics of this file will report NaN for both upper and lower + * bound despite that the column could contain non-NaN data. Thus, in some scenarios explicitly + * checks for NaN is necessary in order to not skip files that may contain matching data. + */ +class InclusiveStatsEvaluator { + private final Expression expr; + private final Set neverNullIds; + + InclusiveStatsEvaluator(Schema schema, Expression unbound) { + this(schema, unbound, true); + } + + InclusiveStatsEvaluator(Schema schema, Expression unbound, boolean caseSensitive) { + Types.StructType struct = schema.asStruct(); + this.expr = Binder.bind(struct, rewriteNot(unbound), caseSensitive); + this.neverNullIds = + neverNullIds( + schema, Binder.boundReferences(struct, Collections.singletonList(expr), caseSensitive)); + } + + /** + * Returns the IDs of the referenced fields that cannot contain null values. + * + *

Stats omit the null count for every required field. The count is zero for a field that + * cannot be null, but it is unknown for a required field that an optional struct contains because + * that field is null whenever the struct is null. + */ + private static Set neverNullIds(Schema schema, Set referencedIds) { + ImmutableSet.Builder neverNull = ImmutableSet.builder(); + + for (int id : referencedIds) { + if (!TypeUtil.isNullable(schema, id)) { + neverNull.add(id); + } + } + + return neverNull.build(); + } + + /** + * Test whether the file may contain records that match the expression. + * + * @param file a tracked file + * @return false if the file cannot contain rows that match the expression, true otherwise. + */ + boolean eval(TrackedFile file) { + return new StatsEvalVisitor().eval(file); + } + + private class StatsEvalVisitor extends InclusiveEvalVisitor { + private ContentStats stats = null; + + private boolean eval(TrackedFile file) { + if (file.recordCount() == 0) { + return ROWS_CANNOT_MATCH; + } + + if (file.recordCount() < 0) { + // imported Avro files may have an incorrect -1 row count + return ROWS_MIGHT_MATCH; + } + + if (null == file.contentStats()) { + return ROWS_MIGHT_MATCH; + } + + this.stats = file.contentStats(); + + return ExpressionVisitors.visitEvaluator(expr, this); + } + + @Override + protected boolean mayContainNull(int id) { + if (neverNullIds.contains(id)) { + return false; + } + + FieldStats fieldStats = stats.statsFor(id); + return fieldStats == null + || !fieldStats.hasNullValueCount() + || fieldStats.nullValueCount() != 0; + } + + @Override + protected boolean containsNullsOnly(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats != null + && fieldStats.hasValueCount() + && fieldStats.hasNullValueCount() + && fieldStats.valueCount() - fieldStats.nullValueCount() == 0; + } + + @Override + protected boolean mayContainNaN(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats == null + || !fieldStats.hasNanValueCount() + || fieldStats.nanValueCount() != 0; + } + + @Override + protected boolean containsNaNsOnly(int id) { + FieldStats fieldStats = stats.statsFor(id); + return fieldStats != null + && fieldStats.hasValueCount() + && fieldStats.hasNanValueCount() + && fieldStats.valueCount() - fieldStats.nanValueCount() == 0; + } + + @Override + protected T lowerBound(BoundReference ref) { + FieldStats fieldStats = stats.statsFor(ref.fieldId()); + return fieldStats != null ? fieldStats.lowerBound() : null; + } + + @Override + protected T upperBound(BoundReference ref) { + FieldStats fieldStats = stats.statsFor(ref.fieldId()); + return fieldStats != null ? fieldStats.upperBound() : null; + } + + @Override + protected T extractLowerBound(BoundExtract bound) { + FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); + if (fieldStats != null && fieldStats.lowerBound() != null) { + VariantObject fieldLowerBounds = fieldStats.lowerBound().value().asObject(); + return castTo(fieldLowerBounds.get(bound.path()), bound.type()); + } + + return null; + } + + @Override + protected T extractUpperBound(BoundExtract bound) { + FieldStats fieldStats = stats.statsFor(bound.ref().fieldId()); + if (fieldStats != null && fieldStats.upperBound() != null) { + VariantObject fieldUpperBounds = fieldStats.upperBound().value().asObject(); + return castTo(fieldUpperBounds.get(bound.path()), bound.type()); + } + + return null; + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java index 1a6ee1923292..32b4c249c2f6 100644 --- a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java +++ b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java @@ -18,12 +18,80 @@ */ package org.apache.iceberg; +import java.util.List; import org.apache.iceberg.types.Types; import org.mockito.Mockito; class StatsTestUtil { + private static final int FORMAT_VERSION_V4 = 4; + private StatsTestUtil() {} + static TrackedFile trackedFile(String location, long recordCount, ContentStats stats) { + return new TrackedFileStruct( + null, + FileContent.DATA, + FORMAT_VERSION_V4, + location, + FileFormat.fromFileName(location), + null, + recordCount, + 1024L, + null, + stats, + null, + null, + null, + null, + null, + null); + } + + static ContentStats contentStats(Types.StructType statsType, FieldStats... fieldStats) { + ContentStatsStruct stats = new ContentStatsStruct(statsType); + for (FieldStats field : fieldStats) { + stats.setStats(field.fieldId(), field); + } + + return stats; + } + + /** Returns the stats for a field, where a null metric is one that the column does not track. */ + static FieldStats fieldStats( + Types.StructType statsType, + int fieldId, + Object lower, + Object upper, + Long valueCount, + Long nullCount, + Long nanCount) { + Types.StructType type = statsType.field(StatsUtil.toBaseId(fieldId)).type().asStructType(); + FieldStatsStruct stats = new FieldStatsStruct<>(type); + set(stats, StatsUtil.LOWER_BOUND_NAME, lower); + set(stats, StatsUtil.UPPER_BOUND_NAME, upper); + set(stats, "value_count", valueCount); + set(stats, "null_value_count", nullCount); + set(stats, "nan_value_count", nanCount); + + return stats; + } + + private static void set(FieldStatsStruct stats, String metric, Object value) { + if (value == null) { + return; + } + + List fields = stats.type().fields(); + for (int pos = 0; pos < fields.size(); pos += 1) { + if (fields.get(pos).name().equals(metric)) { + stats.set(pos, value); + return; + } + } + + throw new IllegalArgumentException("Cannot set " + metric + ": not tracked by " + stats.type()); + } + /** * Mocks a {@link FieldStats} for a stats struct type, stubbing the bounds and only the counts * that are present. A null count leaves the matching {@code has*Count()} reporting false, so a diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java new file mode 100644 index 000000000000..1cf8af4f0093 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluator.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.StatsTestUtil.contentStats; +import static org.apache.iceberg.StatsTestUtil.fieldStats; +import static org.apache.iceberg.StatsTestUtil.trackedFile; +import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.apache.iceberg.expressions.Expressions.notNull; +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +class TestInclusiveStatsEvaluator extends TestInclusiveMetricsEvaluator { + private static final Types.StructType STATS_TYPE = + StatsUtil.statsReadSchema( + SCHEMA, ImmutableList.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)); + + // stats are tracked for the leaf fields, not for the address structs themselves + private static final Types.StructType NESTED_STATS_TYPE = + StatsUtil.statsReadSchema(NESTED_SCHEMA, ImmutableList.of(102, 103, 104, 105)); + + private static final Types.StructType FLOAT_STATS_TYPE = + StatsUtil.statsReadSchema(FLOAT_SCHEMA, ImmutableList.of(1)); + + @Override + protected boolean shouldRead( + Schema schema, Expression expr, boolean caseSensitive, TrackedFile testFile) { + return new InclusiveStatsEvaluator(schema, expr, caseSensitive).eval(testFile); + } + + @Override + protected TrackedFile file() { + return trackedFile( + "file.avro", + 50, + contentStats( + STATS_TYPE, + stats(1, INT_MIN_VALUE, INT_MAX_VALUE, null, null, null), + stats(4, null, null, 50L, 50L, null), + stats(5, null, null, 50L, 10L, null), + stats(6, null, null, 50L, 0L, null), + stats(7, null, null, 50L, null, 50L), + stats(8, null, null, 50L, null, 10L), + stats(9, null, null, 50L, null, 0L), + stats(10, null, null, 50L, 50L, null), + stats(11, Float.NaN, Float.NaN, 50L, 0L, null), + stats(12, Double.NaN, Double.NaN, 50L, 1L, null), + stats(13, null, null, 50L, null, null), + stats(14, "", "房东整租霍营小区二层两居室", 50L, 0L, null))); + } + + @Override + protected TrackedFile file2() { + return trackedFile( + "file_2.avro", 50, contentStats(STATS_TYPE, stats(3, "aa", "dC", 50L, null, null))); + } + + @Override + protected TrackedFile file3() { + return trackedFile( + "file_3.avro", 50, contentStats(STATS_TYPE, stats(3, "1str1", "3str3", 50L, null, null))); + } + + @Override + protected TrackedFile file4() { + return trackedFile( + "file_4.avro", 50, contentStats(STATS_TYPE, stats(3, "abc", "イロハニホヘト", 50L, null, null))); + } + + @Override + protected TrackedFile file5() { + return trackedFile( + "file_5.avro", 50, contentStats(STATS_TYPE, stats(3, "abc", "abcdefghi", 50L, null, null))); + } + + @Override + protected TrackedFile file6() { + return trackedFile( + "file_6.avro", + 10, + contentStats( + NESTED_STATS_TYPE, + stats(NESTED_STATS_TYPE, 102, null, null, 5L, null, null), + stats(NESTED_STATS_TYPE, 103, null, null, 5L, 5L, null), + // required_street2 is required, so its stats do not track a null count + stats(NESTED_STATS_TYPE, 104, null, null, 5L, null, null), + stats(NESTED_STATS_TYPE, 105, null, null, 5L, 5L, null))); + } + + @Override + protected TrackedFile missingStats() { + return trackedFile("file.parquet", 50, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile emptyFile() { + return trackedFile("file.parquet", 0, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile rangeOfValues() { + return trackedFile( + "range_of_values.avro", + 10, + contentStats(STATS_TYPE, stats(3, "aaa", "zzz", 10L, null, null))); + } + + @Override + protected TrackedFile singleValueFile() { + return trackedFile( + "single_value.avro", 10, contentStats(STATS_TYPE, stats(3, "abc", "abc", 10L, null, null))); + } + + @Override + protected TrackedFile singleValueWithNulls() { + return trackedFile( + "single_value_nulls.avro", + 10, + contentStats(STATS_TYPE, stats(14, "abc", "abc", 10L, 2L, null))); + } + + @Override + protected TrackedFile singleValueWithNaN() { + return trackedFile( + "single_value_nan.avro", 10, contentStats(STATS_TYPE, stats(9, 5.0f, 5.0f, 10L, 0L, 2L))); + } + + @Override + protected TrackedFile singleValueNaNBounds() { + return trackedFile( + "single_value_nan_bounds.avro", + 10, + contentStats(STATS_TYPE, stats(9, Float.NaN, Float.NaN, 10L, 0L, 0L))); + } + + @Override + protected TrackedFile singleFloatValueFile() { + return trackedFile( + "single_value_file.avro", + 10, + contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 0L))); + } + + @Override + protected TrackedFile singleFloatValueFileWithNaN() { + return trackedFile( + "single_value_file.avro", + 10, + contentStats(FLOAT_STATS_TYPE, stats(FLOAT_STATS_TYPE, 1, 1.0f, 1.0f, 10L, null, 1L))); + } + + /** + * Content stats omit the null count for a required field, even when an optional struct contains + * it, so a file where every value is null cannot be pruned. + */ + @Override + @Test + public void notNullForRequiredFieldInOptionalStruct() { + boolean shouldRead = + shouldRead(NESTED_SCHEMA, notNull("optional_address.required_street2"), file6()); + assertThat(shouldRead).as("Should read: the null count is not tracked").isTrue(); + } + + @Test + void fileWithoutContentStats() { + TrackedFile file = trackedFile("file.avro", 50, null); + + assertThat(shouldRead(SCHEMA, lessThan("id", INT_MIN_VALUE), file)) + .as("Should read: file does not track content stats") + .isTrue(); + } + + private static FieldStats stats( + int fieldId, Object lower, Object upper, Long valueCount, Long nullCount, Long nanCount) { + return stats(STATS_TYPE, fieldId, lower, upper, valueCount, nullCount, nanCount); + } + + private static FieldStats stats( + Types.StructType statsType, + int fieldId, + Object lower, + Object upper, + Long valueCount, + Long nullCount, + Long nanCount) { + return fieldStats(statsType, fieldId, lower, upper, valueCount, nullCount, nanCount); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java new file mode 100644 index 000000000000..7e82c0af40c1 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithExtract.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.StatsTestUtil.contentStats; +import static org.apache.iceberg.StatsTestUtil.fieldStats; +import static org.apache.iceberg.StatsTestUtil.trackedFile; + +import java.util.Map; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluatorWithExtract; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.variants.Variant; +import org.apache.iceberg.variants.VariantTestUtil; +import org.apache.iceberg.variants.VariantValue; +import org.apache.iceberg.variants.Variants; + +class TestInclusiveStatsEvaluatorWithExtract + extends TestInclusiveMetricsEvaluatorWithExtract { + private static final Types.StructType STATS_TYPE = + StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3)); + + private static final Variant LOWER_BOUND = + VariantTestUtil.variant( + Map.of("$['event_id']", Variants.of(INT_MIN_VALUE), "$['str']", Variants.of("abc"))); + + private static final Variant UPPER_BOUND = + VariantTestUtil.variant( + Map.of("$['event_id']", Variants.of(INT_MAX_VALUE), "$['str']", Variants.of("abe"))); + + @Override + protected boolean shouldRead(Expression expr, TrackedFile testFile, boolean caseSensitive) { + return new InclusiveStatsEvaluator(SCHEMA, expr, caseSensitive).eval(testFile); + } + + @Override + protected TrackedFile file() { + return trackedFile( + "file.avro", + 50, + contentStats( + STATS_TYPE, + fieldStats(STATS_TYPE, 1, null, null, 50L, null, null), + fieldStats(STATS_TYPE, 2, LOWER_BOUND, UPPER_BOUND, 50L, null, null), + fieldStats(STATS_TYPE, 3, null, null, 50L, 50L, null))); + } + + @Override + protected TrackedFile emptyFile() { + return trackedFile("file.parquet", 0, contentStats(STATS_TYPE)); + } + + @Override + protected TrackedFile fileWithVariantBounds(String path, VariantValue lower, VariantValue upper) { + return trackedFile( + "file.parquet", + 50, + contentStats( + STATS_TYPE, + fieldStats( + STATS_TYPE, + 2, + VariantTestUtil.variant(Map.of(path, lower)), + VariantTestUtil.variant(Map.of(path, upper)), + null, + null, + null))); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java new file mode 100644 index 000000000000..a034d9a58ee2 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestInclusiveStatsEvaluatorWithTransforms.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.StatsTestUtil.contentStats; +import static org.apache.iceberg.StatsTestUtil.fieldStats; +import static org.apache.iceberg.StatsTestUtil.trackedFile; + +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.TestInclusiveMetricsEvaluatorWithTransforms; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; + +class TestInclusiveStatsEvaluatorWithTransforms + extends TestInclusiveMetricsEvaluatorWithTransforms { + private static final Types.StructType STATS_TYPE = + StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4, 5, 6)); + + @Override + protected boolean shouldRead(Expression expr, TrackedFile testFile, boolean caseSensitive) { + return new InclusiveStatsEvaluator(SCHEMA, expr, caseSensitive).eval(testFile); + } + + @Override + protected TrackedFile file() { + return trackedFile( + "file.avro", + 50, + contentStats( + STATS_TYPE, + fieldStats(STATS_TYPE, 1, null, null, 50L, null, null), + fieldStats(STATS_TYPE, 2, TS_MIN_VALUE, TS_MAX_VALUE, 50L, null, null), + fieldStats(STATS_TYPE, 3, null, null, 50L, 50L, null), + fieldStats(STATS_TYPE, 4, null, null, 50L, 50L, null), + fieldStats(STATS_TYPE, 6, "abc", "abe", null, null, null))); + } + + @Override + protected TrackedFile emptyFile() { + return trackedFile("file.parquet", 0, contentStats(STATS_TYPE)); + } +} diff --git a/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithExtract.java b/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithExtract.java index 03c1c12f4fba..01944969a9cc 100644 --- a/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithExtract.java +++ b/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithExtract.java @@ -63,15 +63,15 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.FieldSource; -public class TestInclusiveMetricsEvaluatorWithExtract { - private static final Schema SCHEMA = +public class TestInclusiveMetricsEvaluatorWithExtract { + protected static final Schema SCHEMA = new Schema( required(1, "id", IntegerType.get()), required(2, "variant", Types.VariantType.get()), optional(3, "all_nulls", Types.VariantType.get())); - private static final int INT_MIN_VALUE = 30; - private static final int INT_MAX_VALUE = 79; + protected static final int INT_MIN_VALUE = 30; + protected static final int INT_MAX_VALUE = 79; private static final DataFile FILE = new TestDataFile( @@ -108,19 +108,43 @@ public class TestInclusiveMetricsEvaluatorWithExtract { Variants.of("abe"))))); private boolean shouldRead(Expression expr) { - return shouldRead(expr, FILE); + return shouldRead(expr, file()); } - private boolean shouldRead(Expression expr, DataFile file) { + private boolean shouldRead(Expression expr, F file) { return shouldRead(expr, file, true); } private boolean shouldReadCaseInsensitive(Expression expr) { - return shouldRead(expr, FILE, false); + return shouldRead(expr, file(), false); } - private boolean shouldRead(Expression expr, DataFile file, boolean caseSensitive) { - return new InclusiveMetricsEvaluator(SCHEMA, expr, caseSensitive).eval(file); + protected boolean shouldRead(Expression expr, F file, boolean caseSensitive) { + return new InclusiveMetricsEvaluator(SCHEMA, expr, caseSensitive).eval((DataFile) file); + } + + protected F file() { + return asFile(FILE); + } + + protected F emptyFile() { + return asFile(new TestDataFile("file.parquet", Row.of(), 0)); + } + + /** Returns a file with variant bounds for the given path in the {@code variant} column. */ + protected F fileWithVariantBounds(String path, VariantValue lower, VariantValue upper) { + Map lowerBounds = + ImmutableMap.of(2, VariantTestUtil.variantBuffer(Map.of(path, lower))); + Map upperBounds = + ImmutableMap.of(2, VariantTestUtil.variantBuffer(Map.of(path, upper))); + + return asFile( + new TestDataFile("file.parquet", Row.of(), 50, null, null, null, lowerBounds, upperBounds)); + } + + @SuppressWarnings("unchecked") + private F asFile(DataFile dataFile) { + return (F) dataFile; } @Test @@ -223,7 +247,7 @@ public void testMissingStats(Expression expr) { @ParameterizedTest @FieldSource("MISSING_STATS_EXPRESSIONS") public void testZeroRecordFile(Expression expr) { - DataFile empty = new TestDataFile("file.parquet", Row.of(), 0); + F empty = emptyFile(); assertThat(shouldRead(expr, empty)).as("Should never read 0-record file: " + expr).isFalse(); } @@ -851,21 +875,9 @@ public void testIntegerNotIn() { @ParameterizedTest @FieldSource("DATEANDTIMESTAMPTYPESEQPARAMETERS") public void testDateAndTimestampTypesEq(String variantType, Arguments args) { - // lower bounds - Map lowerBounds = - ImmutableMap.of( - 2, - VariantTestUtil.variantBuffer( - Map.of("$['event_timestamp']", (VariantValue) args.get()[1]))); - // upper bounds - Map upperBounds = - ImmutableMap.of( - 2, - VariantTestUtil.variantBuffer( - Map.of("$['event_timestamp']", (VariantValue) args.get()[2]))); - - DataFile file = - new TestDataFile("file.parquet", Row.of(), 50, null, null, null, lowerBounds, upperBounds); + F file = + fileWithVariantBounds( + "$['event_timestamp']", (VariantValue) args.get()[1], (VariantValue) args.get()[2]); Expression expr = equal(extract("variant", "$.event_timestamp", variantType), args.get()[0]); assertThat(shouldRead(expr, file)).isEqualTo(args.get()[3]); } @@ -998,21 +1010,9 @@ public void testDateAndTimestampTypesEq(String variantType, Arguments args) { @ParameterizedTest @FieldSource("DATEANDTIMESTAMPTYPESNOTEQPARAMETERS") public void testDateAndTimestampTypesNotEq(String variantType, Arguments args) { - // lower bounds - Map lowerBounds = - ImmutableMap.of( - 2, - VariantTestUtil.variantBuffer( - Map.of("$['event_timestamp']", (VariantValue) args.get()[1]))); - // upper bounds - Map upperBounds = - ImmutableMap.of( - 2, - VariantTestUtil.variantBuffer( - Map.of("$['event_timestamp']", (VariantValue) args.get()[2]))); - - DataFile file = - new TestDataFile("file.parquet", Row.of(), 50, null, null, null, lowerBounds, upperBounds); + F file = + fileWithVariantBounds( + "$['event_timestamp']", (VariantValue) args.get()[1], (VariantValue) args.get()[2]); Expression expr = notEqual(extract("variant", "$.event_timestamp", variantType), args.get()[0]); assertThat(shouldRead(expr, file)).as("Should read: many possible timestamps" + expr).isTrue(); } @@ -1020,16 +1020,7 @@ public void testDateAndTimestampTypesNotEq(String variantType, Arguments args) { @Test public void testUUIDEq() { UUID uuid = UUID.randomUUID(); - // lower bounds - Map lowerBounds = - ImmutableMap.of( - 2, VariantTestUtil.variantBuffer(Map.of("$['event_uuid']", Variants.ofUUID(uuid)))); - // upper bounds - Map upperBounds = - ImmutableMap.of( - 2, VariantTestUtil.variantBuffer(Map.of("$['event_uuid']", Variants.ofUUID(uuid)))); - DataFile file = - new TestDataFile("file.parquet", Row.of(), 50, null, null, null, lowerBounds, upperBounds); + F file = fileWithVariantBounds("$['event_uuid']", Variants.ofUUID(uuid), Variants.ofUUID(uuid)); Expression expr = equal(extract("variant", "$.event_uuid", PhysicalType.UUID.name()), uuid); assertThat(shouldRead(expr, file)).as("Should read: many possible UUIDs" + expr).isTrue(); } diff --git a/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithTransforms.java b/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithTransforms.java index 41a14667eae1..69db255018de 100644 --- a/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithTransforms.java +++ b/core/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluatorWithTransforms.java @@ -58,8 +58,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.FieldSource; -public class TestInclusiveMetricsEvaluatorWithTransforms { - private static final Schema SCHEMA = +public class TestInclusiveMetricsEvaluatorWithTransforms { + protected static final Schema SCHEMA = new Schema( required(1, "id", IntegerType.get()), required(2, "ts", Types.TimestampType.withZone()), @@ -68,13 +68,13 @@ public class TestInclusiveMetricsEvaluatorWithTransforms { optional(5, "no_stats", IntegerType.get()), optional(6, "str", Types.StringType.get())); - private static final int INT_MIN_VALUE = 30; - private static final int INT_MAX_VALUE = 79; + protected static final int INT_MIN_VALUE = 30; + protected static final int INT_MAX_VALUE = 79; - private static final long TS_MIN_VALUE = + protected static final long TS_MIN_VALUE = DateTimeUtil.microsFromTimestamptz( DateTimeUtil.dateFromDays(30).atStartOfDay().atOffset(ZoneOffset.UTC)); - private static final long TS_MAX_VALUE = + protected static final long TS_MAX_VALUE = DateTimeUtil.microsFromTimestamptz( DateTimeUtil.dateFromDays(79).atStartOfDay().atOffset(ZoneOffset.UTC)); @@ -109,19 +109,32 @@ public class TestInclusiveMetricsEvaluatorWithTransforms { 6, Conversions.toByteBuffer(Types.StringType.get(), "abe"))); private boolean shouldRead(Expression expr) { - return shouldRead(expr, FILE); + return shouldRead(expr, file()); } - private boolean shouldRead(Expression expr, DataFile file) { + private boolean shouldRead(Expression expr, F file) { return shouldRead(expr, file, true); } private boolean shouldReadCaseInsensitive(Expression expr) { - return shouldRead(expr, FILE, false); + return shouldRead(expr, file(), false); } - private boolean shouldRead(Expression expr, DataFile file, boolean caseSensitive) { - return new InclusiveMetricsEvaluator(SCHEMA, expr, caseSensitive).eval(file); + protected boolean shouldRead(Expression expr, F file, boolean caseSensitive) { + return new InclusiveMetricsEvaluator(SCHEMA, expr, caseSensitive).eval((DataFile) file); + } + + protected F file() { + return asFile(FILE); + } + + protected F emptyFile() { + return asFile(new TestDataFile("file.parquet", Row.of(), 0)); + } + + @SuppressWarnings("unchecked") + private F asFile(DataFile dataFile) { + return (F) dataFile; } @Test @@ -289,7 +302,7 @@ public void testMissingStats(Expression expr) { @ParameterizedTest @FieldSource("MISSING_STATS_EXPRESSIONS") public void testZeroRecordFile(Expression expr) { - DataFile empty = new TestDataFile("file.parquet", Row.of(), 0); + F empty = emptyFile(); assertThat(shouldRead(expr, empty)).as("Should never read 0-record file: " + expr).isFalse(); }