SOLR-18251: Introduced docValuesSkipList='true' field property

changelog/unreleased/SOLR-18251-dv-skip-list.yml

@@ -0,0 +1,8 @@
+title: Added `docValuesSkipList` schema field option to enable Lucene DocValues skip lists for sorted DocValues fields (NUMERIC, SORTED_NUMERIC, SORTED, SORTED_SET), improving query efficiency
+type: added
+authors:
+  - name: Vadim Kirilchuk
+  - name: Sanjay Dutt
+links:
+  - name: SOLR-18251
+    url: https://issues.apache.org/jira/browse/SOLR-18251

solr/core/src/java/org/apache/solr/schema/BoolField.java

@@ -23,8 +23,6 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReaderContext;
@@ -47,6 +45,11 @@
 
 /** */
 public class BoolField extends PrimitiveFieldType {
+
+  @Override
+  protected void checkSupportsDocValuesSkipList() { // we support DocValues skip lists
+  }
+
   @Override
   public SortField getSortField(SchemaField field, boolean reverse) {
     field.checkSortability();
@@ -189,9 +192,10 @@ public List<IndexableField> createFields(SchemaField field, Object value) {
       IndexableField docval;
       final BytesRef bytes = new BytesRef(toInternal(value.toString()));
       if (field.multiValued()) {
-        docval = new SortedSetDocValuesField(field.getName(), bytes);
+        docval =
+            createSortedSetDocValuesField(field.getName(), bytes, field.hasDocValuesSkipList());
       } else {
-        docval = new SortedDocValuesField(field.getName(), bytes);
+        docval = createSortedDocValuesField(field.getName(), bytes, field.hasDocValuesSkipList());
       }
 
       // Only create a list of we have 2 values...

solr/core/src/java/org/apache/solr/schema/CollationField.java

@@ -30,7 +30,6 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.collation.CollationKeyAnalyzer;
-import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
@@ -241,6 +240,10 @@ protected Query getSpecializedRangeQuery(
   protected void checkSupportsDocValues() { // we support DocValues
   }
 
+  @Override
+  protected void checkSupportsDocValuesSkipList() { // we support DocValues skip lists
+  }
+
   @Override
   protected boolean enableDocValuesByDefault() {
     return true;
@@ -253,9 +256,11 @@ public List<IndexableField> createFields(SchemaField field, Object value) {
       fields.add(createField(field, value));
       final BytesRef bytes = getCollationKey(field.getName(), value.toString());
       if (field.multiValued()) {
-        fields.add(new SortedSetDocValuesField(field.getName(), bytes));
+        fields.add(
+            createSortedSetDocValuesField(field.getName(), bytes, field.hasDocValuesSkipList()));
       } else {
-        fields.add(new SortedDocValuesField(field.getName(), bytes));
+        fields.add(
+            createSortedDocValuesField(field.getName(), bytes, field.hasDocValuesSkipList()));
       }
       return fields;
     } else {

solr/core/src/java/org/apache/solr/schema/EnumFieldType.java

@@ -71,6 +71,10 @@ protected void init(IndexSchema schema, Map<String, String> args) {
     enumMapping = new EnumMapping(schema, this, args);
   }
 
+  @Override
+  protected void checkSupportsDocValuesSkipList() { // we support DocValues skip lists
+  }
+
   public EnumMapping getEnumMapping() {
     return enumMapping;
   }
@@ -419,9 +423,10 @@ public List<IndexableField> createFields(SchemaField sf, Object value) {
     fields.add(field);
     final long longValue = field.numericValue().longValue();
     if (sf.multiValued()) {
-      fields.add(new SortedNumericDocValuesField(sf.getName(), longValue));
+      fields.add(
+          createSortedNumericDocValuesField(sf.getName(), longValue, sf.hasDocValuesSkipList()));
     } else {
-      fields.add(new NumericDocValuesField(sf.getName(), longValue));
+      fields.add(createNumericDocValuesField(sf.getName(), longValue, sf.hasDocValuesSkipList()));
     }
     return fields;
   }

solr/core/src/java/org/apache/solr/schema/FieldProperties.java

@@ -52,6 +52,7 @@ public abstract class FieldProperties {
   protected static final int USE_DOCVALUES_AS_STORED = 0b100000000000000000;
   protected static final int LARGE_FIELD = 0b1000000000000000000;
   protected static final int UNINVERTIBLE = 0b10000000000000000000;
+  protected static final int DOC_VALUES_SKIP_LIST = 0b100000000000000000000;
 
   static final String[] propertyNames = {
     "indexed",
@@ -73,7 +74,8 @@ public abstract class FieldProperties {
     "termPayloads",
     "useDocValuesAsStored",
     "large",
-    "uninvertible"
+    "uninvertible",
+    "docValuesSkipList"
   };
 
   static final Map<String, Integer> propertyMap = new HashMap<>();

solr/core/src/java/org/apache/solr/schema/FieldType.java

@@ -41,6 +41,9 @@
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
@@ -329,6 +332,66 @@ protected IndexableField createField(String name, String val, IndexableFieldType
     return new Field(name, val, type);
   }
 
+  /**
+   * Creates a sorted DocValues field for a single-value (non-multi-valued) string/bytes field.
+   *
+   * @param fieldName the field name
+   * @param value the bytes value
+   * @param hasDocValuesSkipList whether to include a range skip index
+   * @return the created IndexableField
+   */
+  protected IndexableField createSortedDocValuesField(
+      String fieldName, BytesRef value, boolean hasDocValuesSkipList) {
+    return hasDocValuesSkipList
+        ? SortedDocValuesField.indexedField(fieldName, value)
+        : new SortedDocValuesField(fieldName, value);
+  }
+
+  /**
+   * Creates a sorted set DocValues field for a multi-valued string/bytes field.
+   *
+   * @param fieldName the field name
+   * @param value the bytes value
+   * @param hasDocValuesSkipList whether to include a range skip index
+   * @return the created IndexableField
+   */
+  protected IndexableField createSortedSetDocValuesField(
+      String fieldName, BytesRef value, boolean hasDocValuesSkipList) {
+    return hasDocValuesSkipList
+        ? SortedSetDocValuesField.indexedField(fieldName, value)
+        : new SortedSetDocValuesField(fieldName, value);
+  }
+
+  /**
+   * Creates a numeric DocValues field for a single-value numeric field.
+   *
+   * @param fieldName the field name
+   * @param value the long value
+   * @param hasDocValuesSkipList whether to include a range skip index
+   * @return the created IndexableField
+   */
+  protected IndexableField createNumericDocValuesField(
+      String fieldName, long value, boolean hasDocValuesSkipList) {
+    return hasDocValuesSkipList
+        ? NumericDocValuesField.indexedField(fieldName, value)
+        : new NumericDocValuesField(fieldName, value);
+  }
+
+  /**
+   * Creates a sorted numeric DocValues field for a multi-valued numeric field.
+   *
+   * @param fieldName the field name
+   * @param value the long value
+   * @param hasDocValuesSkipList whether to include a range skip index
+   * @return the created IndexableField
+   */
+  protected IndexableField createSortedNumericDocValuesField(
+      String fieldName, long value, boolean hasDocValuesSkipList) {
+    return hasDocValuesSkipList
+        ? SortedNumericDocValuesField.indexedField(fieldName, value)
+        : new SortedNumericDocValuesField(fieldName, value);
+  }
+
   /**
    * Given a {@link org.apache.solr.schema.SchemaField}, create one or more {@link
    * org.apache.lucene.index.IndexableField} instances
@@ -1148,6 +1211,14 @@ public void checkSchemaField(final SchemaField field) {
     if (field.hasDocValues()) {
       checkSupportsDocValues();
     }
+    if (field.hasDocValuesSkipList()) {
+      if (!field.hasDocValues()) {
+        throw new SolrException(
+            ErrorCode.SERVER_ERROR,
+            "Field type " + this + " can't use doc values skip list without doc values");
+      }
+      checkSupportsDocValuesSkipList();
+    }
     if (field.isLarge() && field.multiValued()) {
       throw new SolrException(
           ErrorCode.SERVER_ERROR, "Field type " + this + " is 'large'; can't support multiValued");
@@ -1167,6 +1238,16 @@ protected void checkSupportsDocValues() {
         ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values");
   }
 
+  /**
+   * Called by {@link #checkSchemaField(SchemaField)} if the field has docValues skip lists. By
+   * default no field types support skip lists. Subclasses that support skip lists should override
+   * this method as a no-op.
+   */
+  protected void checkSupportsDocValuesSkipList() {
+    throw new SolrException(
+        ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values skip lists");
+  }
+
   /**
    * Returns whether this field type should enable docValues by default for schemaVersion >= 1.7.
    * This should not be enabled for fields that did not have docValues implemented by Solr 9.7, as

solr/core/src/java/org/apache/solr/schema/PointField.java

@@ -23,8 +23,6 @@
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.queries.function.ValueSource;
@@ -81,6 +79,10 @@ protected void init(IndexSchema schema, Map<String, String> args) {
     }
   }
 
+  @Override
+  protected void checkSupportsDocValuesSkipList() { // we support DocValues skip lists
+  }
+
   @Override
   public boolean isPointField() {
     return true;
@@ -292,7 +294,7 @@ public List<IndexableField> createFields(SchemaField sf, Object value) {
           assert numericValue instanceof Double;
           bits = Double.doubleToLongBits(numericValue.doubleValue());
         }
-        fields.add(new NumericDocValuesField(sf.getName(), bits));
+        fields.add(createNumericDocValuesField(sf.getName(), bits, sf.hasDocValuesSkipList()));
       } else {
         // MultiValued
         if (numericValue instanceof Integer || numericValue instanceof Long) {
@@ -303,7 +305,8 @@ public List<IndexableField> createFields(SchemaField sf, Object value) {
           assert numericValue instanceof Double;
           bits = NumericUtils.doubleToSortableLong(numericValue.doubleValue());
         }
-        fields.add(new SortedNumericDocValuesField(sf.getName(), bits));
+        fields.add(
+            createSortedNumericDocValuesField(sf.getName(), bits, sf.hasDocValuesSkipList()));
       }
     }
     if (sf.stored()) {

solr/core/src/java/org/apache/solr/schema/SchemaField.java

@@ -112,6 +112,10 @@ public boolean hasDocValues() {
     return (properties & DOC_VALUES) != 0;
   }
 
+  public boolean hasDocValuesSkipList() {
+    return (properties & DOC_VALUES_SKIP_LIST) != 0;
+  }
+
   public boolean storeTermVector() {
     return (properties & STORE_TERMVECTORS) != 0;
   }
@@ -373,6 +377,18 @@ static int calcProps(String name, FieldType ft, Map<String, ?> props) {
       p &= ~pp;
     }
 
+    if (on(falseProps, DOC_VALUES)) {
+      int pp = DOC_VALUES_SKIP_LIST;
+      if (on(DOC_VALUES_SKIP_LIST, trueProps)) {
+        throw new RuntimeException(
+            "SchemaField: "
+                + name
+                + " conflicting 'true' field options for non-docValues field:"
+                + props);
+      }
+      p &= ~pp;
+    }
+
     if (on(falseProps, INDEXED)) {
       int pp = (OMIT_NORMS | OMIT_TF_POSITIONS | OMIT_POSITIONS);
       if (on(pp, falseProps)) {
@@ -466,6 +482,7 @@ public SimpleOrderedMap<Object> getNamedPropertyValues(boolean showDefaults) {
       properties.add(getPropertyName(REQUIRED), isRequired());
       properties.add(getPropertyName(TOKENIZED), isTokenized());
       properties.add(getPropertyName(USE_DOCVALUES_AS_STORED), useDocValuesAsStored());
+      properties.add(getPropertyName(DOC_VALUES_SKIP_LIST), hasDocValuesSkipList());
       // The BINARY property is always false
       // properties.add(getPropertyName(BINARY), isBinary());
     } else {
@@ -532,6 +549,11 @@ public DocValuesType docValuesType() {
     return DocValuesType.NONE;
   }
 
+  /**
+   * For fields with docValues the underlaying lucene field is created without passing these values
+   * as is. Instead the creating class should check on {@link #hasDocValuesSkipList()} and create
+   * the appropriate field type.
+   */
   @Override
   public DocValuesSkipIndexType docValuesSkipIndexType() {
     return DocValuesSkipIndexType.NONE;

solr/core/src/java/org/apache/solr/schema/SortableTextField.java

@@ -19,8 +19,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
@@ -129,12 +127,12 @@ public List<IndexableField> createFields(SchemaField field, Object value) {
     return getIndexableFields(field, f, bytes);
   }
 
-  private static List<IndexableField> getIndexableFields(
+  private List<IndexableField> getIndexableFields(
       SchemaField field, IndexableField f, BytesRef bytes) {
     final IndexableField docval =
         field.multiValued()
-            ? new SortedSetDocValuesField(field.getName(), bytes)
-            : new SortedDocValuesField(field.getName(), bytes);
+            ? createSortedSetDocValuesField(field.getName(), bytes, field.hasDocValuesSkipList())
+            : createSortedDocValuesField(field.getName(), bytes, field.hasDocValuesSkipList());
 
     if (null == f) {
       return List.of(docval);
@@ -148,6 +146,12 @@ protected void checkSupportsDocValues() {
     // No-Op
   }
 
+  /** {@inheritDoc} this field type supports DocValues skip lists, this method is always a No-Op */
+  @Override
+  protected void checkSupportsDocValuesSkipList() {
+    // No-Op
+  }
+
   @Override
   protected boolean enableDocValuesByDefault() {
     return true;

solr/core/src/java/org/apache/solr/schema/StrField.java

@@ -20,8 +20,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
@@ -41,6 +39,10 @@ protected void init(IndexSchema schema, Map<String, String> args) {
     super.init(schema, args);
   }
 
+  @Override
+  protected void checkSupportsDocValuesSkipList() { // we support DocValues skip lists
+  }
+
   @Override
   public List<IndexableField> createFields(SchemaField field, Object value) {
     IndexableField fval = createField(field, value);
@@ -49,9 +51,10 @@ public List<IndexableField> createFields(SchemaField field, Object value) {
       IndexableField docval;
       final BytesRef bytes = getBytesRef(value);
       if (field.multiValued()) {
-        docval = new SortedSetDocValuesField(field.getName(), bytes);
+        docval =
+            createSortedSetDocValuesField(field.getName(), bytes, field.hasDocValuesSkipList());
       } else {
-        docval = new SortedDocValuesField(field.getName(), bytes);
+        docval = createSortedDocValuesField(field.getName(), bytes, field.hasDocValuesSkipList());
       }
 
       // Only create a list of we have 2 values...