perf(next): advanced cost-based join ordering with real in-memory index statistics

src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java

@@ -1,12 +1,17 @@
 package fr.inria.corese.core.next.query.impl.engine.planning;
 
+import fr.inria.corese.core.next.data.Values;
+import fr.inria.corese.core.next.data.api.term.IRI;
 import fr.inria.corese.core.next.query.impl.engine.model.Edge;
 import fr.inria.corese.core.next.query.impl.engine.model.Node;
 import fr.inria.corese.core.next.query.impl.engine.path.PropertyPathEdge;
 import fr.inria.corese.core.next.query.impl.engine.pattern.Exp;
 import fr.inria.corese.core.next.query.impl.engine.pattern.Query;
+import fr.inria.corese.core.next.storage.api.model.PredicateStatistics;
+import fr.inria.corese.core.next.storage.api.model.StorageStatistics;
 
 import java.math.BigDecimal;
+import java.math.RoundingMode;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -41,6 +46,7 @@ static void sort(Query query, Node graph, Exp expression, List<String> variables
             List<Exp> bindings, QueryStatistics statistics) {
         flattenTripleGroups(expression);
         List<Node> bound = new ArrayList<>();
+        PlanningContext context = new PlanningContext(query, graph, statistics, bound, variables, bindings);
         int start = 0;
         while (start < expression.size()) {
             if (!isTriple(expression.get(start))) {
@@ -53,14 +59,23 @@ static void sort(Query query, Node graph, Exp expression, List<String> variables
             }
             if (end - start > 1) {
                 long[] counts = cardinalities(query, graph, expression, start, end, statistics);
-                orderRun(expression, start, end, counts, bound, variables, bindings);
+                orderRun(expression, start, end, counts, context);
             } else {
                 recordBindings(expression.get(start), bound);
             }
             start = end;
         }
     }
 
+    private record PlanningContext(
+            Query query,
+            Node graph,
+            QueryStatistics statistics,
+            List<Node> bound,
+            List<String> variables,
+            List<Exp> bindings) {
+    }
+
     // The parser may emit one BGP per dotted triple. Adjacent pure triple groups
     // are associative joins; explicit groups remain JOIN operators and are retained.
     private static void flattenTripleGroups(Exp expression) {
@@ -105,24 +120,24 @@ private static long[] cardinalities(Query query, Node graph, Exp expression, int
     }
 
     private static void orderRun(Exp expression, int start, int end, long[] counts,
-            List<Node> bound, List<String> variables, List<Exp> bindings) {
+            PlanningContext context) {
         int first = start;
         int seed = SelectiveFilterPushdown.seed(expression, start, end);
         if (seed >= 0) {
             moveFirst(expression, counts, start, first, seed);
-            recordBindings(expression.get(first), bound);
+            recordBindings(expression.get(first), context.bound());
             first++;
         }
         for (int i = first; i < end; i++) {
             int best = i;
             for (int j = i + 1; j < end; j++) {
                 if (isLowerCost(expression.get(j), expression.get(best), counts[j - start], counts[best - start],
-                        bound, variables, bindings)) {
+                        context)) {
                     best = j;
                 }
             }
             moveFirst(expression, counts, start, i, best);
-            recordBindings(expression.get(i), bound);
+            recordBindings(expression.get(i), context.bound());
         }
     }
 
@@ -137,47 +152,144 @@ private static void recordBindings(Exp expression, List<Node> bound) {
     }
 
     private static boolean isLowerCost(Exp candidate, Exp current, long candidateCount, long currentCount,
-            List<Node> bound, List<String> variables, List<Exp> bindings) {
-        boolean candidateConnected = connected(candidate.getEdge(), bound, variables);
-        boolean currentConnected = connected(current.getEdge(), bound, variables);
+            PlanningContext context) {
+        boolean candidateConnected = connected(candidate.getEdge(), context.bound(), context.variables());
+        boolean currentConnected = connected(current.getEdge(), context.bound(), context.variables());
         // Lexicographic cross-product penalty: retain costs within each component,
         // including the initial step when no pattern is connected yet.
         if (candidateConnected != currentConnected) {
             return candidateConnected;
         }
         if (candidateCount >= 0 && currentCount >= 0) {
-            int comparison = effectiveCost(candidate, candidateCount, bound, variables)
-                    .compareTo(effectiveCost(current, currentCount, bound, variables));
+            int comparison = effectiveCost(candidate, candidateCount, context)
+                    .compareTo(effectiveCost(current, currentCount, context));
             if (comparison != 0) {
                 return comparison < 0;
             }
         }
         // Strict comparison plus moveFirst's stable shift preserves source order
         // when both cost and binding score tie. Unavailable counts retain fallback.
-        return candidate.nBind(bound, variables, bindings) > current.nBind(bound, variables, bindings);
+        return candidate.nBind(context.bound(), context.variables(), context.bindings()) > current.nBind(
+                context.bound(), context.variables(), context.bindings());
     }
 
     /**
      * Estimates matches per incoming binding; called only for nonnegative counts.
      * Constants already constrain baseCount, so only newly bound variables reduce it.
-     * Decimal scaling preserves long precision without overflow or rounding.
+     * When index statistics are available, uses exact or sampled distinct counts.
+     * Otherwise, falls back to static decimal scaling.
      */
-    private static BigDecimal effectiveCost(Exp exp, long baseCount,
-            List<Node> bound, List<String> variables) {
+    private static BigDecimal effectiveCost(Exp exp, long baseCount, PlanningContext context) {
         if (baseCount == 0) {
             return BigDecimal.ZERO;
         }
         Edge edge = exp.getEdge();
-        if (fullyBound(edge, bound, variables)) {
+        if (fullyBound(edge, context.bound(), context.variables())) {
             // A probe estimate, not a dataset-wide uniqueness guarantee.
             return BigDecimal.ONE;
         }
-        int scale = reductionScale(edge.getNode(0), bound, variables, SUBJECT_REDUCTION_SCALE)
-                + reductionScale(edge.getNode(1), bound, variables, OBJECT_REDUCTION_SCALE)
-                + reductionScale(edge.getEdgeVariable(), bound, variables, PREDICATE_REDUCTION_SCALE);
+
+        if (context.statistics() != null) {
+            BigDecimal statCost = computeCostFromStatistics(edge, baseCount, context);
+            if (statCost != null) {
+                return statCost;
+            }
+        }
+
+        int scale = reductionScale(edge.getNode(0), context.bound(), context.variables(), SUBJECT_REDUCTION_SCALE)
+                + reductionScale(edge.getNode(1), context.bound(), context.variables(), OBJECT_REDUCTION_SCALE)
+                + reductionScale(edge.getEdgeVariable(), context.bound(), context.variables(), PREDICATE_REDUCTION_SCALE);
         return BigDecimal.valueOf(baseCount).movePointLeft(scale).max(BigDecimal.ONE);
     }
 
+    private static BigDecimal computeCostFromStatistics(Edge edge, long baseCount,
+            PlanningContext context) {
+        Node edgePred = edge.getEdgeVariable() == null ? edge.getEdgeNode() : null;
+        if (edgePred != null && edgePred.isConstant()) {
+            IRI predicateIri = toPredicateIri(edgePred);
+            if (predicateIri != null) {
+                PredicateStatistics stats = context.statistics().getPredicateStatistics(
+                        context.query(), context.graph(), predicateIri);
+                if (stats != null && stats.count() > 0) {
+                    return computeCostFromPredicateStats(edge, baseCount, context.bound(), context.variables(), stats);
+                }
+            }
+        } else if (edge.getEdgeVariable() != null) {
+            StorageStatistics stats = context.statistics().getStorageStatistics();
+            if (stats != null && stats.statementCount() > 0) {
+                return computeCostFromStorageStats(edge, baseCount, context.bound(), context.variables(), stats);
+            }
+        }
+        return null;
+    }
+
+    private static IRI toPredicateIri(Node node) {
+        if (node == null) {
+            return null;
+        }
+        if (node.getDatatypeValue() instanceof IRI iri) {
+            return iri;
+        }
+        String label = node.getLabel();
+        if (label != null && !label.isEmpty()) {
+            try {
+                return Values.factory().createIRI(label);
+            } catch (IllegalArgumentException e) {
+                return null;
+            }
+        }
+        return null;
+    }
+
+    private static BigDecimal computeCostFromPredicateStats(Edge edge, long baseCount,
+            List<Node> bound, List<String> variables, PredicateStatistics stats) {
+        boolean subjBound = bound(edge.getNode(0), bound, variables);
+        boolean objBound = bound(edge.getNode(1), bound, variables);
+
+        if (subjBound && objBound) {
+            BigDecimal reduction = BigDecimal.valueOf(Math.max(1, stats.distinctSubjects()))
+                    .multiply(BigDecimal.valueOf(Math.max(1, stats.distinctObjects())));
+            return BigDecimal.valueOf(baseCount)
+                    .divide(reduction, 4, RoundingMode.HALF_UP)
+                    .max(BigDecimal.ONE);
+        } else if (subjBound) {
+            long distinctSubj = Math.max(1, stats.distinctSubjects());
+            return BigDecimal.valueOf(baseCount)
+                    .divide(BigDecimal.valueOf(distinctSubj), 4, RoundingMode.HALF_UP)
+                    .max(BigDecimal.ONE);
+        } else if (objBound) {
+            long distinctObj = Math.max(1, stats.distinctObjects());
+            return BigDecimal.valueOf(baseCount)
+                    .divide(BigDecimal.valueOf(distinctObj), 4, RoundingMode.HALF_UP)
+                    .max(BigDecimal.ONE);
+        }
+        return BigDecimal.valueOf(baseCount);
+    }
+
+    private static BigDecimal computeCostFromStorageStats(Edge edge, long baseCount,
+            List<Node> bound, List<String> variables, StorageStatistics stats) {
+        boolean subjBound = bound(edge.getNode(0), bound, variables);
+        boolean objBound = bound(edge.getNode(1), bound, variables);
+        boolean predBound = bound(edge.getEdgeVariable(), bound, variables);
+
+        BigDecimal divisor = BigDecimal.ONE;
+        if (subjBound && stats.subjectCount() > 0) {
+            divisor = divisor.multiply(BigDecimal.valueOf(Math.max(1, stats.subjectCount())));
+        }
+        if (objBound && stats.objectCount() > 0) {
+            divisor = divisor.multiply(BigDecimal.valueOf(Math.max(1, stats.objectCount())));
+        }
+        if (predBound && stats.predicateCount() > 0) {
+            divisor = divisor.multiply(BigDecimal.valueOf(Math.max(1, stats.predicateCount())));
+        }
+        if (divisor.compareTo(BigDecimal.ONE) > 0) {
+            return BigDecimal.valueOf(baseCount)
+                    .divide(divisor, 4, RoundingMode.HALF_UP)
+                    .max(BigDecimal.ONE);
+        }
+        return BigDecimal.valueOf(baseCount);
+    }
+
     private static int reductionScale(Node node, List<Node> bound, List<String> variables, int scale) {
         return bound(node, bound, variables) ? scale : 0;
     }

src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/QueryStatistics.java

@@ -1,8 +1,11 @@
 package fr.inria.corese.core.next.query.impl.engine.planning;
 
+import fr.inria.corese.core.next.data.api.term.IRI;
 import fr.inria.corese.core.next.query.impl.engine.model.Edge;
 import fr.inria.corese.core.next.query.impl.engine.model.Node;
 import fr.inria.corese.core.next.query.impl.engine.pattern.Query;
+import fr.inria.corese.core.next.storage.api.model.PredicateStatistics;
+import fr.inria.corese.core.next.storage.api.model.StorageStatistics;
 
 /**
  * Supplies optional planning statistics without opening statement cursors.
@@ -49,4 +52,25 @@ default int getSize(int type) {
     default int getCount(QueryPlanNode node, int type) {
         return -1;
     }
+
+    /**
+     * Returns statistics for a specific predicate, or {@code null} if unavailable.
+     *
+     * @param query query supplying the active dataset
+     * @param graphNode enclosing GRAPH node, or {@code null} for the default graph
+     * @param predicate the predicate IRI to inspect
+     * @return predicate statistics or {@code null} if unavailable
+     */
+    default PredicateStatistics getPredicateStatistics(Query query, Node graphNode, IRI predicate) {
+        return null;
+    }
+
+    /**
+     * Returns global storage statistics, or {@code null} if unavailable.
+     *
+     * @return storage statistics or {@code null} if unavailable
+     */
+    default StorageStatistics getStorageStatistics() {
+        return null;
+    }
 }

src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java

@@ -1,5 +1,6 @@
 package fr.inria.corese.core.next.query.impl.engine.storage;
 
+import fr.inria.corese.core.next.data.Values;
 import fr.inria.corese.core.next.data.api.model.DatatypeValue;
 import fr.inria.corese.core.next.data.api.model.Statement;
 import fr.inria.corese.core.next.data.api.term.IRI;
@@ -16,6 +17,9 @@
 import fr.inria.corese.core.next.query.impl.engine.solution.Mappings;
 import fr.inria.corese.core.next.query.impl.engine.spi.Environment;
 import fr.inria.corese.core.next.storage.api.StorageManager;
+import fr.inria.corese.core.next.storage.api.exception.StorageException;
+import fr.inria.corese.core.next.storage.api.model.PredicateStatistics;
+import fr.inria.corese.core.next.storage.api.model.StorageStatistics;
 import fr.inria.corese.core.next.storage.api.operations.StatementSpliterator;
 import fr.inria.corese.core.next.storage.api.model.StatementPattern;
 import fr.inria.corese.core.next.query.api.exception.QueryEvaluationException;
@@ -25,11 +29,13 @@
 import java.util.IdentityHashMap;
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.Spliterators;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Stream;
 
 /**
@@ -42,7 +48,12 @@
  */
 public class StorageManagerProducer extends ProducerDefault implements QueryStatistics {
 
+    private static final PredicateStatistics MISSING_PREDICATE_STATS =
+            new PredicateStatistics(Values.factory().createIRI("urn:corese:internal:missing-stats"), 0, 0, 0);
+
     private final StorageManager storage;
+    private final Map<IRI, PredicateStatistics> predicateStatsCache = new ConcurrentHashMap<>();
+    private volatile StorageStatistics cachedStorageStats;
     private final Set<EdgeCursor> cursors = Collections.newSetFromMap(new IdentityHashMap<>());
     private boolean closed;
 
@@ -87,6 +98,41 @@ public long estimateCardinality(Query query, Node graphNode, Edge edge) {
         }
     }
 
+    @Override
+    public PredicateStatistics getPredicateStatistics(Query query, Node graphNode, IRI predicate) {
+        if (predicate == null || closed) {
+            return null;
+        }
+        checkInterrupted();
+        PredicateStatistics stats = predicateStatsCache.computeIfAbsent(predicate, p -> {
+            try {
+                var meta = storage.metadata();
+                PredicateStatistics resolved = meta == null ? null : meta.getPredicateStatistics(p);
+                return resolved != null ? resolved : MISSING_PREDICATE_STATS;
+            } catch (StorageException | UnsupportedOperationException | IllegalArgumentException unavailable) {
+                return MISSING_PREDICATE_STATS;
+            }
+        });
+        return stats == MISSING_PREDICATE_STATS ? null : stats;
+    }
+
+    @Override
+    public StorageStatistics getStorageStatistics() {
+        if (closed) {
+            return null;
+        }
+        checkInterrupted();
+        if (cachedStorageStats == null) {
+            try {
+                var meta = storage.metadata();
+                cachedStorageStats = meta == null ? null : meta.getStatistics();
+            } catch (StorageException | UnsupportedOperationException | IllegalArgumentException unavailable) {
+                return null;
+            }
+        }
+        return cachedStorageStats;
+    }
+
     @Override
     public Iterable<Edge> getEdges(Node graphNode, List<Node> from, Edge queryEdge, Environment environment) {
         Objects.requireNonNull(queryEdge, "queryEdge");

src/main/java/fr/inria/corese/core/next/storage/api/model/PredicateStatistics.java

@@ -0,0 +1,33 @@
+package fr.inria.corese.core.next.storage.api.model;
+
+import fr.inria.corese.core.next.data.api.term.IRI;
+import java.util.Objects;
+
+/**
+ * Statistics about a specific predicate within an RDF storage backend.
+ *
+ * @param predicate the predicate IRI
+ * @param count total number of statements with this predicate
+ * @param distinctSubjects number of distinct subjects with this predicate
+ * @param distinctObjects number of distinct objects with this predicate
+ */
+public record PredicateStatistics(
+        IRI predicate,
+        long count,
+        long distinctSubjects,
+        long distinctObjects
+) {
+
+    /**
+     * Compact constructor with validation.
+     *
+     * @throws NullPointerException if {@code predicate} is {@code null}
+     * @throws IllegalArgumentException if any count is negative
+     */
+    public PredicateStatistics {
+        Objects.requireNonNull(predicate, "predicate cannot be null");
+        if (count < 0 || distinctSubjects < 0 || distinctObjects < 0) {
+            throw new IllegalArgumentException("Counts cannot be negative");
+        }
+    }
+}