From 92e5c228ab9fcb6d9b32e829509c0130db347882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Thu, 22 Oct 2026 15:50:14 +0200 Subject: [PATCH 1/2] perf(next): advanced cost-based join ordering with real in-memory index statistics - Add PredicateStatistics record to storage API model - Provide POS-index based distinct subjects and objects extraction in TripleIndex - Expose predicate statistics via MetadataOperations and IndexedStatementStore - Integrate index statistics into QueryStatistics and StorageManagerProducer - Upgrade CardinalityPatternOrdering to use real selectivity ratios from index stats - Refactor ordering methods with PlanningContext to ensure 0 Sonar alerts - Add tests in CardinalityQueryPlannerTest covering index-driven join ordering Closes #637 --- .../planning/CardinalityPatternOrdering.java | 143 +++++++++++++++--- .../impl/engine/planning/QueryStatistics.java | 24 +++ .../storage/StorageManagerProducer.java | 34 +++++ .../api/model/PredicateStatistics.java | 33 ++++ .../api/operations/MetadataOperations.java | 12 ++ .../impl/memory/IndexedStatementStore.java | 31 ++++ .../next/storage/impl/memory/TripleIndex.java | 80 ++++++++++ .../planning/CardinalityQueryPlannerTest.java | 117 ++++++++++++++ 8 files changed, 456 insertions(+), 18 deletions(-) create mode 100644 src/main/java/fr/inria/corese/core/next/storage/api/model/PredicateStatistics.java diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java index 766df2564..d7da5c73d 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java @@ -1,12 +1,17 @@ package fr.inria.corese.core.next.query.impl.engine.planning; +import fr.inria.corese.core.next.data.Values; +import fr.inria.corese.core.next.data.api.term.IRI; import fr.inria.corese.core.next.query.impl.engine.model.Edge; import fr.inria.corese.core.next.query.impl.engine.model.Node; import fr.inria.corese.core.next.query.impl.engine.path.PropertyPathEdge; import fr.inria.corese.core.next.query.impl.engine.pattern.Exp; import fr.inria.corese.core.next.query.impl.engine.pattern.Query; +import fr.inria.corese.core.next.storage.api.model.PredicateStatistics; +import fr.inria.corese.core.next.storage.api.model.StorageStatistics; import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.List; @@ -41,6 +46,7 @@ static void sort(Query query, Node graph, Exp expression, List variables List bindings, QueryStatistics statistics) { flattenTripleGroups(expression); List bound = new ArrayList<>(); + PlanningContext context = new PlanningContext(query, graph, statistics, bound, variables, bindings); int start = 0; while (start < expression.size()) { if (!isTriple(expression.get(start))) { @@ -53,7 +59,7 @@ static void sort(Query query, Node graph, Exp expression, List variables } if (end - start > 1) { long[] counts = cardinalities(query, graph, expression, start, end, statistics); - orderRun(expression, start, end, counts, bound, variables, bindings); + orderRun(expression, start, end, counts, context); } else { recordBindings(expression.get(start), bound); } @@ -61,6 +67,15 @@ static void sort(Query query, Node graph, Exp expression, List variables } } + private record PlanningContext( + Query query, + Node graph, + QueryStatistics statistics, + List bound, + List variables, + List bindings) { + } + // The parser may emit one BGP per dotted triple. Adjacent pure triple groups // are associative joins; explicit groups remain JOIN operators and are retained. private static void flattenTripleGroups(Exp expression) { @@ -105,24 +120,24 @@ private static long[] cardinalities(Query query, Node graph, Exp expression, int } private static void orderRun(Exp expression, int start, int end, long[] counts, - List bound, List variables, List bindings) { + PlanningContext context) { int first = start; int seed = SelectiveFilterPushdown.seed(expression, start, end); if (seed >= 0) { moveFirst(expression, counts, start, first, seed); - recordBindings(expression.get(first), bound); + recordBindings(expression.get(first), context.bound()); first++; } for (int i = first; i < end; i++) { int best = i; for (int j = i + 1; j < end; j++) { if (isLowerCost(expression.get(j), expression.get(best), counts[j - start], counts[best - start], - bound, variables, bindings)) { + context)) { best = j; } } moveFirst(expression, counts, start, i, best); - recordBindings(expression.get(i), bound); + recordBindings(expression.get(i), context.bound()); } } @@ -137,47 +152,139 @@ private static void recordBindings(Exp expression, List bound) { } private static boolean isLowerCost(Exp candidate, Exp current, long candidateCount, long currentCount, - List bound, List variables, List bindings) { - boolean candidateConnected = connected(candidate.getEdge(), bound, variables); - boolean currentConnected = connected(current.getEdge(), bound, variables); + PlanningContext context) { + boolean candidateConnected = connected(candidate.getEdge(), context.bound(), context.variables()); + boolean currentConnected = connected(current.getEdge(), context.bound(), context.variables()); // Lexicographic cross-product penalty: retain costs within each component, // including the initial step when no pattern is connected yet. if (candidateConnected != currentConnected) { return candidateConnected; } if (candidateCount >= 0 && currentCount >= 0) { - int comparison = effectiveCost(candidate, candidateCount, bound, variables) - .compareTo(effectiveCost(current, currentCount, bound, variables)); + int comparison = effectiveCost(candidate, candidateCount, context) + .compareTo(effectiveCost(current, currentCount, context)); if (comparison != 0) { return comparison < 0; } } // Strict comparison plus moveFirst's stable shift preserves source order // when both cost and binding score tie. Unavailable counts retain fallback. - return candidate.nBind(bound, variables, bindings) > current.nBind(bound, variables, bindings); + return candidate.nBind(context.bound(), context.variables(), context.bindings()) > current.nBind( + context.bound(), context.variables(), context.bindings()); } /** * Estimates matches per incoming binding; called only for nonnegative counts. * Constants already constrain baseCount, so only newly bound variables reduce it. - * Decimal scaling preserves long precision without overflow or rounding. + * When index statistics are available, uses exact or sampled distinct counts. + * Otherwise, falls back to static decimal scaling. */ - private static BigDecimal effectiveCost(Exp exp, long baseCount, - List bound, List variables) { + private static BigDecimal effectiveCost(Exp exp, long baseCount, PlanningContext context) { if (baseCount == 0) { return BigDecimal.ZERO; } Edge edge = exp.getEdge(); - if (fullyBound(edge, bound, variables)) { + if (fullyBound(edge, context.bound(), context.variables())) { // A probe estimate, not a dataset-wide uniqueness guarantee. return BigDecimal.ONE; } - int scale = reductionScale(edge.getNode(0), bound, variables, SUBJECT_REDUCTION_SCALE) - + reductionScale(edge.getNode(1), bound, variables, OBJECT_REDUCTION_SCALE) - + reductionScale(edge.getEdgeVariable(), bound, variables, PREDICATE_REDUCTION_SCALE); + + if (context.statistics() != null) { + BigDecimal statCost = computeCostFromStatistics(edge, baseCount, context); + if (statCost != null) { + return statCost; + } + } + + int scale = reductionScale(edge.getNode(0), context.bound(), context.variables(), SUBJECT_REDUCTION_SCALE) + + reductionScale(edge.getNode(1), context.bound(), context.variables(), OBJECT_REDUCTION_SCALE) + + reductionScale(edge.getEdgeVariable(), context.bound(), context.variables(), PREDICATE_REDUCTION_SCALE); return BigDecimal.valueOf(baseCount).movePointLeft(scale).max(BigDecimal.ONE); } + private static BigDecimal computeCostFromStatistics(Edge edge, long baseCount, + PlanningContext context) { + Node edgePred = edge.getEdgeVariable() == null ? edge.getEdgeNode() : null; + if (edgePred != null && edgePred.isConstant()) { + IRI predicateIri = toPredicateIri(edgePred); + if (predicateIri != null) { + PredicateStatistics stats = context.statistics().getPredicateStatistics( + context.query(), context.graph(), predicateIri); + if (stats != null && stats.count() > 0) { + return computeCostFromPredicateStats(edge, baseCount, context.bound(), context.variables(), stats); + } + } + } else if (edge.getEdgeVariable() != null) { + StorageStatistics stats = context.statistics().getStorageStatistics(); + if (stats != null && stats.statementCount() > 0) { + return computeCostFromStorageStats(edge, baseCount, context.bound(), context.variables(), stats); + } + } + return null; + } + + private static IRI toPredicateIri(Node node) { + if (node == null) { + return null; + } + if (node.getDatatypeValue() instanceof IRI iri) { + return iri; + } + String label = node.getLabel(); + if (label != null && !label.isEmpty()) { + return Values.factory().createIRI(label); + } + return null; + } + + private static BigDecimal computeCostFromPredicateStats(Edge edge, long baseCount, + List bound, List variables, PredicateStatistics stats) { + boolean subjBound = bound(edge.getNode(0), bound, variables); + boolean objBound = bound(edge.getNode(1), bound, variables); + + if (subjBound && objBound) { + long reduction = Math.max(1, stats.distinctSubjects()) * Math.max(1, stats.distinctObjects()); + return BigDecimal.valueOf(baseCount) + .divide(BigDecimal.valueOf(reduction), 4, RoundingMode.HALF_UP) + .max(BigDecimal.ONE); + } else if (subjBound) { + long distinctSubj = Math.max(1, stats.distinctSubjects()); + return BigDecimal.valueOf(baseCount) + .divide(BigDecimal.valueOf(distinctSubj), 4, RoundingMode.HALF_UP) + .max(BigDecimal.ONE); + } else if (objBound) { + long distinctObj = Math.max(1, stats.distinctObjects()); + return BigDecimal.valueOf(baseCount) + .divide(BigDecimal.valueOf(distinctObj), 4, RoundingMode.HALF_UP) + .max(BigDecimal.ONE); + } + return BigDecimal.valueOf(baseCount); + } + + private static BigDecimal computeCostFromStorageStats(Edge edge, long baseCount, + List bound, List variables, StorageStatistics stats) { + boolean subjBound = bound(edge.getNode(0), bound, variables); + boolean objBound = bound(edge.getNode(1), bound, variables); + boolean predBound = bound(edge.getEdgeVariable(), bound, variables); + + long divisor = 1; + if (subjBound && stats.subjectCount() > 0) { + divisor *= Math.max(1, stats.subjectCount()); + } + if (objBound && stats.objectCount() > 0) { + divisor *= Math.max(1, stats.objectCount()); + } + if (predBound && stats.predicateCount() > 0) { + divisor *= Math.max(1, stats.predicateCount()); + } + if (divisor > 1) { + return BigDecimal.valueOf(baseCount) + .divide(BigDecimal.valueOf(divisor), 4, RoundingMode.HALF_UP) + .max(BigDecimal.ONE); + } + return BigDecimal.valueOf(baseCount); + } + private static int reductionScale(Node node, List bound, List variables, int scale) { return bound(node, bound, variables) ? scale : 0; } diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/QueryStatistics.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/QueryStatistics.java index df5a72db4..0c604af60 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/QueryStatistics.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/QueryStatistics.java @@ -1,8 +1,11 @@ package fr.inria.corese.core.next.query.impl.engine.planning; +import fr.inria.corese.core.next.data.api.term.IRI; import fr.inria.corese.core.next.query.impl.engine.model.Edge; import fr.inria.corese.core.next.query.impl.engine.model.Node; import fr.inria.corese.core.next.query.impl.engine.pattern.Query; +import fr.inria.corese.core.next.storage.api.model.PredicateStatistics; +import fr.inria.corese.core.next.storage.api.model.StorageStatistics; /** * Supplies optional planning statistics without opening statement cursors. @@ -49,4 +52,25 @@ default int getSize(int type) { default int getCount(QueryPlanNode node, int type) { return -1; } + + /** + * Returns statistics for a specific predicate, or {@code null} if unavailable. + * + * @param query query supplying the active dataset + * @param graphNode enclosing GRAPH node, or {@code null} for the default graph + * @param predicate the predicate IRI to inspect + * @return predicate statistics or {@code null} if unavailable + */ + default PredicateStatistics getPredicateStatistics(Query query, Node graphNode, IRI predicate) { + return null; + } + + /** + * Returns global storage statistics, or {@code null} if unavailable. + * + * @return storage statistics or {@code null} if unavailable + */ + default StorageStatistics getStorageStatistics() { + return null; + } } diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java index 1262e909b..7932ed08e 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java @@ -16,6 +16,8 @@ import fr.inria.corese.core.next.query.impl.engine.solution.Mappings; import fr.inria.corese.core.next.query.impl.engine.spi.Environment; import fr.inria.corese.core.next.storage.api.StorageManager; +import fr.inria.corese.core.next.storage.api.model.PredicateStatistics; +import fr.inria.corese.core.next.storage.api.model.StorageStatistics; import fr.inria.corese.core.next.storage.api.operations.StatementSpliterator; import fr.inria.corese.core.next.storage.api.model.StatementPattern; import fr.inria.corese.core.next.query.api.exception.QueryEvaluationException; @@ -43,6 +45,8 @@ public class StorageManagerProducer extends ProducerDefault implements QueryStatistics { private final StorageManager storage; + private final java.util.Map predicateStatsCache = new java.util.concurrent.ConcurrentHashMap<>(); + private StorageStatistics cachedStorageStats; private final Set cursors = Collections.newSetFromMap(new IdentityHashMap<>()); private boolean closed; @@ -87,6 +91,36 @@ public long estimateCardinality(Query query, Node graphNode, Edge edge) { } } + @Override + public PredicateStatistics getPredicateStatistics(Query query, Node graphNode, IRI predicate) { + if (predicate == null) { + return null; + } + checkInterrupted(); + return predicateStatsCache.computeIfAbsent(predicate, p -> { + try { + var meta = storage.metadata(); + return meta == null ? null : meta.getPredicateStatistics(p); + } catch (UnsupportedOperationException | IllegalArgumentException unavailable) { + return null; + } + }); + } + + @Override + public StorageStatistics getStorageStatistics() { + checkInterrupted(); + if (cachedStorageStats == null) { + try { + var meta = storage.metadata(); + cachedStorageStats = meta == null ? null : meta.getStatistics(); + } catch (UnsupportedOperationException | IllegalArgumentException unavailable) { + return null; + } + } + return cachedStorageStats; + } + @Override public Iterable getEdges(Node graphNode, List from, Edge queryEdge, Environment environment) { Objects.requireNonNull(queryEdge, "queryEdge"); diff --git a/src/main/java/fr/inria/corese/core/next/storage/api/model/PredicateStatistics.java b/src/main/java/fr/inria/corese/core/next/storage/api/model/PredicateStatistics.java new file mode 100644 index 000000000..86e8bc84c --- /dev/null +++ b/src/main/java/fr/inria/corese/core/next/storage/api/model/PredicateStatistics.java @@ -0,0 +1,33 @@ +package fr.inria.corese.core.next.storage.api.model; + +import fr.inria.corese.core.next.data.api.term.IRI; +import java.util.Objects; + +/** + * Statistics about a specific predicate within an RDF storage backend. + * + * @param predicate the predicate IRI + * @param count total number of statements with this predicate + * @param distinctSubjects number of distinct subjects with this predicate + * @param distinctObjects number of distinct objects with this predicate + */ +public record PredicateStatistics( + IRI predicate, + long count, + long distinctSubjects, + long distinctObjects +) { + + /** + * Compact constructor with validation. + * + * @throws NullPointerException if {@code predicate} is {@code null} + * @throws IllegalArgumentException if any count is negative + */ + public PredicateStatistics { + Objects.requireNonNull(predicate, "predicate cannot be null"); + if (count < 0 || distinctSubjects < 0 || distinctObjects < 0) { + throw new IllegalArgumentException("Counts cannot be negative"); + } + } +} diff --git a/src/main/java/fr/inria/corese/core/next/storage/api/operations/MetadataOperations.java b/src/main/java/fr/inria/corese/core/next/storage/api/operations/MetadataOperations.java index 3441005a0..f9d561146 100644 --- a/src/main/java/fr/inria/corese/core/next/storage/api/operations/MetadataOperations.java +++ b/src/main/java/fr/inria/corese/core/next/storage/api/operations/MetadataOperations.java @@ -4,6 +4,7 @@ import fr.inria.corese.core.next.data.api.term.Resource; import fr.inria.corese.core.next.data.api.term.Value; import fr.inria.corese.core.next.storage.api.exception.StorageException; +import fr.inria.corese.core.next.storage.api.model.PredicateStatistics; import fr.inria.corese.core.next.storage.api.model.StorageStatistics; import java.util.Set; @@ -53,4 +54,15 @@ public interface MetadataOperations { * @throws StorageException if statistics cannot be computed */ StorageStatistics getStatistics() throws StorageException; + + /** + * Returns statistics about a specific predicate, or {@code null} if unavailable. + * + * @param predicate the predicate IRI to inspect + * @return predicate statistics, or {@code null} if unavailable or not tracked + * @throws StorageException if statistics cannot be computed + */ + default PredicateStatistics getPredicateStatistics(IRI predicate) throws StorageException { + return null; + } } diff --git a/src/main/java/fr/inria/corese/core/next/storage/impl/memory/IndexedStatementStore.java b/src/main/java/fr/inria/corese/core/next/storage/impl/memory/IndexedStatementStore.java index 5ade691f6..8e8f31409 100644 --- a/src/main/java/fr/inria/corese/core/next/storage/impl/memory/IndexedStatementStore.java +++ b/src/main/java/fr/inria/corese/core/next/storage/impl/memory/IndexedStatementStore.java @@ -25,6 +25,7 @@ import fr.inria.corese.core.next.storage.api.exception.StorageException; import fr.inria.corese.core.next.storage.api.lifecycle.LifecycleState; import fr.inria.corese.core.next.storage.api.lifecycle.StorageLifecycle; +import fr.inria.corese.core.next.storage.api.model.PredicateStatistics; import fr.inria.corese.core.next.storage.api.model.StatementPattern; import fr.inria.corese.core.next.storage.api.model.StorageStatistics; import fr.inria.corese.core.next.storage.api.operations.MetadataOperations; @@ -1623,6 +1624,36 @@ public StorageStatistics getStatistics() { } } + @Override + public PredicateStatistics getPredicateStatistics(IRI predicate) { + if (predicate == null) { + return null; + } + synchronized (database) { + checkAccess(); + long totalCount = 0; + long distinctSubjects = 0; + long distinctObjects = 0; + boolean found = false; + + for (Partition p : state().graphs.values()) { + PredicateStatistics ps = p.pos.estimatePredicateStatistics(predicate, 512); + if (ps != null) { + found = true; + totalCount += ps.count(); + distinctSubjects += ps.distinctSubjects(); + distinctObjects += ps.distinctObjects(); + } + } + if (!found) { + return null; + } + return new PredicateStatistics(predicate, totalCount, + Math.min(totalCount, distinctSubjects), + Math.min(totalCount, distinctObjects)); + } + } + /** * {@inheritDoc} * diff --git a/src/main/java/fr/inria/corese/core/next/storage/impl/memory/TripleIndex.java b/src/main/java/fr/inria/corese/core/next/storage/impl/memory/TripleIndex.java index 7cd41b0ef..acbdd8db4 100644 --- a/src/main/java/fr/inria/corese/core/next/storage/impl/memory/TripleIndex.java +++ b/src/main/java/fr/inria/corese/core/next/storage/impl/memory/TripleIndex.java @@ -1,13 +1,17 @@ package fr.inria.corese.core.next.storage.impl.memory; import fr.inria.corese.core.next.data.api.model.Statement; +import fr.inria.corese.core.next.data.api.term.IRI; import fr.inria.corese.core.next.data.api.term.Resource; import fr.inria.corese.core.next.data.api.term.Value; +import fr.inria.corese.core.next.storage.api.model.PredicateStatistics; import fr.inria.corese.core.next.storage.api.model.StatementPattern; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Predicate; @@ -810,6 +814,82 @@ long count(final StatementPattern pattern) { return count; } + /** + * Estimates predicate statistics (count, distinct subjects, distinct objects) + * using the POS index permutation where the predicate is the root key. + * + * @param predicate the predicate IRI + * @param maxSample maximum number of branch entries to sample for distinct subjects + * @return predicate statistics or null if not found or not in POS order + */ + PredicateStatistics estimatePredicateStatistics(final IRI predicate, final int maxSample) { + if (order != Order.POS || predicate == null) { + return null; + } + final TermKey pKey = new TermKey(predicate); + final Branch branch = roots.get(pKey); + if (branch == null) { + return null; + } + final long distinctObjects = branch.size(); + if (distinctObjects == 0) { + return new PredicateStatistics(predicate, 0, 0, 0); + } + + final PredicateSampleAccumulator accumulator = new PredicateSampleAccumulator(maxSample); + branch.forEach((objKey, entry) -> accumulator.accumulate(entry)); + + final long distinctSubjects = accumulator.estimateDistinctSubjects(distinctObjects); + return new PredicateStatistics(predicate, accumulator.totalCount, distinctSubjects, distinctObjects); + } + + private static final class PredicateSampleAccumulator { + private final Set subjects = new HashSet<>(); + private final int maxSample; + private long totalCount = 0; + private int sampled = 0; + + PredicateSampleAccumulator(final int maxSample) { + this.maxSample = maxSample; + } + + void accumulate(final Object entry) { + if (entry instanceof Leaf leaf) { + accumulateLeaf(leaf); + } else if (entry instanceof Statement statement) { + accumulateStatement(statement); + } + } + + private void accumulateLeaf(final Leaf leaf) { + totalCount += leaf.size(); + if (shouldSample()) { + leaf.forEach((subjKey, st) -> subjects.add(subjKey)); + sampled++; + } + } + + private void accumulateStatement(final Statement statement) { + totalCount++; + if (shouldSample()) { + subjects.add(new TermKey(statement.getSubject())); + sampled++; + } + } + + private boolean shouldSample() { + return maxSample <= 0 || sampled < maxSample; + } + + long estimateDistinctSubjects(final long distinctObjects) { + if (maxSample <= 0 || distinctObjects <= maxSample) { + return subjects.size(); + } + final double ratio = (double) distinctObjects / Math.max(1, sampled); + return Math.min(totalCount, Math.max(1, Math.round(subjects.size() * Math.min(ratio, 2.0)))); + } + } + /** * Counts matching statements in the captured or session-visible state. * diff --git a/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java b/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java index 8e4043c5c..e2f887152 100644 --- a/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java +++ b/src/test/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityQueryPlannerTest.java @@ -18,6 +18,7 @@ import fr.inria.corese.core.next.storage.api.StorageManager; import fr.inria.corese.core.next.storage.api.exception.ErrorCode; import fr.inria.corese.core.next.storage.api.exception.StorageException; +import fr.inria.corese.core.next.storage.api.model.PredicateStatistics; import fr.inria.corese.core.next.storage.api.model.StatementPattern; import fr.inria.corese.core.next.storage.api.operations.QueryOperations; import fr.inria.corese.core.next.storage.impl.memory.MemoryStorageManager; @@ -418,16 +419,132 @@ private static List edges(Exp expression) { return result; } + @Test + void joinOrderingUsesPredicateStatisticsWhenAvailable() { + IRI seedIri = iri("seed"); + IRI highDistinctIri = iri("highDistinct"); + IRI lowDistinctIri = iri("lowDistinct"); + + Map counts = Map.of( + seedIri.stringValue(), 1L, + lowDistinctIri.stringValue(), 1000L, + highDistinctIri.stringValue(), 1000L); + + Map pStats = Map.of( + lowDistinctIri.stringValue(), new PredicateStatistics(lowDistinctIri, 1000, 2, 100), + highDistinctIri.stringValue(), new PredicateStatistics(highDistinctIri, 1000, 500, 100)); + + Query query = query("?s :seed 1 . ?s :lowDistinct ?o1 . ?s :highDistinct ?o2"); + query.complete(new StatisticsProducer(counts, pStats)); + + List plan = predicates(query); + assertEquals(seedIri.stringValue(), plan.get(0)); + assertEquals(highDistinctIri.stringValue(), plan.get(1)); + assertEquals(lowDistinctIri.stringValue(), plan.get(2)); + } + + @Test + void joinOrderingUsesPredicateStatisticsWhenObjectIsBound() { + IRI seedIri = iri("seed"); + IRI highDistinctIri = iri("highDistinctObj"); + IRI lowDistinctIri = iri("lowDistinctObj"); + + Map counts = Map.of( + seedIri.stringValue(), 1L, + lowDistinctIri.stringValue(), 1000L, + highDistinctIri.stringValue(), 1000L); + + Map pStats = Map.of( + lowDistinctIri.stringValue(), new PredicateStatistics(lowDistinctIri, 1000, 100, 2), + highDistinctIri.stringValue(), new PredicateStatistics(highDistinctIri, 1000, 100, 500)); + + Query query = query("?x :seed ?shared . ?s1 :lowDistinctObj ?shared . ?s2 :highDistinctObj ?shared"); + query.complete(new StatisticsProducer(counts, pStats)); + + List plan = predicates(query); + assertEquals(seedIri.stringValue(), plan.get(0)); + assertEquals(highDistinctIri.stringValue(), plan.get(1)); + assertEquals(lowDistinctIri.stringValue(), plan.get(2)); + } + + @Test + void memoryStorageManagerProvidesExactPredicateStatistics() { + IRI type = iri("type"); + var stats = storage.metadata().getPredicateStatistics(type); + org.junit.jupiter.api.Assertions.assertNotNull(stats); + assertEquals(type, stats.predicate()); + assertTrue(stats.count() > 0); + assertTrue(stats.distinctSubjects() > 0); + assertTrue(stats.distinctObjects() > 0); + } + + @Test + void joinOrderingWithIndexStatisticsAvoidsIntermediateCartesianExplosion() { + var benchStorage = MemoryStorageManager.builder().build(); + IRI seed = iri("seed"); + IRI target = iri("target"); + IRI lowSel = iri("lowSel"); + IRI highSel = iri("highSel"); + + var stmts = new ArrayList(); + stmts.add(Values.factory().createStatement(iri("s0"), seed, target)); + + // lowSel: 10,000 statements all pointing to the same target object (1 distinct object) + for (int i = 0; i < 10000; i++) { + stmts.add(Values.factory().createStatement(iri("s_low_" + i), lowSel, target)); + } + // highSel: 10,000 statements each pointing to a distinct object (10,000 distinct objects) + stmts.add(Values.factory().createStatement(iri("s_high_0"), highSel, target)); + for (int i = 1; i < 10000; i++) { + stmts.add(Values.factory().createStatement(iri("s_high_" + i), highSel, iri("other_" + i))); + } + benchStorage.mutations().addAll(stmts); + + var statsLow = benchStorage.metadata().getPredicateStatistics(lowSel); + var statsHigh = benchStorage.metadata().getPredicateStatistics(highSel); + + assertEquals(1, statsLow.distinctObjects()); + assertEquals(10000, statsHigh.distinctObjects()); + + // High selectivity (high distinct objects = small match count per bound object) is placed first + Query plannedQuery = query("?x :seed ?t . ?s1 :lowSel ?t . ?s2 :highSel ?t"); + plannedQuery.complete(new StorageManagerProducer(benchStorage)); + List optimalPlan = predicates(plannedQuery); + assertEquals(List.of(seed.stringValue(), highSel.stringValue(), lowSel.stringValue()), optimalPlan); + + // Verify query produces the exact expected count + String sparql = "PREFIX : SELECT * WHERE { ?x :seed ?t . ?s1 :lowSel ?t . ?s2 :highSel ?t }"; + int countCbo = 0; + try (var res = new SparqlQueryExecutor(benchStorage).evaluateTuple(sparql)) { + while (res.hasNext()) { + res.next(); + countCbo++; + } + } + assertEquals(10000, countCbo); + } + private static class StatisticsProducer extends ProducerDefault implements QueryStatistics { private final Map counts; + private final Map predicateStats; private StatisticsProducer(Map counts) { + this(counts, Map.of()); + } + + private StatisticsProducer(Map counts, Map predicateStats) { this.counts = counts; + this.predicateStats = predicateStats; } @Override public long estimateCardinality(Query query, Node graph, Edge edge) { return counts.getOrDefault(edge.getEdgeLabel(), UNKNOWN_CARDINALITY); } + + @Override + public PredicateStatistics getPredicateStatistics(Query query, Node graphNode, IRI predicate) { + return predicateStats.get(predicate.stringValue()); + } } } From 077c7d58599fbf9844db9b6bb06070261671333e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Thu, 22 Oct 2026 16:01:38 +0200 Subject: [PATCH 2/2] fix(next): address copilot review on cbo index statistics and producer cache - Fix distinct subjects extrapolation formula in TripleIndex - Guard against long arithmetic overflow using BigDecimal in CardinalityPatternOrdering - Safely handle invalid IRI strings in toPredicateIri - Make StorageManagerProducer thread-safe and cache missing predicate stats - Handle StorageException gracefully during statistics inspection --- .../planning/CardinalityPatternOrdering.java | 23 +++++++++------ .../storage/StorageManagerProducer.java | 28 +++++++++++++------ .../next/storage/impl/memory/TripleIndex.java | 3 +- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java index d7da5c73d..0bf8e896a 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/planning/CardinalityPatternOrdering.java @@ -232,7 +232,11 @@ private static IRI toPredicateIri(Node node) { } String label = node.getLabel(); if (label != null && !label.isEmpty()) { - return Values.factory().createIRI(label); + try { + return Values.factory().createIRI(label); + } catch (IllegalArgumentException e) { + return null; + } } return null; } @@ -243,9 +247,10 @@ private static BigDecimal computeCostFromPredicateStats(Edge edge, long baseCoun boolean objBound = bound(edge.getNode(1), bound, variables); if (subjBound && objBound) { - long reduction = Math.max(1, stats.distinctSubjects()) * Math.max(1, stats.distinctObjects()); + BigDecimal reduction = BigDecimal.valueOf(Math.max(1, stats.distinctSubjects())) + .multiply(BigDecimal.valueOf(Math.max(1, stats.distinctObjects()))); return BigDecimal.valueOf(baseCount) - .divide(BigDecimal.valueOf(reduction), 4, RoundingMode.HALF_UP) + .divide(reduction, 4, RoundingMode.HALF_UP) .max(BigDecimal.ONE); } else if (subjBound) { long distinctSubj = Math.max(1, stats.distinctSubjects()); @@ -267,19 +272,19 @@ private static BigDecimal computeCostFromStorageStats(Edge edge, long baseCount, boolean objBound = bound(edge.getNode(1), bound, variables); boolean predBound = bound(edge.getEdgeVariable(), bound, variables); - long divisor = 1; + BigDecimal divisor = BigDecimal.ONE; if (subjBound && stats.subjectCount() > 0) { - divisor *= Math.max(1, stats.subjectCount()); + divisor = divisor.multiply(BigDecimal.valueOf(Math.max(1, stats.subjectCount()))); } if (objBound && stats.objectCount() > 0) { - divisor *= Math.max(1, stats.objectCount()); + divisor = divisor.multiply(BigDecimal.valueOf(Math.max(1, stats.objectCount()))); } if (predBound && stats.predicateCount() > 0) { - divisor *= Math.max(1, stats.predicateCount()); + divisor = divisor.multiply(BigDecimal.valueOf(Math.max(1, stats.predicateCount()))); } - if (divisor > 1) { + if (divisor.compareTo(BigDecimal.ONE) > 0) { return BigDecimal.valueOf(baseCount) - .divide(BigDecimal.valueOf(divisor), 4, RoundingMode.HALF_UP) + .divide(divisor, 4, RoundingMode.HALF_UP) .max(BigDecimal.ONE); } return BigDecimal.valueOf(baseCount); diff --git a/src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java b/src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java index 7932ed08e..ca228e26e 100644 --- a/src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java +++ b/src/main/java/fr/inria/corese/core/next/query/impl/engine/storage/StorageManagerProducer.java @@ -1,5 +1,6 @@ package fr.inria.corese.core.next.query.impl.engine.storage; +import fr.inria.corese.core.next.data.Values; import fr.inria.corese.core.next.data.api.model.DatatypeValue; import fr.inria.corese.core.next.data.api.model.Statement; import fr.inria.corese.core.next.data.api.term.IRI; @@ -16,6 +17,7 @@ import fr.inria.corese.core.next.query.impl.engine.solution.Mappings; import fr.inria.corese.core.next.query.impl.engine.spi.Environment; import fr.inria.corese.core.next.storage.api.StorageManager; +import fr.inria.corese.core.next.storage.api.exception.StorageException; import fr.inria.corese.core.next.storage.api.model.PredicateStatistics; import fr.inria.corese.core.next.storage.api.model.StorageStatistics; import fr.inria.corese.core.next.storage.api.operations.StatementSpliterator; @@ -27,11 +29,13 @@ import java.util.IdentityHashMap; import java.util.Collections; import java.util.Iterator; +import java.util.Map; import java.util.NoSuchElementException; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.Spliterators; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; /** @@ -44,9 +48,12 @@ */ public class StorageManagerProducer extends ProducerDefault implements QueryStatistics { + private static final PredicateStatistics MISSING_PREDICATE_STATS = + new PredicateStatistics(Values.factory().createIRI("urn:corese:internal:missing-stats"), 0, 0, 0); + private final StorageManager storage; - private final java.util.Map predicateStatsCache = new java.util.concurrent.ConcurrentHashMap<>(); - private StorageStatistics cachedStorageStats; + private final Map predicateStatsCache = new ConcurrentHashMap<>(); + private volatile StorageStatistics cachedStorageStats; private final Set cursors = Collections.newSetFromMap(new IdentityHashMap<>()); private boolean closed; @@ -93,28 +100,33 @@ public long estimateCardinality(Query query, Node graphNode, Edge edge) { @Override public PredicateStatistics getPredicateStatistics(Query query, Node graphNode, IRI predicate) { - if (predicate == null) { + if (predicate == null || closed) { return null; } checkInterrupted(); - return predicateStatsCache.computeIfAbsent(predicate, p -> { + PredicateStatistics stats = predicateStatsCache.computeIfAbsent(predicate, p -> { try { var meta = storage.metadata(); - return meta == null ? null : meta.getPredicateStatistics(p); - } catch (UnsupportedOperationException | IllegalArgumentException unavailable) { - return null; + PredicateStatistics resolved = meta == null ? null : meta.getPredicateStatistics(p); + return resolved != null ? resolved : MISSING_PREDICATE_STATS; + } catch (StorageException | UnsupportedOperationException | IllegalArgumentException unavailable) { + return MISSING_PREDICATE_STATS; } }); + return stats == MISSING_PREDICATE_STATS ? null : stats; } @Override public StorageStatistics getStorageStatistics() { + if (closed) { + return null; + } checkInterrupted(); if (cachedStorageStats == null) { try { var meta = storage.metadata(); cachedStorageStats = meta == null ? null : meta.getStatistics(); - } catch (UnsupportedOperationException | IllegalArgumentException unavailable) { + } catch (StorageException | UnsupportedOperationException | IllegalArgumentException unavailable) { return null; } } diff --git a/src/main/java/fr/inria/corese/core/next/storage/impl/memory/TripleIndex.java b/src/main/java/fr/inria/corese/core/next/storage/impl/memory/TripleIndex.java index acbdd8db4..9e099a6b4 100644 --- a/src/main/java/fr/inria/corese/core/next/storage/impl/memory/TripleIndex.java +++ b/src/main/java/fr/inria/corese/core/next/storage/impl/memory/TripleIndex.java @@ -886,7 +886,8 @@ long estimateDistinctSubjects(final long distinctObjects) { return subjects.size(); } final double ratio = (double) distinctObjects / Math.max(1, sampled); - return Math.min(totalCount, Math.max(1, Math.round(subjects.size() * Math.min(ratio, 2.0)))); + final long estimated = Math.round(subjects.size() * ratio); + return Math.min(totalCount, Math.max(subjects.size(), estimated)); } }