Skip to content

Commit 3255b7c

Browse files
hmottestadhmottestad
committed
improvements
1 parent effefd6 commit 3255b7c

6 files changed

Lines changed: 346 additions & 37 deletions

File tree

‎core/sail/base/src/main/java/org/eclipse/rdf4j/sail/base/SketchJoinOrderPlanner.java‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ final class SketchJoinOrderPlanner {
7171
private static final double FINITE_DOMAIN_GUARD_MAX_STEP_WORK_RATIO = 6.0d;
7272
private static final double FILTERED_SEED_MAX_STEP_WORK_RATIO = 12.0d;
7373
private static final double SEED_FILTER_ROW_FLOW_MAX_PASS_RATIO = 0.25d;
74+
private static final double NESTED_NOT_EXISTS_MIN_PASS_RATIO = 0.05d;
7475
private static final double BROAD_BRIDGE_ENDPOINT_PREP_MIN_RATIO = 3.0d;
7576
private static final double FINITE_DOMAIN_PREFIX_GUARD_MIN_ACCESS_ROWS = 1_000.0d;
7677
private static final int FINITE_DOMAIN_FILTER_MAX_ENUMERATED_BINDINGS = 4096;
@@ -1317,6 +1318,13 @@ private SketchBasedJoinEstimator.TuplePlanEstimate filterAdjustedEstimate(JoinOr
13171318

13181319
private double filterPassRatio(JoinOrderPlanner.FilterConstraint filter,
13191320
SketchBasedJoinEstimator.TuplePlanEstimate estimate, long boundVarMask, long factorMask) {
1321+
if (filter.hasNestedTupleExpression()) {
1322+
double nestedPassRatio = nestedTupleFilterPassRatio(filter, estimate);
1323+
if (isValidPassRatio(nestedPassRatio)) {
1324+
return nestedPassRatio;
1325+
}
1326+
return 1.0d;
1327+
}
13201328
if (!canApplyPassRatioToOutputRows(filter)) {
13211329
return 1.0d;
13221330
}
@@ -1335,8 +1343,8 @@ private double filterUncertaintyRows(JoinOrderPlanner.FilterConstraint filter,
13351343
if (isValidPassRatio(finiteDomainFilterPassRatio(filter, factorMask))) {
13361344
return 0.0d;
13371345
}
1338-
if (!isValidPassRatio(filter.getEstimatedPassRatio()) && !isValidPassRatio(nestedExistsPassRatio(filter,
1339-
estimate))) {
1346+
if (!isValidPassRatio(filter.getEstimatedPassRatio())
1347+
&& !isValidPassRatio(nestedTupleFilterPassRatio(filter, estimate))) {
13401348
return estimate.outputRows();
13411349
}
13421350
return 0.0d;
@@ -5225,9 +5233,21 @@ private double nestedTupleWorkRows(TupleExpr nestedTupleExpression, long boundVa
52255233
return isFiniteNonNegative(rows) ? rows : Double.NaN;
52265234
}
52275235

5236+
private double nestedTupleFilterPassRatio(JoinOrderPlanner.FilterConstraint filter,
5237+
SketchBasedJoinEstimator.TuplePlanEstimate rowsEnteringEstimate) {
5238+
double existsPassRatio = nestedExistsPassRatio(filter, rowsEnteringEstimate);
5239+
if (!isValidPassRatio(existsPassRatio)) {
5240+
return Double.NaN;
5241+
}
5242+
if (!filter.isNotExists()) {
5243+
return existsPassRatio;
5244+
}
5245+
return Math.max(NESTED_NOT_EXISTS_MIN_PASS_RATIO, 1.0d - existsPassRatio);
5246+
}
5247+
52285248
private double nestedExistsPassRatio(JoinOrderPlanner.FilterConstraint filter,
52295249
SketchBasedJoinEstimator.TuplePlanEstimate rowsEnteringEstimate) {
5230-
if (!filter.hasNestedTupleExpression() || filter.isNotExists() || rowsEnteringEstimate == null
5250+
if (!filter.hasNestedTupleExpression() || rowsEnteringEstimate == null
52315251
|| !isFiniteNonNegative(rowsEnteringEstimate.outputRows())
52325252
|| rowsEnteringEstimate.outputRows() == 0.0d) {
52335253
return Double.NaN;
@@ -6183,10 +6203,10 @@ private String describeUnlockedFilter(JoinOrderPlanner.FilterConstraint filter,
61836203
String label = singleLineMetricValue(filter.getDebugLabel());
61846204
double passRatio = filter.getEstimatedPassRatio();
61856205
String selectivitySource = filter.getSelectivitySource();
6186-
double nestedExistsPassRatio = nestedExistsPassRatio(filter, rowsEnteringEstimate);
6187-
if (isValidPassRatio(nestedExistsPassRatio)) {
6188-
passRatio = nestedExistsPassRatio;
6189-
selectivitySource = "sketch_nested_exists";
6206+
double nestedPassRatio = nestedTupleFilterPassRatio(filter, rowsEnteringEstimate);
6207+
if (isValidPassRatio(nestedPassRatio)) {
6208+
passRatio = nestedPassRatio;
6209+
selectivitySource = filter.isNotExists() ? "sketch_nested_not_exists" : "sketch_nested_exists";
61906210
}
61916211
if (Double.isFinite(passRatio) && passRatio >= 0.0d && passRatio <= 1.0d) {
61926212
label += " passRatio=" + passRatio;

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbDeferredFilterPlacer.java‎

Lines changed: 83 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,18 @@
2222
import java.util.Optional;
2323
import java.util.Set;
2424

25+
import org.eclipse.rdf4j.model.vocabulary.RDF;
2526
import org.eclipse.rdf4j.query.algebra.Compare;
2627
import org.eclipse.rdf4j.query.algebra.Filter;
2728
import org.eclipse.rdf4j.query.algebra.Not;
2829
import org.eclipse.rdf4j.query.algebra.StatementPattern;
2930
import org.eclipse.rdf4j.query.algebra.TupleExpr;
3031
import org.eclipse.rdf4j.query.algebra.ValueExpr;
32+
import org.eclipse.rdf4j.query.algebra.Var;
3133
import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.JoinOrderPlanner;
3234
import org.eclipse.rdf4j.query.algebra.helpers.AbstractSimpleQueryModelVisitor;
35+
import org.eclipse.rdf4j.query.algebra.helpers.collectors.VarNameCollector;
36+
import org.eclipse.rdf4j.query.explanation.TelemetryMetricNames;
3337

3438
final class LmdbDeferredFilterPlacer {
3539

@@ -295,17 +299,25 @@ private boolean canApplyDeferredFilterToBindingPrefix(DeferredFilter deferredFil
295299
if (LmdbJoinPlanSupport.containsExists(deferredFilter.condition)) {
296300
return false;
297301
}
298-
return !deferredFilter.requiredVars.isEmpty()
302+
Set<String> conditionBindingNames = conditionBindingNames(deferredFilter);
303+
return !conditionBindingNames.isEmpty()
299304
&& (deferredFilter.conditionCost > JoinOrderPlanner.FILTER_COST_CHEAP
300-
|| !Collections.disjoint(prefixBindingNames, deferredFilter.requiredVars))
301-
&& !prefixBindingNames.containsAll(deferredFilter.requiredVars)
302-
&& availableNames.containsAll(deferredFilter.requiredVars)
303-
&& !Collections.disjoint(assignmentBindingNames, deferredFilter.requiredVars)
304-
&& canApplySplitPrefixFilter(deferredFilter, assignmentBindingNames);
305+
|| !Collections.disjoint(prefixBindingNames, conditionBindingNames))
306+
&& !prefixBindingNames.containsAll(conditionBindingNames)
307+
&& availableNames.containsAll(conditionBindingNames)
308+
&& !Collections.disjoint(assignmentBindingNames, conditionBindingNames)
309+
&& canApplySplitPrefixFilter(deferredFilter, assignmentBindingNames, conditionBindingNames);
305310
}
306311

307-
private boolean canApplySplitPrefixFilter(DeferredFilter deferredFilter, Set<String> assignmentBindingNames) {
308-
return assignmentBindingNames.containsAll(deferredFilter.requiredVars)
312+
private Set<String> conditionBindingNames(DeferredFilter deferredFilter) {
313+
Set<String> bindingNames = new HashSet<>(VarNameCollector.process(deferredFilter.condition));
314+
bindingNames.removeIf(bindingName -> bindingName == null || bindingName.startsWith("_const_"));
315+
return bindingNames;
316+
}
317+
318+
private boolean canApplySplitPrefixFilter(DeferredFilter deferredFilter, Set<String> assignmentBindingNames,
319+
Set<String> conditionBindingNames) {
320+
return assignmentBindingNames.containsAll(conditionBindingNames)
309321
|| !containsNotEquals(deferredFilter.condition);
310322
}
311323

@@ -382,6 +394,9 @@ private boolean groupDeferredFilterOnSmallestWindow(List<SegmentFactor> factors,
382394
}
383395
if (window == null && isCorrelatedNotExistsFilter(filter)) {
384396
window = smallestBindingCoveringWindow(factors, filter.requiredVars, boundBeforeSegment);
397+
if (window != null) {
398+
window = expandCorrelatedNotExistsWindowOverCheapGuards(factors, filter, window);
399+
}
385400
}
386401
if (window == null) {
387402
if (filter.originPatterns.isEmpty()) {
@@ -399,6 +414,66 @@ private boolean groupDeferredFilterOnSmallestWindow(List<SegmentFactor> factors,
399414
return groupDeferredFilterOnWindow(factors, filter, window);
400415
}
401416

417+
private int[] expandCorrelatedNotExistsWindowOverCheapGuards(List<SegmentFactor> factors, DeferredFilter filter,
418+
int[] window) {
419+
int end = window[1];
420+
while (end + 1 < factors.size() && isCheapGuardForCorrelatedFilter(factors.get(end + 1), filter)) {
421+
end++;
422+
}
423+
return end == window[1] ? window : new int[] { window[0], end };
424+
}
425+
426+
private boolean isCheapGuardForCorrelatedFilter(SegmentFactor factor, DeferredFilter filter) {
427+
Set<String> bindingNames = plannerBindingNames(factor.bindingNames);
428+
if (bindingNames.isEmpty() || !filter.requiredVars.containsAll(bindingNames)
429+
|| factor.containedPatterns.size() != 1) {
430+
return false;
431+
}
432+
StatementPattern pattern = factor.containedPatterns.iterator()
433+
.next();
434+
return isSubjectTypeGuardForRequiredBinding(pattern, filter.requiredVars) || isExactDirectLookup(pattern);
435+
}
436+
437+
private Set<String> plannerBindingNames(Set<String> bindingNames) {
438+
if (bindingNames == null || bindingNames.isEmpty()) {
439+
return Set.of();
440+
}
441+
Set<String> plannerNames = new HashSet<>();
442+
for (String bindingName : bindingNames) {
443+
if (bindingName != null && !bindingName.startsWith("_const_")) {
444+
plannerNames.add(bindingName);
445+
}
446+
}
447+
return plannerNames;
448+
}
449+
450+
private boolean isSubjectTypeGuardForRequiredBinding(StatementPattern statementPattern, Set<String> requiredVars) {
451+
Var subject = statementPattern.getSubjectVar();
452+
Var predicate = statementPattern.getPredicateVar();
453+
Var object = statementPattern.getObjectVar();
454+
return subject != null
455+
&& !subject.hasValue()
456+
&& requiredVars.contains(subject.getName())
457+
&& predicate != null
458+
&& predicate.hasValue()
459+
&& RDF.TYPE.equals(predicate.getValue())
460+
&& object != null
461+
&& object.hasValue();
462+
}
463+
464+
private boolean isExactDirectLookup(StatementPattern statementPattern) {
465+
if (!"directLookup".equals(
466+
statementPattern.getStringMetricPlanned(TelemetryMetricNames.PLANNED_INDEX_ACCESS_MODE))) {
467+
return false;
468+
}
469+
String lookupComponents = statementPattern
470+
.getStringMetricPlanned(TelemetryMetricNames.PLANNED_LOOKUP_COMPONENTS);
471+
return lookupComponents != null
472+
&& lookupComponents.contains("S")
473+
&& lookupComponents.contains("P")
474+
&& lookupComponents.contains("O");
475+
}
476+
402477
private boolean groupDeferredFilterOnPlannerWindow(List<SegmentFactor> factors, DeferredFilter filter,
403478
Set<String> boundBeforeSegment, Integer targetStep) {
404479
if (targetStep == null || filter.conditionCost == JoinOrderPlanner.FILTER_COST_CHEAP) {

0 commit comments

Comments
 (0)