Skip to content

Commit 8bc250c

Browse files
author
Joe Alphonso
committed
review feedback
1 parent bfb6651 commit 8bc250c

99 files changed

Lines changed: 255 additions & 194 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎core/base-rest-responses/src/main/java/datawave/marking/ColumnVisibilitySecurityMarking.java‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package datawave.marking;
22

3+
import static datawave.marking.AccessExpressionMarkings.ACCESS;
4+
35
import java.util.List;
46
import java.util.Map;
57

@@ -9,10 +11,7 @@
911
import javax.xml.bind.annotation.XmlAccessorType;
1012
import javax.xml.bind.annotation.XmlAttribute;
1113

12-
import org.apache.accumulo.access.Access;
1314
import org.apache.accumulo.access.AccessExpression;
14-
import org.apache.accumulo.access.examples.ParseExamples;
15-
import org.apache.accumulo.access.impl.AccessExpressionImpl;
1615

1716
import com.google.common.base.Preconditions;
1817

@@ -22,8 +21,6 @@ public class ColumnVisibilitySecurityMarking implements SecurityMarking {
2221

2322
public static final String VISIBILITY_MARKING = "columnVisibility";
2423

25-
private static final Access ACCESS = ParseExamples.ACCESS;
26-
2724
@XmlAttribute(name = "columnVisibility")
2825
private String columnVisibility = null;
2926

@@ -51,7 +48,7 @@ public void setColumnVisibility(String columnVisibility) {
5148
@Override
5249
public AccessExpression toAccessExpression() {
5350
if (columnVisibility == null || columnVisibility.isEmpty()) {
54-
return AccessExpressionImpl.EMPTY;
51+
return ACCESS.newExpression("");
5552
}
5653
return ACCESS.newExpression(columnVisibility);
5754
}

‎core/cached-results/pom.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,6 @@
1515
<groupId>gov.nsa.datawave.core</groupId>
1616
<artifactId>base-rest-responses</artifactId>
1717
</dependency>
18-
<dependency>
19-
<groupId>gov.nsa.datawave.core</groupId>
20-
<artifactId>datawave-core-common-util</artifactId>
21-
<version>${project.version}</version>
22-
</dependency>
2318
<dependency>
2419
<groupId>gov.nsa.datawave.core</groupId>
2520
<artifactId>type-utils</artifactId>

‎core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package datawave.core.query.cachedresults;
22

3+
import static datawave.marking.AccessExpressionMarkings.ACCESS;
4+
35
import java.util.ArrayList;
46
import java.util.Collection;
57
import java.util.HashMap;
@@ -9,16 +11,15 @@
911
import java.util.Set;
1012
import java.util.TreeSet;
1113

12-
import org.apache.accumulo.access.impl.AccessExpressionImpl;
1314
import org.apache.commons.lang.StringUtils;
1415
import org.slf4j.Logger;
1516
import org.slf4j.LoggerFactory;
1617

1718
import com.google.common.collect.Sets;
1819

19-
import datawave.core.common.util.AccessExpressionUtil;
2020
import datawave.data.type.Type;
2121
import datawave.marking.AccessExpressionMarkings;
22+
import datawave.marking.AccessExpressionUtil;
2223
import datawave.marking.MarkingFunctions;
2324
import datawave.marking.Markings;
2425
import datawave.webservice.query.cachedresults.CacheableQueryRow;
@@ -28,7 +29,7 @@
2829
public class CacheableQueryRowImpl extends CacheableQueryRow implements ObjectSizeOf {
2930

3031
private static final Logger LOGGER = LoggerFactory.getLogger(CacheableQueryRowImpl.class);
31-
private static final Markings<?> EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(AccessExpressionImpl.EMPTY).build();
32+
private static final Markings<?> EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build();
3233

3334
private String user = null;
3435
private String queryId = null;

‎core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package datawave.core.query.cachedresults;
22

3-
import static org.apache.accumulo.access.examples.ParseExamples.ACCESS;
3+
import static datawave.marking.AccessExpressionMarkings.ACCESS;
44

55
import java.sql.ResultSet;
66
import java.sql.ResultSetMetaData;
@@ -14,7 +14,7 @@
1414
import org.slf4j.Logger;
1515
import org.slf4j.LoggerFactory;
1616

17-
import datawave.core.common.util.AccessExpressionUtil;
17+
import datawave.marking.AccessExpressionUtil;
1818
import datawave.marking.MarkingFunctions;
1919
import datawave.marking.Markings;
2020
import datawave.webservice.query.cachedresults.CacheableQueryRow;

‎core/common-util/pom.xml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,6 @@
1313
<dependencies />
1414
</dependencyManagement>
1515
<dependencies>
16-
<dependency>
17-
<groupId>gov.nsa.datawave.core</groupId>
18-
<artifactId>accumulo-utils</artifactId>
19-
</dependency>
2016
<dependency>
2117
<groupId>gov.nsa.datawave.core</groupId>
2218
<artifactId>datawave-core-connection-pool</artifactId>

‎core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package datawave.modification;
22

3-
import static org.apache.accumulo.access.examples.ParseExamples.ACCESS;
3+
import static datawave.marking.AccessExpressionMarkings.ACCESS;
44

55
import java.util.ArrayList;
66
import java.util.Arrays;
@@ -47,12 +47,12 @@
4747
import com.google.common.collect.Multimap;
4848

4949
import datawave.core.common.connection.AccumuloConnectionFactory;
50-
import datawave.core.common.util.AccessExpressionUtil;
5150
import datawave.core.iterators.FieldIndexDocumentFilter;
5251
import datawave.data.ColumnFamilyConstants;
5352
import datawave.data.type.Type;
5453
import datawave.ingest.protobuf.Uid;
5554
import datawave.ingest.protobuf.Uid.List.Builder;
55+
import datawave.marking.AccessExpressionUtil;
5656
import datawave.marking.MarkingFunctions;
5757
import datawave.marking.Markings;
5858
import datawave.microservice.query.DefaultQueryParameters;

‎core/utils/accumulo-utils/pom.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,6 @@
6767
<artifactId>accumulo-access-core</artifactId>
6868
<version>1.0.0-java-11</version>
6969
</dependency>
70-
<dependency>
71-
<groupId>org.apache.accumulo</groupId>
72-
<artifactId>accumulo-access-examples</artifactId>
73-
<version>1.0.0-java-11</version>
74-
</dependency>
7570
<dependency>
7671
<groupId>org.apache.commons</groupId>
7772
<artifactId>commons-collections4</artifactId>

‎core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionMarkings.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import org.apache.accumulo.access.Access;
44
import org.apache.accumulo.access.AccessExpression;
5-
import org.apache.accumulo.access.examples.ParseExamples;
65

76
import com.fasterxml.jackson.annotation.JsonAutoDetect;
87
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -15,8 +14,8 @@
1514
@Builder
1615
@JsonAutoDetect(getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE)
1716
public class AccessExpressionMarkings implements Markings<AccessExpression> {
17+
public static final Access ACCESS = Access.builder().build();
1818

19-
private static final Access ACCESS = ParseExamples.ACCESS;
2019
public static final String COLUMN_VISIBILITY_KEY = "columnVisibility";
2120

2221
private final AccessExpression accessExpression;

core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java renamed to core/utils/accumulo-utils/src/main/java/datawave/marking/AccessExpressionUtil.java

Lines changed: 136 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,21 @@
1-
package datawave.core.common.util;
1+
package datawave.marking;
22

33
import static java.nio.charset.StandardCharsets.UTF_8;
44

5+
import static datawave.marking.AccessExpressionMarkings.ACCESS;
6+
import static org.apache.accumulo.access.ParsedAccessExpression.ExpressionType.AND;
7+
import static org.apache.accumulo.access.ParsedAccessExpression.ExpressionType.AUTHORIZATION;
8+
59
import java.util.Collection;
610
import java.util.List;
11+
import java.util.SortedSet;
12+
import java.util.TreeSet;
713

814
import org.apache.accumulo.access.Access;
915
import org.apache.accumulo.access.AccessExpression;
1016
import org.apache.accumulo.access.ParsedAccessExpression;
11-
import org.apache.accumulo.access.examples.ParseExamples;
12-
import org.apache.accumulo.access.impl.AccessExpressionImpl;
1317
import org.apache.accumulo.core.security.ColumnVisibility;
1418

15-
import datawave.marking.AccessExpressionMarkings;
16-
import datawave.marking.MarkingFunctions;
17-
import datawave.marking.Markings;
18-
1919
/**
2020
* Utility for bridging between Accumulo's {@link ColumnVisibility} (from accumulo-core) and {@link AccessExpression} (from accumulo-access).
2121
* <p>
@@ -24,9 +24,7 @@
2424
*/
2525
public final class AccessExpressionUtil {
2626

27-
private static final Access ACCESS = ParseExamples.ACCESS;
28-
29-
private static final AccessExpression EMPTY_EXPRESSION = AccessExpressionImpl.EMPTY;
27+
private static final AccessExpression EMPTY_EXPRESSION = ACCESS.newExpression("");
3028

3129
private AccessExpressionUtil() {}
3230

@@ -128,7 +126,7 @@ public static AccessExpression normalize(AccessExpression expression) {
128126
return expression;
129127
}
130128
ParsedAccessExpression parsed = ACCESS.newParsedExpression(expr);
131-
return ACCESS.newExpression(ParseExamples.normalize(parsed).expression);
129+
return ACCESS.newExpression(normalize(parsed).expression);
132130
}
133131

134132
/**
@@ -192,4 +190,131 @@ public static Markings<?> mergeMarkings(MarkingFunctions<?> markingFunctions, Ma
192190
}
193191
return markings1;
194192
}
193+
194+
/**
195+
* As part of normalizing access expression this class is used to sort and dedupe sub-expressions in a tree set.
196+
*/
197+
public static class NormalizedExpression implements Comparable<NormalizedExpression> {
198+
public final String expression;
199+
public final ParsedAccessExpression.ExpressionType type;
200+
201+
NormalizedExpression(String expression, ParsedAccessExpression.ExpressionType type) {
202+
this.expression = expression;
203+
this.type = type;
204+
}
205+
206+
// determines the sort order of different kinds of subexpressions.
207+
private static int typeOrder(ParsedAccessExpression.ExpressionType type) {
208+
switch (type) {
209+
case AUTHORIZATION:
210+
return 1;
211+
case OR:
212+
return 2;
213+
case AND:
214+
return 3;
215+
case EMPTY:
216+
default:
217+
throw new IllegalArgumentException("Unexpected type " + type);
218+
}
219+
}
220+
221+
@Override
222+
public int compareTo(NormalizedExpression o) {
223+
// Changing this comparator would significantly change how expressions are normalized.
224+
int cmp = typeOrder(type) - typeOrder(o.type);
225+
if (cmp == 0) {
226+
if (type == AUTHORIZATION) {
227+
// sort based on the unquoted and unescaped form of the authorization
228+
cmp = ACCESS.unquote(expression).compareTo(ACCESS.unquote(o.expression));
229+
} else {
230+
cmp = expression.compareTo(o.expression);
231+
}
232+
233+
}
234+
return cmp;
235+
}
236+
237+
@Override
238+
public boolean equals(Object o) {
239+
return this == o || (o instanceof NormalizedExpression && compareTo((NormalizedExpression) o) == 0);
240+
}
241+
242+
@Override
243+
public int hashCode() {
244+
return expression.hashCode();
245+
}
246+
}
247+
248+
/**
249+
* This method helps with the flattening aspect of normalization by recursing down as far as possible the parse tree in the case when the expression type is
250+
* the same. As long as the type is the same in the sub expression, keep using the same set.
251+
*/
252+
public static void flatten(ParsedAccessExpression.ExpressionType parentType, ParsedAccessExpression parsed,
253+
SortedSet<NormalizedExpression> normalizedExpressions) {
254+
if (parsed.getType() == parentType) {
255+
for (var child : parsed.getChildren()) {
256+
flatten(parentType, child, normalizedExpressions);
257+
}
258+
} else {
259+
// The type changed, so start again on the subexpression.
260+
normalizedExpressions.add(normalize(parsed));
261+
}
262+
}
263+
264+
/**
265+
* <p>
266+
* For a given access expression this example will deduplicate, sort, flatten, and remove unneeded parentheses or quotes in the expressions. The following
267+
* list gives examples of what each normalization step does.
268+
*
269+
* <ul>
270+
* <li>As an example of flattening, the expression {@code A&(B&C)} flattens to {@code
271+
* A&B&C}.</li>
272+
* <li>As an example of sorting, the expression {@code (Z&Y)|(C&B)} sorts to {@code
273+
* (B&C)|(Y&Z)}</li>
274+
* <li>As an example of deduplication, the expression {@code X&Y&X} normalizes to {@code X&Y}</li>
275+
* <li>As an example of unneeded quotes, the expression {@code "ABC"&"XYZ"} normalizes to {@code ABC&XYZ}</li>
276+
* <li>As an example of unneeded parentheses, the expression {@code (((ABC)|(XYZ)))} normalizes to {@code ABC|XYZ}</li>
277+
* </ul>
278+
*
279+
* <p>
280+
* This algorithm attempts to have the same behavior as the one in the Accumulo 2.1 ColumnVisibility class. However the implementation is very different.
281+
*/
282+
public static NormalizedExpression normalize(ParsedAccessExpression parsed) {
283+
if (parsed.getType() == AUTHORIZATION) {
284+
// If the authorization is quoted and it does not need to be quoted then the following two
285+
// lines will remove the unnecessary quoting.
286+
String unquoted = ACCESS.unquote(parsed.getExpression());
287+
String quoted = ACCESS.quote(unquoted);
288+
return new NormalizedExpression(quoted, parsed.getType());
289+
} else {
290+
// The tree set does the work of sorting and deduplicating sub expressions.
291+
TreeSet<NormalizedExpression> normalizedChildren = new TreeSet<>();
292+
for (var child : parsed.getChildren()) {
293+
flatten(parsed.getType(), child, normalizedChildren);
294+
}
295+
296+
if (normalizedChildren.size() == 1) {
297+
return normalizedChildren.first();
298+
} else {
299+
String operator = parsed.getType() == AND ? "&" : "|";
300+
String sep = "";
301+
302+
StringBuilder builder = new StringBuilder();
303+
304+
for (var child : normalizedChildren) {
305+
builder.append(sep);
306+
if (child.type == AUTHORIZATION) {
307+
builder.append(child.expression);
308+
} else {
309+
builder.append("(");
310+
builder.append(child.expression);
311+
builder.append(")");
312+
}
313+
sep = operator;
314+
}
315+
316+
return new NormalizedExpression(builder.toString(), parsed.getType());
317+
}
318+
}
319+
}
195320
}

‎core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
package datawave.marking;
22

3+
import static datawave.marking.AccessExpressionMarkings.ACCESS;
4+
35
import java.util.Collection;
46
import java.util.HashSet;
57
import java.util.Set;
68
import java.util.stream.Collectors;
79

8-
import org.apache.accumulo.access.Access;
910
import org.apache.accumulo.access.AccessExpression;
1011
import org.apache.accumulo.access.ParsedAccessExpression;
11-
import org.apache.accumulo.access.examples.ParseExamples;
12-
import org.apache.accumulo.access.impl.AccessExpressionImpl;
1312
import org.slf4j.Logger;
1413
import org.slf4j.LoggerFactory;
1514
import org.springframework.context.support.ClassPathXmlApplicationContext;
@@ -49,8 +48,8 @@ public Exception(Throwable cause) {
4948
}
5049

5150
class Default implements MarkingFunctions<AccessExpressionMarkings> {
52-
private static final Access ACCESS = ParseExamples.ACCESS;
53-
private static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(AccessExpressionImpl.EMPTY).build();
51+
52+
private static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build();
5453

5554
@Override
5655
public AccessExpressionMarkings combine(Collection<AccessExpressionMarkings> markings) {
@@ -80,7 +79,7 @@ public AccessExpressionMarkings combine(Collection<AccessExpressionMarkings> mar
8079

8180
// normalize the Parsed Expression and then return a copy without the parse tree
8281
// FIXME: Using a beta of the accumulo-access API so temporarily using the normalize function from the example code
83-
return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(ParseExamples.normalize(expression).expression)).build();
82+
return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(AccessExpressionUtil.normalize(expression).expression)).build();
8483
}
8584
}
8685

0 commit comments

Comments
 (0)