Skip to content

Commit 95f62f1

Browse files
author
Joe Alphonso
committed
Markings refactoring
1 parent 807d0fe commit 95f62f1

62 files changed

Lines changed: 286 additions & 342 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎core/cached-results/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@
1515
<groupId>gov.nsa.datawave.core</groupId>
1616
<artifactId>base-rest-responses</artifactId>
1717
</dependency>
18+
<dependency>
19+
<groupId>gov.nsa.datawave.core</groupId>
20+
<artifactId>datawave-core-common-util</artifactId>
21+
<version>${project.version}</version>
22+
</dependency>
1823
<dependency>
1924
<groupId>gov.nsa.datawave.core</groupId>
2025
<artifactId>type-utils</artifactId>

‎core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowImpl.java‎

Lines changed: 10 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@
99
import java.util.Set;
1010
import java.util.TreeSet;
1111

12-
import org.apache.accumulo.access.AccessExpression;
1312
import org.apache.commons.lang.StringUtils;
1413
import org.slf4j.Logger;
1514
import org.slf4j.LoggerFactory;
1615

1716
import com.google.common.collect.Sets;
1817

18+
import datawave.core.common.util.AccessExpressionUtil;
1919
import datawave.data.type.Type;
2020
import datawave.marking.AccessExpressionMarkings;
2121
import datawave.marking.MarkingFunctions;
@@ -60,22 +60,15 @@ public void addColumn(String columnName, TypedValue columnTypedValue, Markings<?
6060
if (!this.markings.equals(markings)) {
6161
if (this.markings.isEmpty()) {
6262
// validate the markings
63-
try {
64-
markingFunctions.toAccessExpression(markings);
65-
if (this.markings.isEmpty()) {
66-
// markings were empty, so use the one passed in.
67-
this.markings = markings;
68-
}
69-
} catch (MarkingFunctions.Exception e) {
70-
LOGGER.error("Invalid markings {} skipping column {} = {}", markings, columnName, columnTypedValue, e);
71-
return;
63+
AccessExpressionUtil.toAccessExpression(markings);
64+
if (this.markings.isEmpty()) {
65+
// markings were empty, so use the one passed in.
66+
this.markings = markings;
7267
}
7368
} else {
7469
try {
75-
AccessExpression combined = markingFunctions
76-
.combine(List.of(markingFunctions.toAccessExpression(this.markings), markingFunctions.toAccessExpression(markings)));
7770
// use combined marking as new markings
78-
this.markings = markingFunctions.fromAccessExpression(combined);
71+
this.markings = AccessExpressionUtil.combine(markingFunctions, List.of(this.markings, markings));
7972
} catch (MarkingFunctions.Exception e) {
8073
LOGGER.error("Invalid markings {} skipping column {} = {}", markings, columnName, columnTypedValue, e);
8174
return;
@@ -135,11 +128,7 @@ private void manageColumnInsert(Type<?> datawaveType, String columnName, TypedVa
135128

136129
if (!currMarkings.equals(markings)) {
137130
try {
138-
List<AccessExpression> expressions = new ArrayList<>();
139-
expressions.add(markingFunctions.toAccessExpression(currMarkings));
140-
expressions.add(markingFunctions.toAccessExpression(markings));
141-
AccessExpression combined = markingFunctions.combine(expressions);
142-
Markings<?> combinedMarkings = markingFunctions.fromAccessExpression(combined);
131+
Markings<?> combinedMarkings = AccessExpressionUtil.combine(markingFunctions, List.of(currMarkings, markings));
143132

144133
// use combined marking as new markings
145134
columnMarkingsMap.put(columnName, combinedMarkings);
@@ -285,12 +274,8 @@ public void setColFam(String colFam) {
285274

286275
public void setMarkings(Markings<?> markings) {
287276
// validate the markings
288-
try {
289-
markingFunctions.toAccessExpression(markings);
290-
this.markings = markings;
291-
} catch (MarkingFunctions.Exception e) {
292-
LOGGER.error("Invalid markings {}", markings, e);
293-
}
277+
AccessExpressionUtil.toAccessExpression(markings);
278+
this.markings = markings;
294279
}
295280

296281
public void setColumnMarkingsMap(Map<String,Markings<?>> columnMarkingsMap) {
@@ -315,11 +300,7 @@ public String getColumnSecurityMarkingString(Map<String,Integer> columnMap) {
315300
String v = columnColumnVisibilityMap.get(field);
316301
String mStr = "";
317302
if (m != null && !m.isEmpty()) {
318-
try {
319-
mStr = markingFunctions.toAccessExpression(m).getExpression();
320-
} catch (MarkingFunctions.Exception e) {
321-
LOGGER.error("could not serialize markings {}", m, e);
322-
}
303+
mStr = AccessExpressionUtil.toAccessExpression(m).getExpression();
323304
}
324305
if (v == null) {
325306
combinedMap.put(field, mStr);

‎core/cached-results/src/main/java/datawave/core/query/cachedresults/CacheableQueryRowReader.java‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@
99
import java.util.Set;
1010
import java.util.TreeSet;
1111

12-
import org.apache.accumulo.access.AccessExpression;
1312
import org.slf4j.Logger;
1413
import org.slf4j.LoggerFactory;
1514

15+
import datawave.core.common.util.AccessExpressionUtil;
1616
import datawave.marking.MarkingFunctions;
1717
import datawave.marking.Markings;
1818
import datawave.webservice.query.cachedresults.CacheableQueryRow;
@@ -90,12 +90,7 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set<String> fi
9090
if (columnToIndexMap.get("_markings_") != null) {
9191
String mStr = cachedRowSet.getString(columnToIndexMap.get("_markings_"));
9292
if (mStr != null && !mStr.isEmpty()) {
93-
try {
94-
AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(mStr);
95-
cqfc.setMarkings(markingFunctions.fromAccessExpression(ae));
96-
} catch (MarkingFunctions.Exception e) {
97-
log.error("could not deserialize markings {}", mStr, e);
98-
}
93+
cqfc.setMarkings(AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(mStr)));
9994
}
10095
}
10196
if (columnToIndexMap.get("_column_markings_") != null) {
@@ -107,19 +102,14 @@ public static CacheableQueryRow createRow(ResultSet cachedRowSet, Set<String> fi
107102
String columnName = entry.getKey();
108103
String combinedString = entry.getValue();
109104
int x = combinedString.lastIndexOf(':');
110-
try {
111-
if (x >= 0) {
112-
String markingStr = combinedString.substring(0, x);
113-
AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(markingStr);
114-
columnMarkingsMap.put(columnName, markingFunctions.fromAccessExpression(ae));
115-
columnVisibilityMap.put(columnName, combinedString.substring(x + 1));
116-
} else {
117-
AccessExpression ae = MarkingFunctions.Default.ACCESS.newExpression(combinedString);
118-
columnMarkingsMap.put(columnName, markingFunctions.fromAccessExpression(ae));
119-
columnVisibilityMap.put(columnName, "");
120-
}
121-
} catch (MarkingFunctions.Exception e) {
122-
log.error("could not deserialize column markings for {}: {}", columnName, combinedString, e);
105+
if (x >= 0) {
106+
String markingStr = combinedString.substring(0, x);
107+
columnMarkingsMap.put(columnName, AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(markingStr)));
108+
columnVisibilityMap.put(columnName, combinedString.substring(x + 1));
109+
} else {
110+
columnMarkingsMap.put(columnName,
111+
AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(combinedString)));
112+
columnVisibilityMap.put(columnName, "");
123113
}
124114
}
125115
cqfc.setColumnMarkingsMap(columnMarkingsMap);

‎core/common-util/src/main/java/datawave/core/cache/ClassCache.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ public interface ClassCache {
2727
* @param name
2828
* the class name
2929
* @return a {@link Class}
30+
*
31+
* @throws ClassNotFoundException
32+
* if class not found
3033
*/
3134
Class<?> get(String name) throws ClassNotFoundException;
3235
}

‎core/common-util/src/main/java/datawave/core/common/util/AccessExpressionUtil.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,16 @@
22

33
import static java.nio.charset.StandardCharsets.UTF_8;
44

5+
import java.util.Collection;
6+
57
import org.apache.accumulo.access.Access;
68
import org.apache.accumulo.access.AccessExpression;
9+
import org.apache.accumulo.access.ParsedAccessExpression;
10+
import org.apache.accumulo.access.examples.ParseExamples;
711
import org.apache.accumulo.core.security.ColumnVisibility;
812

913
import datawave.marking.AccessExpressionMarkings;
14+
import datawave.marking.MarkingFunctions;
1015
import datawave.marking.Markings;
1116

1217
/**
@@ -38,6 +43,28 @@ public static AccessExpression toAccessExpression(ColumnVisibility cv) {
3843
return ACCESS.newExpression(new String(expr, UTF_8));
3944
}
4045

46+
/**
47+
* Convert a {@link Markings} to an {@link AccessExpression}.
48+
*
49+
* @param markings
50+
* assumed to be of type AccessExpressionMarkings
51+
* @return the embedded access expression
52+
*/
53+
public static AccessExpression toAccessExpression(Markings<?> markings) {
54+
return (AccessExpression) markings.getMarkings();
55+
}
56+
57+
/**
58+
* Convert a {@link AccessExpression} to an {@link AccessExpressionMarkings}.
59+
*
60+
* @param expression
61+
* The AccessExpression to embed in the wrapper class
62+
* @return the wrapped access expression
63+
*/
64+
public static AccessExpressionMarkings fromAccessExpression(AccessExpression expression) {
65+
return AccessExpressionMarkings.builder().accessExpression(expression).build();
66+
}
67+
4168
/**
4269
* Convert an {@link Markings} to a {@link ColumnVisibility}.
4370
*
@@ -85,6 +112,23 @@ public static AccessExpression toAccessExpression(byte[] visibilityBytes) {
85112
return ACCESS.newExpression(new String(visibilityBytes, UTF_8));
86113
}
87114

115+
/**
116+
* Normalize an {@link AccessExpression} by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()}
117+
* behavior.
118+
*
119+
* @param expression
120+
* the expression to normalize
121+
* @return a normalized {@link AccessExpression}
122+
*/
123+
public static AccessExpression normalize(AccessExpression expression) {
124+
String expr = expression.getExpression();
125+
if (expr.isEmpty()) {
126+
return expression;
127+
}
128+
ParsedAccessExpression parsed = MarkingFunctions.Default.ACCESS.newParsedExpression(expr);
129+
return MarkingFunctions.Default.ACCESS.newExpression(ParseExamples.normalize(parsed).expression);
130+
}
131+
88132
/**
89133
* @return the shared {@link Access} instance used by this utility
90134
*/
@@ -98,4 +142,21 @@ public static Access getAccess() {
98142
public static AccessExpression emptyExpression() {
99143
return EMPTY_EXPRESSION;
100144
}
145+
146+
/**
147+
* Helper to call {@link MarkingFunctions#combine(Collection)} with a wildcard-typed {@link MarkingFunctions}. This avoids the wildcard-capture issue that
148+
* arises when callers hold a {@code MarkingFunctions<?>} reference.
149+
*
150+
* @param markingFunctions
151+
* the marking functions instance
152+
* @param markings
153+
* the markings to combine
154+
* @return the combined markings
155+
* @throws MarkingFunctions.Exception
156+
* if the markings cannot be combined
157+
*/
158+
@SuppressWarnings({"unchecked", "rawtypes"})
159+
public static Markings<?> combine(MarkingFunctions<?> markingFunctions, Collection<? extends Markings<?>> markings) throws MarkingFunctions.Exception {
160+
return ((MarkingFunctions) markingFunctions).combine(markings);
161+
}
101162
}

‎core/modification/src/main/java/datawave/modification/MutableMetadataHandler.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ private Markings<?> convertFieldMarkings(Map<String,String> rawMarkings) throws
322322
if (vis.isEmpty()) {
323323
return null;
324324
}
325-
return markingFunctions.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(vis));
325+
return AccessExpressionUtil.fromAccessExpression(MarkingFunctions.Default.ACCESS.newExpression(vis));
326326
}
327327

328328
public void process(AccumuloClient client, ModificationRequestBase request, Map<String,Set<String>> mutableFieldList, Set<Authorizations> userAuths,
@@ -658,7 +658,7 @@ protected void insert(MultiTableBatchWriter writer, String shardId, String datat
658658
if (null == markings || markings.isEmpty())
659659
throw new IllegalArgumentException("No security information specified. Security markings must be supplied");
660660

661-
viz = AccessExpressionUtil.toColumnVisibility(markingFunctions.toAccessExpression(markings));
661+
viz = AccessExpressionUtil.toColumnVisibility(AccessExpressionUtil.toAccessExpression(markings));
662662
}
663663

664664
insert(writer, shardId, datatype, eventUid, viz, fieldName, fieldValue, ts, isIndexOnlyField, isIndexed, isReverseIndexed, dataTypes, false,
@@ -859,14 +859,14 @@ protected List<Pair<Key,Value>> getField(AccumuloClient client, Set<Authorizatio
859859

860860
if (null != oldColumnVisibility) {
861861
// need to compare the normalized values for equivalence. It's possible for the visibility to be in a different order
862-
String oldColViz = MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(oldColumnVisibility)).getExpression();
863-
String thisVis = MarkingFunctions.normalize(AccessExpressionUtil.toAccessExpression(thisViz)).getExpression();
862+
String oldColViz = AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(oldColumnVisibility)).getExpression();
863+
String thisVis = AccessExpressionUtil.normalize(AccessExpressionUtil.toAccessExpression(thisViz)).getExpression();
864864
if (!oldColViz.equals(thisVis)) {
865865
log.trace("Skipping key that does not match with column visibility: {}", e.getKey());
866866
continue;
867867
}
868868
} else {
869-
Markings<?> markings = markingFunctions
869+
Markings<?> markings = AccessExpressionUtil
870870
.fromAccessExpression(AccessExpressionUtil.toAccessExpression(e.getKey().getColumnVisibilityParsed()));
871871
if (null != oldFieldMarkings && !oldFieldMarkings.equals(markings)) {
872872
log.trace("Skipping key that does not match with markings: {}", e.getKey());

‎core/utils/accumulo-utils/src/main/java/datawave/marking/MarkingFunctions.java‎

Lines changed: 12 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package datawave.marking;
22

3-
import java.util.Arrays;
43
import java.util.Collection;
54
import java.util.HashSet;
65
import java.util.Set;
@@ -23,30 +22,7 @@
2322

2423
public interface MarkingFunctions<T extends Markings<?>> {
2524

26-
AccessExpression combine(Collection<AccessExpression> expressions) throws MarkingFunctions.Exception;
27-
28-
T combine(T... markings) throws MarkingFunctions.Exception;
29-
30-
AccessExpression toAccessExpression(Markings<?> markings) throws MarkingFunctions.Exception;
31-
32-
T fromAccessExpression(AccessExpression expression) throws MarkingFunctions.Exception;
33-
34-
/**
35-
* Normalize an {@link AccessExpression} by deduplicating and sorting terms. This is the replacement for the old {@code ColumnVisibility.flatten()}
36-
* behavior.
37-
*
38-
* @param expression
39-
* the expression to normalize
40-
* @return a normalized {@link AccessExpression}
41-
*/
42-
static AccessExpression normalize(AccessExpression expression) {
43-
String expr = expression.getExpression();
44-
if (expr.isEmpty()) {
45-
return expression;
46-
}
47-
ParsedAccessExpression parsed = Default.ACCESS.newParsedExpression(expr);
48-
return Default.ACCESS.newExpression(ParseExamples.normalize(parsed).expression);
49-
}
25+
T combine(Collection<T> markings) throws MarkingFunctions.Exception;
5026

5127
class Exception extends java.lang.Exception {
5228

@@ -73,23 +49,25 @@ public Exception(Throwable cause) {
7349

7450
class Default implements MarkingFunctions<AccessExpressionMarkings> {
7551
public static final Access ACCESS = Access.builder().build();
76-
public static final AccessExpression EMPTY_MARKINGS = ACCESS.newExpression("");
52+
public static final AccessExpressionMarkings EMPTY_MARKINGS = AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression("")).build();
7753

7854
@Override
79-
public AccessExpression combine(Collection<AccessExpression> expressions) {
80-
// filter out any empty expressions and concatenate with '&'
81-
// @formatter:off
55+
public AccessExpressionMarkings combine(Collection<AccessExpressionMarkings> markings) {
8256

83-
Set<AccessExpression> uniqueExpressions = new HashSet<>(expressions);
57+
Set<AccessExpressionMarkings> uniqueMarkings = new HashSet<>(markings);
8458

85-
if (uniqueExpressions.isEmpty()) {
59+
if (uniqueMarkings.isEmpty()) {
8660
return EMPTY_MARKINGS;
8761
}
8862

89-
if (uniqueExpressions.size() == 1) {
90-
return uniqueExpressions.stream().findFirst().get();
63+
if (uniqueMarkings.size() == 1) {
64+
return uniqueMarkings.stream().findFirst().get();
9165
}
9266

67+
Set<AccessExpression> uniqueExpressions = markings.stream().map(AccessExpressionMarkings::getMarkings).collect(Collectors.toSet());
68+
69+
// filter out any empty expressions and concatenate with '&'
70+
// @formatter:off
9371
ParsedAccessExpression expression = ACCESS.newParsedExpression(uniqueExpressions
9472
.stream()
9573
.map(AccessExpression::getExpression)
@@ -101,25 +79,8 @@ public AccessExpression combine(Collection<AccessExpression> expressions) {
10179

10280
// normalize the Parsed Expression and then return a copy without the parse tree
10381
// FIXME: Using a beta of the accumulo-access API so temporarily using the normalize function from the example code
104-
return ACCESS.newExpression(ParseExamples.normalize(expression).expression);
105-
}
106-
107-
@Override
108-
public final AccessExpressionMarkings combine(AccessExpressionMarkings... markings) {
109-
return AccessExpressionMarkings.builder().accessExpression(combine(Arrays.stream(markings).map(Markings::getMarkings).collect(Collectors.toSet())))
110-
.build();
82+
return AccessExpressionMarkings.builder().accessExpression(ACCESS.newExpression(ParseExamples.normalize(expression).expression)).build();
11183
}
112-
113-
@Override
114-
public AccessExpression toAccessExpression(Markings<?> markings) {
115-
return (AccessExpression) markings.getMarkings();
116-
}
117-
118-
@Override
119-
public AccessExpressionMarkings fromAccessExpression(AccessExpression expression) {
120-
return AccessExpressionMarkings.builder().accessExpression(expression).build();
121-
}
122-
12384
}
12485

12586
/**

0 commit comments

Comments
 (0)