Skip to content

Commit e048e89

Browse files
JuicyDragonJuicyDragon
committed
tests for prod set and export
1 parent a517783 commit e048e89

3 files changed

Lines changed: 234 additions & 8 deletions

File tree

‎IntelliJ/src/main/java/com/nuix/enginebaseline/ThrowCapableConsumer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package com.nuix.enginebaseline;
22

33
/***
4-
* An interface similar to {@link java.util.function.Consumer} but allows for throwing checked exceptions.
4+
* An interface similar to {@link java.util.function.Consumer}, but allows for throwing checked exceptions.
55
* @param <T> The type of the argument accepted.
66
*/
77
@FunctionalInterface

‎IntelliJ/src/test/java/BasicTests.java‎

Lines changed: 232 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@
33
import org.junit.jupiter.api.Test;
44

55
import java.io.File;
6-
import java.util.*;
6+
import java.util.HashMap;
7+
import java.util.List;
8+
import java.util.Map;
9+
import java.util.Set;
710
import java.util.concurrent.atomic.AtomicBoolean;
811
import java.util.concurrent.atomic.AtomicInteger;
912
import java.util.concurrent.atomic.AtomicLong;
@@ -28,7 +31,7 @@ public void GetLicenseAutomaticCleanup() throws Exception {
2831
public void GetLicenseTryWithResourcesCleanup() throws Exception {
2932
// Create engine instance using try-with-resources, get utilities, use, closes
3033
// at end of try-with-resources
31-
try(NuixEngine nuixEngine = constructNuixEngine()) {
34+
try (NuixEngine nuixEngine = constructNuixEngine()) {
3235
Utilities utilities = nuixEngine.getUtilities();
3336
utilities.getItemTypeUtility().getAllTypes();
3437
}
@@ -135,7 +138,7 @@ public void SearchAndTag() throws Exception {
135138
AtomicLong itemCount = new AtomicLong(0);
136139
processor.whenItemProcessed(info -> {
137140
long currentItemCount = itemCount.addAndGet(1);
138-
if(System.currentTimeMillis() - lastProgressTime[0] > updateIntervalSeconds * 1000) {
141+
if (System.currentTimeMillis() - lastProgressTime[0] > updateIntervalSeconds * 1000) {
139142
lastProgressTime[0] = System.currentTimeMillis();
140143
log.info(String.format("%s items processed", currentItemCount));
141144
}
@@ -147,7 +150,7 @@ public void SearchAndTag() throws Exception {
147150

148151
log.info("Applying Tags...");
149152
for (TermCount termCount : termCounts) {
150-
String tag = "Terms|"+termCount.term;
153+
String tag = "Terms|" + termCount.term;
151154
Set<Item> responsiveItems = nuixCase.searchUnsorted(termCount.term);
152155
log.info(String.format("Tagging %s items with tag '%s'",
153156
responsiveItems.size(), tag));
@@ -156,8 +159,8 @@ public void SearchAndTag() throws Exception {
156159

157160
log.info("Validating tag counts...");
158161
for (TermCount termCount : termCounts) {
159-
String tag = "Terms|"+termCount.term;
160-
String query = "tag:\""+tag+"\"";
162+
String tag = "Terms|" + termCount.term;
163+
String query = "tag:\"" + tag + "\"";
161164
long hitCount = nuixCase.count(query);
162165
assertEquals(termCount.count, hitCount, String.format("For term %s, expect %s tagged items, but got %s",
163166
termCount.term, termCount.count, hitCount));
@@ -167,4 +170,227 @@ public void SearchAndTag() throws Exception {
167170
nuixCase.close();
168171
}));
169172
}
173+
174+
@Test
175+
public void CreateProductionSet() throws Exception {
176+
File caseDirectory = new File(testOutputDirectory, "CreateProductionSet_Case");
177+
File dataDirectory = new File(testOutputDirectory, "CreateProductionSet_Natives");
178+
179+
List<TermCount> termCounts = createSearchableTestData(dataDirectory, 5000);
180+
181+
NuixEngine nuixEngine = constructNuixEngine();
182+
nuixEngine.run((utilities -> {
183+
// Create a new case
184+
Map<String, Object> caseSettings = Map.of(
185+
"compound", false,
186+
"name", "CreateProductionSet",
187+
"description", "A Nuix case created using the Nuix Java Engine API",
188+
"investigator", "Test"
189+
);
190+
SimpleCase nuixCase = (SimpleCase) utilities.getCaseFactory().create(caseDirectory, caseSettings);
191+
192+
log.info("Queuing data for processing...");
193+
Processor processor = nuixCase.createProcessor();
194+
EvidenceContainer evidenceContainer = processor.newEvidenceContainer("SearchTestData");
195+
evidenceContainer.addFile(dataDirectory);
196+
evidenceContainer.save();
197+
198+
// Periodically log progress
199+
final long[] lastProgressTime = {0};
200+
int updateIntervalSeconds = 10;
201+
AtomicLong itemCount = new AtomicLong(0);
202+
processor.whenItemProcessed(info -> {
203+
long currentItemCount = itemCount.addAndGet(1);
204+
if (System.currentTimeMillis() - lastProgressTime[0] > updateIntervalSeconds * 1000) {
205+
lastProgressTime[0] = System.currentTimeMillis();
206+
log.info(String.format("%s items processed", currentItemCount));
207+
}
208+
});
209+
210+
log.info("Processing starting...");
211+
processor.process();
212+
log.info("Processing completed");
213+
214+
// Create and configure our production set
215+
ProductionSet productionSet = nuixCase.newProductionSet("TestProductionSet_" + System.currentTimeMillis());
216+
Map<String, Object> numberingOptions = Map.of(
217+
"prefix", "ABC-",
218+
"documentId", Map.of(
219+
"minWidth", 9,
220+
"startAt", 1
221+
)
222+
);
223+
productionSet.setNumberingOptions(numberingOptions);
224+
225+
// Obtain items that we will be adding to the production set
226+
List<Item> items = nuixCase.search("flag:audited");
227+
228+
// Add these items to the production set
229+
productionSet.addItems(items);
230+
231+
// Let's report the first and last number in this production set
232+
List<ProductionSetItem> productionSetItems = productionSet.getProductionSetItems();
233+
String firstNumber = productionSetItems.get(0).getDocumentNumber().toString();
234+
String lastNumber = productionSetItems.get(productionSetItems.size() - 1).getDocumentNumber().toString();
235+
log.info(String.format("Production Set Created with Numbers %s to %s", firstNumber, lastNumber));
236+
237+
log.info("Closing case");
238+
nuixCase.close();
239+
}));
240+
}
241+
242+
@Test
243+
public void Export() throws Exception {
244+
File caseDirectory = new File(testOutputDirectory, "ExportTest_Case");
245+
File dataDirectory = new File(testOutputDirectory, "ExportTest_Natives");
246+
File exportDirectory = new File(testOutputDirectory, "ExportTest_Export");
247+
248+
List<TermCount> termCounts = createSearchableTestData(dataDirectory, 5000);
249+
250+
NuixEngine nuixEngine = constructNuixEngine();
251+
nuixEngine.run((utilities -> {
252+
// Create a new case
253+
Map<String, Object> caseSettings = Map.of(
254+
"compound", false,
255+
"name", "ExportTest",
256+
"description", "A Nuix case created using the Nuix Java Engine API",
257+
"investigator", "Test"
258+
);
259+
SimpleCase nuixCase = (SimpleCase) utilities.getCaseFactory().create(caseDirectory, caseSettings);
260+
261+
log.info("Queuing data for processing...");
262+
Processor processor = nuixCase.createProcessor();
263+
EvidenceContainer evidenceContainer = processor.newEvidenceContainer("SearchTestData");
264+
evidenceContainer.addFile(dataDirectory);
265+
evidenceContainer.save();
266+
267+
// Periodically log progress
268+
final long[] lastProgressTime = {0};
269+
int updateIntervalSeconds = 10;
270+
AtomicLong itemCount = new AtomicLong(0);
271+
processor.whenItemProcessed(info -> {
272+
long currentItemCount = itemCount.addAndGet(1);
273+
if (System.currentTimeMillis() - lastProgressTime[0] > updateIntervalSeconds * 1000) {
274+
lastProgressTime[0] = System.currentTimeMillis();
275+
log.info(String.format("%s items processed", currentItemCount));
276+
}
277+
});
278+
279+
log.info("Processing starting...");
280+
processor.process();
281+
log.info("Processing completed");
282+
283+
BatchExporter exporter = utilities.createBatchExporter(exportDirectory);
284+
285+
// We will using the same naming type for all products, possible choices:
286+
// - "document_id" (e.g. "ABC-000000001.pdf"), requires license feature EXPORT_LEGAL
287+
// - "document_id_with_page" (e.g. "ABC-000000001_11.pdf"), requires license feature EXPORT_LEGAL
288+
// - "page_only" (e.g. "0001.pdf"), requires license feature EXPORT_LEGAL
289+
// - "full" (e.g. "ABC0010010001.pdf"), requires license feature EXPORT_LEGAL
290+
// - "full_with_periods" (e.g. "ABC.001.001.0001.pdf"), requires license feature EXPORT_LEGAL
291+
// - "item_name" (e.g. "original-name.pdf"), requires license feature EXPORT_ITEMS
292+
// - "item_name_with_path" (e.g. "mailbox/inbox/original-name.pdf"), requires license feature EXPORT_ITEMS
293+
// - "guid" (e.g. "04d/04dd8e72-f087-4f66-848a-6585bce732d5.pdf"), requires license feature EXPORT_ITEMS
294+
// - "md5" (e.g. "790/79054025255fb1a26e4bc422aef54eb4.pdf"), requires license feature EXPORT_ITEMS
295+
String productNaming = "guid";
296+
297+
// ======================================
298+
// * Add and Configure Native Exporting *
299+
// ======================================
300+
301+
// Add native product to our exporter
302+
exporter.addProduct("native", Map.of(
303+
"naming", productNaming,
304+
"path", "NATIVES",
305+
"mailFormat", "eml",
306+
"includeAttachments", true
307+
));
308+
309+
// Add native product to our exporter
310+
exporter.addProduct("text", Map.of(
311+
"naming", productNaming,
312+
"path", "TEXT"
313+
));
314+
315+
// Add native product to our exporter
316+
exporter.addProduct("pdf", Map.of(
317+
"naming", productNaming,
318+
"path", "PDF",
319+
"regenerateStored", false
320+
));
321+
322+
// Add native product to our exporter
323+
exporter.addProduct("tiff", Map.of(
324+
"naming", productNaming,
325+
"path", "TIFF",
326+
"regenerateStored", false,
327+
"multiPageTiff", false,
328+
"tiffDpi", 300,
329+
"tiffFormat", "MONOCHROME_CCITT_T6_G4"
330+
));
331+
332+
// Add Concordance DAT load file
333+
exporter.addLoadFile("concordance", Map.of(
334+
"metadataProfile", "Default",
335+
"encoding", "UTF-8"
336+
));
337+
338+
// Now we are going to instruct the processor how many workers to use to load the data. See API documentation for
339+
// ParallelProcessingConfigurable.setParallelProcessingSettings for list of settings and what they do.
340+
exporter.setParallelProcessingSettings(Map.of(
341+
"workerCount", utilities.getLicence().getWorkers(),
342+
"workerTemp", new File(testOutputDirectory,"WorkerTemp").getAbsolutePath()
343+
));
344+
345+
// Track error count
346+
AtomicInteger errorCount = new AtomicInteger();
347+
348+
exporter.whenItemEventOccurs(new ItemEventCallback() {
349+
// Use this to track when we reported progress last
350+
long lastProgressMillis = System.currentTimeMillis();
351+
@Override
352+
public void itemProcessed(ItemEventInfo info) {
353+
// Report progress if it has been at least 5 seconds (5000 milliseconds) since
354+
// the last time we reported progress
355+
long currentTimeMillis = System.currentTimeMillis();
356+
if (currentTimeMillis - lastProgressMillis > 5 * 1000) {
357+
String progressMessage = String.format(
358+
"Stage: %s, Progress: %s, Errors: %s",
359+
info.getStage(), info.getStageCount(), errorCount.get());
360+
log.info(progressMessage);
361+
lastProgressMillis = System.currentTimeMillis();
362+
}
363+
364+
// If this particular item had an error we always record this
365+
Exception possibleItemException = info.getFailure();
366+
if (possibleItemException != null) {
367+
Item item = info.getItem();
368+
errorCount.incrementAndGet();
369+
String errorMessage = String.format(
370+
"Error while exporting item %s/%s: %s",
371+
item.getGuid(), item.getLocalisedName(), possibleItemException.getMessage());
372+
log.error(errorMessage);
373+
}
374+
}
375+
});
376+
377+
String itemsToExportQuery = "flag:audited";
378+
379+
log.info(String.format("Searching: %s", itemsToExportQuery));
380+
List<Item> itemsToExport = nuixCase.search(itemsToExportQuery);
381+
log.info(String.format("Responsive Items: %s", itemsToExport.size()));
382+
383+
log.info("Beginning export...");
384+
exporter.exportItems(itemsToExport);
385+
log.info("Export completed");
386+
387+
log.info(String.format("Errors: %s", errorCount.get()));
388+
if (errorCount.get() > 0) {
389+
log.info("Review logs for more details regarding export errors");
390+
}
391+
392+
log.info("Closing case");
393+
nuixCase.close();
394+
}));
395+
}
170396
}

‎IntelliJ/src/test/java/CommonTestFunctionality.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public TermCount(String term, long count) {
5050
// When true, the testOutputDirectory used during tests will be deleted
5151
// upon test completion. Set this to false if you wish to manually review the output
5252
// of tests afterwards.
53-
protected static boolean deleteTestOutputOnCompletion = false;
53+
protected static boolean deleteTestOutputOnCompletion = true;
5454

5555

5656
@BeforeAll

0 commit comments

Comments
 (0)