WIP sync up

JuicyDragon · JuicyDragon · commit fb299500e985 · 2023-05-22T08:09:53.000-07:00
diff --git a/IntelliJ/build.gradle.kts b/IntelliJ/build.gradle.kts
@@ -41,6 +41,7 @@ dependencies {
                 "**/*joda*.jar",
                 "**/*commons*.jar",
                 "**/*guava*.jar",
+                "**/*gson*.jar",
                 // All Nuix API jars
                 "**/nuix-*.jar"
         )
diff --git a/IntelliJ/src/test/java/BasicTests.java b/IntelliJ/src/test/java/BasicTests.java
@@ -3,7 +3,6 @@
 import org.junit.jupiter.api.Test;
 
 import java.io.File;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -72,9 +71,8 @@ public void CreateAndOpenSimpleCase() throws Exception {
     @Test
     public void LoadDataIntoSimpleCase() throws Exception {
         File caseDirectory = new File(testOutputDirectory, "LoadDataIntoSimpleCase_Case");
-        File dataDirectory = new File(testOutputDirectory, "LoadDataIntoSimpleCase_Natives");
-
-        List<TermCount> termCounts = createSearchableTestData(dataDirectory, 1000);
+        File textFilesDirectory = TestData.getTestDataTextFilesDirectory();
+        Map<String, Long> termCounts = TestData.getTestDataTextFileTermCounts();
 
         NuixEngine nuixEngine = constructNuixEngine();
         nuixEngine.run((utilities -> {
@@ -90,17 +88,19 @@ public void LoadDataIntoSimpleCase() throws Exception {
             log.info("Queuing data for processing...");
             Processor processor = nuixCase.createProcessor();
             EvidenceContainer evidenceContainer = processor.newEvidenceContainer("SearchTestData");
-            evidenceContainer.addFile(dataDirectory);
+            evidenceContainer.addFile(textFilesDirectory);
             evidenceContainer.save();
             log.info("Processing starting...");
             processor.process();
             log.info("Processing completed");
 
             log.info("Validating search counts...");
-            for (TermCount termCount : termCounts) {
-                long hitCount = nuixCase.count(termCount.term);
-                assertEquals(termCount.count, hitCount, String.format("For term %s, expect %s but got %s",
-                        termCount.term, termCount.count, hitCount));
+            for (Map.Entry<String, Long> termCount : termCounts.entrySet()) {
+                String term = termCount.getKey();
+                Long count = termCount.getValue() + 1; // Add 1 for hit on term counts JSON
+                long hitCount = nuixCase.count(term);
+                assertEquals(count, hitCount, String.format("For term %s, expect %s but got %s",
+                        term, count, hitCount));
             }
 
             log.info("Closing case");
@@ -111,9 +111,8 @@ public void LoadDataIntoSimpleCase() throws Exception {
     @Test
     public void SearchAndTag() throws Exception {
         File caseDirectory = new File(testOutputDirectory, "SearchAndTag_Case");
-        File dataDirectory = new File(testOutputDirectory, "SearchAndTag_Natives");
-
-        List<TermCount> termCounts = createSearchableTestData(dataDirectory, 5000);
+        File textFilesDirectory = TestData.getTestDataTextFilesDirectory();
+        Map<String, Long> termCounts = TestData.getTestDataTextFileTermCounts();
 
         NuixEngine nuixEngine = constructNuixEngine();
         nuixEngine.run((utilities -> {
@@ -129,7 +128,7 @@ public void SearchAndTag() throws Exception {
             log.info("Queuing data for processing...");
             Processor processor = nuixCase.createProcessor();
             EvidenceContainer evidenceContainer = processor.newEvidenceContainer("SearchTestData");
-            evidenceContainer.addFile(dataDirectory);
+            evidenceContainer.addFile(textFilesDirectory);
             evidenceContainer.save();
 
             // Periodically log progress
@@ -149,21 +148,24 @@ public void SearchAndTag() throws Exception {
             log.info("Processing completed");
 
             log.info("Applying Tags...");
-            for (TermCount termCount : termCounts) {
-                String tag = "Terms|" + termCount.term;
-                Set<Item> responsiveItems = nuixCase.searchUnsorted(termCount.term);
+            for (Map.Entry<String, Long> termCount : termCounts.entrySet()) {
+                String term = termCount.getKey();
+                String tag = "Terms|" + term;
+                Set<Item> responsiveItems = nuixCase.searchUnsorted(term);
                 log.info(String.format("Tagging %s items with tag '%s'",
                         responsiveItems.size(), tag));
                 utilities.getBulkAnnotater().addTag(tag, responsiveItems);
             }
 
             log.info("Validating tag counts...");
-            for (TermCount termCount : termCounts) {
-                String tag = "Terms|" + termCount.term;
+            for (Map.Entry<String, Long> termCount : termCounts.entrySet()) {
+                String term = termCount.getKey();
+                Long count = termCount.getValue() + 1; // Add 1 for hit on term counts JSON
+                String tag = "Terms|" + term;
                 String query = "tag:\"" + tag + "\"";
                 long hitCount = nuixCase.count(query);
-                assertEquals(termCount.count, hitCount, String.format("For term %s, expect %s tagged items, but got %s",
-                        termCount.term, termCount.count, hitCount));
+                assertEquals(count, hitCount, String.format("For term %s, expect %s tagged items, but got %s",
+                        term, count, hitCount));
             }
 
             log.info("Closing case");
@@ -174,9 +176,8 @@ public void SearchAndTag() throws Exception {
     @Test
     public void CreateProductionSet() throws Exception {
         File caseDirectory = new File(testOutputDirectory, "CreateProductionSet_Case");
-        File dataDirectory = new File(testOutputDirectory, "CreateProductionSet_Natives");
-
-        List<TermCount> termCounts = createSearchableTestData(dataDirectory, 5000);
+        File textFilesDirectory = TestData.getTestDataTextFilesDirectory();
+        Map<String, Long> termCounts = TestData.getTestDataTextFileTermCounts();
 
         NuixEngine nuixEngine = constructNuixEngine();
         nuixEngine.run((utilities -> {
@@ -192,7 +193,7 @@ public void CreateProductionSet() throws Exception {
             log.info("Queuing data for processing...");
             Processor processor = nuixCase.createProcessor();
             EvidenceContainer evidenceContainer = processor.newEvidenceContainer("SearchTestData");
-            evidenceContainer.addFile(dataDirectory);
+            evidenceContainer.addFile(textFilesDirectory);
             evidenceContainer.save();
 
             // Periodically log progress
@@ -242,11 +243,10 @@ public void CreateProductionSet() throws Exception {
     @Test
     public void Export() throws Exception {
         File caseDirectory = new File(testOutputDirectory, "ExportTest_Case");
-        File dataDirectory = new File(testOutputDirectory, "ExportTest_Natives");
+        File textFilesDirectory = TestData.getTestDataTextFilesDirectory();
+        Map<String, Long> termCounts = TestData.getTestDataTextFileTermCounts();
         File exportDirectory = new File(testOutputDirectory, "ExportTest_Export");
 
-        List<TermCount> termCounts = createSearchableTestData(dataDirectory, 5000);
-
         NuixEngine nuixEngine = constructNuixEngine();
         nuixEngine.run((utilities -> {
             // Create a new case
@@ -261,7 +261,7 @@ public void Export() throws Exception {
             log.info("Queuing data for processing...");
             Processor processor = nuixCase.createProcessor();
             EvidenceContainer evidenceContainer = processor.newEvidenceContainer("SearchTestData");
-            evidenceContainer.addFile(dataDirectory);
+            evidenceContainer.addFile(textFilesDirectory);
             evidenceContainer.save();
 
             // Periodically log progress
@@ -339,7 +339,7 @@ public void Export() throws Exception {
             // ParallelProcessingConfigurable.setParallelProcessingSettings for list of settings and what they do.
             exporter.setParallelProcessingSettings(Map.of(
                     "workerCount", utilities.getLicence().getWorkers(),
-                    "workerTemp", new File(testOutputDirectory,"WorkerTemp").getAbsolutePath()
+                    "workerTemp", new File(testOutputDirectory, "WorkerTemp").getAbsolutePath()
             ));
 
             // Track error count
@@ -348,6 +348,7 @@ public void Export() throws Exception {
             exporter.whenItemEventOccurs(new ItemEventCallback() {
                 // Use this to track when we reported progress last
                 long lastProgressMillis = System.currentTimeMillis();
+
                 @Override
                 public void itemProcessed(ItemEventInfo info) {
                     // Report progress if it has been at least 5 seconds (5000 milliseconds) since
diff --git a/IntelliJ/src/test/java/CommonTestFunctionality.java b/IntelliJ/src/test/java/CommonTestFunctionality.java
@@ -1,7 +1,5 @@
-import com.esotericsoftware.minlog.Log;
 import com.nuix.enginebaseline.NuixEngine;
 import com.nuix.enginebaseline.NuixLicenseResolver;
-import net.datafaker.Faker;
 import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -12,7 +10,6 @@
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.RuntimeMXBean;
-import java.nio.charset.StandardCharsets;
 import java.util.*;
 
 public class CommonTestFunctionality {
@@ -38,30 +35,16 @@ public TermCount(String term, long count) {
     // place inputs for tests.
     protected static File testDataDirectory;
 
-    // A seeded Random instance so that we get repeatable random results in a few places
-    // where randomness is used, for example while generating test source data.
-    protected static Random rand = new Random(1234567890);
-
-    // Instance of Faker library to assist with fake data generation for tests.  We provide it
-    // seeded Random instance so that when repeatedly running the same test or tests, the results will hopefully
-    // be consistent between runs.
-    protected static Faker faker = new Faker(rand);
-
     // When true, the testOutputDirectory used during tests will be deleted
     // upon test completion.  Set this to false if you wish to manually review the output
     // of tests afterwards.
     protected static boolean deleteTestOutputOnCompletion = true;
 
-
     @BeforeAll
     public static void setup() throws Exception {
+        TestData.init();
         log = LogManager.getLogger("Tests");
-
         testOutputDirectory = new File(System.getenv("TEST_OUTPUT_DIRECTORY"));
-        testDataDirectory = new File(System.getenv("TEST_DATA_DIRECTORY"));
-        log.info("TEST_OUTPUT_DIRECTORY: " + testOutputDirectory.getAbsolutePath());
-        log.info("TEST_DATA_DIRECTORY: " + testDataDirectory.getAbsolutePath());
-
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             if (deleteTestOutputOnCompletion) {
                 try {
@@ -117,69 +100,4 @@ public NuixEngine constructNuixEngine(String... additionalRequiredFeatures) {
                 .setEngineDistributionDirectoryFromEnvVar()
                 .setLogDirectory(new File(testOutputDirectory, "Logs_"+System.currentTimeMillis()));
     }
-
-    /***
-     * Creates a series of random text files at a specified location, returning details about the expected terms
-     * and their counts so a test may later ingest and verify the counts.
-     * @param outputDirectory Where the text files should be written to
-     * @param itemsToGenerate The number of text files desired
-     * @return A List of {@link TermCount} objects
-     */
-    public List<TermCount> createSearchableTestData(File outputDirectory, int itemsToGenerate) {
-        log.info(String.format("Generating %s random text files, for use as test data, to directory %s",
-                itemsToGenerate, outputDirectory));
-        Map<String, TermCount> overallTermCounts = new HashMap<>();
-
-        List<String> termPool = new ArrayList<>();
-
-        // Pre generate pool of 2000 terms
-        for (int i = 0; i < 2000; i++) {
-            String term = faker.text().text(4, 8);
-            termPool.add(term);
-        }
-
-        // Will hold terms to be written to each generated text file
-        Set<String> textFileTerms = new HashSet<>();
-
-        // Iteratively generate test text files
-        for (int i = 0; i < itemsToGenerate; i++) {
-            if (i + 1 % 1000 == 0) {
-                Log.info(String.format("Generated %s fake text files so far...", i + 1));
-            }
-            textFileTerms.clear();
-
-            // Randomly determine how many terms to write
-            int targetTermCount = faker.random().nextInt(5, 10);
-
-            // Determine output file name
-            File textFile = new File(outputDirectory, String.format("%08d.txt", i));
-
-            for (int t = 0; t < targetTermCount; t++) {
-                // Grab term from pool
-                String term = termPool.get(faker.random().nextInt(0, termPool.size() - 1));
-
-                // We will only write each term once per file for simplicity
-                if (textFileTerms.contains(term)) {
-                    continue;
-                } else {
-                    textFileTerms.add(term);
-                }
-
-                // Track this term in overall counts for later verification
-                if (!overallTermCounts.containsKey(term)) {
-                    overallTermCounts.put(term, new TermCount(term, 1));
-                } else {
-                    overallTermCounts.get(term).count++;
-                }
-            }
-
-            try {
-                FileUtils.writeStringToFile(textFile, String.join(" ", textFileTerms), StandardCharsets.UTF_8);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-        return new ArrayList<>(overallTermCounts.values());
-    }
 }
diff --git a/IntelliJ/src/test/java/ExampleUsingText.java b/IntelliJ/src/test/java/ExampleUsingText.java
@@ -6,7 +6,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.Reader;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
@@ -21,9 +20,8 @@ public class ExampleUsingText extends CommonTestFunctionality {
     @Test
     public void ExampleUsingText() throws Exception {
         File caseDirectory = new File(testOutputDirectory, "ExampleUsingText_Case");
-        File dataDirectory = new File(testOutputDirectory, "ExampleUsingText_Natives");
-
-        List<CommonTestFunctionality.TermCount> termCounts = createSearchableTestData(dataDirectory, 5000);
+        File textFilesDirectory = TestData.getTestDataTextFilesDirectory();
+        Map<String,Long> termCounts = TestData.getTestDataTextFileTermCounts();
 
         NuixEngine nuixEngine = constructNuixEngine();
         nuixEngine.run((utilities -> {
@@ -39,7 +37,7 @@ public void ExampleUsingText() throws Exception {
             log.info("Queuing data for processing...");
             Processor processor = nuixCase.createProcessor();
             EvidenceContainer evidenceContainer = processor.newEvidenceContainer("SearchTestData");
-            evidenceContainer.addFile(dataDirectory);
+            evidenceContainer.addFile(textFilesDirectory);
             evidenceContainer.save();
 
             // Periodically log progress
diff --git a/IntelliJ/src/test/java/FakeDataGenerator.java b/IntelliJ/src/test/java/FakeDataGenerator.java
diff --git a/IntelliJ/src/test/java/TestData.java b/IntelliJ/src/test/java/TestData.java

Original file line number	Diff line number	Diff line change
`@@ -41,6 +41,7 @@ dependencies {`
`41`	`41`	`"*/joda*.jar",`
`42`	`42`	`"*/commons*.jar",`
`43`	`43`	`"*/guava*.jar",`
	`44`	`+ "*/gson*.jar",`
`44`	`45`	`// All Nuix API jars`
`45`	`46`	`"*/nuix-.jar"`
`46`	`47`	`)`