Merge pull request #104 from IBM-Cloud/fix-numeric-keys

yumaoka · web-flow · commit 5936b7c2d7ca · 2018-03-12T13:26:36.000-04:00
Fix numeric keys
diff --git a/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/impl/JsonResource.java b/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/impl/JsonResource.java
@@ -27,7 +27,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import com.google.gson.GsonBuilder;
@@ -54,7 +53,7 @@
  */
 public class JsonResource extends ResourceFilter {
 
-    private class KeyPiece {
+    static class KeyPiece {
         String keyValue;
         JsonToken keyType;
 
@@ -248,55 +247,59 @@ public void write(OutputStream outStream, LanguageBundle languageBundle,
         }
     }
 
-    private List<KeyPiece> splitKeyPieces(String key) {
+    static List<KeyPiece> splitKeyPieces(String key) {
         if (USE_JSONPATH_PATTERN.matcher(key).matches()) {
             List<KeyPiece> result = new ArrayList<KeyPiece>();
-            Matcher onlyDigits = Pattern.compile("^\\d+$").matcher("");
+            boolean inQuotes = false;
+            StringBuilder currentToken = new StringBuilder();
             // Disregard $ at the beginning - it's not really part of the key...
-            List<String> tokens = findTokens(key.substring(JSONPATH_ROOT.length()));
-            for (String s : tokens) {
-                if (s.startsWith("'")) {
-                    // Turn any "\u0027" in the key back into '
-                    String modifiedKeyPiece = s.substring(1, s.length() - 1).replaceAll("\\\\u0027", "'");
-                    result.add(new KeyPiece(modifiedKeyPiece, JsonToken.BEGIN_OBJECT));
-                } else if (onlyDigits.reset(s).matches()) {
-                    result.add(new KeyPiece(s, JsonToken.BEGIN_ARRAY));
-                } else {
-                    for (String s2 : s.split("\\.")) {
-                        if (!s2.isEmpty()) {
-                            result.add(new KeyPiece(s2, JsonToken.BEGIN_OBJECT));
+            StringCharacterIterator i = new StringCharacterIterator(key.substring(JSONPATH_ROOT.length()));
+            boolean inSubscript = false;
+            while (i.current() != StringCharacterIterator.DONE) {
+                char c = i.current();
+                if (c == '\'') {
+                    inQuotes = !inQuotes;
+                }
+                if (!inQuotes && (c == '.' || c == '[' || c == ']')) {
+                    if (currentToken.length() > 0) {
+                        addToken(result, currentToken.toString(), inSubscript);
+                        currentToken.setLength(0);
+                        if (inSubscript) {
+                            inSubscript = false;
                         }
                     }
+                    if (c == '[') {
+                        inSubscript = true; // Record that the next token had an
+                                        // array subscript on it.
+                    }
+                } else {
+                    currentToken.append(c);
                 }
+                i.next();
             }
+            addToken(result, currentToken.toString(), inSubscript);
+
             return Collections.unmodifiableList(result);
         }
         // Otherwise, this is a plain JSON object label
         return Collections.singletonList(new KeyPiece(key, JsonToken.BEGIN_OBJECT));
     }
 
-    private static List<String> findTokens(String data) {
-        List<String> tokens = new ArrayList<String>();
-        boolean inQuotes = false;
-        StringBuilder currentToken = new StringBuilder();
-        StringCharacterIterator i = new StringCharacterIterator(data);
-        while (i.current() != StringCharacterIterator.DONE) {
-            char c = i.current();
-            if (c == '\'') {
-                inQuotes = !inQuotes;
-            }
-            if (!inQuotes && (c == '.' || c == '[' || c == ']')) {
-                if (currentToken.length() > 0) {
-                    tokens.add(currentToken.toString());
-                    currentToken.setLength(0);
+    static void addToken(List<KeyPiece> result, String s, boolean inSubscript) {
+        if (s.startsWith("'")) {
+            // Turn any "\u0027" in the key back into '
+            String modifiedKeyPiece = s.substring(1, s.length() - 1).replaceAll("\\\\u0027", "'");
+            result.add(new KeyPiece(modifiedKeyPiece, JsonToken.BEGIN_OBJECT));
+        } else if (inSubscript) {
+            // [0] produces an array
+            result.add(new KeyPiece(s, JsonToken.BEGIN_ARRAY));
+        } else {
+            for (String s2 : s.split("\\.")) {
+                if (!s2.isEmpty()) {
+                    result.add(new KeyPiece(s2, JsonToken.BEGIN_OBJECT));
                 }
-            } else {
-                currentToken.append(c);
             }
-            i.next();
         }
-        tokens.add(currentToken.toString());
-        return Collections.unmodifiableList(tokens);
     }
 
     // TODO: Implement merge method
diff --git a/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/impl/JsonResourceTest.java b/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/impl/JsonResourceTest.java
@@ -15,11 +15,13 @@
  */
 package com.ibm.g11n.pipeline.resfilter.impl;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -29,21 +31,28 @@
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map.Entry;
 
 import org.junit.Test;
 
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
 import com.ibm.g11n.pipeline.resfilter.LanguageBundle;
 import com.ibm.g11n.pipeline.resfilter.LanguageBundleBuilder;
 import com.ibm.g11n.pipeline.resfilter.ResourceFilterException;
 import com.ibm.g11n.pipeline.resfilter.ResourceString;
 import com.ibm.g11n.pipeline.resfilter.ResourceString.ResourceStringComparator;
+import com.ibm.g11n.pipeline.resfilter.impl.JsonResource.KeyPiece;
 
 /**
  * @author farhan
  *
  */
 public class JsonResourceTest {
     private static final File INPUT_FILE = new File("src/test/resource/resfilter/json/input.json");
+    private static final File INPUT_FILE2 = new File("src/test/resource/resfilter/json/other-input.json");
+    private static final File SPLITKEYS = new File("src/test/resource/resfilter/json/testSplitKeys.json");
 
     private static final File EXPECTED_WRITE_FILE = new File("src/test/resource/resfilter/json/write-output.json");
 
@@ -62,8 +71,10 @@ public class JsonResourceTest {
         lst.add(ResourceString.with("$.countries[1].Asia[1]", "Japan").sequenceNumber(9).build());
         lst.add(ResourceString.with("$.countries[1].Asia[2]", "India").sequenceNumber(10).build());
         lst.add(ResourceString.with("$.countries[2].Americas['S. America'][0]", "Brazil").sequenceNumber(11).build());
-        lst.add(ResourceString.with("$.countries[2].Americas['S. America'][1]", "Venezuela").sequenceNumber(12).build());
-        lst.add(ResourceString.with("$.countries[2].Americas['N. America'][0]", "United States [USA]").sequenceNumber(13).build());
+        lst.add(ResourceString.with("$.countries[2].Americas['S. America'][1]", "Venezuela").sequenceNumber(12)
+                .build());
+        lst.add(ResourceString.with("$.countries[2].Americas['N. America'][0]", "United States [USA]")
+                .sequenceNumber(13).build());
         lst.add(ResourceString.with("$.countries[2].Americas['N. America'][1]", "Canada").sequenceNumber(14).build());
         lst.add(ResourceString.with("$.countries[2].Americas['N. America'][2]", "Mexico").sequenceNumber(15).build());
         lst.add(ResourceString.with("$.countries[3].Africa[0]", "Egypt").sequenceNumber(16).build());
@@ -77,10 +88,14 @@ public class JsonResourceTest {
         lst.add(ResourceString.with("another.text", "Another plain old string").sequenceNumber(24).build());
         lst.add(ResourceString.with("frog['2']", "Red-eyed Tree Frog").sequenceNumber(25).build());
         lst.add(ResourceString.with("owl[3]", "Great Horned Owl").sequenceNumber(26).build());
-        lst.add(ResourceString.with("$['$.xxx']", "Looks like JSONPATH, but actually plain old string").sequenceNumber(27).build());
-        lst.add(ResourceString.with("$['$.']", "Looks like JSONPATH prefix, but actually plain old string").sequenceNumber(28).build());
-        lst.add(ResourceString.with("$abc", "Starts with JSONPATH root char, but just a string").sequenceNumber(29).build());
-        lst.add(ResourceString.with("$['ibm.com']['g11n.pipeline.title']", "Globalization Pipeline").sequenceNumber(30).build());
+        lst.add(ResourceString.with("$['$.xxx']", "Looks like JSONPATH, but actually plain old string")
+                .sequenceNumber(27).build());
+        lst.add(ResourceString.with("$['$.']", "Looks like JSONPATH prefix, but actually plain old string")
+                .sequenceNumber(28).build());
+        lst.add(ResourceString.with("$abc", "Starts with JSONPATH root char, but just a string").sequenceNumber(29)
+                .build());
+        lst.add(ResourceString.with("$['ibm.com']['g11n.pipeline.title']", "Globalization Pipeline").sequenceNumber(30)
+                .build());
 
         Collections.sort(lst, new ResourceStringComparator());
         EXPECTED_INPUT_RES_LIST = lst;
@@ -117,7 +132,8 @@ public class JsonResourceTest {
         bundleBuilder.addResourceString("frog['2']", "Red-eyed Tree Frog - XL", 25);
         bundleBuilder.addResourceString("owl[3]", "Great Horned Owl - XL", 26);
         bundleBuilder.addResourceString("$['$.xxx']", "Looks like JSONPATH, but actually plain old string - XL", 27);
-        bundleBuilder.addResourceString("$['$.']", "Looks like JSONPATH prefix, but actually plain old string - XL", 28);
+        bundleBuilder.addResourceString("$['$.']", "Looks like JSONPATH prefix, but actually plain old string - XL",
+                28);
         bundleBuilder.addResourceString("$abc", "Starts with JSONPATH root char, but just a string - XL", 29);
         bundleBuilder.addResourceString("$['ibm.com']['g11n.pipeline.title']", "Globalization Pipeline - XL", 30);
         WRITE_BUNDLE = bundleBuilder.build();
@@ -133,7 +149,8 @@ public void testParse() throws IOException, ResourceFilterException {
             LanguageBundle bundle = res.parse(is, null);
             List<ResourceString> resStrList = new ArrayList<>(bundle.getResourceStrings());
             Collections.sort(resStrList, new ResourceStringComparator());
-            assertEquals("ResourceStrings did not match.", EXPECTED_INPUT_RES_LIST, resStrList);
+            assertArrayEquals("ResourceStrings did not match.", EXPECTED_INPUT_RES_LIST.toArray(),
+                    resStrList.toArray());
         }
     }
 
@@ -145,12 +162,108 @@ public void testWrite() throws IOException, ResourceFilterException {
         try (OutputStream os = new FileOutputStream(tempFile)) {
             res.write(os, WRITE_BUNDLE, null);
             os.flush();
-            assertTrue(ResourceTestUtil.compareFiles(EXPECTED_WRITE_FILE, tempFile));
+            ResourceTestUtil.compareFilesJson(EXPECTED_WRITE_FILE, tempFile);
+        }
+    }
+
+    // @Test
+    // public void testTestFiles() throws IOException, ResourceFilterException {
+    // // just test the test files
+    // ResourceTestUtil.compareFilesJson(INPUT_FILE, EXPECTED_WRITE_FILE);
+    // }
+
+    @Test
+    public void testReWrite() throws IOException, ResourceFilterException {
+        // First parse
+        assertTrue("The input test file <" + INPUT_FILE + "> does not exist.", INPUT_FILE.exists());
+
+        try (InputStream is = new FileInputStream(INPUT_FILE)) {
+            JsonResource res2 = new JsonResource();
+            LanguageBundle bundle = res2.parse(is, null);
+            List<ResourceString> resStrList = new ArrayList<>(bundle.getResourceStrings());
+            Collections.sort(resStrList, new ResourceStringComparator());
+            assertArrayEquals("ResourceStrings did not match.", EXPECTED_INPUT_RES_LIST.toArray(),
+                    resStrList.toArray());
+
+            // Now write
+            File tempFile = File.createTempFile(this.getClass().getSimpleName(), "2.json");
+            // File tempFile = new File("/tmp/2.json");
+            tempFile.deleteOnExit();
+
+            try (OutputStream os = new FileOutputStream(tempFile)) {
+                res.write(os, bundle, null);
+                os.flush();
+                ResourceTestUtil.compareFilesJson(INPUT_FILE, tempFile);
+            }
+        }
+    }
+
+    @Test
+    public void testReWriteOther() throws IOException, ResourceFilterException {
+        // First parse
+        assertTrue("The input test file <" + INPUT_FILE + "> does not exist.", INPUT_FILE2.exists());
+
+        try (InputStream is = new FileInputStream(INPUT_FILE2)) {
+            JsonResource res2 = new JsonResource();
+            LanguageBundle bundle = res2.parse(is, null);
+            List<ResourceString> resStrList = new ArrayList<>(bundle.getResourceStrings());
+            Collections.sort(resStrList, new ResourceStringComparator());
+            // assertEquals("ResourceStrings did not match.",
+            // EXPECTED_INPUT_RES_LIST, resStrList);
+
+            // Now write
+            File tempFile = File.createTempFile(this.getClass().getSimpleName(), "3.json");
+            // File tempFile = new File("/tmp/3.json");
+            tempFile.deleteOnExit();
+
+            try (OutputStream os = new FileOutputStream(tempFile)) {
+                res.write(os, bundle, null);
+                os.flush();
+                System.out.println(ResourceTestUtil.fileToString(tempFile));
+                ResourceTestUtil.compareFilesJson(INPUT_FILE2, tempFile);
+            }
+        }
+    }
+    
+    /**
+     * @throws IOException
+     * @throws FileNotFoundException
+     * @See {@link JsonResource#splitKeyPieces(String)}
+     */
+    @Test
+    public void testSplitKeyPieces() throws FileNotFoundException, IOException {
+        /*
+         * Initial data generated with: Gson g = new GsonBuilder().create();
+         * TreeMap<String, List<KeyPiece>> tm = new TreeMap<String,
+         * List<KeyPiece>>();
+         * 
+         * for (final ResourceString s : EXPECTED_INPUT_RES_LIST) {
+         * tm.put(s.getKey(), JsonResource.splitKeyPieces(s.getKey())); }
+         * System.out.println(g.toJson(tm));
+         */
+
+        JsonObject SPLITKEY_DATA = ResourceTestUtil.parseJson(SPLITKEYS).getAsJsonObject();
+        for (final Entry<String, JsonElement> e : SPLITKEY_DATA.entrySet()) {
+            final String key = e.getKey();
+            final JsonArray expectList = e.getValue().getAsJsonArray();
+
+            List<KeyPiece> actualList = JsonResource.splitKeyPieces(key);
+
+            String prefix = "‘" + key + "’: ";
+            assertEquals(prefix + "key count", expectList.size(), actualList.size());
+            for (int n = 0; n < expectList.size(); n++) {
+                JsonObject expectObject = expectList.get(n).getAsJsonObject();
+                KeyPiece actualObject = actualList.get(n);
+                String subPrefix = prefix + " key " + n;
+                assertEquals(subPrefix + " value ", expectObject.get("keyValue").getAsString(), actualObject.keyValue);
+                assertEquals(subPrefix + " type ", expectObject.get("keyType").getAsString(),
+                        actualObject.keyType.name());
+            }
         }
     }
 
     // TODO: Not ready yet
-//    @Test
-//    public void testMerge() throws IOException, ResourceFilterException {
-//    }
+    // @Test
+    // public void testMerge() throws IOException, ResourceFilterException {
+    // }
 }
diff --git a/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/impl/ResourceTestUtil.java b/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/impl/ResourceTestUtil.java
@@ -16,16 +16,21 @@
 
 package com.ibm.g11n.pipeline.resfilter.impl;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.Reader;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParser;
+
 /**
  * @author farhan, JCEmmons
  *
@@ -76,11 +81,13 @@ public static boolean compareFiles(File expected, File actual, int n) throws Fil
 
         return true;
     }
+
     /**
      * Returns true if the two files match exactly up to the number of lines
-     * specified in n 
+     * specified in n
      */
-    public static boolean compareFilesUpTo(File expected, File actual, int n) throws FileNotFoundException, IOException {
+    public static boolean compareFilesUpTo(File expected, File actual, int n)
+            throws FileNotFoundException, IOException {
         try (BufferedReader expectedRdr = new BufferedReader(new FileReader(expected));
                 BufferedReader actualRdr = new BufferedReader(new FileReader(actual))) {
 
@@ -124,4 +131,26 @@ public static String fileToString(File file) throws FileNotFoundException, IOExc
         byte[] encoded = Files.readAllBytes(Paths.get(file.getPath()));
         return new String(encoded, "UTF-8");
     }
+
+    /**
+     * @param expectedWriteFile
+     * @param tempFile
+     * @return
+     * @throws IOException
+     * @throws FileNotFoundException
+     */
+    public static void compareFilesJson(File expectedWriteFile, File tempFile)
+            throws FileNotFoundException, IOException {
+        JsonElement expected = parseJson(expectedWriteFile);
+        JsonElement actual = parseJson(tempFile);
+        assertEquals("JSON mismatch: " + tempFile.getName() + " did not match " + expectedWriteFile.getName(), expected,
+                actual);
+    }
+
+    public static JsonElement parseJson(final File f) throws FileNotFoundException, IOException {
+        try (final Reader reader = new FileReader(f)) {
+            JsonElement parse = new JsonParser().parse(reader);
+            return parse;
+        }
+    }
 }
diff --git a/gp-res-filter/src/test/resource/resfilter/json/other-input.json b/gp-res-filter/src/test/resource/resfilter/json/other-input.json
@@ -0,0 +1,6 @@
+{
+	"hello": {
+		"1": "Firstly",
+		"two": "Secondly"
+	}
+}
diff --git a/gp-res-filter/src/test/resource/resfilter/json/testSplitKeys.json b/gp-res-filter/src/test/resource/resfilter/json/testSplitKeys.json
