Handles backslashes in java props comments

Properly handles backslashes in Java properties comments - only
unescaping backslash ‘u’ encoded Unicode code points, but not decoding
`\t` `\r` `\n` `\\` and other ASCII escapes.

Author: yoshito-umaoka

gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/impl/JavaPropertiesResource.java

@@ -130,9 +130,17 @@ public LanguageBundle parse(InputStream inStream, FilterOptions options)
             // either
             // a blank line (global comment) or a key/value pair
             if (line.startsWith("#") || line.startsWith("!")) {
-                // Strip off the leading comment marker, and perform any
-                // necessary unescaping here.
-                currentNotes.add(unescape(line.substring(1)));
+                // Strip off the leading comment marker, and decode Unicode escape
+                // if necessary.
+                if (enc == Encoding.UTF_8) {
+                    // Do not unescape unicode - because if backslash u encoding is used
+                    // in comment, it is on purpose and better not to decode it.
+                    currentNotes.add(line.substring(1));
+                } else {
+                    // Unescape unicode - if not UTF-8 props, backslash u encoding must be
+                    // used always.
+                    currentNotes.add(unescapeOnlyUnicode(line.substring(1)));
+                }
             } else if (line.isEmpty()) {
                 // We are following the convention that the first blank line in
                 // a properties
@@ -206,7 +214,13 @@ public void write(OutputStream outStream, LanguageBundle languageBundle,
 
         PrintWriter pw = new PrintWriter(new OutputStreamWriter(outStream, getCharset()));
         for (String note : languageBundle.getNotes()) {
-            pw.println("#"+note);
+            if (enc == Encoding.UTF_8) {
+                // No needs to escape raw Unicode charters
+                pw.println("#" + note);
+            } else {
+                // Needs to escape Unicode characters
+                pw.println("#" + escapeOnlyUnicode(note));
+            }
         }
         if (!languageBundle.getNotes().isEmpty()) {
             pw.println();
@@ -641,7 +655,66 @@ private static String unescape(String str) {
 
         return buf.toString();
     }
-    
+
+    /**
+     * Escape non-ASCII code points to backslash u encoded UTF-16 code points. This method
+     * does not escape any other control characters and compatible with native2ascii.
+     * 
+     * @param str An input string
+     * @return A string without raw non-ASCII code points.
+     */
+    private static String escapeOnlyUnicode(String str) {
+        final StringBuilder buf = new StringBuilder();
+        for (int i = 0; i < str.length(); i++) {
+            final char c = str.charAt(i);
+            if (c > 0x7F) {
+                appendUnicodeEscape(buf, c);
+            } else {
+                buf.append(c);
+            }
+        }
+        return buf.toString();
+    }
+
+    /**
+     * Unescape backslash u encoded UTF-16 code points. This method does not
+     * unescape any other backslash escape sequence and compatible with native2ascii with
+     * -reverse option.
+     * 
+     * @param str An input string
+     * @return A string without backslash u encoded code points.
+     */
+    private static String unescapeOnlyUnicode(String str) {
+        final StringBuilder buf = new StringBuilder();
+        for (int i = 0; i < str.length(); i++) {
+            final char c = str.charAt(i);
+            if (c == BACKSLASH && i + 5 >= str.length()) {
+                boolean isUniEsc = false;
+                i++;
+                final char c1 = str.charAt(i);
+                if (c1 == 'u') {
+                    final String hstr = str.substring(i + 1, i + 5);
+                    try {
+                        final char codeUnit = (char)Integer.parseInt(hstr, 16);
+                        buf.append(Character.valueOf(codeUnit));
+                        i += 4;
+                        isUniEsc = true;
+                    } catch (NumberFormatException e) {
+                        // Ignore malformed pattern and just emit the sequence
+                        // starting backslash 'u'
+                    }
+                }
+                if (!isUniEsc) {
+                    // emit a backslash and following character together
+                    buf.append(c).append(c1);
+                }
+            } else {
+                buf.append(c);
+            }
+        }
+        return buf.toString();
+    }
+
     /***
      * For MessageFormat with number args, convert double single quote back to single quote during import
      * @param inputStr  The message pattern string with single quotes escaped

gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/impl/JavaPropertiesResourceTest.java

@@ -99,6 +99,9 @@ public class JavaPropertiesResourceTest {
         lst.add(ResourceString.with("non-param", "This {} is not a parameter.")
                 .sequenceNumber(11).notes(Arrays.asList(" Not a Java MessageFormat param")).build());
 
+        lst.add(ResourceString.with("backslashes", "a\\b\\c")
+                .sequenceNumber(12).notes(Arrays.asList(" A comment with backslashes - a\\b\\c")).build());
+
         Collections.sort(lst, new ResourceStringComparator());
         EXPECTED_INPUT_RES_LIST = lst;
     }
@@ -129,6 +132,8 @@ public class JavaPropertiesResourceTest {
                 Arrays.asList(" Quote"));
         bundleBuilder.addResourceString("non-param", "This {} is not a parameter.", 11,
                 Arrays.asList(" Not a Java MessageFormat param"));
+        bundleBuilder.addResourceString("backslashes", "a\\b\\c", 12,
+                Arrays.asList(" A comment with backslashes - a\\b\\c"));
         bundleBuilder.addNotes(Arrays.asList(
                 " You are reading the \".properties\" entry.",
                 " The exclamation mark can also mark text as comments.",
@@ -154,6 +159,7 @@ public class JavaPropertiesResourceTest {
         // PropDef does not detect message pattern - message pattern handling is done by the logic in JavaPropertiesResource class
         EXPECTED_PROP_DEF_LIST.add(new PropDef("withQuote", "You''re about to delete '{1}' rows in Mike''s file {0}.", PropSeparator.EQUAL));
         EXPECTED_PROP_DEF_LIST.add(new PropDef("non-param", "This {} is not a parameter.", PropSeparator.EQUAL));
+        EXPECTED_PROP_DEF_LIST.add(new PropDef("backslashes", "a\\b\\c", PropSeparator.EQUAL));
     }
 
     private static final JavaPropertiesResource res = new JavaPropertiesResource();

gp-res-filter/src/test/resource/resfilter/properties/input.properties

@@ -24,4 +24,6 @@ withTabs = Tab1\tTab2\tTab3\t
 # Quote
 withQuote = You''re about to delete '{0}' rows in Mike''s file {0}.
 # Not a Java MessageFormat param
-non-param = This {} is not a parameter.
+non-param = This {} is not a parameter.
+# A comment with backslashes - a\b\c
+backslashes = a\\b\\c

gp-res-filter/src/test/resource/resfilter/properties/merge-output.properties

@@ -24,4 +24,6 @@ withTabs = localized Tab1\tTab2\tTab3\t
 # Quote
 withQuote = You''re about to delete '{1}' rows in Mike''s file {0}.
 # Not a Java MessageFormat param
-non-param = This {} is not a parameter.
+non-param = This {} is not a parameter.
+# A comment with backslashes - a\b\c
+backslashes = a\\b\\c

gp-res-filter/src/test/resource/resfilter/properties/parse-test-input.properties

@@ -8,4 +8,5 @@ tab : pick up the\u00A5 tab
 trailSPs = trailing SPs  
 withTabs = Tab1\tTab2\tTab3\t
 withQuote = You''re about to delete '{1}' rows in Mike''s file {0}.
-non-param = This {} is not a parameter.
+non-param = This {} is not a parameter.
+backslashes = a\\b\\c

gp-res-filter/src/test/resource/resfilter/properties/parseline-test-input.properties

@@ -8,4 +8,5 @@ leadTabs = leading tabs
 trailSPs = trailing SPs  
 withTabs = Tab1\tTab2\tTab3\t
 withQuote = You''re about to delete '{1}' rows in Mike''s file {0}.
-non-param = This {} is not a parameter.
+non-param = This {} is not a parameter.
+backslashes = a\\b\\c

gp-res-filter/src/test/resource/resfilter/properties/write-output.properties

@@ -25,4 +25,6 @@ withTabs = localized Tab1\tTab2\tTab3\t
 # Quote
 withQuote = You''re about to delete '{1}' rows in Mike''s file {0}.
 # Not a Java MessageFormat param
-non-param = This {} is not a parameter.
+non-param = This {} is not a parameter.
+# A comment with backslashes - a\b\c
+backslashes = a\\b\\c