Merge branch 'fix-unicode-replacements'

Author: bruderstein

PythonScript.Tests/tests/TestReplacer.cpp

@@ -106,5 +106,20 @@ TEST_F(ReplacerTest, ReplaceSimpleAnsi) {
     for_each(entries.begin(), entries.end(), deleteEntry);
 }
 
+/** Tests UTF8 replacement with a UTF-8 string replacement
+ */
+TEST_F(ReplacerTest, ReplaceUtf8ReplacementString) {
 
+    NppPythonScript::Replacer<NppPythonScript::Utf8CharTraits> replacer;
+    std::list<NppPythonScript::ReplaceEntry* > entries;
+    bool moreEntries = replacer.startReplace("aaa\xC3\xB4" "ZZZ" , 8, 0, 0, "aaa\xC3\xB4", "x" "\xC3\xB5" "z", NppPythonScript::python_re_flag_normal,entries);
+    ASSERT_EQ(1, entries.size());
+    std::list<NppPythonScript::ReplaceEntry*>::const_iterator it = entries.begin();
+    ASSERT_EQ(0, (*it)->getStart());
+    ASSERT_STREQ("x" "\xC3\xB5" "z", (*it)->getReplacement());
+    ASSERT_EQ(4, (*it)->getReplacementLength());
+
+
+    for_each(entries.begin(), entries.end(), deleteEntry);
+}
 }

PythonScript/python_tests/tests/ReplaceAnsiTestCase.py

@@ -3,28 +3,33 @@
 from Npp import *
 
 class ReplaceAnsiTestCase(unittest.TestCase):
-	def setUp(self):
-		notepad.new()
-		notepad.runMenuCommand("Encoding", "Encode in ANSI")
-		editor.write(u'Here is some text\r\nWith some umlauts XäXüXö\r\n'.encode('windows-1252'));
-		
-	def tearDown(self):
-		editor.setSavePoint()
-		notepad.close()
-		
-	def test_simple_replace(self):
-		editor.rereplace(r'some\s([a-z]+)', 'TEST');
-		text = editor.getText()
-		self.assertEqual(text, u'Here is TEST\r\nWith TEST XäXüXö\r\n'.encode('windows-1252'));
-		
-	def test_ansi_replace(self):
-		editor.rereplace(u'X[äö]'.encode('windows-1252'), 'YY');
-		text = editor.getText()
-		self.assertEqual(text, u'Here is some text\r\nWith some umlauts YYXüYY\r\n'.encode('windows-1252'));
+    def setUp(self):
+        notepad.new()
+        notepad.runMenuCommand("Encoding", "Encode in ANSI")
+        editor.write(u'Here is some text\r\nWith some umlauts XäXüXö\r\n'.encode('windows-1252'));
+        
+    def tearDown(self):
+        editor.setSavePoint()
+        notepad.close()
+        
+    def test_simple_replace(self):
+        editor.rereplace(r'some\s([a-z]+)', 'TEST');
+        text = editor.getText()
+        self.assertEqual(text, u'Here is TEST\r\nWith TEST XäXüXö\r\n'.encode('windows-1252'));
+        
+    def test_ansi_replace(self):
+        editor.rereplace(u'X[äö]'.encode('windows-1252'), 'YY');
+        text = editor.getText()
+        self.assertEqual(text, u'Here is some text\r\nWith some umlauts YYXüYY\r\n'.encode('windows-1252'));
 
-	def test_unicode_replace(self):
-		editor.rereplace(u'X[äö]', 'PP');
-		text = editor.getText()
-		self.assertEqual(text, u'Here is some text\r\nWith some umlauts PPXüPP\r\n'.encode('windows-1252'));	
+    def test_unicode_replace(self):
+        editor.rereplace(u'X[äö]', 'PP');
+        text = editor.getText()
+        self.assertEqual(text, u'Here is some text\r\nWith some umlauts PPXüPP\r\n'.encode('windows-1252'));	
+
+    def test_replace_with_unicode(self):
+        editor.rereplace('Here|With', u'XäöüY')
+        text = editor.getText()
+        self.assertEqual(text, u'XäöüY is some text\r\nXäöüY some umlauts XäXüXö\r\n'.encode('windows-1252'))
 
 suite = unittest.TestLoader().loadTestsFromTestCase(ReplaceAnsiTestCase)

PythonScript/python_tests/tests/ReplaceUTF8TestCase.py

@@ -26,4 +26,16 @@ def test_replace_condition(self):
         editor.rereplace('(Here)|(Xä)', '(?1Cheese)(?2Y)')
         text = editor.getText()
         self.assertEqual(text, 'Cheese is some text\r\nWith some umlauts YXüXö\r\n')
+
+
+    def test_replace_with_unicode(self):
+        editor.rereplace('Here|With', u'XäöüY')
+        text = editor.getText()
+        self.assertEqual(text, 'XäöüY is some text\r\nXäöüY some umlauts XäXüXö\r\n')
+
+    def test_replace_with_unicode_using_standard_string(self):
+        editor.rereplace('Here|With', 'XäöüY')  # note the standard replacement string here
+        text = editor.getText()
+        self.assertEqual(text, 'XäöüY is some text\r\nXäöüY some umlauts XäXüXö\r\n')
+
 suite = unittest.TestLoader().loadTestsFromTestCase(ReplaceUTF8TestCase)

PythonScript/src/Replacer.h

@@ -138,7 +138,8 @@ int BoostRegexMatch<CharTraitsT>::groupIndexFromName(const char *groupName)
 template <class CharTraitsT>
 void BoostRegexMatch<CharTraitsT>::expand(const char *format, char **result, int *resultLength)
 {
-    CharTraitsT::string_type resultString = m_match->format(format, boost::regex_constants::format_all);
+    CharTraitsT::string_type formatString = CharTraitsT::fromChars(format);
+    CharTraitsT::string_type resultString = m_match->format(formatString, boost::regex_constants::format_all);
 
     std::string charResult(CharTraitsT::toCharString(resultString));
 
@@ -166,6 +167,10 @@ typename std::string BoostRegexMatch<CharTraitsT>::getTextForGroup(GroupDetail*
             // We just want a u32string to utf8 char*
 		    return std::basic_string<char>(UtfConversion::toUtf8(ConstString<U32>(source)));
 		}
+
+        static string_type fromChars(const char *source) {
+            return string_type(UtfConversion::toUtf32(ConstString<U8>(source)));
+		}
 	};
 
     class AnsiCharTraits {
@@ -178,6 +183,10 @@ typename std::string BoostRegexMatch<CharTraitsT>::getTextForGroup(GroupDetail*
         static std::basic_string<char> toCharString(const string_type& source) { 
 		    return source;
 		}
+
+        static string_type fromChars(const char *source) {
+            return string_type(source);
+		}
 	};
 
     template <class CharTraitsT>