Merge branch 'upstream-candidate-1.9.4' into upstream-master

Author: dcamper

Bundle.ecl

@@ -6,5 +6,5 @@ EXPORT Bundle := MODULE(Std.BundleBase)
     EXPORT License := 'http://www.apache.org/licenses/LICENSE-2.0';
     EXPORT Copyright := 'Copyright (C) 2024 HPCC Systems';
     EXPORT DependsOn := [];
-    EXPORT Version := '1.9.3';
+    EXPORT Version := '1.9.4';
 END;

Profile.ecl

@@ -529,7 +529,9 @@ EXPORT Profile(inFile,
 
         // Determine if a UTF-8 string really contains UTF-8 characters
         #UNIQUENAME(IsUTF8);
-        LOCAL BOOLEAN %IsUTF8%(UTF8 str) := EMBED(C++)
+        LOCAL BOOLEAN %IsUTF8%(DATA str) := EMBED(C++)
+            #option pure;
+
             if (lenStr == 0)
                 return false;
 
@@ -543,37 +545,37 @@ EXPORT Profile(inFile,
                     // ASCII; continue scan
                     bytes += 1;
                 }
-                else if ((0xC2 <= bytes[0] && bytes[0] <= 0xDF) && (0x80 <= bytes[1] && bytes[1] <= 0xBF))
+                else if ((0xC2 <= bytes[0] && bytes[0] <= 0xDF) && (bytes+1 < endPtr) && (0x80 <= bytes[1] && bytes[1] <= 0xBF))
                 {
                     // Valid non-overlong 2-byte
                     return true;
                 }
-                else if (bytes[0] == 0xE0 && (0xA0 <= bytes[1] && bytes[1] <= 0xBF) && (0x80 <= bytes[2] && bytes[2] <= 0xBF))
+                else if (bytes[0] == 0xE0 && (bytes+2 < endPtr) && (0xA0 <= bytes[1] && bytes[1] <= 0xBF) && (0x80 <= bytes[2] && bytes[2] <= 0xBF))
                 {
                     // Valid excluding overlongs
                     return true;
                 }
-                else if (((0xE1 <= bytes[0] && bytes[0] <= 0xEC) || bytes[0] == 0xEE || bytes[0] == 0xEF) && (0x80 <= bytes[1] && bytes[1] <= 0xBF) && (0x80 <= bytes[2] && bytes[2] <= 0xBF))
+                else if (((0xE1 <= bytes[0] && bytes[0] <= 0xEC) || bytes[0] == 0xEE || bytes[0] == 0xEF) && (bytes+2 < endPtr) && (0x80 <= bytes[1] && bytes[1] <= 0xBF) && (0x80 <= bytes[2] && bytes[2] <= 0xBF))
                 {
                     // Valid straight 3-byte
                     return true;
                 }
-                else if (bytes[0] == 0xED && (0x80 <= bytes[1] && bytes[1] <= 0x9F) && (0x80 <= bytes[2] && bytes[2] <= 0xBF))
+                else if (bytes[0] == 0xED && (bytes+2 < endPtr) && (0x80 <= bytes[1] && bytes[1] <= 0x9F) && (0x80 <= bytes[2] && bytes[2] <= 0xBF))
                 {
                     // Valid excluding surrogates
                     return true;
                 }
-                else if (bytes[0] == 0xF0 && (0x90 <= bytes[1] && bytes[1] <= 0xBF) && (0x80 <= bytes[2] && bytes[2] <= 0xBF) && (0x80 <= bytes[3] && bytes[3] <= 0xBF))
+                else if (bytes[0] == 0xF0 && (bytes+3 < endPtr) && (0x90 <= bytes[1] && bytes[1] <= 0xBF) && (0x80 <= bytes[2] && bytes[2] <= 0xBF) && (0x80 <= bytes[3] && bytes[3] <= 0xBF))
                 {
                     // Valid planes 1-3
                     return true;
                 }
-                else if ((0xF1 <= bytes[0] && bytes[0] <= 0xF3) && (0x80 <= bytes[1] && bytes[1] <= 0xBF) && (0x80 <= bytes[2] && bytes[2] <= 0xBF) && (0x80 <= bytes[3] && bytes[3] <= 0xBF))
+                else if ((0xF1 <= bytes[0] && bytes[0] <= 0xF3) && (bytes+3 < endPtr) && (0x80 <= bytes[1] && bytes[1] <= 0xBF) && (0x80 <= bytes[2] && bytes[2] <= 0xBF) && (0x80 <= bytes[3] && bytes[3] <= 0xBF))
                 {
                     // Valid planes 4-15
                     return true;
                 }
-                else if (bytes[0] == 0xF4 && (0x80 <= bytes[1] && bytes[1] <= 0x8F) && (0x80 <= bytes[2] && bytes[2] <= 0xBF) && (0x80 <= bytes[3] && bytes[3] <= 0xBF))
+                else if (bytes[0] == 0xF4 && (bytes+3 < endPtr) && (0x80 <= bytes[1] && bytes[1] <= 0x8F) && (0x80 <= bytes[2] && bytes[2] <= 0xBF) && (0x80 <= bytes[3] && bytes[3] <= 0xBF))
                 {
                     // Valid plane 16
                     return true;
@@ -613,9 +615,9 @@ EXPORT Profile(inFile,
         // Pattern mapping a UNICODE datatype; using regex due to the complexity
         // of the character set
         #UNIQUENAME(_MapUpperCharUni);
-        LOCAL %_MapUpperCharUni%(UNICODE s) := REGEXREPLACE(u'\\p{Uppercase_Letter}', s, u'A');
+        LOCAL %_MapUpperCharUni%(UNICODE s) := REGEXREPLACE(u'\\p{Lu}', s, u'A');
         #UNIQUENAME(_MapLowerCharUni);
-        LOCAL %_MapLowerCharUni%(UNICODE s) := REGEXREPLACE(u'[[\\p{Lowercase_Letter}][\\p{Titlecase_Letter}][\\p{Modifier_Letter}][\\p{Other_Letter}]]', s, u'a');
+        LOCAL %_MapLowerCharUni%(UNICODE s) := REGEXREPLACE(u'[[\\p{Ll}][\\p{Lt}][\\p{Lm}][\\p{Lo}]]', s, u'a');
         #UNIQUENAME(_MapDigitUni);
         LOCAL %_MapDigitUni%(UNICODE s) := REGEXREPLACE(u'[1-9]', s, u'9'); // Leave '0' as-is and replace with '9' later
         #UNIQUENAME(_MapAllUni);
@@ -824,7 +826,7 @@ EXPORT Profile(inFile,
                                                                             #IF(%_IsSetType%(%'@type'%))
                                                                                 FALSE
                                                                             #ELSEIF(REGEXFIND('(unicode)|(utf)', %'@type'%))
-                                                                                %IsUTF8%((UTF8)_inFile.#EXPAND(%'namePrefix'% + %'@name'%))
+                                                                                %IsUTF8%((DATA)_inFile.#EXPAND(%'namePrefix'% + %'@name'%))
                                                                             #ELSE
                                                                                 FALSE
                                                                             #END

README.md

@@ -21,18 +21,19 @@ research tools to an ECL programmer.
 <a name="installation"></a>
 ### Installation
 
-**Note:**  `DataPatterns.Profile()` and `DataPatterns.BestRecordStructure()` are
-now included in HPCC version 7.4.0!  They have been added to the ECL Standard
-Library (within `Std.DataPatterns`) and also integrated with ECL Watch so you can
-create a profile from a saved logical file using only a web browser.  Note that
-the Std library version of Profile() will create a visualization of the results
-only when executed from ECL Watch; visualizations will not be generated if
-Profile() is called from ECL code.  If that is important to you, install this
-bundle version instead (they coexist peacefully).
+**Note:**  `Profile()`, `BestRecordStructure()` and `Benford()` are
+now included in the HPCC Systems platform!  They have been added to the ECL
+Standard Library (within `Std.DataPatterns`) and `Profile()` has also been
+integrated within ECL Watch so you can create a profile from a saved logical file
+using only a web browser.  Note that the Std library version of `Profile()` will
+create a visualization of the results only when executed from ECL Watch;
+visualizations will not be generated if `Std.DataPatterns.Profile()` is
+called from ECL code.  If that is important to you, install this bundle
+version instead (they coexist peacefully).
 
 This code is installed as an ECL Bundle.  Complete instructions for managing ECL
 Bundles can be found in [The ECL IDE and HPCC Client
-Tools](https://cdn.hpccsystems.com/releases/CE-Candidate-7.12.0/docs/EN_US/TheECLIDEandHPCCClientTools_EN_US-7.12.0-1.pdf)
+Tools](https://cdn.hpccsystems.com/releases/CE-Candidate-9.4.2/docs/EN_US/TheECLIDEandHPCCClientTools_EN_US-9.4.2-1.pdf)
 documentation.
 
 Use the ecl command line tool to install this bundle:
@@ -101,6 +102,7 @@ level, such as within your "My Files" folder.
 |1.9.1|Fix IMPORT in (Profile) Tests module; support UTF-8 strings in Mode values and example text patterns|
 |1.9.2|Security updates|
 |1.9.3|Better identify upper- and lower-case Unicode characters in text patterns; scan Unicode and UTF-8 strings to see if they can be represented with a STRING data type instead|
+|1.9.4|README fixes and updates; improve UTF-8 detection and prevent buffer overruns during character scans; use short form of Unicode property names in regex|
 </details>
 
 ---