Use Unicode classes for word boundary markers.

Author: scossu

requirements.txt

@@ -5,5 +5,6 @@ flask>=2.3,<3
 flask-cors>=4.0,<5
 python-dotenv>=1.0,<2
 pyyaml>=6.0,<7
+regex>=2023.8.8
 uwsgi>=2.0,<2.1
 yiddish==0.0.21

scriptshifter/trans.py

@@ -1,7 +1,8 @@
 import logging
 
 from importlib import import_module
-from re import Pattern, compile
+from re import Pattern
+from regex import compile
 from unicodedata import normalize as precomp_normalize
 
 from scriptshifter.exceptions import BREAK, CONT
@@ -13,8 +14,10 @@
 
 logger = logging.getLogger(__name__)
 
-WORD_PTN = compile(r"\w")
-WB_PTN = compile(r"\W")
+# Beginning-of-word pattern.
+BOW_PTN = compile(r"(?<=[\p{P}\p{Z}]|^)[\p{L}\p{M}\p{S}]")
+# End-of-word pattern.
+EOW_PTN = compile(r"[\p{L}\p{M}\p{S}](?=[\p{P}\p{Z}]|$)")
 
 
 class Transliterator:
@@ -107,33 +110,10 @@ def normalize_src(self):
         for nk, nv in norm_rules.items():
             self.src = self.src.replace(nk, nv)
 
-        return self.run_hook("post_normalize")
-
-    def cur_at_bow(self, cur=None):
-        """
-        Check if cursor is at the beginning of a word.
-
-        @param cur(int): Position to check. By default, the current cursor.
-        """
-        if cur is None:
-            cur = self.cur
-        return (
-            self.cur == 0
-            or WB_PTN.match(self.src[cur - 1])
-        ) and WORD_PTN.match(self.src[cur])
-
-    def cur_at_eow(self, cur=None):
-        """
-        Check if cursor is at the end of a word.
+        self.bow_coords = {m.span()[0] for m in BOW_PTN.finditer(self.src)}
+        self.eow_coords = {m.span()[0] for m in EOW_PTN.finditer(self.src)}
 
-        @param cur(int): Position to check. By default, the current cursor.
-        """
-        if cur is None:
-            cur = self.cur
-        return (
-            cur == len(self.src) - 1
-            or WB_PTN.match(self.src[cur + 1])
-        ) and WORD_PTN.match(self.src[cur])
+        return self.run_hook("post_normalize")
 
 
 def transliterate(src, lang, t_dir="s2r", capitalize=False, options={}):
@@ -209,11 +189,11 @@ def transliterate(src, lang, t_dir="s2r", capitalize=False, options={}):
             ctx.cur_flags = 0
 
             # Look for a word boundary and flag word beginning/end it if found.
-            if ctx.cur_at_bow():
+            if ctx.cur in ctx.bow_coords:
                 # Beginning of word.
                 logger.debug(f"Beginning of word at position {ctx.cur}.")
                 ctx.cur_flags |= BOW
-            if ctx.cur_at_eow():
+            if ctx.cur in ctx.eow_coords:
                 # End of word.
                 logger.debug(f"End of word at position {ctx.cur}.")
                 ctx.cur_flags |= EOW
@@ -319,7 +299,7 @@ def transliterate(src, lang, t_dir="s2r", capitalize=False, options={}):
                         # Can't rely on EOW flag, we must check on the last
                         # character of the potential match.
                         ctx.src_tk.flags & EOW
-                        and not ctx.cur_at_eow(ctx.cur + step - 1)
+                        and ctx.cur + step - 1 not in ctx.eow_coords
                     )
                 ):
                     continue

test/data/script_samples/tibetan.csv

@@ -25,3 +25,4 @@
 "tibetan","Rgyud don rig-ʼdzin dgyes paʾi źal luṅ ","རྒྱུད་དོན་རིག་འཛིན་དགྱེས་པའི་ཞལ་ལུང་།","r2s","{""capitalize"": ""first""}","From Lobsang"
 "tibetan","Gnas brtan chen po bcu drug gi mchod pa rgyal bstan mdzad med nor bu zhes bya ba bzhugs so","གནས་བརྟན་ཆེན་པོ་བཅུ་དྲུག་གི་མཆོད་པ་རྒྱལ་བསྟན་མཛད་མེད་ནོར་བུ་ཞེས་བྱ་བ་བཞུགས་སོ།","r2s","{""capitalize"": ""first""}","From Lobsang"
 "tibetan","སྐྱབས་འགྲོ་ཡན་ལག་དྲུག་པ་བཤད་པའི་བསྐྱུད་བྱང་ཟིན་མ་ཊཱིཀ་བཞུགས་སོ།","skyabs ’gro yan lag drug pa bshad pa’i bskyud byang zin ma ṭīka bzhugs so","s2r",,"From Lobsang"
+"tibetan","བོད་ཀྱི་ཆོས་འབྱུང་དང་རྒྱལ་རབས་ཀྱི་སྐོར: སྦ་བཞེད་དེབ་ཐེར་དམར་པོ: དེབ་ཐེར་དཀར་པོ: གངས༌ཅན་བོད་ཀྱི་རྒྱལ་བསྟན་ཕྱི་མོ་སྔ་འགྱུར་རྙིང་མའི་བྱུང་བ་མདོ་ཙམ་བརྗོད་པ་པདྨ་དམ་རྭ་གའི་དོ་ཤལ་གཞོན་ནུ་དགྱེས་པའི་མགུལ་རྒྱན / སྦ༌གསལ༌སྣང, ་ཚལ༌པ་ཀུན༌དགའ༌རྡོ༌རྗེ, ་དགེ༌འདུན༌ཆོས༌འཕེལ, ་ཁམས༌སྤྲུལ་བསོད༌ནམས༌དོན༌གྲུབ / པའི་བེའུ་མིའི་དགེའི་ཤུའི་མོའི་ཐུའུ་གསོའི་པའོ","Bod kyi chos ʾbyung dang rgyal rabs kyi skor : Sba bzhed Deb ther dmar po : Deb ther dkar po : Gangs-can Bod kyi rgyal bstan phyi mo snga ʾgyur rnying maʾi byung ba mdo tsam brjod pa Padma dma rwa gaʾi do shal gzhon nu dgyes paʾi mgul rgyan / Sba-gsal-snang, Tshal-pa Kun-dgaʾ-rdo-rje, Dge-ʾdun-chos-ʾphel, Khams-sprul Bsod-nams-don-grub / paʼi beʼu miʼi dgeʼi shuʼi moʼi Thuʼu gsoʼi paʼo",,,