Merge pull request #202 from lcnetdev/main

Update test branch

Author: scossu

doc/supported_scripts.md

@@ -63,7 +63,7 @@ third-party library.
 |  [macedonian](../scriptshifter/tables/data/macedonian.yml)  |  Macedonian  |  Y  |  Y  |  stable  |  
 |  [malayalam](../scriptshifter/tables/data/malayalam.yml)  |  Malayalam  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [mansi_cyrillic](../scriptshifter/tables/data/mansi_cyrillic.yml)  |  Mansi (Cyrillic)  |  Y  |  Y  |  stable  |  
-|  [marathi](../scriptshifter/tables/data/marathi.yml)  |  Marathi  |  Y  |  Y  |    |  s-to-r lacks capitalization
+|  [marathi](../scriptshifter/tables/data/marathi_devanagari.yml)  |  Marathi  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [mari_cyrillic](../scriptshifter/tables/data/mari_cyrillic.yml)  |  Mari (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [moldovan_cyrillic](../scriptshifter/tables/data/moldovan_cyrillic.yml)  |  Moldovan (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [mongolian_cyrillic](../scriptshifter/tables/data/mongolian_cyrillic.yml)  |  Mongolian (Cyrillic)  |  Y  |  Y  |  stable  |  
@@ -86,13 +86,13 @@ third-party library.
 |  [sanskrit_devanagari](../scriptshifter/tables/data/sanskrit_devanagari.yml)  |  Sanskrit (Devanagari)  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [serbian](../scriptshifter/tables/data/serbian.yml)  |  Serbian  |  Y  |  Y  |  stable  |  
 |  [shor_cyrillic](../scriptshifter/tables/data/shor_cyrillic.yml)  |  Shor (Cyrillic)  |  Y  |  Y  |  stable  |  
-|  [sinhalese_sinhala](../scriptshifter/tables/data/sinhalese_sinhala.yml)  |  Sinhalese (Sinhala)  |  Y  |  Y  |    |  s-to-r lacks capitalization
+|  [sinhalese_sinhala](../scriptshifter/tables/data/sinhalese.yml)  |  Sinhalese (Sinhala)  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [syriac_cyrillic](../scriptshifter/tables/data/syriac_cyrillic.yml)  |  Syriac (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [tajik_cyrillic](../scriptshifter/tables/data/tajik_cyrillic.yml)  |  Tajik (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [tamil](../scriptshifter/tables/data/tamil.yml)  |  Tamil  |  Y  |  Y  |  beta  |  
 |  [tamil_brahmi](../scriptshifter/tables/data/tamil_brahmi.yml)  |  Tamil Brahmi  |  Y  |  Y  |    |  
 |  [tamil_extended](../scriptshifter/tables/data/tamil_extended.yml)  |  Tamil (extended)  |  Y  |  Y  |    |  
-|  [tatar-kryashen_cyrillic](../scriptshifter/tables/data/tatar.yml)  |  Tatar-Kryashen (Cyrillic)  |  Y  |  Y  |  stable  |  
+|  [tatar-kryashen_cyrillic](../scriptshifter/tables/data/tatar-kryashen_cyrillic.yml)  |  Tatar-Kryashen (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [tatar_cyrillic](../scriptshifter/tables/data/tatar_cyrillic.yml)  |  Tatar (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [telugu](../scriptshifter/tables/data/telugu.yml)  |  Telugu  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [thai](../scriptshifter/tables/data/thai.yml)  |  Thai  |  Y  |  Y  |    |  

scriptshifter/hooks/general/__init__.py

@@ -27,6 +27,25 @@
 logger = getLogger(__name__)
 
 
+def capitalize_pre_assembly(ctx):
+    """
+    Capitalize a not-yet-assembled result list according to user options.
+    """
+    ctx.dest_ls = _capitalize(ctx.dest_ls, ctx.options.get("capitalize"))
+
+
+def capitalize_post_assembly(ctx):
+    """
+    Capitalize an already assembled result string according to user options.
+    """
+    dest_ls = ctx.dest.split(" ")
+
+    dest_ls = _capitalize(dest_ls, ctx.options.get("capitalize"))
+
+    return " ".join(dest_ls)
+
+
+
 def normalize_spacing_post_assembly(ctx):
     """
     Remove duplicate and unwanted whitespace around punctuation.
@@ -53,3 +72,18 @@ def normalize_spacing_post_assembly(ctx):
     # norm = NORM8_RE.sub(r"\1\2", norm)
 
     return norm
+
+
+def _capitalize(src, which):
+    """
+    Only capitalize first word and words preceded by space.
+
+    NOTE: this function is only used for capitalizing hook-generated
+    transliterations, which are not normally processed. Double cap rules are
+    not applicable here.
+    """
+    if which == "first":
+        ctx.dest_ls[0] = ctx.dest_ls[0].upper()
+
+    elif which == "all":
+        ctx.dest_ls = [tk[0].upper() + tk[1:] for tk in ctx.dest_ls]

scriptshifter/hooks/hebrew/dicta_api.py

@@ -3,7 +3,7 @@
 from requests import post
 
 from scriptshifter.exceptions import BREAK, UpstreamError
-from scriptshifter.tools import capitalize
+from scriptshifter.hooks.general import capitalize_post_assembly
 
 EP = environ.get("TXL_DICTA_EP")
 DEFAULT_GENRE = "rabbinic"
@@ -25,16 +25,8 @@ def s2r_post_config(ctx):
     except Exception:
         raise UpstreamError("Error received from Dicta service.")
 
-    rom = rsp.json().get("transliteration")
-
-    if rom:
-        if ctx.options["capitalize"] == "all":
-            rom = capitalize(rom)
-        elif ctx.options["capitalize"] == "first":
-            rom = rom[0].upper() + rom[1:]
-    else:
-        ctx.warnings.append("Upstream service returned empty result.")
-
-    ctx.dest = rom
+    ctx.dest = rsp.json().get("transliteration")
+    if ctx.dest:
+        ctx.dest = capitalize_post_assembly(ctx)
 
     return BREAK

scriptshifter/hooks/korean/romanizer.py

@@ -28,7 +28,7 @@
 
 from scriptshifter.exceptions import BREAK
 from scriptshifter.hooks.korean import KCONF
-from scriptshifter.tools import capitalize
+from scriptshifter.hooks.general import capitalize_post_assembly
 
 
 PWD = path.dirname(path.realpath(__file__))
@@ -62,6 +62,12 @@ def s2r_nonames_post_config(ctx):
     ctx.dest, ctx.warnings = _romanize_nonames(
             ctx.src, ctx.options)
 
+    if ctx.dest:
+        # FKR042: Capitalize all first letters
+        # FKR043: Capitalize the first letter
+        logger.debug(f"Before capitalization: {ctx.dest}")
+        ctx.dest = capitalize_post_assembly(ctx)
+
     return BREAK
 
 
@@ -74,6 +80,12 @@ def s2r_names_post_config(ctx):
     """
     ctx.dest, ctx.warnings = _romanize_names(ctx.src, ctx.options)
 
+    if ctx.dest:
+        # FKR042: Capitalize all first letters
+        # FKR043: Capitalize the first letter
+        logger.debug(f"Before capitalization: {ctx.dest}")
+        ctx.dest = capitalize_post_assembly(ctx)
+
     return BREAK
 
 
@@ -105,19 +117,9 @@ def _romanize_nonames(src, options):
 
     rom = _romanize_oclc_auto(kor)
 
-    logger.debug(f"Before capitalization: {rom}")
-    # FKR042: Capitalize all first letters
-    if options["capitalize"] == "all":
-        rom = capitalize(rom)
-    # FKR043: Capitalize the first letter
-    elif options["capitalize"] == "first":
-        rom = rom[0].upper() + rom[1:]
-
     # FKR044: Ambiguities
     ambi = re.sub("[,.\";: ]+", " ", rom)
 
-    # TODO Decide what to do with these. There is no facility for outputting
-    # warnings or notes to the user yet.
     warnings = []
     _fkr_log(45)
     for exp, warn in KCONF["fkr045"].items():
@@ -308,10 +310,11 @@ def _kor_corp_name_rom(src):
         src = src[:-4]
         yu = "R"
 
-    rom_tok = []
-    for tok in src.split(" "):
-        rom_tok.append(_romanize_oclc_auto(tok))
-    rom = capitalize(" ".join(rom_tok))
+    rom_tok = [
+        _romanize_oclc_auto(tok)
+        for tok in src.split(" ")
+    ]
+    rom = " ".join(rom_tok)
 
     if chu == "L":
         rom = "(Chu) " + rom

scriptshifter/hooks/yiddish_/__init__.py

@@ -16,24 +16,16 @@
 from yiddish import detransliterate, transliterate
 
 from scriptshifter.exceptions import BREAK
-from scriptshifter.tools import capitalize
 
 
 def s2r_post_config(ctx):
     """
     Script to Roman.
     """
-    rom = transliterate(
+    ctx.dest = transliterate(
             ctx.src, loc=True,
             loshn_koydesh=ctx.options.get("loshn_koydesh"))
 
-    if ctx.options["capitalize"] == "all":
-        rom = capitalize(rom)
-    elif ctx.options["capitalize"] == "first":
-        rom = rom[0].upper() + rom[1:]
-
-    ctx.dest = rom
-
     return BREAK
 
 

scriptshifter/tables/data/_ignore_base.yml

@@ -31,8 +31,10 @@ roman_to_script:
     - "X{1,3}I{,3}"
     - "X{1,3}I(V|X)"
     - "X{1,3}VI{,3}"
+    - "[\u2021$][0-9a-z] *"
 
 script_to_roman:
   ignore:
     - " "
-
+  ignore_ptn:
+    - "[\u2021$][0-9a-z] *"