Merge branch 'main' into test

Author: scossu

doc/supported_scripts.md

@@ -63,7 +63,7 @@ third-party library.
 |  [macedonian](../scriptshifter/tables/data/macedonian.yml)  |  Macedonian  |  Y  |  Y  |  stable  |  
 |  [malayalam](../scriptshifter/tables/data/malayalam.yml)  |  Malayalam  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [mansi_cyrillic](../scriptshifter/tables/data/mansi_cyrillic.yml)  |  Mansi (Cyrillic)  |  Y  |  Y  |  stable  |  
-|  [marathi](../scriptshifter/tables/data/marathi.yml)  |  Marathi  |  Y  |  Y  |    |  s-to-r lacks capitalization
+|  [marathi](../scriptshifter/tables/data/marathi_devanagari.yml)  |  Marathi  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [mari_cyrillic](../scriptshifter/tables/data/mari_cyrillic.yml)  |  Mari (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [moldovan_cyrillic](../scriptshifter/tables/data/moldovan_cyrillic.yml)  |  Moldovan (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [mongolian_cyrillic](../scriptshifter/tables/data/mongolian_cyrillic.yml)  |  Mongolian (Cyrillic)  |  Y  |  Y  |  stable  |  
@@ -86,13 +86,13 @@ third-party library.
 |  [sanskrit_devanagari](../scriptshifter/tables/data/sanskrit_devanagari.yml)  |  Sanskrit (Devanagari)  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [serbian](../scriptshifter/tables/data/serbian.yml)  |  Serbian  |  Y  |  Y  |  stable  |  
 |  [shor_cyrillic](../scriptshifter/tables/data/shor_cyrillic.yml)  |  Shor (Cyrillic)  |  Y  |  Y  |  stable  |  
-|  [sinhalese_sinhala](../scriptshifter/tables/data/sinhalese_sinhala.yml)  |  Sinhalese (Sinhala)  |  Y  |  Y  |    |  s-to-r lacks capitalization
+|  [sinhalese_sinhala](../scriptshifter/tables/data/sinhalese.yml)  |  Sinhalese (Sinhala)  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [syriac_cyrillic](../scriptshifter/tables/data/syriac_cyrillic.yml)  |  Syriac (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [tajik_cyrillic](../scriptshifter/tables/data/tajik_cyrillic.yml)  |  Tajik (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [tamil](../scriptshifter/tables/data/tamil.yml)  |  Tamil  |  Y  |  Y  |  beta  |  
 |  [tamil_brahmi](../scriptshifter/tables/data/tamil_brahmi.yml)  |  Tamil Brahmi  |  Y  |  Y  |    |  
 |  [tamil_extended](../scriptshifter/tables/data/tamil_extended.yml)  |  Tamil (extended)  |  Y  |  Y  |    |  
-|  [tatar-kryashen_cyrillic](../scriptshifter/tables/data/tatar.yml)  |  Tatar-Kryashen (Cyrillic)  |  Y  |  Y  |  stable  |  
+|  [tatar-kryashen_cyrillic](../scriptshifter/tables/data/tatar-kryashen_cyrillic.yml)  |  Tatar-Kryashen (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [tatar_cyrillic](../scriptshifter/tables/data/tatar_cyrillic.yml)  |  Tatar (Cyrillic)  |  Y  |  Y  |  stable  |  
 |  [telugu](../scriptshifter/tables/data/telugu.yml)  |  Telugu  |  Y  |  Y  |    |  s-to-r lacks capitalization
 |  [thai](../scriptshifter/tables/data/thai.yml)  |  Thai  |  Y  |  Y  |    |  

scriptshifter/hooks/general/__init__.py

@@ -27,6 +27,24 @@
 logger = getLogger(__name__)
 
 
+def capitalize_pre_assembly(ctx):
+    """
+    Capitalize a not-yet-assembled result list according to user options.
+    """
+    ctx.dest_ls = _capitalize(ctx.dest_ls, ctx.options.get("capitalize"))
+
+
+def capitalize_post_assembly(ctx):
+    """
+    Capitalize an already assembled result string according to user options.
+    """
+    dest_ls = ctx.dest.split(" ")
+
+    dest_ls = _capitalize(dest_ls, ctx.options.get("capitalize"))
+
+    return " ".join(dest_ls)
+
+
 def normalize_spacing_post_assembly(ctx):
     """
     Remove duplicate and unwanted whitespace around punctuation.
@@ -53,3 +71,21 @@ def normalize_spacing_post_assembly(ctx):
     # norm = NORM8_RE.sub(r"\1\2", norm)
 
     return norm
+
+
+def _capitalize(src, which):
+    """
+    capitalize first word only or all words.
+
+    NOTE: this function is only used for capitalizing hook-generated
+    transliterations, which are not normally processed. Double cap rules are
+    not applicable here.
+    """
+    if which == "first":
+        src[0] = src[0].capitalize()
+        return src
+
+    if which == "all":
+        return [tk[0].upper() + tk[1:] for tk in src]
+
+    return src

scriptshifter/hooks/hebrew/dicta_api.py

@@ -3,7 +3,7 @@
 from requests import post
 
 from scriptshifter.exceptions import BREAK, UpstreamError
-from scriptshifter.tools import capitalize
+from scriptshifter.hooks.general import capitalize_post_assembly
 
 EP = environ.get("TXL_DICTA_EP")
 DEFAULT_GENRE = "rabbinic"
@@ -25,16 +25,8 @@ def s2r_post_config(ctx):
     except Exception:
         raise UpstreamError("Error received from Dicta service.")
 
-    rom = rsp.json().get("transliteration")
-
-    if rom:
-        if ctx.options["capitalize"] == "all":
-            rom = capitalize(rom)
-        elif ctx.options["capitalize"] == "first":
-            rom = rom[0].upper() + rom[1:]
-    else:
-        ctx.warnings.append("Upstream service returned empty result.")
-
-    ctx.dest = rom
+    ctx.dest = rsp.json().get("transliteration")
+    if ctx.dest:
+        ctx.dest = capitalize_post_assembly(ctx)
 
     return BREAK

scriptshifter/hooks/korean/romanizer.py

@@ -28,7 +28,7 @@
 
 from scriptshifter.exceptions import BREAK
 from scriptshifter.hooks.korean import KCONF
-from scriptshifter.tools import capitalize
+from scriptshifter.hooks.general import capitalize_post_assembly
 
 
 PWD = path.dirname(path.realpath(__file__))
@@ -62,6 +62,12 @@ def s2r_nonames_post_config(ctx):
     ctx.dest, ctx.warnings = _romanize_nonames(
             ctx.src, ctx.options)
 
+    if ctx.dest:
+        # FKR042: Capitalize all first letters
+        # FKR043: Capitalize the first letter
+        logger.debug(f"Before capitalization: {ctx.dest}")
+        ctx.dest = capitalize_post_assembly(ctx)
+
     return BREAK
 
 
@@ -74,6 +80,12 @@ def s2r_names_post_config(ctx):
     """
     ctx.dest, ctx.warnings = _romanize_names(ctx.src, ctx.options)
 
+    if ctx.dest:
+        # FKR042: Capitalize all first letters
+        # FKR043: Capitalize the first letter
+        logger.debug(f"Before capitalization: {ctx.dest}")
+        ctx.dest = capitalize_post_assembly(ctx)
+
     return BREAK
 
 
@@ -105,19 +117,9 @@ def _romanize_nonames(src, options):
 
     rom = _romanize_oclc_auto(kor)
 
-    logger.debug(f"Before capitalization: {rom}")
-    # FKR042: Capitalize all first letters
-    if options["capitalize"] == "all":
-        rom = capitalize(rom)
-    # FKR043: Capitalize the first letter
-    elif options["capitalize"] == "first":
-        rom = rom[0].upper() + rom[1:]
-
     # FKR044: Ambiguities
     ambi = re.sub("[,.\";: ]+", " ", rom)
 
-    # TODO Decide what to do with these. There is no facility for outputting
-    # warnings or notes to the user yet.
     warnings = []
     _fkr_log(45)
     for exp, warn in KCONF["fkr045"].items():
@@ -308,10 +310,11 @@ def _kor_corp_name_rom(src):
         src = src[:-4]
         yu = "R"
 
-    rom_tok = []
-    for tok in src.split(" "):
-        rom_tok.append(_romanize_oclc_auto(tok))
-    rom = capitalize(" ".join(rom_tok))
+    rom_tok = [
+        _romanize_oclc_auto(tok)
+        for tok in src.split(" ")
+    ]
+    rom = " ".join(rom_tok)
 
     if chu == "L":
         rom = "(Chu) " + rom

scriptshifter/hooks/yiddish_/__init__.py

@@ -16,24 +16,16 @@
 from yiddish import detransliterate, transliterate
 
 from scriptshifter.exceptions import BREAK
-from scriptshifter.tools import capitalize
 
 
 def s2r_post_config(ctx):
     """
     Script to Roman.
     """
-    rom = transliterate(
+    ctx.dest = transliterate(
             ctx.src, loc=True,
             loshn_koydesh=ctx.options.get("loshn_koydesh"))
 
-    if ctx.options["capitalize"] == "all":
-        rom = capitalize(rom)
-    elif ctx.options["capitalize"] == "first":
-        rom = rom[0].upper() + rom[1:]
-
-    ctx.dest = rom
-
     return BREAK
 
 

scriptshifter/tables/data/_cyrillic_base.yml

@@ -9,30 +9,37 @@ roman_to_script:
     - "date of publication not identified"
     - "place of publication not identified"
     - "publisher not identified"
+    - "and one other"
+    - "et al."
+  ignore_ptn:
+    - "and ([a-z0-9]+ )?others"
+
+    # Incorrectly entered (but frequently found) Roman numerals.
     # NOTE There is ambiguity about ignoring these
     # words. Note that the single-character Roman
     # numerals are not included on purpose.
     # Ideally the source editors should use the
     # dedicated U+2160÷U+216F (uppercase Roman
     # numerals) and/or U+2170÷U+217F (lower case Roman
     # numerals) ranges to avoid this ambiguity.
-    - "and one other"
-    - "et al."
-  ignore_ptn:
-    - "and ([a-z0-9]+ )?others"
-    - "I{2,3}"
-    - "I(V|X)"
-    - "LI{,3}"
-    - "LI?(V|X)"
-    - "L(V|X{1,3})I{,3}"
-    - "LX{1,3}I?V"
-    - "LX{1,3}VI{,3}"
-    - "(V|X{1,3})I{,3}"
-    - "X{1,3}I{,3}"
-    - "X{1,3}I(V|X)"
-    - "X{1,3}VI{,3}"
+    - "\\bI{2,3}\\b"
+    - "\\bI(V|X)\\b"
+    - "\\bLI{,3}\\b"
+    - "\\bLI?(V|X)\\b"
+    - "\\bL(V|X{1,3})I{,3}\\b"
+    - "\\bLX{1,3}I?V\\b"
+    - "\\bLX{1,3}VI{,3}\\b"
+    - "\\b(V|X{1,3})I{,3}\\b"
+    - "\\bX{1,3}I{,3}\\b"
+    - "\\bX{1,3}I(V|X)\\b"
+    - "\\bX{1,3}VI{,3}\\b"
+
+    # MARC sub-field markers.
+    - "\\b[\u2021$][0-9a-z]\\b"
 
 script_to_roman:
   ignore:
     - " "
-
+  ignore_ptn:
+    # MARC sub-field markers.
+    - "\\b[\u2021$][0-9a-z]\\b"

scriptshifter/tables/data/_ignore_base.yml

@@ -1,7 +1,7 @@
 general:
   name: Abkhaz (Cyrillic)
   parents:
-    - _cyrillic_base
+    - cyrillic_generic
 
 roman_to_script:
   map: