Merge pull request #245 from lcnetdev/dec_config

Decouple configs

Author: scossu

doc/config.md

@@ -13,9 +13,7 @@ Configuration files, also called transliteration tables, are contained in the
 The configuration file names are key to most operations in the software. They
 are all-lowercase and use underscores to separate words, e.g.
 `church_slavonic`. They have the `.yml` extension and are written in the
-[YAML](https://yaml.org/) configuration language. Hence, a transliteration
-request to the `/trans` REST API endpoint providing `church_slavonic` as the
-transliteration language, uses the `church_slavonic.yml` configuration file.
+[YAML](https://yaml.org/) configuration language.
 
 Other files are present in the `data` directory that are not exposed to the end
 user via Web UI or REST API. These files may be incomplete transliteration
@@ -35,10 +33,49 @@ transliteration table key names as described previously, and the values are
 key-value pairs which can have arbitrary contents. These contents are displayed
 to the user in the `/languages` API endpoint.
 
-The only mandatory key for each key-value pair is `name`, which is the
-human-readable label that is displayed in the Web UI. Other keys, such as
-`description`, may be used to inform the user about the scope of a particular
-table.
+Each entry of the index file are the following:
+
+### `<entry_name>`
+
+The key for the language/script. This is referred in multiple places across
+the application, e.g. the `/trans/mongolian_cyrillic` API method transliterates
+a sentence using the `mongolian_cyrillic` index entry. By default, the
+`mongolian_cyrillic` entry in the index file (see below) uses the
+`mongolian_cyrillic.yml` configuration file in the `data/` folder. This can be
+overridden by the `conf` key (see below).
+
+By convention, an entry name uses the name of the language, followed by the
+name of the script, separated by an underscore, *only if that language is known
+to exist in multiple scripts*. For example, `mongolian_cyrillic` is used for
+Mongolian written in Cyrillic, and `mongolian_mongol_bichig` for the native
+Mongol Bichig script; while `persian` is only found in Arabic script, so
+`arabic` is not added.
+
+### `<entry_name>.name`
+
+Human-readable label that is displayed in the Web UI. Mandatory.
+
+### `<entry_name>.conf`
+
+Override the default configuration file lookup. By default,
+the configuration file is inferred from the key name, e.g. `chinese` looks up
+`data/chinese.yml`. However, some entries in the index may not have a distinct
+configuration and reuse an existing configuration that works for that language.
+Several languages in the Cyrillic script use this method.
+
+The value is the full file name relative to the `data/` directory, e.g.
+`cyrillic_generic.yml`.
+
+### `<entry_name>.marc_code`
+
+MARC code from the [MARC Standards Office registry
+](https://www.loc.gov/marc/languages/language_name.html). This may be used by
+external applications to more easily look up entries. Optional.
+
+### `<entry_name>.description`
+
+Additional description that  may be used to inform the user about the scope of
+a particular table. Optional.
 
 ## Inheritance
 

scriptshifter/tables/__init__.py

@@ -66,6 +66,8 @@
 
 logger = logging.getLogger(__name__)
 
+tbl_index = None  # Module-level index of all scripts.
+
 
 class Token(str):
     """
@@ -165,11 +167,12 @@ def init_db():
             conn.executescript(fh.read())
 
     # Populate tables.
+    global tbl_index
     with open(path.join(path.dirname(TABLE_DIR), "index.yml")) as fh:
-        tlist = load(fh, Loader=Loader)
+        tbl_index = load(fh, Loader=Loader)
     try:
         with conn:
-            for tname, tdata in tlist.items():
+            for tname, tdata in tbl_index.items():
                 populate_table(conn, tname, tdata)
 
         # If the DB already exists, it will be overwritten ONLY on success at
@@ -340,9 +343,14 @@ def load_table(tname):
     the language & script metadata and parsing rules.
     """
 
-    fname = path.join(TABLE_DIR, tname + ".yml")
+    try:
+        fname = path.join(TABLE_DIR, tbl_index[tname]["conf"])
+    except KeyError:
+        # If no `conf` key is provided, use the conventional table name + .yml.
+        fname = path.join(TABLE_DIR, tname + ".yml")
     if not access(fname, R_OK):
-        raise ValueError(f"No transliteration table for {tname}!")
+        raise ValueError(
+                f"No transliteration table `{fname}` found for {tname}!")
 
     with open(fname) as fh:
         tdata = load(fh, Loader=Loader)
@@ -400,7 +408,7 @@ def load_table(tname):
 
         # Inherit normalization rules.
         for parent in parents:
-            parent_langsec = load_table(parent)["script_to_roman"]
+            parent_langsec = load_table(parent).get("script_to_roman", {})
             normalize |= parent_langsec.get("normalize", {})
 
         for k, v in tdata["script_to_roman"].get("normalize", {}).items():

scriptshifter/tables/data/_ignore_base.yml

@@ -22,22 +22,17 @@ roman_to_script:
     # dedicated U+2160÷U+216F (uppercase Roman
     # numerals) and/or U+2170÷U+217F (lower case Roman
     # numerals) ranges to avoid this ambiguity.
-    - "\\bI{2,3}\\b"
-    - "\\bI(V|X)\\b"
-    - "\\bLI{,3}\\b"
-    - "\\bLI?(V|X)\\b"
-    - "\\bL(V|X{1,3})I{,3}\\b"
-    - "\\bLX{1,3}I?V\\b"
-    - "\\bLX{1,3}VI{,3}\\b"
-    - "\\b(V|X{1,3})I{,3}\\b"
-    - "\\bX{1,3}I{,3}\\b"
-    - "\\bX{1,3}I(V|X)\\b"
-    - "\\bX{1,3}VI{,3}\\b"
+    - "I{2,3}\\b"
+    - "I(V|X)\\b"
+    - "LI{,3}\\b"
+    - "LI?(V|X)\\b"
+    - "L(V|X{1,3})I{,3}\\b"
+    - "LX{1,3}I?V\\b"
+    - "LX{1,3}VI{,3}\\b"
+    - "(V|X{1,3})I{,3}\\b"
+    - "X{1,3}I{,3}\\b"
+    - "X{1,3}I(V|X)\\b"
+    - "X{1,3}VI{,3}\\b"
 
     # MARC sub-field markers.
-    - "\\b[\u2021\u01C2\\$][0-9a-z]\\b"
-
-script_to_roman:
-  ignore_ptn:
-    # MARC sub-field markers.
-    - "\\b[\u2021\u01C2\\$][0-9a-z]\\b"
+    - "[\u2021\u01C2\\$][0-9a-z]\\b"