Merge pull request #250 from lcnetdev/tibetan

Update Tibetan; use Unicode word boundaries instead of hardcoded ones.

Author: scossu

.gitignore

@@ -140,3 +140,4 @@ ext/arabic_rom/data
 scriptshifter/data/*.db
 !.keep
 VERSION
+.~lock.*

ext/arabic_rom

@@ -5,5 +5,6 @@ flask>=2.3,<3
 flask-cors>=4.0,<5
 python-dotenv>=1.0,<2
 pyyaml>=6.0,<7
+regex>=2023.8.8
 uwsgi>=2.0,<2.1
 yiddish==0.0.21

requirements.txt

@@ -38,6 +38,6 @@ python3 src/loc_transcribe.py predict mle dev --mle_model models/mle/size1.0.tsv
 
 # Seq2Seq
 echo "Preparing Seq2seq."
-make prep_seq2seq
+python3 src/data/make_seq2seq_dataset.py -l ${SS_LANG}
 echo "Training models."
-python3 src/loc_transcribe.py train seq2seq --train --size {1.0,0.5,0.25,0.125,0.0625,0.03125,0.015625}
+python3 src/data/make_seq2seq_dataset.py  --size {1.0,0.5,0.25,0.125,0.0625,0.03125,0.015625}

scriptshifter/hooks/arabic/build_model.sh

@@ -47,8 +47,6 @@
 )
 # Package path where hook functions are kept.
 HOOK_PKG_PATH = "scriptshifter.hooks"
-# Default characters defining a word boundary. This is configurable per-table.
-WORD_BOUNDARY = " \n\t:;.,\"'-()[]{}"
 
 # Token word boundary marker. Used in maps to distinguish special
 # transliterations for initial, final, and standalone tokens.