Update row data type naming

Enkidu93 · Enkidu93 · commit f25702e79400 · 2026-01-21T17:43:54.000-05:00
diff --git a/machine/corpora/__init__.py b/machine/corpora/__init__.py
@@ -4,7 +4,6 @@
 from .alignment_row import AlignmentRow
 from .corpora_utils import batch
 from .corpus import Corpus
-from .data_type import DataType
 from .dbl_bundle_text_corpus import DblBundleTextCorpus
 from .dictionary_alignment_corpus import DictionaryAlignmentCorpus
 from .dictionary_text_corpus import DictionaryTextCorpus
@@ -49,6 +48,7 @@
 from .text_file_text import TextFileText
 from .text_file_text_corpus import TextFileTextCorpus
 from .text_row import TextRow, TextRowFlags
+from .text_row_content_type import TextRowContentType
 from .token_processors import (
     escape_spaces,
     lowercase,
@@ -103,7 +103,7 @@
     "batch",
     "Corpus",
     "create_versification_ref_corpus",
-    "DataType",
+    "TextRowContentType",
     "DblBundleTextCorpus",
     "DictionaryAlignmentCorpus",
     "DictionaryTextCorpus",
diff --git a/machine/corpora/data_type.py b/machine/corpora/data_type.py
diff --git a/machine/corpora/memory_text.py b/machine/corpora/memory_text.py
@@ -1,13 +1,15 @@
 from typing import Generator, Iterable
 
 from .corpora_utils import gen
-from .data_type import DataType
 from .text import Text
 from .text_row import TextRow
+from .text_row_content_type import TextRowContentType
 
 
 class MemoryText(Text):
-    def __init__(self, id: str, rows: Iterable[TextRow] = [], data_type: DataType = DataType.SENTENCE) -> None:
+    def __init__(
+        self, id: str, rows: Iterable[TextRow] = [], data_type: TextRowContentType = TextRowContentType.SEGMENT
+    ) -> None:
         self._id = id
         self._rows = list(rows)
         if any([r.data_type != data_type for r in self._rows]):
@@ -23,7 +25,7 @@ def sort_key(self) -> str:
         return self._id
 
     @property
-    def data_type(self) -> DataType:
+    def data_type(self) -> TextRowContentType:
         return self._data_type
 
     def _get_rows(self) -> Generator[TextRow, None, None]:
diff --git a/machine/corpora/n_parallel_text_corpus.py b/machine/corpora/n_parallel_text_corpus.py
@@ -2,20 +2,20 @@
 from typing import Any, Callable, Iterable, List, Optional, Sequence, Set, cast
 
 from ..scripture.verse_ref import Versification
-from .data_type import DataType
 from .n_parallel_text_corpus_base import NParallelTextCorpusBase
 from .n_parallel_text_row import NParallelTextRow
 from .scripture_ref import ScriptureRef
 from .text_corpus import TextCorpus
 from .text_corpus_enumerator import TextCorpusEnumerator
 from .text_row import TextRow, TextRowFlags
+from .text_row_content_type import TextRowContentType
 
 
 class _RangeRow:
     refs: List[Any]
     segment: List[str]
     is_sentence_start: bool = False
-    data_type: DataType = DataType.SENTENCE
+    data_type: TextRowContentType = TextRowContentType.SEGMENT
 
     @property
     def is_in_range(self):
@@ -38,7 +38,7 @@ def __init__(self, n: int):
         self.text_id = ""
         self.versifications: Optional[List[Versification]] = None
         self.row_ref_comparer = None
-        self.data_type = DataType.SENTENCE
+        self.data_type = TextRowContentType.SEGMENT
 
     @property
     def is_in_range(self) -> bool:
@@ -293,7 +293,7 @@ def _create_rows(
             yield range_info.create_row()
 
         default_refs = [[r.ref for r in rows if r is not None][0]]
-        data_type = DataType.SENTENCE
+        data_type = TextRowContentType.SEGMENT
 
         text_id: Optional[str] = None
         refs: List[List[Any]] = []
diff --git a/machine/corpora/n_parallel_text_row.py b/machine/corpora/n_parallel_text_row.py
@@ -1,11 +1,13 @@
 from typing import Any, Sequence
 
-from .data_type import DataType
 from .text_row import TextRowFlags
+from .text_row_content_type import TextRowContentType
 
 
 class NParallelTextRow:
-    def __init__(self, text_id: str, n_refs: Sequence[Sequence[Any]], data_type: DataType = DataType.SENTENCE):
+    def __init__(
+        self, text_id: str, n_refs: Sequence[Sequence[Any]], data_type: TextRowContentType = TextRowContentType.SEGMENT
+    ):
         if len([n_ref for n_ref in n_refs if n_ref is not None and len(n_ref) > 0]) == 0:
             raise ValueError(f"Refs must be provided but n_refs={n_refs}")
         self._text_id = text_id
@@ -24,7 +26,7 @@ def ref(self) -> Any:
         return self._n_refs[0][0]
 
     @property
-    def data_type(self) -> DataType:
+    def data_type(self) -> TextRowContentType:
         return self._data_type
 
     @property
diff --git a/machine/corpora/parallel_text_corpus.py b/machine/corpora/parallel_text_corpus.py
@@ -26,8 +26,8 @@
 from .aligned_word_pair import AlignedWordPair
 from .corpora_utils import get_split_indices
 from .corpus import Corpus
-from .data_type import DataType
 from .parallel_text_row import ParallelTextRow
+from .text_row_content_type import TextRowContentType
 from .token_processors import escape_spaces, lowercase, normalize, unescape_spaces
 
 if TYPE_CHECKING:
@@ -419,7 +419,7 @@ def to_hf_dataset(
         if alignment_column is not None:
             features_dict[alignment_column] = Sequence({source_lang: Value("int32"), target_lang: Value("int32")})
         if data_type_column is not None:
-            features_dict[data_type_column] = ClassLabel(names=[e.name for e in DataType])
+            features_dict[data_type_column] = ClassLabel(names=[e.name for e in TextRowContentType])
         features = Features(features_dict)
 
         def iterable() -> Iterable[dict]:
diff --git a/machine/corpora/parallel_text_row.py b/machine/corpora/parallel_text_row.py
@@ -3,8 +3,8 @@
 from typing import Any, Collection, Optional, Sequence
 
 from .aligned_word_pair import AlignedWordPair
-from .data_type import DataType
 from .text_row import TextRowFlags
+from .text_row_content_type import TextRowContentType
 
 
 class ParallelTextRow(Sequence[Sequence[str]]):
@@ -18,7 +18,7 @@ def __init__(
         aligned_word_pairs: Optional[Collection[AlignedWordPair]] = None,
         source_flags: TextRowFlags = TextRowFlags.SENTENCE_START,
         target_flags: TextRowFlags = TextRowFlags.SENTENCE_START,
-        data_type: DataType = DataType.SENTENCE,
+        data_type: TextRowContentType = TextRowContentType.SEGMENT,
     ) -> None:
         if not text_id:
             raise ValueError("A text_id must be set.")
@@ -55,7 +55,7 @@ def refs(self) -> Sequence[Any]:
         return self.target_refs if len(self.source_refs) == 0 else self.source_refs
 
     @property
-    def data_type(self) -> DataType:
+    def data_type(self) -> TextRowContentType:
         return self._data_type
 
     @property
diff --git a/machine/corpora/paratext_backup_terms_corpus.py b/machine/corpora/paratext_backup_terms_corpus.py
@@ -2,11 +2,11 @@
 from zipfile import ZipFile
 
 from ..utils.typeshed import StrPath
-from .data_type import DataType
 from .dictionary_text_corpus import DictionaryTextCorpus
 from .key_term import KeyTerm
 from .memory_text import MemoryText
 from .text_row import TextRow
+from .text_row_content_type import TextRowContentType
 from .zip_paratext_project_settings_parser import ZipParatextProjectSettingsParser
 from .zip_paratext_project_terms_parser import ZipParatextProjectTermsParser
 
@@ -29,9 +29,9 @@ def __init__(self, filename: StrPath, term_categories: Sequence[str], use_term_g
             text = MemoryText(
                 text_id,
                 [
-                    TextRow(text_id, key_term.id, key_term.renderings, data_type=DataType.GLOSS)
+                    TextRow(text_id, key_term.id, key_term.renderings, data_type=TextRowContentType.WORD)
                     for key_term in key_terms
                 ],
-                data_type=DataType.GLOSS,
+                data_type=TextRowContentType.WORD,
             )
             self._add_text(text)
diff --git a/machine/corpora/scripture_text.py b/machine/corpora/scripture_text.py
@@ -4,15 +4,15 @@
 from ..scripture.verse_ref import VerseRef, Versification
 from ..utils.context_managed_generator import ContextManagedGenerator
 from .corpora_utils import gen, get_scripture_text_sort_key
-from .data_type import DataType
 from .scripture_ref import ScriptureElement, ScriptureRef
 from .text_base import TextBase
 from .text_row import TextRow, TextRowFlags
+from .text_row_content_type import TextRowContentType
 
 
 class ScriptureText(TextBase):
     def __init__(self, id: str, versification: Optional[Versification] = None) -> None:
-        super().__init__(id, get_scripture_text_sort_key(id), data_type=DataType.SENTENCE)
+        super().__init__(id, get_scripture_text_sort_key(id), data_type=TextRowContentType.SEGMENT)
         self._versification = ENGLISH_VERSIFICATION if versification is None else versification
 
     @property
diff --git a/machine/corpora/text.py b/machine/corpora/text.py
@@ -1,8 +1,8 @@
 from abc import abstractmethod
 
 from .corpus import Corpus
-from .data_type import DataType
 from .text_row import TextRow
+from .text_row_content_type import TextRowContentType
 
 
 class Text(Corpus[TextRow]):
@@ -16,4 +16,4 @@ def sort_key(self) -> str: ...
 
     @property
     @abstractmethod
-    def data_type(self) -> DataType: ...
+    def data_type(self) -> TextRowContentType: ...
diff --git a/machine/corpora/text_base.py b/machine/corpora/text_base.py
@@ -1,12 +1,12 @@
 from typing import Any
 
-from .data_type import DataType
 from .text import Text
 from .text_row import TextRow, TextRowFlags
+from .text_row_content_type import TextRowContentType
 
 
 class TextBase(Text):
-    def __init__(self, id: str, sort_key: str, data_type: DataType = DataType.SENTENCE) -> None:
+    def __init__(self, id: str, sort_key: str, data_type: TextRowContentType = TextRowContentType.SEGMENT) -> None:
         self._id = id
         self._sort_key = sort_key
         self._data_type = data_type
@@ -20,7 +20,7 @@ def sort_key(self) -> str:
         return self._sort_key
 
     @property
-    def data_type(self) -> DataType:
+    def data_type(self) -> TextRowContentType:
         return self._data_type
 
     def _create_row(self, text: str, ref: Any, flags: TextRowFlags = TextRowFlags.SENTENCE_START) -> TextRow:
diff --git a/machine/corpora/text_corpus_enumerator.py b/machine/corpora/text_corpus_enumerator.py
@@ -2,9 +2,9 @@
 from typing import Any, ContextManager, Generator, List, Optional, Tuple, cast
 
 from ..scripture.verse_ref import Versification
-from .data_type import DataType
 from .scripture_ref import EMPTY_SCRIPTURE_REF, ScriptureRef
 from .text_row import TextRow, TextRowFlags
+from .text_row_content_type import TextRowContentType
 
 
 class TextCorpusEnumerator(ContextManager["TextCorpusEnumerator"], Generator[TextRow, None, None]):
@@ -78,10 +78,10 @@ def _collect_verses(self):
                     range_start_row.ref,
                     segment=list(range_start_row.segment) + list(row.segment),
                     flags=flags,
-                    data_type=DataType.SENTENCE,
+                    data_type=TextRowContentType.SEGMENT,
                 )
                 rows[range_start_offset] = range_start_ref, new_text_row
-                row = TextRow(row.text_id, row.ref, flags=TextRowFlags.IN_RANGE, data_type=DataType.SENTENCE)
+                row = TextRow(row.text_id, row.ref, flags=TextRowFlags.IN_RANGE, data_type=TextRowContentType.SEGMENT)
                 range_start_offset -= 1
             else:
                 range_start_offset = -1
diff --git a/machine/corpora/text_file_text.py b/machine/corpora/text_file_text.py
@@ -4,14 +4,14 @@
 
 from ..utils.string_utils import parse_integer
 from ..utils.typeshed import StrPath
-from .data_type import DataType
 from .multi_key_ref import MultiKeyRef
 from .text_base import TextBase
 from .text_row import TextRow, TextRowFlags
+from .text_row_content_type import TextRowContentType
 
 
 class TextFileText(TextBase):
-    def __init__(self, id: str, filename: StrPath, data_type: DataType = DataType.SENTENCE) -> None:
+    def __init__(self, id: str, filename: StrPath, data_type: TextRowContentType = TextRowContentType.SEGMENT) -> None:
         super().__init__(id, id, data_type)
         self._filename = Path(filename)
 
diff --git a/machine/corpora/text_file_text_corpus.py b/machine/corpora/text_file_text_corpus.py
@@ -3,17 +3,17 @@
 
 from ..utils.typeshed import StrPath
 from .corpora_utils import get_files
-from .data_type import DataType
 from .dictionary_text_corpus import DictionaryTextCorpus
 from .text_file_text import TextFileText
+from .text_row_content_type import TextRowContentType
 
 
 class TextFileTextCorpus(DictionaryTextCorpus):
     @overload
-    def __init__(self, file_patterns: Iterable[StrPath], data_types: Iterable[DataType] = []) -> None: ...
+    def __init__(self, file_patterns: Iterable[StrPath], data_types: Iterable[TextRowContentType] = []) -> None: ...
 
     @overload
-    def __init__(self, *file_patterns: StrPath, data_types: Iterable[DataType] = []) -> None: ...
+    def __init__(self, *file_patterns: StrPath, data_types: Iterable[TextRowContentType] = []) -> None: ...
 
     def __init__(self, *args, **kwargs) -> None:
         file_patterns: Iterable[str]
@@ -23,10 +23,12 @@ def __init__(self, *args, **kwargs) -> None:
             file_patterns = (str(p) for p in args)
         else:
             file_patterns = (str(p) for p in args[0])
-        data_types = list(cast(Iterable[DataType], kwargs.get("data_types", [])))
+        data_types = list(cast(Iterable[TextRowContentType], kwargs.get("data_types", [])))
         texts = (
             TextFileText(
-                id, filename, data_types[pattern_index] if pattern_index < len(data_types) else DataType.SENTENCE
+                id,
+                filename,
+                data_types[pattern_index] if pattern_index < len(data_types) else TextRowContentType.SEGMENT,
             )
             for id, filename, pattern_index in get_files(file_patterns)
         )
diff --git a/machine/corpora/text_row.py b/machine/corpora/text_row.py
@@ -1,7 +1,7 @@
 from enum import Flag, auto
 from typing import Any, Sequence
 
-from .data_type import DataType
+from .text_row_content_type import TextRowContentType
 
 
 class TextRowFlags(Flag):
@@ -18,7 +18,7 @@ def __init__(
         ref: Any,
         segment: Sequence[str] = [],
         flags: TextRowFlags = TextRowFlags.SENTENCE_START,
-        data_type: DataType = DataType.SENTENCE,
+        data_type: TextRowContentType = TextRowContentType.SEGMENT,
     ) -> None:
         self._text_id = text_id
         self._ref = ref
@@ -36,7 +36,7 @@ def ref(self) -> Any:
         return self._ref
 
     @property
-    def data_type(self) -> DataType:
+    def data_type(self) -> TextRowContentType:
         return self._data_type
 
     @property
diff --git a/machine/corpora/text_row_content_type.py b/machine/corpora/text_row_content_type.py
@@ -0,0 +1,6 @@
+from enum import Enum, auto
+
+
+class TextRowContentType(Enum):
+    WORD = auto()
+    SEGMENT = auto()
diff --git a/machine/jobs/translation_file_service.py b/machine/jobs/translation_file_service.py
@@ -4,9 +4,9 @@
 
 import json_stream
 
-from ..corpora.data_type import DataType
 from ..corpora.text_corpus import TextCorpus
 from ..corpora.text_file_text_corpus import TextFileTextCorpus
+from ..corpora.text_row_content_type import TextRowContentType
 from ..utils.context_managed_generator import ContextManagedGenerator
 from .shared_file_service_base import DictToJsonWriter, SharedFileServiceBase
 from .shared_file_service_factory import SharedFileServiceType, get_shared_file_service
@@ -54,7 +54,7 @@ def create_source_corpus(self) -> TextCorpus:
                     f"{self.shared_file_service.build_path}/{self._source_terms_filename}"
                 ),
             ],
-            data_types=[DataType.SENTENCE, DataType.GLOSS],
+            data_types=[TextRowContentType.SEGMENT, TextRowContentType.WORD],
         )
 
     def create_target_corpus(self) -> TextCorpus:
@@ -67,7 +67,7 @@ def create_target_corpus(self) -> TextCorpus:
                     f"{self.shared_file_service.build_path}/{self._target_terms_filename}"
                 ),
             ],
-            data_types=[DataType.SENTENCE, DataType.GLOSS],
+            data_types=[TextRowContentType.SEGMENT, TextRowContentType.WORD],
         )
 
     def exists_source_corpus(self) -> bool:
diff --git a/machine/translation/huggingface/hugging_face_nmt_model_trainer.py b/machine/translation/huggingface/hugging_face_nmt_model_trainer.py
diff --git a/tests/translation/huggingface/test_hugging_face_nmt_model_trainer.py b/tests/translation/huggingface/test_hugging_face_nmt_model_trainer.py
