Port NParallelText corpus (#247)

Author: Enkidu93

machine/corpora/__init__.py

@@ -16,6 +16,8 @@
 from .memory_stream_container import MemoryStreamContainer
 from .memory_text import MemoryText
 from .multi_key_ref import MultiKeyRef
+from .n_parallel_text_corpus import NParallelTextCorpus
+from .n_parallel_text_row import NParallelTextRow
 from .parallel_text_corpus import ParallelTextCorpus
 from .parallel_text_row import ParallelTextRow
 from .paratext_backup_terms_corpus import ParatextBackupTermsCorpus
@@ -121,6 +123,8 @@
     "nfkc_normalize",
     "nfkd_normalize",
     "normalize",
+    "NParallelTextCorpus",
+    "NParallelTextRow",
     "ParallelTextCorpus",
     "ParallelTextRow",
     "ParatextBackupTermsCorpus",

machine/corpora/corpus.py

@@ -5,10 +5,11 @@
 from ..utils.context_managed_generator import ContextManagedGenerator
 from .alignment_row import AlignmentRow
 from .corpora_utils import batch, get_split_indices
+from .n_parallel_text_row import NParallelTextRow
 from .parallel_text_row import ParallelTextRow
 from .text_row import TextRow
 
-Row = TypeVar("Row", TextRow, ParallelTextRow, AlignmentRow)
+Row = TypeVar("Row", TextRow, ParallelTextRow, AlignmentRow, NParallelTextRow)
 Item = TypeVar("Item")
 
 

machine/corpora/n_parallel_text_corpus.py

@@ -0,0 +1,20 @@
+from abc import ABC, abstractmethod
+from typing import Iterable, Sequence
+
+from .corpus import Corpus
+from .n_parallel_text_row import NParallelTextRow
+from .text_corpus import TextCorpus
+
+
+class NParallelTextCorpusBase(Corpus[NParallelTextRow], ABC):
+
+    @property
+    @abstractmethod
+    def n(self) -> int: ...
+
+    @property
+    @abstractmethod
+    def corpora(self) -> Sequence[TextCorpus]: ...
+
+    @abstractmethod
+    def get_rows(self, text_ids: Iterable[str]) -> Sequence[NParallelTextRow]: ...

machine/corpora/n_parallel_text_corpus_base.py

@@ -0,0 +1,47 @@
+from typing import Any, Sequence
+
+from .text_row import TextRowFlags
+
+
+class NParallelTextRow:
+    def __init__(self, text_id: str, n_refs: Sequence[Sequence[Any]]):
+        if len([n_ref for n_ref in n_refs if n_ref is not None and len(n_ref) > 0]) == 0:
+            raise ValueError(f"Refs must be provided but n_refs={n_refs}")
+        self._text_id = text_id
+        self._n_refs = n_refs
+        self._n = len(n_refs)
+        self.n_segments: Sequence[Sequence[str]] = [[] for _ in range(0, self._n)]
+        self.n_flags: Sequence[TextRowFlags] = [TextRowFlags.SENTENCE_START for _ in range(0, self._n)]
+
+    @property
+    def text_id(self) -> str:
+        return self._text_id
+
+    @property
+    def ref(self) -> Any:
+        return self._n_refs[0][0]
+
+    @property
+    def n_refs(self) -> Sequence[Sequence[Any]]:
+        return self._n_refs
+
+    def is_sentence_start(self, i: int) -> bool:
+        return TextRowFlags.SENTENCE_START in self.n_flags[i]
+
+    def is_in_range(self, i: int) -> bool:
+        return TextRowFlags.IN_RANGE in self.n_flags[i]
+
+    def is_range_start(self, i: int) -> bool:
+        return TextRowFlags.RANGE_START in self.n_flags[i]
+
+    @property
+    def is_empty(self):
+        return sum([1 for s in self.n_segments if len(s) == 0]) == 0
+
+    def text(self, i: int) -> str:
+        return " ".join(self.n_segments[i])
+
+    def invert(self) -> "NParallelTextRow":
+        inverted_row = NParallelTextRow(self._text_id, list(reversed(self._n_refs)))
+        inverted_row.n_flags = list(reversed(self.n_flags))
+        return inverted_row