Use Any for refs; make all_rows 'read-only'

Author: Enkidu93

machine/corpora/n_parallel_text_corpus.py

@@ -11,7 +11,7 @@
 
 
 class _RangeRow:
-    refs: List[object]
+    refs: List[Any]
     segment: List[str]
     is_sentence_start: bool = False
 
@@ -49,8 +49,8 @@ def add_text_row(self, row: TextRow, index: int):
         self.rows[index].segment.extend(row.segment)
 
     def create_row(self) -> NParallelTextRow:
-        refs: List[List[object]] = [[] for _ in range(self.n)]
-        reference_refs: List[object] = [r.refs[0] if len(r.refs) > 0 else None for r in self.rows if len(r.refs) > 0]
+        refs: List[List[Any]] = [[] for _ in range(self.n)]
+        reference_refs: List[Any] = [r.refs[0] if len(r.refs) > 0 else None for r in self.rows if len(r.refs) > 0]
         for i in range(len(self.rows)):
             row = self.rows[i]
 
@@ -76,12 +76,10 @@ def create_row(self) -> NParallelTextRow:
 
 
 class NParallelTextCorpus(NParallelTextCorpusBase):
-    def __init__(
-        self, corpora: Sequence[TextCorpus], row_ref_comparer: Optional[Callable[[object, object], int]] = None
-    ):
+    def __init__(self, corpora: Sequence[TextCorpus], row_ref_comparer: Optional[Callable[[Any, Any], int]] = None):
         self._corpora = corpora
         self._row_ref_comparer = row_ref_comparer if row_ref_comparer is not None else default_row_ref_comparer
-        self.all_rows: Sequence[bool] = tuple(False for _ in range(len(corpora)))
+        self._all_rows: Sequence[bool] = [False for _ in range(len(corpora))]
 
     def is_tokenized(self, i: int) -> bool:
         return self.corpora[i].is_tokenized
@@ -95,9 +93,13 @@ def corpora(self) -> Sequence[TextCorpus]:
         return list(self._corpora)
 
     @property
-    def row_ref_comparer(self) -> Callable[[object, object], int]:
+    def row_ref_comparer(self) -> Callable[[Any, Any], int]:
         return self._row_ref_comparer
 
+    @property
+    def all_rows(self) -> Sequence[bool]:
+        return self._all_rows
+
     def _get_text_ids_from_corpora(self) -> Set[str]:
         text_ids: Set[str] = set()
         all_rows_text_ids: Set[str] = set()
@@ -107,7 +109,7 @@ def _get_text_ids_from_corpora(self) -> Set[str]:
             else:
                 text_ids = text_ids.intersection({text.id for text in self.corpora[i].texts})
 
-            if self.all_rows[i]:
+            if self._all_rows[i]:
                 all_rows_text_ids = all_rows_text_ids.union({text.id for text in self.corpora[i].texts})
         text_ids = text_ids.union(all_rows_text_ids)
         return text_ids
@@ -129,7 +131,7 @@ def get_rows(self, text_ids: Optional[Iterable[str]] = None) -> Iterable[NParall
     def _all_ranges_are_new_ranges(rows: List[TextRow]):
         return all([True if row.is_range_start or not row.is_in_range else False for row in rows])
 
-    def _min_ref_indexes(self, refs: Sequence[object]) -> Sequence[int]:
+    def _min_ref_indexes(self, refs: Sequence[Any]) -> Sequence[int]:
         min_ref = refs[0]
         min_ref_indexes = [0]
         for i in range(len(refs)):
@@ -181,7 +183,7 @@ def _get_rows(self, generators: List[TextCorpusEnumerator]) -> Iterable[NParalle
                     or len([i for i in min_ref_indexes if not completed[i]]) == 1
                 ):
                     # then there are some non-min refs or only one incomplete generator
-                    if any([not self.all_rows[i] for i in non_min_ref_indexes]) and any(
+                    if any([not self._all_rows[i] for i in non_min_ref_indexes]) and any(
                         [not completed[i] and current_rows[i].is_in_range for i in min_ref_indexes]
                     ):
                         # At least one of the non-min rows has not been marked as 'all rows'
@@ -231,7 +233,7 @@ def _get_rows(self, generators: List[TextCorpusEnumerator]) -> Iterable[NParalle
                     if any(
                         [
                             current_rows[i].is_in_range
-                            and all([j == i or not self.all_rows[j] for j in min_ref_indexes])
+                            and all([j == i or not self._all_rows[j] for j in min_ref_indexes])
                             for i in min_ref_indexes
                         ]
                     ):
@@ -271,7 +273,7 @@ def _get_rows(self, generators: List[TextCorpusEnumerator]) -> Iterable[NParalle
             if range_info.is_in_range:
                 yield range_info.create_row()
 
-    def _correct_versification(self, refs: List[object], i: int) -> List[object]:
+    def _correct_versification(self, refs: List[Any], i: int) -> List[Any]:
         if any([not c.is_scripture for c in self.corpora]) or len(refs) == 0:
             return refs
         return [
@@ -288,7 +290,7 @@ def _create_rows(
         default_refs = [[r.ref for r in rows if r is not None][0]]
 
         text_id: Optional[str] = None
-        refs: List[List[object]] = []
+        refs: List[List[Any]] = []
         flags: List[TextRowFlags] = []
         for i in range(self.n):
             refs.append([])
@@ -342,7 +344,7 @@ def _create_min_ref_rows(
         text_rows = [None for _ in range(self.n)]
         rows_have_content = False
         for i in min_ref_indexes:
-            if not self.all_rows[i] or i in already_yielded:
+            if not self._all_rows[i] or i in already_yielded:
                 continue
             text_row = current_rows[i]
             text_rows[i] = text_row

machine/corpora/n_parallel_text_row.py

@@ -1,10 +1,10 @@
-from typing import Sequence
+from typing import Any, Sequence
 
 from .text_row import TextRowFlags
 
 
 class NParallelTextRow:
-    def __init__(self, text_id: str, n_refs: Sequence[Sequence[object]]):
+    def __init__(self, text_id: str, n_refs: Sequence[Sequence[Any]]):
         if len([n_ref for n_ref in n_refs if n_ref is not None and len(n_ref) > 0]) == 0:
             raise ValueError(f"Refs must be provided but n_refs={n_refs}")
         self._text_id = text_id
@@ -18,11 +18,11 @@ def text_id(self) -> str:
         return self._text_id
 
     @property
-    def ref(self) -> object:
+    def ref(self) -> Any:
         return self._n_refs[0][0]
 
     @property
-    def n_refs(self) -> Sequence[Sequence[object]]:
+    def n_refs(self) -> Sequence[Sequence[Any]]:
         return self._n_refs
 
     def is_sentence_start(self, i: int) -> bool:

machine/corpora/standard_parallel_text_corpus.py

@@ -27,7 +27,7 @@ def __init__(
         self._all_source_rows = all_source_rows
         self._all_target_rows = all_target_rows
         self._n_parallel_text_corpus = NParallelTextCorpus([source_corpus, target_corpus])
-        self._n_parallel_text_corpus.all_rows = [self._all_source_rows, self.all_target_rows]
+        self._n_parallel_text_corpus._all_rows = [self._all_source_rows, self.all_target_rows]
         self._row_ref_comparer = row_ref_comparer or default_row_ref_comparer
 
     @property

tests/corpora/test_n_parallel_text_corpus.py

@@ -115,7 +115,7 @@ def test_get_rows_three_corpora_missing_rows_all_all_rows():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1, corpus2, corpus3])
-    n_parallel_corpus.all_rows = [True, True, True]
+    n_parallel_corpus._all_rows = [True, True, True]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 3
     assert all([r[0] == 3 for r in rows[2].n_refs])
@@ -152,7 +152,7 @@ def test_get_rows_three_corpora_missing_rows_some_all_rows():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1, corpus2, corpus3])
-    n_parallel_corpus.all_rows = [True, False, True]
+    n_parallel_corpus._all_rows = [True, False, True]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 2
     assert all([r[0] == 3 for r in rows[1].n_refs])
@@ -192,7 +192,7 @@ def test_get_rows_three_corpora_missing_rows_all_all_rows_missing_middle():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1, corpus2, corpus3])
-    n_parallel_corpus.all_rows = [True, True, True]
+    n_parallel_corpus._all_rows = [True, True, True]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 3
     assert all([len(r) == 0 or r[0] == 2 for r in rows[1].n_refs])
@@ -228,7 +228,7 @@ def test_get_rows_three_corpora_missing_rows_missing_last_rows():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1, corpus2, corpus3])
-    n_parallel_corpus.all_rows = [True, False, False]
+    n_parallel_corpus._all_rows = [True, False, False]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 3
     assert all([r[0] == 2 for r in rows[1].n_refs])
@@ -247,7 +247,7 @@ def test_get_rows_three_corpora_one_corpus():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1])
-    n_parallel_corpus.all_rows = [True]
+    n_parallel_corpus._all_rows = [True]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 2
     assert all([r[0] == 1 for r in rows[0].n_refs])
@@ -389,7 +389,7 @@ def test_get_rows_three_corpora_overlapping_ranges_all_individual_rows():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1, corpus2, corpus3])
-    n_parallel_corpus.all_rows = [False, False, True]
+    n_parallel_corpus._all_rows = [False, False, True]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 3
     assert rows[0].n_refs[0] == [1]
@@ -437,7 +437,7 @@ def test_get_rows_three_corpora_overlapping_ranges_all_one_through_two_rows():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1, corpus2, corpus3])
-    n_parallel_corpus.all_rows = [False, True, False]
+    n_parallel_corpus._all_rows = [False, True, False]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 2
     assert rows[0].n_refs[0] == [1, 2]
@@ -485,7 +485,7 @@ def test_get_rows_three_corpora_overlapping_ranges_all_two_through_three_rows():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1, corpus2, corpus3])
-    n_parallel_corpus.all_rows = [True, False, False]
+    n_parallel_corpus._all_rows = [True, False, False]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 2
     assert rows[0].n_refs[0] == [1]
@@ -559,7 +559,7 @@ def test_get_rows_three_corpora_same_ref_corpora_of_different_sizes():
         )
     )
     n_parallel_corpus = NParallelTextCorpus([corpus1, corpus2, corpus3])
-    n_parallel_corpus.all_rows = [True, True, True]
+    n_parallel_corpus._all_rows = [True, True, True]
     rows = list(n_parallel_corpus.get_rows())
     assert len(rows) == 4
     assert rows[0].n_refs[1] == [1]