add test case for chapter filtering

Author: mshannon-sil

machine/corpora/__init__.py

@@ -27,7 +27,7 @@
 from .paratext_project_settings import ParatextProjectSettings
 from .paratext_project_settings_parser_base import ParatextProjectSettingsParserBase
 from .paratext_project_terms_parser_base import KeyTerm, ParatextProjectTermsParserBase
-from .paratext_project_text_updater_base import ParatextProjectTextUpdaterBase
+from .paratext_project_text_updater_base import ParatextProjectTextUpdaterBase, filter_tokens_by_chapter
 from .paratext_project_versification_error_detector_base import ParatextProjectVersificationErrorDetectorBase
 from .paratext_text_corpus import ParatextTextCorpus
 from .place_markers_usfm_update_block_handler import PlaceMarkersAlignmentInfo, PlaceMarkersUsfmUpdateBlockHandler

machine/corpora/paratext_project_text_updater_base.py

@@ -31,8 +31,8 @@ def __init__(
     def update_usfm(
         self,
         book_id: str,
-        chapters: Optional[Sequence[int]] = None,
         rows: Optional[Sequence[UpdateUsfmRow]] = None,
+        chapters: Optional[Sequence[int]] = None,
         full_name: Optional[str] = None,
         text_behavior: UpdateUsfmTextBehavior = UpdateUsfmTextBehavior.PREFER_EXISTING,
         paragraph_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.PRESERVE,
@@ -65,7 +65,7 @@ def update_usfm(
         try:
             tokenizer = UsfmTokenizer(self._settings.stylesheet)
             tokens = tokenizer.tokenize(usfm)
-            tokens = self.filter_tokens_by_chapter(tokens, chapters)
+            tokens = filter_tokens_by_chapter(tokens, chapters)
             parser = UsfmParser(tokens, handler, self._settings.stylesheet, self._settings.versification)
             parser.process_tokens()
             return handler.get_usfm(self._settings.stylesheet)
@@ -77,24 +77,25 @@ def update_usfm(
             )
             raise RuntimeError(error_message) from e
 
-    def filter_tokens_by_chapter(
-        self, tokens: Sequence[UsfmToken], chapters: Optional[Sequence[int]] = None
-    ) -> Sequence[UsfmToken]:
-        if chapters is None:
-            return tokens
-        tokens_within_chapters: List[UsfmToken] = []
-        in_chapter: bool = False
-        for index, token in enumerate(tokens):
-            if index == 0 and token.marker == "id":
-                tokens_within_chapters.append(token)
-                if 1 in chapters:
-                    in_chapter = True
-            elif token.type == UsfmTokenType.CHAPTER:
-                if token.data and int(token.data) in chapters:
-                    in_chapter = True
-                    tokens_within_chapters.append(token)
-                else:
-                    in_chapter = False
-            elif in_chapter:
+
+def filter_tokens_by_chapter(
+    tokens: Sequence[UsfmToken], chapters: Optional[Sequence[int]] = None
+) -> Sequence[UsfmToken]:
+    if chapters is None:
+        return tokens
+    tokens_within_chapters: List[UsfmToken] = []
+    in_chapter: bool = False
+    for index, token in enumerate(tokens):
+        if index == 0 and token.marker == "id":
+            tokens_within_chapters.append(token)
+            if 1 in chapters:
+                in_chapter = True
+        elif token.type == UsfmTokenType.CHAPTER:
+            if token.data and int(token.data) in chapters:
+                in_chapter = True
                 tokens_within_chapters.append(token)
-        return tokens_within_chapters
+            else:
+                in_chapter = False
+        elif in_chapter:
+            tokens_within_chapters.append(token)
+    return tokens_within_chapters

tests/corpora/test_update_usfm_parser_handler.py

@@ -9,10 +9,12 @@
     UpdateUsfmParserHandler,
     UpdateUsfmRow,
     UpdateUsfmTextBehavior,
+    UsfmParser,
+    UsfmTokenizer,
     UsfmUpdateBlock,
     UsfmUpdateBlockElementType,
     UsfmUpdateBlockHandler,
-    parse_usfm,
+    filter_tokens_by_chapter,
 )
 
 
@@ -1494,13 +1496,39 @@ def test_update_block_footnote_at_start_of_chapter_with_preceding_text():
     )
 
 
+def test_filter_chapters() -> None:
+    usfm = r"""\id MAT - Test
+\h Matthew
+\c 1
+\v 1 Some text
+\v 2
+\v 3 Other text
+\c 2
+\v 1 Some text
+\c 3
+\v 1 Some text
+\c 4
+\v 1 Some text
+"""
+    chapters = [2, 4]
+    target = update_usfm(chapters=chapters, source=usfm)
+    result = r"""\id MAT
+\c 2
+\v 1 Some text
+\c 4
+\v 1 Some text
+"""
+    assert_usfm_equals(target, result)
+
+
 def scr_ref(*refs: str) -> List[ScriptureRef]:
     return [ScriptureRef.parse(ref) for ref in refs]
 
 
 def update_usfm(
     rows: Optional[Sequence[UpdateUsfmRow]] = None,
     source: Optional[str] = None,
+    chapters: Optional[Sequence[int]] = None,
     id_text: Optional[str] = None,
     text_behavior: UpdateUsfmTextBehavior = UpdateUsfmTextBehavior.PREFER_NEW,
     paragraph_behavior: UpdateUsfmMarkerBehavior = UpdateUsfmMarkerBehavior.PRESERVE,
@@ -1516,6 +1544,7 @@ def update_usfm(
         return updater.update_usfm(
             "MAT",
             rows,
+            chapters,
             id_text,
             text_behavior,
             paragraph_behavior,
@@ -1542,7 +1571,11 @@ def update_usfm(
             lambda _: False,
             compare_segments,
         )
-        parse_usfm(source, updater)
+        tokenizer = UsfmTokenizer()
+        tokens = tokenizer.tokenize(source)
+        tokens = filter_tokens_by_chapter(tokens, chapters)
+        parser = UsfmParser(tokens, updater)
+        parser.process_tokens()
         return updater.get_usfm()