move filtering before token processing

Author: mshannon-sil

machine/corpora/paratext_project_text_updater_base.py

@@ -1,5 +1,5 @@
 from abc import ABC
-from typing import Callable, Iterable, Optional, Sequence, Union
+from typing import Callable, Iterable, List, Optional, Sequence, Union
 
 from .paratext_project_file_handler import ParatextProjectFileHandler
 from .paratext_project_settings import ParatextProjectSettings
@@ -10,7 +10,9 @@
     UpdateUsfmRow,
     UpdateUsfmTextBehavior,
 )
-from .usfm_parser import parse_usfm
+from .usfm_parser import UsfmParser
+from .usfm_token import UsfmTokenType
+from .usfm_tokenizer import UsfmToken, UsfmTokenizer
 from .usfm_update_block_handler import UsfmUpdateBlockHandler, UsfmUpdateBlockHandlerError
 
 
@@ -61,12 +63,38 @@ def update_usfm(
             compare_segments=compare_segments,
         )
         try:
-            parse_usfm(usfm, handler, self._settings.stylesheet, self._settings.versification)
-            return handler.get_usfm(self._settings.stylesheet, chapters)
+            tokenizer = UsfmTokenizer(self._settings.stylesheet)
+            tokens = tokenizer.tokenize(usfm)
+            tokens = self.filter_tokens_by_chapter(tokens, chapters)
+            parser = UsfmParser(tokens, handler, self._settings.stylesheet, self._settings.versification)
+            parser.process_tokens()
+            return handler.get_usfm(self._settings.stylesheet)
         except Exception as e:
             error_message = (
                 f"An error occurred while parsing the usfm for '{book_id}'"
                 f"{f' in project {self._settings.name}' if self._settings.name else ''}"
                 f". Error: '{e}'"
             )
             raise RuntimeError(error_message) from e
+
+    def filter_tokens_by_chapter(
+        self, tokens: Sequence[UsfmToken], chapters: Optional[Sequence[int]] = None
+    ) -> Sequence[UsfmToken]:
+        if chapters is None:
+            return tokens
+        tokens_within_chapters: List[UsfmToken] = []
+        in_chapter: bool = False
+        for index, token in enumerate(tokens):
+            if index == 0 and token.marker == "id":
+                tokens_within_chapters.append(token)
+                if 1 in chapters:
+                    in_chapter = True
+            elif token.type == UsfmTokenType.CHAPTER:
+                if token.data and int(token.data) in chapters:
+                    in_chapter = True
+                    tokens_within_chapters.append(token)
+                else:
+                    in_chapter = False
+            elif in_chapter:
+                tokens_within_chapters.append(token)
+        return tokens_within_chapters

machine/corpora/update_usfm_parser_handler.py

@@ -334,15 +334,11 @@ def _end_embed_text(self, state: UsfmParserState, scripture_ref: ScriptureRef) -
         if embed_outside_of_block:
             self._end_update_block(state, [scripture_ref])
 
-    def get_usfm(
-        self, stylesheet: Union[str, UsfmStylesheet] = "usfm.sty", chapters: Optional[Sequence[int]] = None
-    ) -> str:
+    def get_usfm(self, stylesheet: Union[str, UsfmStylesheet] = "usfm.sty") -> str:
         if isinstance(stylesheet, str):
             stylesheet = UsfmStylesheet(stylesheet)
         tokenizer = UsfmTokenizer(stylesheet)
         tokens = list(self._tokens)
-        if chapters is not None:
-            tokens = self._get_incremental_draft_tokens(tokens, chapters)
         if len(self._remarks) > 0:
             remark_tokens: List[UsfmToken] = []
             for remark in self._remarks:
@@ -354,23 +350,6 @@ def get_usfm(
                         tokens[index + 1 : index + 1] = remark_tokens
         return tokenizer.detokenize(tokens)
 
-    def _get_incremental_draft_tokens(self, tokens: List[UsfmToken], chapters: Sequence[int]) -> List[UsfmToken]:
-        incremental_draft_tokens: List[UsfmToken] = []
-        in_chapter: bool = False
-        for index, token in enumerate(tokens):
-            if index == 0 and token.marker == "id":
-                incremental_draft_tokens.append(token)
-                continue
-            elif token.type == UsfmTokenType.CHAPTER:
-                if token.data and int(token.data) in chapters:
-                    in_chapter = True
-                    incremental_draft_tokens.append(token)
-                else:
-                    in_chapter = False
-            elif in_chapter:
-                incremental_draft_tokens.append(token)
-        return incremental_draft_tokens
-
     def _advance_rows(self, seg_scr_refs: Sequence[ScriptureRef]) -> Tuple[List[str], Optional[dict[str, object]]]:
         row_texts: List[str] = []
         row_metadata = None