fix(content filter): preserve inline text in retained paragraphs

crawl4ai/content_filter_strategy.py

@@ -651,6 +651,40 @@ def __init__(
             "h6": 0.7,
         }
 
+        # Phrasing elements keep the parent block's decision. Expose this on
+        # each filter so callers can customize which tags inherit that rule.
+        self.inline_tags = frozenset(
+            {
+                "a",
+                "abbr",
+                "b",
+                "bdi",
+                "bdo",
+                "cite",
+                "code",
+                "data",
+                "del",
+                "dfn",
+                "em",
+                "i",
+                "ins",
+                "kbd",
+                "mark",
+                "q",
+                "ruby",
+                "s",
+                "samp",
+                "small",
+                "span",
+                "strong",
+                "sub",
+                "sup",
+                "time",
+                "u",
+                "var",
+            }
+        )
+
     def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]:
         """
         Implements content filtering using pruning algorithm with dynamic threshold.
@@ -726,47 +760,51 @@ def _prune_tree(self, node):
         if self._is_preserved(node):
             return
 
-        text_len = len(node.get_text(strip=True))
-        tag_len = len(node.encode_contents().decode("utf-8"))
-        link_text_len = sum(
-            len(s.strip())
-            for s in (a.string for a in node.find_all("a", recursive=False))
-            if s
-        )
-
-        metrics = {
-            "node": node,
-            "tag_name": node.name,
-            "text_len": text_len,
-            "tag_len": tag_len,
-            "link_text_len": link_text_len,
-        }
-
-        score = self._compute_composite_score(metrics, text_len, tag_len, link_text_len)
+        if node.name not in self.inline_tags:
+            text_len = len(node.get_text(strip=True))
+            tag_len = len(node.encode_contents().decode("utf-8"))
+            link_text_len = sum(
+                len(s.strip())
+                for s in (a.string for a in node.find_all("a", recursive=False))
+                if s
+            )
 
-        if self.threshold_type == "fixed":
-            should_remove = score < self.threshold
-        else:  # dynamic
-            tag_importance = self.tag_importance.get(node.name, 0.7)
-            text_ratio = text_len / tag_len if tag_len > 0 else 0
-            link_ratio = link_text_len / text_len if text_len > 0 else 1
+            metrics = {
+                "node": node,
+                "tag_name": node.name,
+                "text_len": text_len,
+                "tag_len": tag_len,
+                "link_text_len": link_text_len,
+            }
 
-            threshold = self.threshold  # base threshold
-            if tag_importance > 1:
-                threshold *= 0.8
-            if text_ratio > 0.4:
-                threshold *= 0.9
-            if link_ratio > 0.6:
-                threshold *= 1.2
+            score = self._compute_composite_score(
+                metrics, text_len, tag_len, link_text_len
+            )
 
-            should_remove = score < threshold
+            if self.threshold_type == "fixed":
+                should_remove = score < self.threshold
+            else:  # dynamic
+                tag_importance = self.tag_importance.get(node.name, 0.7)
+                text_ratio = text_len / tag_len if tag_len > 0 else 0
+                link_ratio = link_text_len / text_len if text_len > 0 else 1
+
+                threshold = self.threshold  # base threshold
+                if tag_importance > 1:
+                    threshold *= 0.8
+                if text_ratio > 0.4:
+                    threshold *= 0.9
+                if link_ratio > 0.6:
+                    threshold *= 1.2
+
+                should_remove = score < threshold
+
+            if should_remove:
+                node.decompose()
+                return
 
-        if should_remove:
-            node.decompose()
-        else:
-            children = [child for child in node.children if hasattr(child, "name")]
-            for child in children:
-                self._prune_tree(child)
+        children = [child for child in node.children if hasattr(child, "name")]
+        for child in children:
+            self._prune_tree(child)
 
     def _compute_composite_score(self, metrics, text_len, tag_len, link_text_len):
         """Computes the composite score"""

crawl4ai/content_filter_strategy_lxml.py

@@ -313,6 +313,9 @@ def _outer_len(self, el, inner_len: int) -> int:
     # Scoring (top-down pass)
     # ------------------------------------------------------------------ #
     def _should_remove(self, node, m: _NodeMetrics, metrics: Dict[object, _NodeMetrics]) -> bool:
+        if node.tag in self.inline_tags:
+            return False
+
         text_len = m.text_len
         tag_len = m.inner_len
 

tests/async/test_content_filter_prune.py

@@ -5,6 +5,7 @@
 sys.path.append(parent_dir)
 
 from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy_lxml import PruningContentFilterLXML
 
 
 @pytest.fixture
@@ -63,7 +64,34 @@ def mixed_content_html():
     """
 
 
+@pytest.fixture
+def nested_inline_html():
+    return """<html><body><article>
+    <p>After the Meiji Restoration, the imperial capital in Kyoto was moved to Edo, and the city was renamed Tokyo
+    (<abbr title="literal translation">lit.</abbr> <span class="gloss-quot">'</span><span class="gloss-text">
+    <a href="/wiki/Capital_of_Japan">Eastern Capital</a></span><span class="gloss-quot">'</span>).
+    The city was greatly damaged by the 1923 earthquake and rebuilt in the following decades.</p>
+    <p>Tokyo (<span title="Japanese-language text"><span lang="ja">東京</span></span>) is the capital of Japan
+    and one of the most populous cities in the world, with a population of about fourteen million.</p>
+    </article></body></html>"""
+
+
 class TestPruningContentFilter:
+    @pytest.mark.parametrize(
+        "filter_class",
+        [PruningContentFilter, PruningContentFilterLXML],
+        ids=["bs4", "lxml"],
+    )
+    @pytest.mark.filterwarnings("ignore::DeprecationWarning")
+    def test_pruning_preserves_nested_inline_content_in_kept_paragraphs(
+        self, nested_inline_html, filter_class
+    ):
+        contents = filter_class().filter_content(nested_inline_html)
+        actual = " ".join(contents)
+
+        assert "Eastern Capital" in actual
+        assert "東京" in actual
+
     def test_basic_pruning(self, basic_html):
         """Test basic content pruning functionality"""
         filter = PruningContentFilter(min_word_threshold=5)