diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py index 24634ac32..eadb4f430 100644 --- a/crawl4ai/content_filter_strategy.py +++ b/crawl4ai/content_filter_strategy.py @@ -651,6 +651,40 @@ def __init__( "h6": 0.7, } + # Phrasing elements keep the parent block's decision. Expose this on + # each filter so callers can customize which tags inherit that rule. + self.inline_tags = frozenset( + { + "a", + "abbr", + "b", + "bdi", + "bdo", + "cite", + "code", + "data", + "del", + "dfn", + "em", + "i", + "ins", + "kbd", + "mark", + "q", + "ruby", + "s", + "samp", + "small", + "span", + "strong", + "sub", + "sup", + "time", + "u", + "var", + } + ) + def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]: """ Implements content filtering using pruning algorithm with dynamic threshold. @@ -726,47 +760,51 @@ def _prune_tree(self, node): if self._is_preserved(node): return - text_len = len(node.get_text(strip=True)) - tag_len = len(node.encode_contents().decode("utf-8")) - link_text_len = sum( - len(s.strip()) - for s in (a.string for a in node.find_all("a", recursive=False)) - if s - ) - - metrics = { - "node": node, - "tag_name": node.name, - "text_len": text_len, - "tag_len": tag_len, - "link_text_len": link_text_len, - } - - score = self._compute_composite_score(metrics, text_len, tag_len, link_text_len) + if node.name not in self.inline_tags: + text_len = len(node.get_text(strip=True)) + tag_len = len(node.encode_contents().decode("utf-8")) + link_text_len = sum( + len(s.strip()) + for s in (a.string for a in node.find_all("a", recursive=False)) + if s + ) - if self.threshold_type == "fixed": - should_remove = score < self.threshold - else: # dynamic - tag_importance = self.tag_importance.get(node.name, 0.7) - text_ratio = text_len / tag_len if tag_len > 0 else 0 - link_ratio = link_text_len / text_len if text_len > 0 else 1 + metrics = { + "node": node, + "tag_name": node.name, + "text_len": text_len, + "tag_len": tag_len, + "link_text_len": link_text_len, + } - threshold = self.threshold # base threshold - if tag_importance > 1: - threshold *= 0.8 - if text_ratio > 0.4: - threshold *= 0.9 - if link_ratio > 0.6: - threshold *= 1.2 + score = self._compute_composite_score( + metrics, text_len, tag_len, link_text_len + ) - should_remove = score < threshold + if self.threshold_type == "fixed": + should_remove = score < self.threshold + else: # dynamic + tag_importance = self.tag_importance.get(node.name, 0.7) + text_ratio = text_len / tag_len if tag_len > 0 else 0 + link_ratio = link_text_len / text_len if text_len > 0 else 1 + + threshold = self.threshold # base threshold + if tag_importance > 1: + threshold *= 0.8 + if text_ratio > 0.4: + threshold *= 0.9 + if link_ratio > 0.6: + threshold *= 1.2 + + should_remove = score < threshold + + if should_remove: + node.decompose() + return - if should_remove: - node.decompose() - else: - children = [child for child in node.children if hasattr(child, "name")] - for child in children: - self._prune_tree(child) + children = [child for child in node.children if hasattr(child, "name")] + for child in children: + self._prune_tree(child) def _compute_composite_score(self, metrics, text_len, tag_len, link_text_len): """Computes the composite score""" diff --git a/crawl4ai/content_filter_strategy_lxml.py b/crawl4ai/content_filter_strategy_lxml.py index 24671945b..f7a4eba86 100644 --- a/crawl4ai/content_filter_strategy_lxml.py +++ b/crawl4ai/content_filter_strategy_lxml.py @@ -313,6 +313,9 @@ def _outer_len(self, el, inner_len: int) -> int: # Scoring (top-down pass) # ------------------------------------------------------------------ # def _should_remove(self, node, m: _NodeMetrics, metrics: Dict[object, _NodeMetrics]) -> bool: + if node.tag in self.inline_tags: + return False + text_len = m.text_len tag_len = m.inner_len diff --git a/tests/async/test_content_filter_prune.py b/tests/async/test_content_filter_prune.py index 1f75a9e1e..f17ffd541 100644 --- a/tests/async/test_content_filter_prune.py +++ b/tests/async/test_content_filter_prune.py @@ -5,6 +5,7 @@ sys.path.append(parent_dir) from crawl4ai.content_filter_strategy import PruningContentFilter +from crawl4ai.content_filter_strategy_lxml import PruningContentFilterLXML @pytest.fixture @@ -63,7 +64,34 @@ def mixed_content_html(): """ +@pytest.fixture +def nested_inline_html(): + return """
+

After the Meiji Restoration, the imperial capital in Kyoto was moved to Edo, and the city was renamed Tokyo + (lit. ' + Eastern Capital'). + The city was greatly damaged by the 1923 earthquake and rebuilt in the following decades.

+

Tokyo (東京) is the capital of Japan + and one of the most populous cities in the world, with a population of about fourteen million.

+
""" + + class TestPruningContentFilter: + @pytest.mark.parametrize( + "filter_class", + [PruningContentFilter, PruningContentFilterLXML], + ids=["bs4", "lxml"], + ) + @pytest.mark.filterwarnings("ignore::DeprecationWarning") + def test_pruning_preserves_nested_inline_content_in_kept_paragraphs( + self, nested_inline_html, filter_class + ): + contents = filter_class().filter_content(nested_inline_html) + actual = " ".join(contents) + + assert "Eastern Capital" in actual + assert "東京" in actual + def test_basic_pruning(self, basic_html): """Test basic content pruning functionality""" filter = PruningContentFilter(min_word_threshold=5)