+ After the Meiji Restoration, the imperial capital in Kyoto was moved to Edo, and the city was renamed Tokyo
+ (lit. '
+ Eastern Capital').
+ The city was greatly damaged by the 1923 earthquake and rebuilt in the following decades.
+ Tokyo (東京) is the capital of Japan
+ and one of the most populous cities in the world, with a population of about fourteen million.
+ """
+
+
class TestPruningContentFilter:
+ @pytest.mark.filterwarnings("ignore::DeprecationWarning")
+ def test_pruning_preserves_nested_inline_content_in_kept_paragraphs(
+ self, nested_inline_html
+ ):
+ contents = PruningContentFilter().filter_content(nested_inline_html)
+ actual = " ".join(contents)
+
+ assert "Eastern Capital" in actual
+ assert "東京" in actual
+
+ def test_lxml_pruning_preserves_nested_inline_content_in_kept_paragraphs(
+ self, nested_inline_html
+ ):
+ contents = PruningContentFilterLXML().filter_content(nested_inline_html)
+ actual = " ".join(contents)
+
+ assert "Eastern Capital" in actual
+ assert "東京" in actual
+
def test_basic_pruning(self, basic_html):
"""Test basic content pruning functionality"""
filter = PruningContentFilter(min_word_threshold=5)
From 40927b95eab50610d1c5776a35f9b1e5304ff091 Mon Sep 17 00:00:00 2001
From: Ha1baraA11 <243105435+Ha1baraA11@users.noreply.github.com>
Date: Thu, 22 Oct 2026 22:03:25 +0800
Subject: [PATCH 2/2] fix(content filter): narrow inline pruning tags
---
crawl4ai/content_filter_strategy.py | 182 ++++++++++-------------
crawl4ai/content_filter_strategy_lxml.py | 8 +-
tests/async/test_content_filter_prune.py | 18 +--
3 files changed, 85 insertions(+), 123 deletions(-)
diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py
index f6c5405f7..eadb4f430 100644
--- a/crawl4ai/content_filter_strategy.py
+++ b/crawl4ai/content_filter_strategy.py
@@ -31,67 +31,6 @@
from .async_logger import AsyncLogger, LogLevel, LogColor
-# Phrasing elements carry inline meaning within a block. Their text density is
-# not useful on its own: nested tags can make a short span look like noise even
-# when the containing paragraph is retained.
-_PRUNING_INLINE_TAGS = frozenset(
- {
- "a",
- "abbr",
- "audio",
- "b",
- "bdi",
- "bdo",
- "br",
- "button",
- "canvas",
- "cite",
- "code",
- "data",
- "datalist",
- "del",
- "dfn",
- "em",
- "embed",
- "i",
- "iframe",
- "img",
- "input",
- "ins",
- "kbd",
- "label",
- "map",
- "mark",
- "math",
- "meter",
- "noscript",
- "object",
- "output",
- "picture",
- "progress",
- "q",
- "ruby",
- "s",
- "samp",
- "script",
- "select",
- "slot",
- "small",
- "span",
- "strong",
- "sub",
- "sup",
- "svg",
- "textarea",
- "time",
- "u",
- "var",
- "video",
- "wbr",
- }
-)
-
-
class RelevantContentFilter(ABC):
"""Abstract base class for content filtering strategies"""
@@ -712,6 +651,40 @@ def __init__(
"h6": 0.7,
}
+ # Phrasing elements keep the parent block's decision. Expose this on
+ # each filter so callers can customize which tags inherit that rule.
+ self.inline_tags = frozenset(
+ {
+ "a",
+ "abbr",
+ "b",
+ "bdi",
+ "bdo",
+ "cite",
+ "code",
+ "data",
+ "del",
+ "dfn",
+ "em",
+ "i",
+ "ins",
+ "kbd",
+ "mark",
+ "q",
+ "ruby",
+ "s",
+ "samp",
+ "small",
+ "span",
+ "strong",
+ "sub",
+ "sup",
+ "time",
+ "u",
+ "var",
+ }
+ )
+
def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]:
"""
Implements content filtering using pruning algorithm with dynamic threshold.
@@ -787,56 +760,51 @@ def _prune_tree(self, node):
if self._is_preserved(node):
return
- # Inline nodes inherit the keep/remove decision of their containing
- # block. Scoring a short nested span or link independently can remove
- # words from a paragraph that otherwise passes the filter.
- if node.name in _PRUNING_INLINE_TAGS:
- children = [child for child in node.children if hasattr(child, "name")]
- for child in children:
- self._prune_tree(child)
- return
-
- text_len = len(node.get_text(strip=True))
- tag_len = len(node.encode_contents().decode("utf-8"))
- link_text_len = sum(
- len(s.strip())
- for s in (a.string for a in node.find_all("a", recursive=False))
- if s
- )
-
- metrics = {
- "node": node,
- "tag_name": node.name,
- "text_len": text_len,
- "tag_len": tag_len,
- "link_text_len": link_text_len,
- }
-
- score = self._compute_composite_score(metrics, text_len, tag_len, link_text_len)
+ if node.name not in self.inline_tags:
+ text_len = len(node.get_text(strip=True))
+ tag_len = len(node.encode_contents().decode("utf-8"))
+ link_text_len = sum(
+ len(s.strip())
+ for s in (a.string for a in node.find_all("a", recursive=False))
+ if s
+ )
- if self.threshold_type == "fixed":
- should_remove = score < self.threshold
- else: # dynamic
- tag_importance = self.tag_importance.get(node.name, 0.7)
- text_ratio = text_len / tag_len if tag_len > 0 else 0
- link_ratio = link_text_len / text_len if text_len > 0 else 1
+ metrics = {
+ "node": node,
+ "tag_name": node.name,
+ "text_len": text_len,
+ "tag_len": tag_len,
+ "link_text_len": link_text_len,
+ }
- threshold = self.threshold # base threshold
- if tag_importance > 1:
- threshold *= 0.8
- if text_ratio > 0.4:
- threshold *= 0.9
- if link_ratio > 0.6:
- threshold *= 1.2
+ score = self._compute_composite_score(
+ metrics, text_len, tag_len, link_text_len
+ )
- should_remove = score < threshold
+ if self.threshold_type == "fixed":
+ should_remove = score < self.threshold
+ else: # dynamic
+ tag_importance = self.tag_importance.get(node.name, 0.7)
+ text_ratio = text_len / tag_len if tag_len > 0 else 0
+ link_ratio = link_text_len / text_len if text_len > 0 else 1
+
+ threshold = self.threshold # base threshold
+ if tag_importance > 1:
+ threshold *= 0.8
+ if text_ratio > 0.4:
+ threshold *= 0.9
+ if link_ratio > 0.6:
+ threshold *= 1.2
+
+ should_remove = score < threshold
+
+ if should_remove:
+ node.decompose()
+ return
- if should_remove:
- node.decompose()
- else:
- children = [child for child in node.children if hasattr(child, "name")]
- for child in children:
- self._prune_tree(child)
+ children = [child for child in node.children if hasattr(child, "name")]
+ for child in children:
+ self._prune_tree(child)
def _compute_composite_score(self, metrics, text_len, tag_len, link_text_len):
"""Computes the composite score"""
diff --git a/crawl4ai/content_filter_strategy_lxml.py b/crawl4ai/content_filter_strategy_lxml.py
index ee7e228b7..f7a4eba86 100644
--- a/crawl4ai/content_filter_strategy_lxml.py
+++ b/crawl4ai/content_filter_strategy_lxml.py
@@ -55,7 +55,7 @@
from lxml import etree
from lxml import html as lhtml
-from .content_filter_strategy import PruningContentFilter, _PRUNING_INLINE_TAGS
+from .content_filter_strategy import PruningContentFilter
# HTML void elements, taken verbatim from BeautifulSoup's LXMLTreeBuilder
# ``empty_element_tags``. These serialize self-closed (``