From a594e44bc58fc404ca8537b54925defa5dea00ee Mon Sep 17 00:00:00 2001 From: xcosmosbox <2162381070@qq.com> Date: Tue, 30 Jun 2026 00:15:18 +0800 Subject: [PATCH] feat(parser): add support for XML file parsing --- .../document_parser/formats/xml/parser.py | 221 +++++++++++++++++ .../orchestration/format_adapters.py | 20 ++ .../orchestration/format_router.py | 5 + .../contract/test_xml_parser_contract.py | 226 ++++++++++++++++++ 4 files changed, 472 insertions(+) create mode 100644 apps/worker/app/services/document_parser/formats/xml/parser.py create mode 100644 apps/worker/tests/contract/test_xml_parser_contract.py diff --git a/apps/worker/app/services/document_parser/formats/xml/parser.py b/apps/worker/app/services/document_parser/formats/xml/parser.py new file mode 100644 index 000000000..bd880c606 --- /dev/null +++ b/apps/worker/app/services/document_parser/formats/xml/parser.py @@ -0,0 +1,221 @@ +# pyright: reportArgumentType=false, reportAttributeAccessIssue=false, reportCallIssue=false, reportReturnType=false +"""XML document parser. + +Converts .xml files into markdown-like text lines by recursively walking +the XML element tree with Python's built-in xml.etree.ElementTree, then +delegates to the standard parse_md() pipeline for hierarchy reconstruction, +heading detection, and LLM-based enrichment. + +The conversion is intentionally lightweight — complex XML schemas and +namespaces are ignored in favor of text extraction. Element tag names +are used as lightweight structural hints. +""" + +from xml.etree import ElementTree + + +def parse_xml( + output_dir: str, + source_type: str, + file_path: str, + base_llm_paras=None, + relative_root: str | None = None, +): + """Parse an XML file into a hierarchical document DataFrame. + + Walks the XML element tree to extract text content from all elements, + using tag names to create markdown-like headings for structural + containers. The extracted lines are then fed through parse_md() for + heading detection, hierarchy reconstruction, and LLM enrichment — + the same pipeline used by the text and HTML adapters. + + Args: + output_dir: Full output directory path for parsed artifacts. + source_type: Source type label (always "xml" from the adapter). + file_path: Absolute path to the .xml file on disk. + base_llm_paras: LLM parameter dict for summary enrichment. + relative_root: Root path segment for hierarchical path construction. + + Returns: + pd.DataFrame with columns [path, content, type, summary, keywords] + representing the parsed document hierarchy. + """ + from app.services.common.file_loading import load_file_bytes + from app.services.document_parser.formats.markdown.parser import parse_md + + # Load and decode the XML file as UTF-8 text. + xml_bytes = load_file_bytes(file_path, file_url="") + xml_text = xml_bytes.decode("utf-8") + + # Convert XML document into markdown-like text lines via recursive + # ElementTree traversal. We keep this lightweight — no schema + # awareness, no namespace handling beyond what ElementTree provides. + md_lines = _xml_to_md_lines(xml_text) + + # Delegate to the standard markdown parsing pipeline. + parsed_df = parse_md( + output_dir, + source_type=source_type, + md_lines=md_lines, + base_llm_paras=base_llm_paras, + relative_root=relative_root, + ) + return parsed_df + + +def _xml_to_md_lines(xml_text: str) -> list[str]: + """Convert XML document into markdown-like text lines. + + Parses the XML string with ElementTree, then recursively walks + the element tree to extract text content. Container-like elements + (based on common naming patterns) get heading-style prefixes; + leaf elements produce plain text lines. + + Args: + xml_text: Raw XML document as a UTF-8 string. + + Returns: + List of text lines suitable for the parse_md() pipeline. + """ + try: + root = ElementTree.fromstring(xml_text) + except ElementTree.ParseError: + # If the XML is malformed, return the raw text as fallback so + # parse_md() can still attempt to extract meaningful content. + return xml_text.splitlines() + + lines: list[str] = [] + _walk_xml_elements(root, lines, depth=0) + + # Strip trailing blank lines. + while lines and not lines[-1].strip(): + lines.pop() + + return lines + + +def _walk_xml_elements( + element: ElementTree.Element, + lines: list[str], + depth: int = 0, +) -> None: + """Recursively walk XML elements, appending markdown-like lines. + + This is the core extraction loop. It processes each XML element: + - Elements with only text content (leaf elements) → text line. + - Elements with child elements → recurse, optionally emitting a + heading line for container-like tag names. + - Mixed content (text + children) → text line then recurse. + + Tag names that look like section/document containers (section, + chapter, article, item, entry, record, document, part, module, + component, chapter, topic) get a heading-style prefix based on + nesting depth. + + Args: + element: An xml.etree.ElementTree Element. + lines: Accumulator list for output lines. + depth: Current nesting depth (controls heading level). + """ + tag = _local_name(element.tag) + text = (element.text or "").strip() + + children = list(element) + + if not children: + # ── Leaf element: emit tag as heading + text ─────────── + if text: + heading = _tag_to_heading(tag, depth) + if heading: + lines.append(heading) + lines.append(text) + return + + # ── Container element: optional heading, then recurse ────── + if text: + # Mixed content: element has both direct text and children. + heading = _tag_to_heading(tag, depth) + if heading: + lines.append(heading) + lines.append(text) + elif _is_container_tag(tag): + # Pure container: emit a heading for structural navigation. + heading = _tag_to_heading(tag, depth) + if heading: + lines.append(heading) + + for child in children: + _walk_xml_elements(child, lines, depth + 1) + + # Emit tail text (text after the closing tag). + tail = (element.tail or "").strip() + if tail: + lines.append(tail) + + +def _local_name(tag: str) -> str: + """Strip XML namespace from a tag, returning just the local name. + + ElementTree represents namespaced tags as '{uri}localname'; + this extracts only the 'localname' portion for cleaner output. + + Args: + tag: Full ElementTree tag string, possibly with namespace. + + Returns: + The local part of the tag name, lowercased. + """ + if "}" in tag: + return tag.split("}", 1)[1].lower() + return tag.lower() + + +def _is_container_tag(tag: str) -> bool: + """Return True if the tag name looks like a document structure container. + + Heuristic based on common XML vocabulary patterns. This is + intentionally simple — we match against a fixed set of known + container tag names rather than trying to infer structure from + the schema. + + Args: + tag: Lowercased local tag name. + + Returns: + True if the tag is recognized as a structural container. + """ + _container_tags = frozenset({ + "section", "chapter", "article", "item", "entry", + "record", "document", "part", "module", "component", + "topic", "group", "block", "segment", "division", + "body", "header", "footer", "sidebar", "content", + "abstract", "description", "summary", "details", + }) + return tag in _container_tags + + +def _tag_to_heading(tag: str, depth: int) -> str | None: + """Convert a tag name and depth into a markdown heading line. + + Container tags at depth 0–1 get h1/h2; deeper nesting produces + progressively lower heading levels, clamped at h6. + + Args: + tag: Lowercased local tag name. + depth: Nesting depth in the XML tree. + + Returns: + A markdown heading string like '## Section Title', or None + if the tag should not produce a heading. + """ + if not _is_container_tag(tag): + return None + + # Map depth to heading level: depth 0 → h1, depth 1 → h2, etc. + # Clamp to h6 maximum. + level = min(depth + 1, 6) + prefix = "#" * level + + # Use a human-readable version of the tag as the heading text. + label = tag.replace("_", " ").title() + return f"{prefix} {label}" diff --git a/apps/worker/app/services/document_parser/orchestration/format_adapters.py b/apps/worker/app/services/document_parser/orchestration/format_adapters.py index e6af2f8e5..cef91b6d7 100644 --- a/apps/worker/app/services/document_parser/orchestration/format_adapters.py +++ b/apps/worker/app/services/document_parser/orchestration/format_adapters.py @@ -179,6 +179,26 @@ def parse(self, session: ParseSession) -> ParseOutput: return ParseOutput(output_dir=session.full_output_dir, parsed_df=None) +@dataclass(frozen=True) +class XmlParseAdapter: + """Adapter that parses .xml files through the markdown pipeline. + XML content is recursively walked to extract text from elements, + with tag names mapped to markdown headings where appropriate. + The extracted lines are then routed through parse_md().""" + document_format: object + + def parse(self, session: ParseSession) -> ParseOutput: + from app.services.document_parser.formats.xml.parser import parse_xml + parsed_df = parse_xml( + session.full_output_dir, + source_type="xml", + file_path=session.file_full_path, + base_llm_paras=session.base_llm_paras, + relative_root=session.relative_root, + ) + return ParseOutput(output_dir=session.full_output_dir, parsed_df=parsed_df) + + def _parse_docx_path( docx_path: str, session: ParseSession, diff --git a/apps/worker/app/services/document_parser/orchestration/format_router.py b/apps/worker/app/services/document_parser/orchestration/format_router.py index b94cf2787..75fa91481 100644 --- a/apps/worker/app/services/document_parser/orchestration/format_router.py +++ b/apps/worker/app/services/document_parser/orchestration/format_router.py @@ -22,6 +22,7 @@ class DocumentFormat(str, Enum): PPTX = "pptx" MARKDOWN = "markdown" JSON = "json" + XML = "xml" SUPPORTED_FILE_TYPES: tuple[str, ...] = ( @@ -37,6 +38,7 @@ class DocumentFormat(str, Enum): ".xlsx", ".pptx", ".md", + ".xml", ".json", ) @@ -65,6 +67,8 @@ def resolve_document_format(file_path: str) -> DocumentFormat: return DocumentFormat.MARKDOWN if extension == ".json": return DocumentFormat.JSON + if extension == ".xml": + return DocumentFormat.XML raise ValidationException( user_message=f"Unsupported file type: {extension}", @@ -89,6 +93,7 @@ def get_document_parse_adapter(document_format: DocumentFormat) -> DocumentParse DocumentFormat.XLSX: format_adapters.XlsxParseAdapter(document_format), DocumentFormat.PPTX: format_adapters.PptxParseAdapter(document_format), DocumentFormat.MARKDOWN: format_adapters.MarkdownParseAdapter(document_format), + DocumentFormat.XML: format_adapters.XmlParseAdapter(document_format), DocumentFormat.JSON: format_adapters.JsonParseAdapter(document_format), } return adapter_by_format[document_format] diff --git a/apps/worker/tests/contract/test_xml_parser_contract.py b/apps/worker/tests/contract/test_xml_parser_contract.py new file mode 100644 index 000000000..8319c192d --- /dev/null +++ b/apps/worker/tests/contract/test_xml_parser_contract.py @@ -0,0 +1,226 @@ +"""Contract tests for the XML format parser. + +Verifies that .xml files are correctly routed through the +XmlParseAdapter and produce valid ParseOutput with the expected +DataFrame contract: columns [path, content, type, summary, keywords]. + +Test pattern follows test_html_parser_contract.py and +test_excel_parser_contract.py. +""" + +from __future__ import annotations + +from pathlib import Path + + +def _write_contract_xml(test_xml_path: Path) -> None: + """Write a realistic XML document to the given path for contract testing. + + The document mimics a technical specification with nested sections, + mixed content (text + child elements), and various container types. + """ + xml_content = """\ + + + API Specification v2.1 + + This document defines the public API contract for the Knowhere + document processing service, covering authentication, endpoints, + and error handling. + +
+ Authentication + + All API requests require a valid API key passed in the + Authorization header using the Bearer token scheme. + + + API Key Format + Keys are 256-bit hex-encoded strings prefixed with "kh_". + + + Key Rotation + Keys may be rotated every 90 days. Old keys remain valid for a 24-hour grace period. + +
+
+ Endpoints + + All endpoints are available under the base URL https://api.knowhereto.ai/v1. + + + POST /documents/upload + Upload a new document for processing. Accepts multipart form data with the file and optional metadata JSON. + + + GET /documents/{id}/status + Retrieve the current processing status of a document by its unique identifier. + +
+
+ Error Handling + + The API uses standard HTTP status codes and returns error details in a consistent JSON envelope. + +
+""" + test_xml_path.write_text(xml_content, encoding="utf-8") + + +def test_xml_parser_contract_produces_valid_parse_output( + worker_contract_environment: None, + tmp_path: Path, +) -> None: + """Verify that a .xml file is routed through the XmlParseAdapter + and produces a ParseOutput with the expected DataFrame contract.""" + from app.services.document_parser.parse_service import checkerboard_parse_output + + xml_path = tmp_path / "api_spec.xml" + output_root = tmp_path / "parser-output" + _write_contract_xml(xml_path) + + # Run through the public parser seam with LLM features disabled. + parse_output = checkerboard_parse_output( + file_full_path=str(xml_path), + filename="api_spec.xml", + output_dir=str(output_root), + internal_output_filename="api_spec.xml", + summary_image=False, + summary_table=False, + summary_txt=False, + smart_title_parse=False, + stopwords=[], + ) + + full_output_dir = parse_output.output_dir + parsed_df = parse_output.parsed_df + + # ── Output directory contract ─────────────────────────────── + assert full_output_dir.endswith("api_spec.xml"), ( + f"Expected output directory to end with 'api_spec.xml', " + f"got: {full_output_dir}" + ) + assert parsed_df is not None, ( + "Expected parsed_df to be non-None for XML input" + ) + + # ── DataFrame column contract ─────────────────────────────── + expected_columns = ["path", "content", "type", "summary", "keywords"] + for col in expected_columns: + assert col in parsed_df.columns, ( + f"Expected column '{col}' in parsed_df, " + f"got columns: {list(parsed_df.columns)}" + ) + + # ── Content extraction ────────────────────────────────────── + all_content = " ".join(str(c) for c in parsed_df["content"].tolist() if c) + assert "API Specification" in all_content, ( + "Expected document title in parsed output" + ) + assert "Authentication" in all_content, ( + "Expected section heading content in parsed output" + ) + assert "Bearer token" in all_content, ( + "Expected paragraph text in parsed output" + ) + assert "kh_" in all_content, ( + "Expected nested item text in parsed output" + ) + + # ── Path contract ─────────────────────────────────────────── + paths = parsed_df["path"].tolist() + assert any(p.startswith("api_spec.xml") for p in paths), ( + f"Expected at least one path starting with 'api_spec.xml', " + f"got paths: {paths}" + ) + + +def test_xml_parser_handles_malformed_xml_gracefully( + worker_contract_environment: None, + tmp_path: Path, +) -> None: + """Verify that malformed XML is handled gracefully — the parser + should not crash but return a best-effort parse via fallback mode.""" + from app.services.document_parser.parse_service import checkerboard_parse_output + + # Write intentionally malformed XML (missing closing tag). + bad_xml_path = tmp_path / "bad.xml" + bad_xml_path.write_text( + "Valid contentUnclosed", + encoding="utf-8", + ) + output_root = tmp_path / "parser-output" + + # The parser must not raise an unhandled exception on malformed input. + parse_output = checkerboard_parse_output( + file_full_path=str(bad_xml_path), + filename="bad.xml", + output_dir=str(output_root), + internal_output_filename="bad.xml", + summary_image=False, + summary_table=False, + summary_txt=False, + smart_title_parse=False, + stopwords=[], + ) + + # Should still produce a valid ParseOutput (best-effort). + assert parse_output.parsed_df is not None, ( + "Expected parsed_df to be non-None even for malformed XML" + ) + + +def test_xml_to_md_lines_extracts_text_hierarchy() -> None: + """Unit test: verify that _xml_to_md_lines correctly extracts text + from a structured XML document and preserves the hierarchy.""" + from app.services.document_parser.formats.xml.parser import _xml_to_md_lines + + xml = """\ + + + Test Document +
+ Overview + This is a test paragraph with important information. +
+
+ Details + + Item One + Description of the first item. + + + Item Two + Description of the second item. + +
+""" + + lines = _xml_to_md_lines(xml) + + all_text = "\n".join(lines) + assert "Test Document" in all_text, f"Expected 'Test Document' in {lines}" + assert "Overview" in all_text, f"Expected 'Overview' in {lines}" + assert "important information" in all_text, ( + f"Expected paragraph text in {lines}" + ) + assert "Item One" in all_text, f"Expected 'Item One' in {lines}" + assert "Item Two" in all_text, f"Expected 'Item Two' in {lines}" + + +def test_xml_to_md_lines_handles_malformed() -> None: + """Unit test: malformed XML should fall back to raw text lines.""" + from app.services.document_parser.formats.xml.parser import _xml_to_md_lines + + lines = _xml_to_md_lines("text 0, "Expected at least one line from fallback" + + +def test_local_name_strips_namespace() -> None: + """Unit test: verify _local_name correctly strips XML namespace URIs.""" + from app.services.document_parser.formats.xml.parser import _local_name + + assert _local_name("{http://example.com/ns}section") == "section" + assert _local_name("paragraph") == "paragraph" + assert _local_name("{urn:oasis:names:tc:opendocument}body") == "body"