✨ feat(mq-crawler): add sitemap.xml ingestion as a seed-URL source

Cargo.toml

@@ -76,6 +76,7 @@ opentelemetry_sdk = {version = "0.32", features = ["rt-tokio"]}
 opfs = "0.2.0"
 percent-encoding = "2.3.2"
 proptest = "1.11"
+quick-xml = "0.41.0"
 rand = "0.10.1"
 rayon = "1.11.0"
 regex-lite = "0.1.9"

crates/mq-crawler/Cargo.toml

@@ -21,6 +21,7 @@ futures = {workspace = true}
 miette = {workspace = true, features = ["fancy"]}
 mq-lang = {workspace = true}
 mq-markdown = {workspace = true}
+quick-xml = {workspace = true}
 reqwest = {workspace = true, features = ["json"]}
 robots_txt = {workspace = true}
 scraper = {workspace = true}

crates/mq-crawler/README.md

@@ -26,6 +26,7 @@ Make web scraping and content extraction effortless with intelligent Markdown co
 - **Headless Chrome**: Built-in headless Chrome for JavaScript-heavy sites (no external server needed)
 - **WebDriver Support**: Use Selenium WebDriver for browser-based crawling
 - **Domain Filtering**: Restrict crawling to specific domains
+- **Sitemap Ingestion**: Seed the crawl frontier from a `sitemap.xml` (or sitemap index) up front
 
 ## Installation
 
@@ -108,6 +109,20 @@ mq-crawl -c 3 https://example.com
 mq-crawl -c 10 -d 0.1 https://example.com
 ```
 
+### Sitemap Ingestion
+
+```bash
+# Seed the crawl frontier with every URL listed in a sitemap.xml,
+# in addition to the start URL. Sitemap index files (<sitemapindex>)
+# are followed recursively. Discovered URLs still respect robots.txt,
+# --allowed-domains, and --depth.
+mq-crawl --sitemap https://example.com/sitemap.xml https://example.com
+
+# Useful with --depth 0 to crawl exactly the pages listed in the sitemap
+# without following links at all.
+mq-crawl --depth 0 --sitemap https://example.com/sitemap.xml https://example.com
+```
+
 ### Custom Robots.txt
 
 ```bash
@@ -223,6 +238,8 @@ Options:
           WebDriver URL for browser-based crawling (e.g., http://localhost:4444)
   -f, --format <FORMAT>
           Output format: text or json [default: text]
+      --sitemap <SITEMAP_URL>
+          URL of a sitemap.xml (or sitemap index) to enumerate additional seed URLs from
       --extract-scripts-as-code-blocks
           Extract <script> tags as code blocks in Markdown
       --generate-front-matter

crates/mq-crawler/src/crawler.rs

@@ -179,6 +179,7 @@ impl Crawler {
         conversion_options: mq_markdown::ConversionOptions,
         depth_limit: Option<usize>,
         allowed_domains: Option<Vec<String>>,
+        additional_seed_urls: Vec<Url>,
     ) -> Result<Self, String> {
         let initial_domain = start_url
             .domain()
@@ -188,6 +189,9 @@ impl Crawler {
 
         let to_visit = SegQueue::new();
         to_visit.push((start_url.clone(), 0));
+        for seed_url in additional_seed_urls {
+            to_visit.push((seed_url, 0));
+        }
 
         Ok(Self {
             allowed_domains,
@@ -704,6 +708,7 @@ mod tests {
             mq_markdown::ConversionOptions::default(),
             depth_limit,
             allowed_domains,
+            Vec::new(),
         )
         .await
         .unwrap()
@@ -767,6 +772,7 @@ mod tests {
             mq_markdown::ConversionOptions::default(),
             Some(2),
             None,
+            Vec::new(),
         )
         .await
         .unwrap();
@@ -790,10 +796,48 @@ mod tests {
             mq_markdown::ConversionOptions::default(),
             None,
             None,
+            Vec::new(),
         )
         .await
         .unwrap();
 
         assert_eq!(crawler.depth_limit, None);
     }
+
+    #[tokio::test]
+    async fn test_crawler_new_with_additional_seed_urls() {
+        let start_url = Url::parse("http://start.invalid/").unwrap();
+        let http_client = HttpClient::new_reqwest(30.0).unwrap();
+        let seed_urls = vec![
+            Url::parse("http://start.invalid/from-sitemap-1").unwrap(),
+            Url::parse("http://start.invalid/from-sitemap-2").unwrap(),
+        ];
+        let crawler = Crawler::new(
+            http_client,
+            start_url.clone(),
+            0.0,
+            None,
+            None,
+            None,
+            1,
+            OutputFormat::Text,
+            mq_markdown::ConversionOptions::default(),
+            None,
+            None,
+            seed_urls.clone(),
+        )
+        .await
+        .unwrap();
+
+        let mut queued = Vec::new();
+        while let Some((url, depth)) = crawler.to_visit.pop() {
+            queued.push((url, depth));
+        }
+
+        assert_eq!(queued.len(), 3);
+        assert!(queued.contains(&(start_url, 0)));
+        for seed_url in seed_urls {
+            assert!(queued.contains(&(seed_url, 0)));
+        }
+    }
 }

crates/mq-crawler/src/lib.rs

@@ -10,6 +10,7 @@
 //! - robots.txt compliance
 //! - HTML to markdown conversion
 //! - Link discovery and following
+//! - sitemap.xml ingestion as a seed-URL source
 //! - Crawl statistics and result tracking
 //! - Support for custom HTTP headers and user agents
 //! - Rate limiting and politeness delays
@@ -43,3 +44,4 @@
 pub mod crawler;
 pub mod http_client;
 pub mod robots;
+pub mod sitemap;

crates/mq-crawler/src/main.rs

@@ -98,6 +98,11 @@ struct CliArgs {
     /// Output format for results and statistics
     #[clap(short = 'f', long, default_value_t = OutputFormat::Text)]
     format: OutputFormat,
+    /// Optional URL of a sitemap.xml (or sitemap index) to enumerate additional seed URLs from.
+    /// Discovered URLs are added to the crawl frontier alongside the start URL and are still
+    /// subject to robots.txt, domain filtering, and depth limits.
+    #[clap(long, value_name = "SITEMAP_URL")]
+    sitemap: Option<Url>,
     #[clap(flatten)]
     pub conversion: ConversionArgs,
 }
@@ -213,6 +218,22 @@ async fn main() {
         OutputFormat::Json => mq_crawler::crawler::OutputFormat::Json,
     };
 
+    let sitemap_seed_urls = if let Some(ref sitemap_url) = args.sitemap {
+        tracing::info!("Fetching seed URLs from sitemap: {}", sitemap_url);
+        match mq_crawler::sitemap::fetch_sitemap_urls(&client, sitemap_url).await {
+            Ok(urls) => {
+                tracing::info!("Discovered {} seed URL(s) from sitemap.", urls.len());
+                urls
+            }
+            Err(e) => {
+                tracing::error!("Failed to fetch sitemap {}: {}", sitemap_url, e);
+                return;
+            }
+        }
+    } else {
+        Vec::new()
+    };
+
     match Crawler::new(
         client,
         args.url.clone(),
@@ -229,6 +250,7 @@ async fn main() {
         },
         args.depth,
         effective_allowed,
+        sitemap_seed_urls,
     )
     .await
     {