diff --git a/Cargo.lock b/Cargo.lock index 0d931f2f0..29a72237f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -492,9 +492,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.66" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -2520,6 +2520,7 @@ dependencies = [ "miette", "mq-lang", "mq-markdown", + "quick-xml", "reqwest", "robots_txt", "rstest", @@ -3076,7 +3077,7 @@ dependencies = [ "opentelemetry", "percent-encoding", "portable-atomic", - "rand 0.9.4", + "rand 0.9.5", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -3360,7 +3361,7 @@ dependencies = [ "bit-vec", "bitflags", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_chacha", "rand_xorshift", "regex-syntax", @@ -3506,9 +3507,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -5095,7 +5096,7 @@ dependencies = [ "http 1.4.2", "httparse", "log", - "rand 0.9.4", + "rand 0.9.5", "sha1", "thiserror 2.0.18", "utf-8", @@ -5288,9 +5289,9 @@ checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" [[package]] name = "uuid" -version = "1.23.4" +version = "1.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -6245,9 +6246,9 @@ checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "bd2f034a4bebf216c9e4b7083603e024cf930873fd67830cfb083c9fa33129d9" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index 33af0cb5f..b92d4c93f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ opentelemetry_sdk = {version = "0.32", features = ["rt-tokio"]} opfs = "0.2.0" percent-encoding = "2.3.2" proptest = "1.11" +quick-xml = "0.41.0" rand = "0.10.1" rayon = "1.11.0" regex-lite = "0.1.9" diff --git a/crates/mq-crawler/Cargo.toml b/crates/mq-crawler/Cargo.toml index 642454cfc..e73d1b627 100644 --- a/crates/mq-crawler/Cargo.toml +++ b/crates/mq-crawler/Cargo.toml @@ -21,6 +21,7 @@ futures = {workspace = true} miette = {workspace = true, features = ["fancy"]} mq-lang = {workspace = true} mq-markdown = {workspace = true} +quick-xml = {workspace = true} reqwest = {workspace = true, features = ["json"]} robots_txt = {workspace = true} scraper = {workspace = true} diff --git a/crates/mq-crawler/README.md b/crates/mq-crawler/README.md index 0808e2711..57743a6fb 100644 --- a/crates/mq-crawler/README.md +++ b/crates/mq-crawler/README.md @@ -26,6 +26,7 @@ Make web scraping and content extraction effortless with intelligent Markdown co - **Headless Chrome**: Built-in headless Chrome for JavaScript-heavy sites (no external server needed) - **WebDriver Support**: Use Selenium WebDriver for browser-based crawling - **Domain Filtering**: Restrict crawling to specific domains +- **Sitemap Ingestion**: Seed the crawl frontier from a `sitemap.xml` (or sitemap index) up front ## Installation @@ -108,6 +109,20 @@ mq-crawl -c 3 https://example.com mq-crawl -c 10 -d 0.1 https://example.com ``` +### Sitemap Ingestion + +```bash +# Seed the crawl frontier with every URL listed in a sitemap.xml, +# in addition to the start URL. Sitemap index files () +# are followed recursively. Discovered URLs still respect robots.txt, +# --allowed-domains, and --depth. +mq-crawl --sitemap https://example.com/sitemap.xml https://example.com + +# Useful with --depth 0 to crawl exactly the pages listed in the sitemap +# without following links at all. +mq-crawl --depth 0 --sitemap https://example.com/sitemap.xml https://example.com +``` + ### Custom Robots.txt ```bash @@ -223,6 +238,8 @@ Options: WebDriver URL for browser-based crawling (e.g., http://localhost:4444) -f, --format Output format: text or json [default: text] + --sitemap + URL of a sitemap.xml (or sitemap index) to enumerate additional seed URLs from --extract-scripts-as-code-blocks Extract