Support multiple config file formats and refactor config module (#2104)

Resolves #1930 by adding support for loading configuration from pyproject.toml and Cargo.toml. Support for package.json has also been added, as it is a common way to configure tools in the JavaScript ecosystem.
During implementation, the config module was split into logical submodules to keep the core high-level structs clean.

To handle the different config formats, a ConfigLoader trait was introduced. After a few iterations, a simple imperative approach -- where the entire file is strictly deserialized and an option returned -- was ruled out: because the config uses strict validation (denying unknown fields), a single typo would cause parsing to fail silently rather than surfacing the error to the user.

To fix this, the trait was split into is_match and load. The is_match method does a lightweight check to see if the lychee section exists in the file. If it does, load enforces strict schema deserialization and properly bubbles up validation errors.

Heavyweight dependencies like the cargo_toml crate (which builds massive ASTs for entire manifests) were intentionally avoided. Instead, minimal custom envelope structs with serde extract only the lychee blocks, keeping binary size small and compilation fast. Cargo package metadata takes precedence over workspace metadata if both are present.

Tests cover each loader to verify that valid blocks are mapped correctly, missing keys are ignored, precedence works as intended, and malformed schemas produce the expected errors.

Author: mre

lychee-bin/src/client.rs

@@ -1,4 +1,4 @@
-use crate::options::{Config, HeaderMapExt};
+use crate::config::{Config, HeaderMapExt};
 use crate::parse::parse_remaps;
 use anyhow::{Context, Result};
 use http::{HeaderMap, StatusCode};

lychee-bin/src/commands/mod.rs

@@ -12,7 +12,7 @@ use std::io::{self, Write};
 use std::path::PathBuf;
 
 use crate::cache::Cache;
-use crate::options::Config;
+use crate::config::Config;
 use lychee_lib::RequestError;
 use lychee_lib::{Client, Request};
 

lychee-bin/src/config/header.rs

@@ -0,0 +1,165 @@
+//! Parse and handle custom HTTP headers.
+//!
+//! Provides utilities for taking user-provided HTTP header strings
+//! (e.g. from the CLI or config files) and converting them into strongly
+//! typed `reqwest` headers.
+
+use anyhow::{Error, Result, anyhow};
+use clap::builder::TypedValueParser;
+use http::{
+    HeaderMap,
+    header::{HeaderName, HeaderValue},
+};
+use std::{collections::HashMap, str::FromStr};
+
+/// Parse a single header into a [`HeaderName`] and [`HeaderValue`]
+///
+/// Headers are expected to be in format `Header-Name: Header-Value`.
+/// The header name and value are trimmed of whitespace.
+///
+/// If the header contains multiple colons, the part after the first colon is
+/// considered the value.
+///
+/// # Errors
+///
+/// This fails if the header does not contain exactly one `:` character or
+/// if the header name contains non-ASCII characters.
+pub(crate) fn parse_single_header(header: &str) -> Result<(HeaderName, HeaderValue)> {
+    let parts: Vec<&str> = header.splitn(2, ':').collect();
+    match parts.as_slice() {
+        [name, value] => {
+            let name = name.trim();
+            let name = HeaderName::from_str(name)
+                .map_err(|e| anyhow!("Unable to convert header name '{name}': {e}"))?;
+            let value = HeaderValue::from_str(value.trim())
+                .map_err(|e| anyhow!("Unable to read value of header with name '{name}': {e}"))?;
+            Ok((name, value))
+        }
+        _ => Err(anyhow!(
+            "Invalid header format. Expected colon-separated string in the format 'HeaderName: HeaderValue'"
+        )),
+    }
+}
+
+/// Parses a single HTTP header into a tuple of (String, String)
+///
+/// This does NOT merge multiple headers into one.
+#[derive(Clone, Debug)]
+pub(crate) struct HeaderParser;
+
+impl TypedValueParser for HeaderParser {
+    type Value = (String, String);
+
+    fn parse_ref(
+        &self,
+        _cmd: &clap::Command,
+        _arg: Option<&clap::Arg>,
+        value: &std::ffi::OsStr,
+    ) -> Result<Self::Value, clap::Error> {
+        let header_str = value.to_str().ok_or_else(|| {
+            clap::Error::raw(
+                clap::error::ErrorKind::InvalidValue,
+                "Header value contains invalid UTF-8",
+            )
+        })?;
+
+        match parse_single_header(header_str) {
+            Ok((name, value)) => {
+                let Ok(value) = value.to_str() else {
+                    return Err(clap::Error::raw(
+                        clap::error::ErrorKind::InvalidValue,
+                        "Header value contains invalid UTF-8",
+                    ));
+                };
+
+                Ok((name.to_string(), value.to_string()))
+            }
+            Err(e) => Err(clap::Error::raw(
+                clap::error::ErrorKind::InvalidValue,
+                e.to_string(),
+            )),
+        }
+    }
+}
+
+impl clap::builder::ValueParserFactory for HeaderParser {
+    type Parser = HeaderParser;
+    fn value_parser() -> Self::Parser {
+        HeaderParser
+    }
+}
+
+/// Extension trait for converting a map of header pairs to a `HeaderMap`
+pub(crate) trait HeaderMapExt {
+    /// Convert a collection of header key-value pairs to a `HeaderMap`
+    ///
+    /// # Errors
+    ///
+    /// This fails if any header name or value cannot be parsed into a valid
+    /// `HeaderName` or `HeaderValue` respectively.
+    fn from_header_pairs(headers: &HashMap<String, String>) -> Result<HeaderMap, Error>;
+}
+
+impl HeaderMapExt for HeaderMap {
+    fn from_header_pairs(headers: &HashMap<String, String>) -> Result<HeaderMap, Error> {
+        let mut header_map = HeaderMap::new();
+        for (name, value) in headers {
+            let header_name = HeaderName::from_bytes(name.as_bytes())
+                .map_err(|e| anyhow!("Invalid header name '{name}': {e}"))?;
+            let header_value = HeaderValue::from_str(value)
+                .map_err(|e| anyhow!("Invalid header value '{value}': {e}"))?;
+            header_map.insert(header_name, header_value);
+        }
+        Ok(header_map)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_custom_headers() {
+        assert_eq!(
+            parse_single_header("accept:text/html").unwrap(),
+            (
+                HeaderName::from_static("accept"),
+                HeaderValue::from_static("text/html")
+            )
+        );
+    }
+
+    #[test]
+    fn test_parse_custom_header_multiple_colons() {
+        assert_eq!(
+            parse_single_header("key:x-test:check=this").unwrap(),
+            (
+                HeaderName::from_static("key"),
+                HeaderValue::from_static("x-test:check=this")
+            )
+        );
+    }
+
+    #[test]
+    fn test_parse_custom_headers_with_equals() {
+        assert_eq!(
+            parse_single_header("key:x-test=check=this").unwrap(),
+            (
+                HeaderName::from_static("key"),
+                HeaderValue::from_static("x-test=check=this")
+            )
+        );
+    }
+
+    #[test]
+    /// We should not reveal potentially sensitive data contained in the headers.
+    /// See: [#1297](https://github.com/lycheeverse/lychee/issues/1297)
+    fn test_does_not_echo_sensitive_data() {
+        let error = parse_single_header("My-Header💣: secret")
+            .expect_err("Should not allow unicode as key");
+        assert!(!error.to_string().contains("secret"));
+
+        let error = parse_single_header("secret").expect_err("Should fail when no `:` given");
+        assert!(!error.to_string().contains("secret"));
+    }
+}

lychee-bin/src/config/loaders/cargo_toml.rs

@@ -0,0 +1,137 @@
+//! `Cargo.toml` configuration loader.
+//!
+//! This module allows configuring lychee via a `[package.metadata.lychee]` or
+//! `[workspace.metadata.lychee]` section in a `Cargo.toml` file.
+//!
+//! # Tradeoffs
+//!
+//! While there are crates like `cargo_toml` available for parsing `Cargo.toml`
+//! files, we deliberately avoid using them. Such crates bring in heavy dependencies
+//! and extensive struct hierarchies to represent the entire Cargo schema (dependencies,
+//! targets, profiles, etc.).
+//!
+//! Instead, we define a lightweight, custom set of structs to extract exactly
+//! what we need using `serde` and `toml`.
+//!
+//! # Example
+//!
+//! ```toml
+//! [package.metadata.lychee]
+//! exclude = ["foo", "bar"]
+//! max_redirects = 5
+//! ```
+
+use super::{ConfigLoader, ConfigMatch};
+use crate::config::Config;
+use anyhow::{Context, Result};
+use serde::Deserialize;
+
+pub(crate) const CARGO_CONFIG_FILE: &str = "Cargo.toml";
+
+/// The lychee config can be defined in either
+/// `[package.metadata.lychee]` or `[workspace.metadata.lychee]`.
+#[derive(Deserialize)]
+struct CargoToml {
+    package: Option<CargoSection>,
+    workspace: Option<CargoSection>,
+}
+
+#[derive(Deserialize)]
+struct CargoSection {
+    metadata: Option<Metadata>,
+}
+
+#[derive(Deserialize)]
+struct Metadata {
+    lychee: Option<Config>,
+}
+
+pub(crate) struct CargoTomlLoader;
+
+impl ConfigLoader for CargoTomlLoader {
+    fn filename(&self) -> &str {
+        CARGO_CONFIG_FILE
+    }
+
+    fn load(&self, contents: &str) -> Result<ConfigMatch> {
+        let cargo = toml::from_str::<CargoToml>(contents)
+            .with_context(|| "Failed to parse lychee config from Cargo.toml")?;
+
+        // Package metadata fully replaces workspace metadata (instead of merging).
+        // That's useful, because it allows users to define a workspace-wide
+        // default config and then override it in specific packages.
+        let config = [cargo.package, cargo.workspace]
+            .into_iter()
+            .flatten()
+            .find_map(|s| s.metadata.and_then(|m| m.lychee));
+
+        match config {
+            Some(config) => Ok(ConfigMatch::Found(Box::new(config))),
+            None => Ok(ConfigMatch::NotFound),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_load_package_config() {
+        let toml = r#"
+        [package.metadata.lychee]
+        exclude = ["foo"]
+        "#;
+        let result = CargoTomlLoader.load(toml).unwrap();
+        match result {
+            ConfigMatch::Found(config) => assert_eq!(config.exclude, vec!["foo".to_string()]),
+            ConfigMatch::NotFound => panic!("Expected config to be found"),
+        }
+    }
+
+    #[test]
+    fn test_load_workspace_config() {
+        let toml = r#"
+        [workspace.metadata.lychee]
+        exclude = ["bar"]
+        "#;
+        let result = CargoTomlLoader.load(toml).unwrap();
+        match result {
+            ConfigMatch::Found(config) => assert_eq!(config.exclude, vec!["bar".to_string()]),
+            ConfigMatch::NotFound => panic!("Expected config to be found"),
+        }
+    }
+
+    #[test]
+    fn test_load_package_takes_precedence() {
+        let toml = r#"
+        [workspace.metadata.lychee]
+        exclude = ["bar"]
+
+        [package.metadata.lychee]
+        exclude = ["foo"]
+        "#;
+        let result = CargoTomlLoader.load(toml).unwrap();
+        match result {
+            ConfigMatch::Found(config) => assert_eq!(config.exclude, vec!["foo".to_string()]),
+            ConfigMatch::NotFound => panic!("Expected config to be found"),
+        }
+    }
+
+    #[test]
+    fn test_load_no_lychee_config() {
+        let toml = r#"
+        [package]
+        name = "lychee"
+        version = "1.0.0"
+
+        [workspace]
+        members = ["lychee-bin"]
+        "#;
+        let result = CargoTomlLoader.load(toml).unwrap();
+        match result {
+            ConfigMatch::NotFound => (),
+            ConfigMatch::Found(_) => panic!("Expected no config to be found"),
+        }
+    }
+}

lychee-bin/src/config/loaders/lychee_toml.rs

@@ -0,0 +1,62 @@
+//! `lychee.toml` configuration loader.
+//!
+//! This module allows configuring lychee via a standard `lychee.toml` file.
+//! This is the default configuration file format for lychee.
+//!
+//! Unlike `Cargo.toml` and `pyproject.toml` which require the configuration
+//! to be scoped under a specific section (like `[package.metadata.lychee]`),
+//! `lychee.toml` defines the configuration at the root level of the document.
+//!
+//! # Example
+//!
+//! ```toml
+//! exclude = ["foo", "bar"]
+//! timeout = 10
+//! ```
+
+use super::{ConfigLoader, ConfigMatch};
+use anyhow::{Context, Result};
+
+pub(crate) const LYCHEE_CONFIG_FILE: &str = "lychee.toml";
+
+pub(crate) struct LycheeTomlLoader;
+
+impl ConfigLoader for LycheeTomlLoader {
+    fn filename(&self) -> &str {
+        LYCHEE_CONFIG_FILE
+    }
+
+    /// We strictly deserialize the entire file directly into our `Config` struct.
+    /// Any failure here is a genuine configuration error that we want to bubble up.
+    /// A dedicated `lychee.toml` file is assumed to always contain lychee configuration.
+    fn load(&self, contents: &str) -> Result<ConfigMatch> {
+        let config =
+            toml::from_str(contents).with_context(|| "Failed to parse configuration file")?;
+        Ok(ConfigMatch::Found(Box::new(config)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_load_config() {
+        let toml = r#"
+        exclude = ["foo"]
+        "#;
+        let result = LycheeTomlLoader.load(toml).unwrap();
+        match result {
+            ConfigMatch::Found(config) => assert_eq!(config.exclude, vec!["foo".to_string()]),
+            ConfigMatch::NotFound => panic!("Expected config to be found"),
+        }
+    }
+
+    #[test]
+    fn test_load_invalid_config() {
+        let toml = r#"
+        exclude = "foo" # should be an array
+        "#;
+        assert!(LycheeTomlLoader.load(toml).is_err());
+    }
+}