fix static file serving in local dev, fix edge case in prod

Cargo.toml

@@ -1,7 +1,7 @@
 [workspace]
 resolver = "2" 
 
-members = ["crates/bin/*", "crates/lib/*"]
+members = ["crates/bin/*", "crates/lib/*", "integration_tests"]
 
 default-members = [
     "crates/bin/docs_rs_admin",

README.md

@@ -66,19 +66,19 @@ docker compose up --wait db s3
 # allow downloads from the s3 container to support the /crate/.../download endpoint
 mcli policy set download docsrs/rust-docs-rs
 # Setup the database you just created
-cargo run -- database migrate
+cargo run --bin docs_rs_admin -- database migrate
 # Update the currently used toolchain to the latest nightly
 # This also sets up the docs.rs build environment.
 # This will take a while the first time but will be cached afterwards.
-cargo run -- build update-toolchain
+cargo run --bin docs_rs_builder -- build update-toolchain
 # Build a sample crate to make sure it works
-cargo run -- build crate regex 1.3.1
+cargo run --bin docs_rs_builder -- build crate regex 1.3.1
 # This starts the web server but does not build any crates.
 # It does not automatically run the migrations, so you need to do that manually (see above).
-cargo run -- start-web-server
+cargo run --bin docs_rs_web
 # If you want the server to automatically restart when code or templates change
 # you can use `cargo-watch`:
-cargo watch -x "run -- start-web-server"
+cargo watch -x "run --bin docs_rs_web"
 ```
 
 If you need to store big files in the repository's directory it's recommended to
@@ -243,7 +243,7 @@ See `cargo run -- --help` for a full list of commands.
 
 ```sh
 # This command will start web interface of docs.rs on http://localhost:3000
-cargo run -- start-web-server
+cargo run --bin docs_rs_webserver start-web-server
 ```
 
 #### `build` subcommand
@@ -252,14 +252,14 @@ cargo run -- start-web-server
 # Builds <CRATE_NAME> <CRATE_VERSION> and adds it into database
 # This is the main command to build and add a documentation into docs.rs.
 # For example, `docker compose run --rm builder-a build crate regex 1.1.6`
-cargo run -- build crate <CRATE_NAME> <CRATE_VERSION>
+cargo run --bin docs_rs_builder -- build crate <CRATE_NAME> <CRATE_VERSION>
 
 # alternatively, within docker-compose containers
 docker compose run --rm builder-a build crate <CRATE_NAME> <CRATE_VERSION>
 
 # Builds every crate on crates.io and adds them into database
 # (beware: this may take months to finish)
-cargo run -- build world
+cargo run --bin docs_rs_builder -- build world
 
 # Builds a local package you have at <SOURCE> and adds it to the database.
 # The package does not have to be on crates.io.
@@ -268,21 +268,18 @@ cargo run -- build world
 # In certain scenarios it might be necessary to first package the respective
 # crate by using the `cargo package` command.
 # See also /docs/build-workspaces.md
-cargo run -- build crate --local /path/to/source
+cargo run --bin docs_rs_builder -- build crate --local /path/to/source
 ```
 
 #### `database` subcommand
 
 ```sh
-# Adds a directory into database to serve with `staticfile` crate.
-cargo run -- database add-directory <DIRECTORY> [PREFIX]
-
 # Updates repository stats for crates.
 # You need to set the DOCSRS_GITHUB_ACCESSTOKEN
 # environment variable in order to run this command.
 # Set DOCSRS_GITLAB_ACCESSTOKEN to raise the rate limit for GitLab repositories,
 # or leave it blank to fetch repositories at a slower rate.
-cargo run -- database update-repository-fields
+cargo run --bin docs_rs_admin -- database update-repository-fields
 ```
 
 If you want to explore or edit database manually, you can connect to the database
@@ -297,29 +294,21 @@ The database contains a blacklist of crates that should not be built.
 
 ```sh
 # List the crates on the blacklist
-cargo run -- database blacklist list
+cargo run --bin docs_rs_admin -- database blacklist list
 
 # Adds <CRATE_NAME> to the blacklist
-cargo run -- database blacklist add <CRATE_NAME>
+cargo run --bin docs_rs_admin -- database blacklist add <CRATE_NAME>
 
 # Removes <CRATE_NAME> from the blacklist
-cargo run -- database blacklist remove <CRATE_NAME>
+cargo run --bin docs_rs_admin -- database blacklist remove <CRATE_NAME>
 ```
 
 If you want to revert to a precise migration, you can run:
 
 ```sh
-cargo run -- database migrate <migration number>
+cargo run --bin docs_rs_admin -- database migrate <migration number>
 ```
 
-#### `daemon` subcommand
-
-```sh
-# Run a persistent daemon which queues builds and starts a web server.
-cargo run -- daemon --registry-watcher=disabled
-# Add crates to the queue
-cargo run -- queue add <CRATE> <VERSION>
-```
 
 ### Updating vendored sources
 

crates/bin/docs_rs_web/Cargo.toml

@@ -97,5 +97,6 @@ kuchikiki = "0.8"
 mockito = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 pretty_assertions = { workspace = true }
+tempfile = { workspace = true }
 test-case = { workspace = true }
 walkdir = { workspace = true }

crates/bin/docs_rs_web/src/context.rs

@@ -0,0 +1,20 @@
+use anyhow::Result;
+use docs_rs_context::Context;
+use std::sync::Arc;
+
+pub async fn build_context() -> Result<Arc<Context>> {
+    Ok(Arc::new(
+        Context::builder()
+            .with_runtime()
+            .await?
+            .with_meter_provider()?
+            .with_pool()
+            .await?
+            .with_build_queue()?
+            .with_storage()
+            .await?
+            .with_registry_api()?
+            .with_build_limits()?
+            .build()?,
+    ))
+}

crates/bin/docs_rs_web/src/handlers/mod.rs

@@ -118,7 +118,7 @@ pub(crate) async fn build_axum_app(
     template_data: Arc<TemplateData>,
 ) -> Result<AxumRouter, Error> {
     apply_middleware(
-        routes::build_axum_routes(),
+        routes::build_axum_routes()?,
         config,
         context,
         Some(template_data),

crates/bin/docs_rs_web/src/handlers/statics.rs

@@ -2,6 +2,7 @@ use crate::{
     cache::CachePolicy, cache::STATIC_ASSET_CACHE_POLICY, metrics::request_recorder,
     routes::get_static,
 };
+use anyhow::{Result, bail};
 use axum::{
     Router as AxumRouter,
     extract::{Extension, Request},
@@ -16,8 +17,46 @@ use axum_extra::{
 use docs_rs_headers::IfNoneMatch;
 use docs_rs_mimes::APPLICATION_OPENSEARCH_XML;
 use http::{StatusCode, Uri};
+use std::{
+    env,
+    path::{Path, PathBuf},
+};
 use tower_http::services::ServeDir;
 
+const MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
+const STATIC_DIR_NAME: &str = "static";
+const VENDOR_DIR_NAME: &str = "vendor";
+const STATIC_DIR_NAMES: &[&str] = &[STATIC_DIR_NAME, VENDOR_DIR_NAME];
+
+/// Find the root directory for serving our static assets.
+///
+/// We have two directories we expect: `vendor` and `static`.
+///
+/// First we check if they exist in the current working directory:
+/// this works
+/// * inside the docker container, or
+/// * any production deploy,
+/// * when running `cargo run` from inside the `docs_rs_web` subcrate.
+///
+/// If they don't exist there, we try to find the folders in
+/// `CARGO_MANIFEST_DIR`.
+/// This allows running the server from the project root.
+pub(crate) fn static_root_dir() -> Result<PathBuf> {
+    let manifest_dir = PathBuf::from(MANIFEST_DIR);
+    for candidate in [env::current_dir()?, manifest_dir] {
+        if STATIC_DIR_NAMES
+            .iter()
+            .all(|name| candidate.join(name).is_dir())
+        {
+            return Ok(candidate);
+        }
+    }
+
+    bail!(
+        "Could not find static root directory containing '{STATIC_DIR_NAME}' and '{VENDOR_DIR_NAME}' folders"
+    );
+}
+
 const VENDORED_CSS: &str = include_str!(concat!(env!("OUT_DIR"), "/vendored.css"));
 const STYLE_CSS: &str = include_str!(concat!(env!("OUT_DIR"), "/style.css"));
 const RUSTDOC_CSS: &str = include_str!(concat!(env!("OUT_DIR"), "/rustdoc.css"));
@@ -93,11 +132,22 @@ async fn conditional_get(
     }
 
     let mut res = next.run(req).await;
-    res.headers_mut().typed_insert(etag);
+    let status = res.status();
+    // Typically we only end up here when we have a successful response.
+    //
+    // But there is an edge case, that only happens when there is a path
+    // where we were able to statically generate an ETag in the
+    // build-script, but the file can't be found later.
+    // Until now, happend only in local dev, but would also happen
+    // if the static file was deleted from the server after deployment.
+    if status.is_success() {
+        res.headers_mut().typed_insert(etag);
+    }
     res
 }
 
-pub(crate) fn build_static_router() -> AxumRouter {
+pub(crate) fn build_static_router(root: impl AsRef<Path>) -> AxumRouter {
+    let root = root.as_ref();
     AxumRouter::new()
         .route(
             "/vendored.css",
@@ -120,33 +170,39 @@ pub(crate) fn build_static_router() -> AxumRouter {
             get_static(|| async { build_static_css_response(RUSTDOC_2025_08_20_CSS) }),
         )
         .fallback_service(
-            get_service(ServeDir::new("static").fallback(ServeDir::new("vendor")))
-                .layer(middleware::from_fn(set_needed_static_headers))
-                .layer(middleware::from_fn(|request, next| async {
-                    request_recorder(request, next, Some("static resource")).await
-                })),
+            get_service(
+                ServeDir::new(root.join(STATIC_DIR_NAME))
+                    .fallback(ServeDir::new(root.join(VENDOR_DIR_NAME))),
+            )
+            .layer(middleware::from_fn(set_needed_static_headers))
+            .layer(middleware::from_fn(|request, next| async {
+                request_recorder(request, next, Some("static resource")).await
+            })),
         )
         .layer(middleware::from_fn(conditional_get))
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::testing::{
-        AxumResponseTestExt, AxumRouterTestExt, TestEnvironmentExt as _, async_wrapper,
+    use crate::{
+        handlers::apply_middleware,
+        page::TemplateData,
+        testing::{
+            AxumResponseTestExt, AxumRouterTestExt, TestEnvironment, TestEnvironmentExt as _,
+            async_wrapper,
+        },
     };
     use axum::{Router, body::Body};
     use docs_rs_headers::compute_etag;
     use http::{
         HeaderMap,
         header::{CONTENT_LENGTH, CONTENT_TYPE, ETAG},
     };
-    use std::fs;
+    use std::{fs, sync::Arc};
     use test_case::test_case;
     use tower::ServiceExt as _;
 
-    const STATIC_SEARCH_PATHS: &[&str] = &["static", "vendor"];
-
     fn content_length(resp: &Response) -> u64 {
         resp.headers()
             .get(CONTENT_LENGTH)
@@ -302,13 +358,14 @@ mod tests {
         async_wrapper(|env| async move {
             let web = env.web_app().await;
 
-            for root in STATIC_SEARCH_PATHS {
-                for entry in walkdir::WalkDir::new(root) {
+            for root in STATIC_DIR_NAMES {
+                let root = static_root_dir()?.join(root);
+                for entry in walkdir::WalkDir::new(&root) {
                     let entry = entry?;
                     if !entry.file_type().is_file() {
                         continue;
                     }
-                    let file = entry.path().strip_prefix(root).unwrap();
+                    let file = entry.path().strip_prefix(&root).unwrap();
                     let path = entry.path();
 
                     let url = format!("/-/static/{}", file.to_str().unwrap());
@@ -340,6 +397,48 @@ mod tests {
         });
     }
 
+    #[tokio::test(flavor = "multi_thread")]
+    async fn static_files_should_exist_but_is_locally_deleted() -> Result<()> {
+        let env = TestEnvironment::new().await?;
+
+        /// build a small axum app with middleware, but just with the static router only.
+        async fn build_static_app(env: &TestEnvironment, root: impl AsRef<Path>) -> Result<Router> {
+            let template_data = Arc::new(TemplateData::new(1).unwrap());
+            apply_middleware(
+                build_static_router(root),
+                env.config().clone(),
+                env.context().clone(),
+                Some(template_data),
+            )
+            .await
+        }
+
+        const PATH: &str = "/menu.js";
+        {
+            // Sanity check if we have a path that should exist
+            let web = env.web_app().await;
+            web.assert_success(&format!("/-/static{PATH}")).await?;
+
+            // and if our static router thing works theoretically
+            let static_app = build_static_app(&env, &static_root_dir()?).await?;
+            static_app.assert_success(PATH).await?;
+        }
+
+        // set up a broken static router.
+        // The compile-time generated etag map says `menu.js` should exist,
+        // but in the given root for static files, it's missing.
+        let tempdir = tempfile::tempdir()?;
+        let static_app = build_static_app(&env, &tempdir).await?;
+
+        // before bugfix, this would add caching headers, and
+        // trigger a `debug_assert`.
+        // The 404 is what we expect.
+        // `assert_not_found` also asserts if no-caching headers are set.
+        static_app.assert_not_found(PATH).await?;
+
+        Ok(())
+    }
+
     #[test]
     fn static_mime_types() {
         async_wrapper(|env| async move {

crates/bin/docs_rs_web/src/lib.rs

@@ -6,6 +6,7 @@
 
 mod cache;
 mod config;
+mod context;
 mod error;
 mod extractors;
 mod file;
@@ -21,6 +22,7 @@ pub(crate) mod testing;
 mod utils;
 
 pub use config::Config;
+pub use context::build_context;
 pub use docs_rs_build_limits::DEFAULT_MAX_TARGETS;
 pub use docs_rs_utils::{APP_USER_AGENT, BUILD_VERSION, RUSTDOC_STATIC_STORAGE_PREFIX};
 pub use font_awesome_as_a_crate::icons;