From d73b1fd5127a9f194efa25c20d00b2192abdf571 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 16 Jan 2026 02:41:14 +0100 Subject: [PATCH 1/3] web: move about-pages into separate module, remove duplicate test --- crates/bin/docs_rs_web/src/handlers/about.rs | 108 +++++++++++++++++ crates/bin/docs_rs_web/src/handlers/mod.rs | 1 + .../bin/docs_rs_web/src/handlers/sitemap.rs | 110 +----------------- crates/bin/docs_rs_web/src/routes.rs | 60 ++++------ 4 files changed, 134 insertions(+), 145 deletions(-) create mode 100644 crates/bin/docs_rs_web/src/handlers/about.rs diff --git a/crates/bin/docs_rs_web/src/handlers/about.rs b/crates/bin/docs_rs_web/src/handlers/about.rs new file mode 100644 index 000000000..05ce53313 --- /dev/null +++ b/crates/bin/docs_rs_web/src/handlers/about.rs @@ -0,0 +1,108 @@ +use crate::{ + error::{AxumErrorPage, AxumResult}, + extractors::{DbConnection, Path}, + impl_axum_webpage, + page::templates::{RenderBrands, RenderSolid, filters}, +}; +use askama::Template; +use axum::{extract::Extension, http::StatusCode, response::IntoResponse}; +use docs_rs_build_limits::Limits; +use docs_rs_context::Context; +use docs_rs_database::service_config::{ConfigName, get_config}; +use std::sync::Arc; + +#[derive(Template)] +#[template(path = "core/about/builds.html")] +#[derive(Debug, Clone, PartialEq, Eq)] +struct AboutBuilds { + /// The current version of rustc that docs.rs is using to build crates + rustc_version: Option, + /// The default crate build limits + limits: Limits, + /// Just for the template, since this isn't shared with AboutPage + active_tab: &'static str, +} + +impl_axum_webpage!(AboutBuilds); + +pub(crate) async fn about_builds_handler( + mut conn: DbConnection, + Extension(context): Extension>, +) -> AxumResult { + Ok(AboutBuilds { + rustc_version: get_config::(&mut conn, ConfigName::RustcVersion).await?, + limits: Limits::new(context.config().build_limits()?), + active_tab: "builds", + }) +} + +macro_rules! about_page { + ($ty:ident, $template:literal) => { + #[derive(Template)] + #[template(path = $template)] + struct $ty; + + impl_axum_webpage! { $ty } + }; +} + +about_page!(AboutPage, "core/about/index.html"); +about_page!(AboutPageBadges, "core/about/badges.html"); +about_page!(AboutPageMetadata, "core/about/metadata.html"); +about_page!(AboutPageRedirection, "core/about/redirections.html"); +about_page!(AboutPageDownload, "core/about/download.html"); +about_page!(AboutPageRustdocJson, "core/about/rustdoc-json.html"); + +pub(crate) async fn about_handler(subpage: Option>) -> AxumResult { + let subpage = match subpage { + Some(subpage) => subpage.0, + None => "index".to_string(), + }; + + let response = match &subpage[..] { + "about" | "index" => AboutPage.into_response(), + "badges" => AboutPageBadges.into_response(), + "metadata" => AboutPageMetadata.into_response(), + "redirections" => AboutPageRedirection.into_response(), + "download" => AboutPageDownload.into_response(), + "rustdoc-json" => AboutPageRustdocJson.into_response(), + _ => { + let msg = "This /about page does not exist. \ + Perhaps you are interested in creating it?"; + let page = AxumErrorPage { + title: "The requested page does not exist", + message: msg.into(), + status: StatusCode::NOT_FOUND, + }; + page.into_response() + } + }; + Ok(response) +} + +#[cfg(test)] +mod tests { + use crate::testing::{AxumRouterTestExt, TestEnvironment, TestEnvironmentExt as _}; + use anyhow::Result; + + #[tokio::test(flavor = "multi_thread")] + async fn about_page() -> Result<()> { + let env = TestEnvironment::new().await?; + let web = env.web_app().await; + for file in std::fs::read_dir("templates/core/about")? { + use std::ffi::OsStr; + + let file_path = file?.path(); + if file_path.extension() != Some(OsStr::new("html")) + || file_path.file_stem() == Some(OsStr::new("index")) + { + continue; + } + let filename = file_path.file_stem().unwrap().to_str().unwrap(); + let path = format!("/about/{filename}"); + web.assert_success(&path).await?; + } + web.assert_success("/about").await?; + Ok(()) + } +} diff --git a/crates/bin/docs_rs_web/src/handlers/mod.rs b/crates/bin/docs_rs_web/src/handlers/mod.rs index a9a7ccc09..cdc21d03d 100644 --- a/crates/bin/docs_rs_web/src/handlers/mod.rs +++ b/crates/bin/docs_rs_web/src/handlers/mod.rs @@ -1,5 +1,6 @@ //! Web interface of docs.rs +pub(crate) mod about; pub(crate) mod build_details; pub(crate) mod builds; pub(crate) mod crate_details; diff --git a/crates/bin/docs_rs_web/src/handlers/sitemap.rs b/crates/bin/docs_rs_web/src/handlers/sitemap.rs index 7c60e485b..b77fa3725 100644 --- a/crates/bin/docs_rs_web/src/handlers/sitemap.rs +++ b/crates/bin/docs_rs_web/src/handlers/sitemap.rs @@ -1,25 +1,20 @@ use crate::{ - error::{AxumErrorPage, AxumNope, AxumResult}, + error::{AxumNope, AxumResult}, extractors::{DbConnection, Path}, impl_axum_webpage, - page::templates::{RenderBrands, RenderSolid, filters}, + page::templates::filters, }; use askama::Template; use async_stream::stream; use axum::{ body::{Body, Bytes}, - extract::Extension, http::StatusCode, response::IntoResponse, }; use axum_extra::{TypedHeader, headers::ContentType}; use chrono::{TimeZone, Utc}; -use docs_rs_build_limits::Limits; -use docs_rs_context::Context; -use docs_rs_database::service_config::{ConfigName, get_config}; use docs_rs_mimes as mimes; use futures_util::{StreamExt as _, pin_mut}; -use std::sync::Arc; use tracing::{Span, error}; use tracing_futures::Instrument as _; @@ -149,75 +144,6 @@ pub(crate) async fn sitemap_handler( )) } -#[derive(Template)] -#[template(path = "core/about/builds.html")] -#[derive(Debug, Clone, PartialEq, Eq)] -struct AboutBuilds { - /// The current version of rustc that docs.rs is using to build crates - rustc_version: Option, - /// The default crate build limits - limits: Limits, - /// Just for the template, since this isn't shared with AboutPage - active_tab: &'static str, -} - -impl_axum_webpage!(AboutBuilds); - -pub(crate) async fn about_builds_handler( - mut conn: DbConnection, - Extension(context): Extension>, -) -> AxumResult { - Ok(AboutBuilds { - rustc_version: get_config::(&mut conn, ConfigName::RustcVersion).await?, - limits: Limits::new(context.config().build_limits()?), - active_tab: "builds", - }) -} - -macro_rules! about_page { - ($ty:ident, $template:literal) => { - #[derive(Template)] - #[template(path = $template)] - struct $ty; - - impl_axum_webpage! { $ty } - }; -} - -about_page!(AboutPage, "core/about/index.html"); -about_page!(AboutPageBadges, "core/about/badges.html"); -about_page!(AboutPageMetadata, "core/about/metadata.html"); -about_page!(AboutPageRedirection, "core/about/redirections.html"); -about_page!(AboutPageDownload, "core/about/download.html"); -about_page!(AboutPageRustdocJson, "core/about/rustdoc-json.html"); - -pub(crate) async fn about_handler(subpage: Option>) -> AxumResult { - let subpage = match subpage { - Some(subpage) => subpage.0, - None => "index".to_string(), - }; - - let response = match &subpage[..] { - "about" | "index" => AboutPage.into_response(), - "badges" => AboutPageBadges.into_response(), - "metadata" => AboutPageMetadata.into_response(), - "redirections" => AboutPageRedirection.into_response(), - "download" => AboutPageDownload.into_response(), - "rustdoc-json" => AboutPageRustdocJson.into_response(), - _ => { - let msg = "This /about page does not exist. \ - Perhaps you are interested in creating it?"; - let page = AxumErrorPage { - title: "The requested page does not exist", - message: msg.into(), - status: StatusCode::NOT_FOUND, - }; - page.into_response() - } - }; - Ok(response) -} - #[cfg(test)] mod tests { use crate::testing::{ @@ -317,36 +243,4 @@ mod tests { Ok(()) }) } - - #[test] - fn about_page() { - async_wrapper(|env| async move { - let web = env.web_app().await; - for file in std::fs::read_dir("templates/core/about")? { - use std::ffi::OsStr; - - let file_path = file?.path(); - if file_path.extension() != Some(OsStr::new("html")) - || file_path.file_stem() == Some(OsStr::new("index")) - { - continue; - } - let filename = file_path.file_stem().unwrap().to_str().unwrap(); - let path = format!("/about/{filename}"); - web.assert_success(&path).await?; - } - web.assert_success("/about").await?; - Ok(()) - }) - } - - #[test] - fn robots_txt() { - async_wrapper(|env| async move { - let web = env.web_app().await; - web.assert_redirect("/robots.txt", "/-/static/robots.txt") - .await?; - Ok(()) - }) - } } diff --git a/crates/bin/docs_rs_web/src/routes.rs b/crates/bin/docs_rs_web/src/routes.rs index 35725c366..22e4011ac 100644 --- a/crates/bin/docs_rs_web/src/routes.rs +++ b/crates/bin/docs_rs_web/src/routes.rs @@ -2,7 +2,7 @@ use crate::{ cache::CachePolicy, error::AxumNope, handlers::{ - build_details, builds, crate_details, features, releases, rustdoc, sitemap, source, + about, build_details, builds, crate_details, features, releases, rustdoc, sitemap, source, statics::build_static_router, status, }, metrics::request_recorder, @@ -137,9 +137,9 @@ pub(crate) fn build_axum_routes() -> AxumRouter { "/-/sitemap/{letter}/sitemap.xml", get_internal(sitemap::sitemap_handler), ) - .route_with_tsr("/about/builds", get_internal(sitemap::about_builds_handler)) - .route_with_tsr("/about", get_internal(sitemap::about_handler)) - .route_with_tsr("/about/{subpage}", get_internal(sitemap::about_handler)) + .route_with_tsr("/about/builds", get_internal(about::about_builds_handler)) + .route_with_tsr("/about", get_internal(about::about_handler)) + .route_with_tsr("/about/{subpage}", get_internal(about::about_handler)) .route("/", get_internal(releases::home_page)) .route_with_tsr("/releases", get_internal(releases::recent_releases_handler)) .route_with_tsr( @@ -356,45 +356,31 @@ async fn fallback() -> impl IntoResponse { mod tests { use crate::cache::CachePolicy; use crate::testing::{ - AxumResponseTestExt, AxumRouterTestExt, TestEnvironmentExt as _, async_wrapper, + AxumResponseTestExt, AxumRouterTestExt, TestEnvironment, TestEnvironmentExt as _, + async_wrapper, }; + use anyhow::Result; use reqwest::StatusCode; + use test_case::test_case; - #[test] - fn test_root_redirects() { - async_wrapper(|env| async move { - let web = env.web_app().await; - let config = env.config(); - // These are "well-known" resources that will be requested from the root, but support - // redirection - web.assert_redirect_cached( - "/favicon.ico", - "/-/static/favicon.ico", - CachePolicy::ForeverInCdnAndBrowser, - config, - ) - .await?; - web.assert_redirect_cached( - "/robots.txt", - "/-/static/robots.txt", - CachePolicy::ForeverInCdnAndBrowser, - config, - ) - .await?; + // These are "well-known" resources that will be requested from the root, but support + // redirection + #[test_case("/favicon.ico", "/-/static/favicon.ico")] + #[test_case("/robots.txt", "/-/static/robots.txt")] + // This has previously been served with a url pointing to the root, it may be + // plausible to remove the redirects in the future, but for now we need to keep serving + // it. + #[test_case("/opensearch.xml", "/-/static/opensearch.xml")] + #[tokio::test(flavor = "multi_thread")] + async fn test_root_redirects(path: &str, target: &str) -> Result<()> { + let env = TestEnvironment::new().await?; + let web = env.web_app().await; + let config = env.config(); - // This has previously been served with a url pointing to the root, it may be - // plausible to remove the redirects in the future, but for now we need to keep serving - // it. - web.assert_redirect_cached( - "/opensearch.xml", - "/-/static/opensearch.xml", - CachePolicy::ForeverInCdnAndBrowser, - config, - ) + web.assert_redirect_cached(path, target, CachePolicy::ForeverInCdnAndBrowser, config) .await?; - Ok(()) - }); + Ok(()) } #[test] From fa6028f0dc296271cb76c607538ebabc2d0bacfe Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 16 Jan 2026 02:49:38 +0100 Subject: [PATCH 2/3] web: add short cache to about/builds page --- crates/bin/docs_rs_web/src/handlers/about.rs | 24 ++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/bin/docs_rs_web/src/handlers/about.rs b/crates/bin/docs_rs_web/src/handlers/about.rs index 05ce53313..fd3636faf 100644 --- a/crates/bin/docs_rs_web/src/handlers/about.rs +++ b/crates/bin/docs_rs_web/src/handlers/about.rs @@ -1,4 +1,5 @@ use crate::{ + cache::CachePolicy, error::{AxumErrorPage, AxumResult}, extractors::{DbConnection, Path}, impl_axum_webpage, @@ -23,7 +24,12 @@ struct AboutBuilds { active_tab: &'static str, } -impl_axum_webpage!(AboutBuilds); +impl_axum_webpage!( + AboutBuilds, + // NOTE: potential future improvement: serve a special surrogate key, and + // purge that after we updated the local toolchain. + cache_policy = |_| CachePolicy::ShortInCdnAndBrowser, +); pub(crate) async fn about_builds_handler( mut conn: DbConnection, @@ -82,9 +88,23 @@ pub(crate) async fn about_handler(subpage: Option>) -> AxumResult Result<()> { + let env = TestEnvironment::new().await?; + let web = env.web_app().await; + + let response = web.assert_success("/about/builds").await?; + response.assert_cache_control(CachePolicy::ShortInCdnAndBrowser, env.config()); + + Ok(()) + } + #[tokio::test(flavor = "multi_thread")] async fn about_page() -> Result<()> { let env = TestEnvironment::new().await?; From 25919d2ea835f184a05866f44e2fd0a5cdb8f800 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 16 Jan 2026 03:15:41 +0100 Subject: [PATCH 3/3] web: cache about-pages forever, invalidate on docs.rs deploy --- Cargo.lock | 2 + crates/bin/docs_rs_admin/Cargo.toml | 2 + .../bin/docs_rs_admin/bin/docs_rs_release.sh | 18 +++++++++ crates/bin/docs_rs_admin/src/main.rs | 39 ++++++++++++++++++- crates/bin/docs_rs_web/src/cache.rs | 4 ++ crates/bin/docs_rs_web/src/handlers/about.rs | 29 +++++++------- dockerfiles/Dockerfile | 2 + 7 files changed, 81 insertions(+), 15 deletions(-) create mode 100755 crates/bin/docs_rs_admin/bin/docs_rs_release.sh diff --git a/Cargo.lock b/Cargo.lock index 4fd4d6b0f..559f7965c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1940,6 +1940,8 @@ dependencies = [ "docs_rs_config", "docs_rs_context", "docs_rs_database", + "docs_rs_fastly", + "docs_rs_headers", "docs_rs_logging", "docs_rs_opentelemetry", "docs_rs_storage", diff --git a/crates/bin/docs_rs_admin/Cargo.toml b/crates/bin/docs_rs_admin/Cargo.toml index ca029ba7b..29629f25d 100644 --- a/crates/bin/docs_rs_admin/Cargo.toml +++ b/crates/bin/docs_rs_admin/Cargo.toml @@ -14,6 +14,8 @@ docs_rs_build_limits = { path = "../../lib/docs_rs_build_limits" } docs_rs_build_queue = { path = "../../lib/docs_rs_build_queue" } docs_rs_context = { path = "../../lib/docs_rs_context" } docs_rs_database = { path = "../../lib/docs_rs_database" } +docs_rs_fastly = { path = "../../lib/docs_rs_fastly" } +docs_rs_headers = { path = "../../lib/docs_rs_headers" } docs_rs_logging = { path = "../../lib/docs_rs_logging" } docs_rs_types = { path = "../../lib/docs_rs_types" } docs_rs_utils = { path = "../../lib/docs_rs_utils" } diff --git a/crates/bin/docs_rs_admin/bin/docs_rs_release.sh b/crates/bin/docs_rs_admin/bin/docs_rs_release.sh new file mode 100755 index 000000000..6dd09bc2e --- /dev/null +++ b/crates/bin/docs_rs_admin/bin/docs_rs_release.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# all things that should happen after we deploy a new version. +# at some point, should be integrated into the docker containers, +# AWS deploy, etc. +# +# Should only be run once per release, more than once doesn't hurt, +# but don't run in parallel. + +# run database migrations. +DOCSRS_MIN_POOL_IDLE=1 DOCSRS_MAX_POOL_SIZE=10 docs_rs_admin database migrate + +# purge static content that can only change on release. +# See `crates/bin/docs_rs_web` `cache::SURROGATE_KEY_DOCSRS_STATIC` and its +# usages. +docs_rs_admin cdn purge docs-rs-static diff --git a/crates/bin/docs_rs_admin/src/main.rs b/crates/bin/docs_rs_admin/src/main.rs index 975031cdd..8719d0050 100644 --- a/crates/bin/docs_rs_admin/src/main.rs +++ b/crates/bin/docs_rs_admin/src/main.rs @@ -2,7 +2,7 @@ mod rebuilds; #[cfg(test)] pub(crate) mod testing; -use anyhow::{Context as _, Result}; +use anyhow::{Context as _, Result, bail}; use chrono::NaiveDate; use clap::{Parser, Subcommand}; use docs_rs_build_limits::{Overrides, blacklist}; @@ -15,9 +15,12 @@ use docs_rs_database::{ crate_details, service_config::{ConfigName, set_config}, }; +use docs_rs_fastly::CdnBehaviour as _; +use docs_rs_headers::SurrogateKey; use docs_rs_types::{CrateId, KrateName, Version}; use futures_util::StreamExt; use rebuilds::queue_rebuilds_faulty_rustdoc; +use std::iter; #[tokio::main] async fn main() -> Result<()> { @@ -55,6 +58,11 @@ enum CommandLine { #[command(subcommand)] subcommand: QueueSubcommand, }, + + Cdn { + #[command(subcommand)] + subcommand: CdnSubcommand, + }, } impl CommandLine { @@ -68,12 +76,14 @@ impl CommandLine { .with_build_queue()? .with_repository_stats()? .with_registry_api()? + .with_maybe_cdn()? .build()?; match self { Self::Build { subcommand } => subcommand.handle_args(ctx).await?, Self::Database { subcommand } => subcommand.handle_args(ctx).await?, Self::Queue { subcommand } => subcommand.handle_args(ctx).await?, + Self::Cdn { subcommand } => subcommand.handle_args(ctx).await?, } Ok(()) @@ -471,3 +481,30 @@ impl BlacklistSubcommand { Ok(()) } } + +#[derive(Debug, Clone, PartialEq, Eq, Subcommand)] +enum CdnSubcommand { + /// purge pages with a surrogate key from the CDN + Purge { + /// Name of crate to build + #[arg(name = "SURROGATE_KEY")] + surrogate_key: SurrogateKey, + }, +} + +impl CdnSubcommand { + async fn handle_args(self, ctx: Context) -> Result<()> { + match self { + Self::Purge { surrogate_key } => { + if let Some(cdn) = ctx.cdn() { + cdn.purge_surrogate_keys(iter::once(surrogate_key)) + .await + .context("failed to purge CDN by surrogate key")?; + } else { + bail!("CDN is not configured, cannot purge"); + } + } + } + Ok(()) + } +} diff --git a/crates/bin/docs_rs_web/src/cache.rs b/crates/bin/docs_rs_web/src/cache.rs index e8ee61031..ae05c7772 100644 --- a/crates/bin/docs_rs_web/src/cache.rs +++ b/crates/bin/docs_rs_web/src/cache.rs @@ -16,6 +16,10 @@ use tracing::error; /// This enables us to use the fastly "soft purge" for everything. pub const SURROGATE_KEY_ALL: SurrogateKey = SurrogateKey::from_static("all"); +/// A surrogate key that we apply to content that is static and should be +/// invalidated everything we deploy a new version of docs.rs. +pub const SURROGATE_KEY_DOCSRS_STATIC: SurrogateKey = SurrogateKey::from_static("docs-rs-static"); + /// cache poicy for static assets like rustdoc files or build assets. pub const STATIC_ASSET_CACHE_POLICY: CachePolicy = CachePolicy::ForeverInCdnAndBrowser; diff --git a/crates/bin/docs_rs_web/src/handlers/about.rs b/crates/bin/docs_rs_web/src/handlers/about.rs index fd3636faf..10584abb8 100644 --- a/crates/bin/docs_rs_web/src/handlers/about.rs +++ b/crates/bin/docs_rs_web/src/handlers/about.rs @@ -1,5 +1,5 @@ use crate::{ - cache::CachePolicy, + cache::{CachePolicy, SURROGATE_KEY_DOCSRS_STATIC}, error::{AxumErrorPage, AxumResult}, extractors::{DbConnection, Path}, impl_axum_webpage, @@ -48,7 +48,10 @@ macro_rules! about_page { #[template(path = $template)] struct $ty; - impl_axum_webpage! { $ty } + impl_axum_webpage! { + $ty, + cache_policy = |_| CachePolicy::ForeverInCdn(SURROGATE_KEY_DOCSRS_STATIC.into()) + } }; } @@ -94,17 +97,6 @@ mod tests { }; use anyhow::Result; - #[tokio::test(flavor = "multi_thread")] - async fn about_build_is_cached() -> Result<()> { - let env = TestEnvironment::new().await?; - let web = env.web_app().await; - - let response = web.assert_success("/about/builds").await?; - response.assert_cache_control(CachePolicy::ShortInCdnAndBrowser, env.config()); - - Ok(()) - } - #[tokio::test(flavor = "multi_thread")] async fn about_page() -> Result<()> { let env = TestEnvironment::new().await?; @@ -120,7 +112,16 @@ mod tests { } let filename = file_path.file_stem().unwrap().to_str().unwrap(); let path = format!("/about/{filename}"); - web.assert_success(&path).await?; + let response = web.assert_success(&path).await?; + + if filename == "builds" { + response.assert_cache_control(CachePolicy::ShortInCdnAndBrowser, env.config()); + } else { + response.assert_cache_control( + CachePolicy::ForeverInCdn(SURROGATE_KEY_DOCSRS_STATIC.into()), + env.config(), + ); + } } web.assert_success("/about").await?; Ok(()) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 2dfe28ff7..ad41f82fe 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -204,6 +204,8 @@ RUN apt-get update \ WORKDIR /srv/docsrs +COPY crates/bin/docs_rs_admin/bin/docs_rs_release.sh /usr/local/bin/ + COPY --from=build /artifacts/docs_rs_admin /usr/local/bin/ COPY --from=build /artifacts/cratesfyi /usr/local/bin