move & cache about pages where possible

crates/bin/docs_rs_admin/Cargo.toml

@@ -14,6 +14,8 @@ docs_rs_build_limits = { path = "../../lib/docs_rs_build_limits" }
 docs_rs_build_queue = { path = "../../lib/docs_rs_build_queue" }
 docs_rs_context = { path = "../../lib/docs_rs_context" }
 docs_rs_database = { path = "../../lib/docs_rs_database" }
+docs_rs_fastly = { path = "../../lib/docs_rs_fastly" }
+docs_rs_headers = { path = "../../lib/docs_rs_headers" }
 docs_rs_logging = { path = "../../lib/docs_rs_logging" }
 docs_rs_types = { path = "../../lib/docs_rs_types" }
 docs_rs_utils = { path = "../../lib/docs_rs_utils" }

crates/bin/docs_rs_admin/bin/docs_rs_release.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# all things that should happen after we deploy a new version.
+# at some point, should be integrated into the docker containers, 
+# AWS deploy, etc.  
+#
+# Should only be run once per release, more than once doesn't hurt, 
+# but don't run in parallel.
+
+# run database migrations.
+DOCSRS_MIN_POOL_IDLE=1 DOCSRS_MAX_POOL_SIZE=10 docs_rs_admin database migrate
+
+# purge static content that can only change on release. 
+# See `crates/bin/docs_rs_web` `cache::SURROGATE_KEY_DOCSRS_STATIC` and its 
+# usages.
+docs_rs_admin cdn purge docs-rs-static

crates/bin/docs_rs_admin/src/main.rs

@@ -2,7 +2,7 @@ mod rebuilds;
 #[cfg(test)]
 pub(crate) mod testing;
 
-use anyhow::{Context as _, Result};
+use anyhow::{Context as _, Result, bail};
 use chrono::NaiveDate;
 use clap::{Parser, Subcommand};
 use docs_rs_build_limits::{Overrides, blacklist};
@@ -15,9 +15,12 @@ use docs_rs_database::{
     crate_details,
     service_config::{ConfigName, set_config},
 };
+use docs_rs_fastly::CdnBehaviour as _;
+use docs_rs_headers::SurrogateKey;
 use docs_rs_types::{CrateId, KrateName, Version};
 use futures_util::StreamExt;
 use rebuilds::queue_rebuilds_faulty_rustdoc;
+use std::iter;
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -55,6 +58,11 @@ enum CommandLine {
         #[command(subcommand)]
         subcommand: QueueSubcommand,
     },
+
+    Cdn {
+        #[command(subcommand)]
+        subcommand: CdnSubcommand,
+    },
 }
 
 impl CommandLine {
@@ -68,12 +76,14 @@ impl CommandLine {
             .with_build_queue()?
             .with_repository_stats()?
             .with_registry_api()?
+            .with_maybe_cdn()?
             .build()?;
 
         match self {
             Self::Build { subcommand } => subcommand.handle_args(ctx).await?,
             Self::Database { subcommand } => subcommand.handle_args(ctx).await?,
             Self::Queue { subcommand } => subcommand.handle_args(ctx).await?,
+            Self::Cdn { subcommand } => subcommand.handle_args(ctx).await?,
         }
 
         Ok(())
@@ -471,3 +481,30 @@ impl BlacklistSubcommand {
         Ok(())
     }
 }
+
+#[derive(Debug, Clone, PartialEq, Eq, Subcommand)]
+enum CdnSubcommand {
+    /// purge pages with a surrogate key from the CDN
+    Purge {
+        /// Name of crate to build
+        #[arg(name = "SURROGATE_KEY")]
+        surrogate_key: SurrogateKey,
+    },
+}
+
+impl CdnSubcommand {
+    async fn handle_args(self, ctx: Context) -> Result<()> {
+        match self {
+            Self::Purge { surrogate_key } => {
+                if let Some(cdn) = ctx.cdn() {
+                    cdn.purge_surrogate_keys(iter::once(surrogate_key))
+                        .await
+                        .context("failed to purge CDN by surrogate key")?;
+                } else {
+                    bail!("CDN is not configured, cannot purge");
+                }
+            }
+        }
+        Ok(())
+    }
+}

crates/bin/docs_rs_web/src/cache.rs

@@ -16,6 +16,10 @@ use tracing::error;
 /// This enables us to use the fastly "soft purge" for everything.
 pub const SURROGATE_KEY_ALL: SurrogateKey = SurrogateKey::from_static("all");
 
+/// A surrogate key that we apply to content that is static and should be
+/// invalidated everything we deploy a new version of docs.rs.
+pub const SURROGATE_KEY_DOCSRS_STATIC: SurrogateKey = SurrogateKey::from_static("docs-rs-static");
+
 /// cache poicy for static assets like rustdoc files or build assets.
 pub const STATIC_ASSET_CACHE_POLICY: CachePolicy = CachePolicy::ForeverInCdnAndBrowser;
 

crates/bin/docs_rs_web/src/handlers/about.rs

@@ -0,0 +1,129 @@
+use crate::{
+    cache::{CachePolicy, SURROGATE_KEY_DOCSRS_STATIC},
+    error::{AxumErrorPage, AxumResult},
+    extractors::{DbConnection, Path},
+    impl_axum_webpage,
+    page::templates::{RenderBrands, RenderSolid, filters},
+};
+use askama::Template;
+use axum::{extract::Extension, http::StatusCode, response::IntoResponse};
+use docs_rs_build_limits::Limits;
+use docs_rs_context::Context;
+use docs_rs_database::service_config::{ConfigName, get_config};
+use std::sync::Arc;
+
+#[derive(Template)]
+#[template(path = "core/about/builds.html")]
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct AboutBuilds {
+    /// The current version of rustc that docs.rs is using to build crates
+    rustc_version: Option<String>,
+    /// The default crate build limits
+    limits: Limits,
+    /// Just for the template, since this isn't shared with AboutPage
+    active_tab: &'static str,
+}
+
+impl_axum_webpage!(
+    AboutBuilds,
+    // NOTE: potential future improvement: serve a special surrogate key, and
+    // purge that after we updated the local toolchain.
+    cache_policy = |_| CachePolicy::ShortInCdnAndBrowser,
+);
+
+pub(crate) async fn about_builds_handler(
+    mut conn: DbConnection,
+    Extension(context): Extension<Arc<Context>>,
+) -> AxumResult<impl IntoResponse> {
+    Ok(AboutBuilds {
+        rustc_version: get_config::<String>(&mut conn, ConfigName::RustcVersion).await?,
+        limits: Limits::new(context.config().build_limits()?),
+        active_tab: "builds",
+    })
+}
+
+macro_rules! about_page {
+    ($ty:ident, $template:literal) => {
+        #[derive(Template)]
+        #[template(path = $template)]
+        struct $ty;
+
+        impl_axum_webpage! {
+            $ty,
+            cache_policy = |_| CachePolicy::ForeverInCdn(SURROGATE_KEY_DOCSRS_STATIC.into())
+        }
+    };
+}
+
+about_page!(AboutPage, "core/about/index.html");
+about_page!(AboutPageBadges, "core/about/badges.html");
+about_page!(AboutPageMetadata, "core/about/metadata.html");
+about_page!(AboutPageRedirection, "core/about/redirections.html");
+about_page!(AboutPageDownload, "core/about/download.html");
+about_page!(AboutPageRustdocJson, "core/about/rustdoc-json.html");
+
+pub(crate) async fn about_handler(subpage: Option<Path<String>>) -> AxumResult<impl IntoResponse> {
+    let subpage = match subpage {
+        Some(subpage) => subpage.0,
+        None => "index".to_string(),
+    };
+
+    let response = match &subpage[..] {
+        "about" | "index" => AboutPage.into_response(),
+        "badges" => AboutPageBadges.into_response(),
+        "metadata" => AboutPageMetadata.into_response(),
+        "redirections" => AboutPageRedirection.into_response(),
+        "download" => AboutPageDownload.into_response(),
+        "rustdoc-json" => AboutPageRustdocJson.into_response(),
+        _ => {
+            let msg = "This /about page does not exist. \
+                Perhaps you are interested in <a href=\"https://github.com/rust-lang/docs.rs/tree/master/templates/core/about\">creating</a> it?";
+            let page = AxumErrorPage {
+                title: "The requested page does not exist",
+                message: msg.into(),
+                status: StatusCode::NOT_FOUND,
+            };
+            page.into_response()
+        }
+    };
+    Ok(response)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::testing::{
+        AxumResponseTestExt as _, AxumRouterTestExt, TestEnvironment, TestEnvironmentExt as _,
+    };
+    use anyhow::Result;
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn about_page() -> Result<()> {
+        let env = TestEnvironment::new().await?;
+        let web = env.web_app().await;
+        for file in std::fs::read_dir("templates/core/about")? {
+            use std::ffi::OsStr;
+
+            let file_path = file?.path();
+            if file_path.extension() != Some(OsStr::new("html"))
+                || file_path.file_stem() == Some(OsStr::new("index"))
+            {
+                continue;
+            }
+            let filename = file_path.file_stem().unwrap().to_str().unwrap();
+            let path = format!("/about/{filename}");
+            let response = web.assert_success(&path).await?;
+
+            if filename == "builds" {
+                response.assert_cache_control(CachePolicy::ShortInCdnAndBrowser, env.config());
+            } else {
+                response.assert_cache_control(
+                    CachePolicy::ForeverInCdn(SURROGATE_KEY_DOCSRS_STATIC.into()),
+                    env.config(),
+                );
+            }
+        }
+        web.assert_success("/about").await?;
+        Ok(())
+    }
+}

crates/bin/docs_rs_web/src/handlers/mod.rs

@@ -1,5 +1,6 @@
 //! Web interface of docs.rs
 
+pub(crate) mod about;
 pub(crate) mod build_details;
 pub(crate) mod builds;
 pub(crate) mod crate_details;