diff --git a/.sqlx/query-029cd6f3f3e44686a9259a8aa2155c044e87f75430006b1c01069d350a3e0838.json b/.sqlx/query-029cd6f3f3e44686a9259a8aa2155c044e87f75430006b1c01069d350a3e0838.json new file mode 100644 index 000000000..037c1c25d --- /dev/null +++ b/.sqlx/query-029cd6f3f3e44686a9259a8aa2155c044e87f75430006b1c01069d350a3e0838.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE config SET value = $2 WHERE name = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Json" + ] + }, + "nullable": [] + }, + "hash": "029cd6f3f3e44686a9259a8aa2155c044e87f75430006b1c01069d350a3e0838" +} diff --git a/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/Cargo.lock b/Cargo.lock index 8c6ad2459..021efc5f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1972,6 +1972,7 @@ dependencies = [ "docs_rs_storage", "docs_rs_test_fakes", "docs_rs_types", + "docs_rs_uri", "docs_rs_utils", "futures-util", "pretty_assertions", @@ -2015,6 +2016,7 @@ dependencies = [ "docs_rs_storage", "docs_rs_test_fakes", "docs_rs_types", + "docs_rs_uri", "docs_rs_utils", "futures-util", "opentelemetry", @@ -2117,6 +2119,7 @@ dependencies = [ "docs_rs_opentelemetry", "docs_rs_registry_api", "docs_rs_types", + "docs_rs_uri", "docs_rs_utils", "futures-util", "hex", diff --git a/crates/bin/cratesfyi/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/bin/cratesfyi/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/bin/cratesfyi/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/bin/docs_rs_admin/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/bin/docs_rs_admin/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/bin/docs_rs_admin/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/bin/docs_rs_admin/Cargo.toml b/crates/bin/docs_rs_admin/Cargo.toml index cf7d7e48f..c7c2a55ae 100644 --- a/crates/bin/docs_rs_admin/Cargo.toml +++ b/crates/bin/docs_rs_admin/Cargo.toml @@ -20,6 +20,7 @@ docs_rs_logging = { path = "../../lib/docs_rs_logging" } docs_rs_repository_stats = { path = "../../lib/docs_rs_repository_stats" } docs_rs_storage = { path = "../../lib/docs_rs_storage" } docs_rs_types = { path = "../../lib/docs_rs_types" } +docs_rs_uri = { path = "../../lib/docs_rs_uri" } docs_rs_utils = { path = "../../lib/docs_rs_utils" } futures-util = { workspace = true } sqlx = { workspace = true } diff --git a/crates/bin/docs_rs_admin/src/main.rs b/crates/bin/docs_rs_admin/src/main.rs index 54821f537..97f5969a8 100644 --- a/crates/bin/docs_rs_admin/src/main.rs +++ b/crates/bin/docs_rs_admin/src/main.rs @@ -14,12 +14,13 @@ use docs_rs_build_queue::priority::{ use docs_rs_context::Context; use docs_rs_database::{ crate_details, - service_config::{ConfigName, set_config}, + service_config::{Abnormality, ConfigName, remove_config, set_config}, }; use docs_rs_fastly::CdnBehaviour as _; use docs_rs_headers::SurrogateKey; use docs_rs_repository_stats::workspaces; use docs_rs_types::{CrateId, KrateName, ReleaseId, Version}; +use docs_rs_uri::EscapedURI; use futures_util::StreamExt; use rebuilds::queue_rebuilds_faulty_rustdoc; use std::iter; @@ -37,7 +38,7 @@ async fn main() -> Result<()> { Ok(()) } -#[derive(Debug, Clone, PartialEq, Eq, Parser)] +#[derive(Debug, Clone, PartialEq, Parser)] #[command( about = env!("CARGO_PKG_DESCRIPTION"), version = docs_rs_utils::BUILD_VERSION, @@ -350,7 +351,7 @@ impl BuildSubcommand { } } -#[derive(Debug, Clone, PartialEq, Eq, Subcommand)] +#[derive(Debug, Clone, PartialEq, Subcommand)] enum DatabaseSubcommand { /// Run database migration Migrate { @@ -359,6 +360,12 @@ enum DatabaseSubcommand { version: Option, }, + /// Manage the abnormality shown in the site header + Abnormality { + #[command(subcommand)] + command: AbnormalitySubcommand, + }, + /// temporary command to repackage missing crates into archive storage. /// starts at the earliest release and works forwards. Repackage { @@ -404,6 +411,8 @@ impl DatabaseSubcommand { } .context("Failed to run database migrations")?, + Self::Abnormality { command } => command.handle_args(ctx).await?, + Self::Repackage { limit } => { let pool = ctx.pool()?; let storage = ctx.storage()?; @@ -504,6 +513,60 @@ impl DatabaseSubcommand { } } +#[derive(Debug, Clone, PartialEq, Subcommand)] +enum AbnormalitySubcommand { + /// Set the abnormality shown in the site header + Set { + #[arg(long)] + url: EscapedURI, + #[arg(long)] + text: String, + /// explanation to be shown on the status page, can be HTML + #[arg(long)] + explanation: Option, + }, + + /// Remove the abnormality shown in the site header + Remove, +} + +impl AbnormalitySubcommand { + async fn handle_args(self, ctx: Context) -> Result<()> { + let mut conn = ctx + .pool()? + .get_async() + .await + .context("failed to get a database connection")?; + + match self { + Self::Set { + url, + text, + explanation, + } => { + set_config( + &mut conn, + ConfigName::Abnormality, + Abnormality { + url, + text, + explanation, + }, + ) + .await + .context("failed to set abnormality in database")?; + } + Self::Remove => { + remove_config(&mut conn, ConfigName::Abnormality) + .await + .context("failed to remove abnormality from database")?; + } + } + + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Subcommand)] enum LimitsSubcommand { /// Get sandbox limit overrides for a crate diff --git a/crates/bin/docs_rs_builder/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/bin/docs_rs_builder/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/bin/docs_rs_builder/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/bin/docs_rs_import_release/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/bin/docs_rs_watcher/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/bin/docs_rs_watcher/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/bin/docs_rs_watcher/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/bin/docs_rs_web/.sqlx/query-029cd6f3f3e44686a9259a8aa2155c044e87f75430006b1c01069d350a3e0838.json b/crates/bin/docs_rs_web/.sqlx/query-029cd6f3f3e44686a9259a8aa2155c044e87f75430006b1c01069d350a3e0838.json new file mode 100644 index 000000000..037c1c25d --- /dev/null +++ b/crates/bin/docs_rs_web/.sqlx/query-029cd6f3f3e44686a9259a8aa2155c044e87f75430006b1c01069d350a3e0838.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE config SET value = $2 WHERE name = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Json" + ] + }, + "nullable": [] + }, + "hash": "029cd6f3f3e44686a9259a8aa2155c044e87f75430006b1c01069d350a3e0838" +} diff --git a/crates/bin/docs_rs_web/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/bin/docs_rs_web/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/bin/docs_rs_web/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/bin/docs_rs_web/src/cache.rs b/crates/bin/docs_rs_web/src/cache.rs index 195198172..ccb84195d 100644 --- a/crates/bin/docs_rs_web/src/cache.rs +++ b/crates/bin/docs_rs_web/src/cache.rs @@ -73,6 +73,16 @@ static SHORT: ResponseCacheHeaders = ResponseCacheHeaders { is_caching_something: true, }; +/// Cache for a little longer time in the browser & in the CDN. +/// Helps protecting against traffic spikes. +static LONGER: ResponseCacheHeaders = ResponseCacheHeaders { + cache_control: Some(HeaderValue::from_static("public, max-age=600")), + surrogate_control: None, + surrogate_keys: None, + needs_cdn_invalidation: false, + is_caching_something: true, +}; + /// don't cache, don't even store. Never. Ever. static NO_STORE_MUST_REVALIDATE: ResponseCacheHeaders = ResponseCacheHeaders { cache_control: Some(HeaderValue::from_static( @@ -133,6 +143,11 @@ pub enum CachePolicy { /// Can be used when the content can be a _little_ outdated, /// while protecting against spikes in traffic. ShortInCdnAndBrowser, + /// cache for a little longer short time in the browser & CDN. + /// right now: 10 minutes + /// Can be used when the content can be a _little_ outdated, + /// while protecting against spikes in traffic. + LongerInCdnAndBrowser, /// cache forever in browser & CDN. /// Valid when you have hashed / versioned filenames and every rebuild would /// change the filename. @@ -160,6 +175,7 @@ impl CachePolicy { CachePolicy::NoCaching => NO_CACHING.clone(), CachePolicy::NoStoreMustRevalidate => NO_STORE_MUST_REVALIDATE.clone(), CachePolicy::ShortInCdnAndBrowser => SHORT.clone(), + CachePolicy::LongerInCdnAndBrowser => LONGER.clone(), CachePolicy::ForeverInCdnAndBrowser => FOREVER_IN_CDN_AND_BROWSER.clone(), CachePolicy::ForeverInCdn(surrogate_keys) => { if config.cache_invalidatable_responses { @@ -325,6 +341,7 @@ mod tests { NoCaching, NoStoreMustRevalidate, ShortInCdnAndBrowser, + LongerInCdnAndBrowser, ForeverInCdnAndBrowser, ForeverInCdn(key.clone().into()), ForeverInCdnAndStaleInBrowser(key.clone().into()), diff --git a/crates/bin/docs_rs_web/src/handlers/about.rs b/crates/bin/docs_rs_web/src/handlers/about.rs index 10584abb8..928d2de60 100644 --- a/crates/bin/docs_rs_web/src/handlers/about.rs +++ b/crates/bin/docs_rs_web/src/handlers/about.rs @@ -107,6 +107,7 @@ mod tests { let file_path = file?.path(); if file_path.extension() != Some(OsStr::new("html")) || file_path.file_stem() == Some(OsStr::new("index")) + || file_path.file_stem() == Some(OsStr::new("status")) { continue; } diff --git a/crates/bin/docs_rs_web/src/handlers/build_status.rs b/crates/bin/docs_rs_web/src/handlers/build_status.rs new file mode 100644 index 000000000..ac560a4c2 --- /dev/null +++ b/crates/bin/docs_rs_web/src/handlers/build_status.rs @@ -0,0 +1,214 @@ +use crate::{ + cache::CachePolicy, + error::{AxumNope, AxumResult}, + extractors::{DbConnection, rustdoc::RustdocParams}, + match_release::match_version, +}; +use axum::{ + Json, extract::Extension, http::header::ACCESS_CONTROL_ALLOW_ORIGIN, response::IntoResponse, +}; + +pub(crate) async fn status_handler( + params: RustdocParams, + mut conn: DbConnection, +) -> impl IntoResponse { + ( + Extension(CachePolicy::NoStoreMustRevalidate), + [(ACCESS_CONTROL_ALLOW_ORIGIN, "*")], + // We use an async block to emulate a try block so that we can apply the above CORS header + // and cache policy to both successful and failed responses + async move { + let matched_release = match_version(&mut conn, params.name(), params.req_version()) + .await? + .assume_exact_name()?; + + let rustdoc_status = matched_release.rustdoc_status(); + + let version = matched_release + .into_canonical_req_version_or_else(|confirmed_name, version| { + AxumNope::Redirect( + params + .clone() + .with_name(confirmed_name) + .with_req_version(version) + .build_status_url(), + CachePolicy::NoCaching, + ) + })? + .into_version(); + + let json = Json(serde_json::json!({ + "version": version.to_string(), + "doc_status": rustdoc_status, + })); + + AxumResult::Ok(json.into_response()) + } + .await, + ) +} + +#[cfg(test)] +mod tests { + use crate::{ + cache::CachePolicy, + testing::{AxumResponseTestExt, AxumRouterTestExt, TestEnvironmentExt as _, async_wrapper}, + }; + use docs_rs_types::ReqVersion; + use reqwest::StatusCode; + use test_case::test_case; + + #[test_case("latest")] + #[test_case("0.1")] + #[test_case("0.1.0")] + #[test_case("=0.1.0"; "exact_version")] + fn status(req_version: &str) { + async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + + env.fake_release() + .await + .name("foo") + .version("0.1.0") + .create() + .await?; + + let response = env + .web_app() + .await + .get_and_follow_redirects(&format!("/crate/foo/{req_version}/status.json")) + .await?; + response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); + assert_eq!(response.headers()["access-control-allow-origin"], "*"); + assert_eq!(response.status(), StatusCode::OK); + let value: serde_json::Value = serde_json::from_str(&response.text().await?)?; + + assert_eq!( + value, + serde_json::json!({ + "version": "0.1.0", + "doc_status": true, + }) + ); + + Ok(()) + }); + } + + #[test] + fn redirect_latest() { + async_wrapper(|env| async move { + env.fake_release() + .await + .name("foo") + .version("0.1.0") + .create() + .await?; + + let web = env.web_app().await; + let redirect = web + .assert_redirect("/crate/foo/*/status.json", "/crate/foo/latest/status.json") + .await?; + redirect.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); + assert_eq!(redirect.headers()["access-control-allow-origin"], "*"); + + Ok(()) + }); + } + + #[test_case("0.1")] + #[test_case("~0.1"; "semver")] + fn redirect(req_version: &str) { + async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + + env.fake_release() + .await + .name("foo") + .version("0.1.0") + .create() + .await?; + + let web = env.web_app().await; + let redirect = web + .assert_redirect( + &format!("/crate/foo/{req_version}/status.json"), + "/crate/foo/0.1.0/status.json", + ) + .await?; + redirect.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); + assert_eq!(redirect.headers()["access-control-allow-origin"], "*"); + + Ok(()) + }); + } + + #[test_case("latest")] + #[test_case("0.1")] + #[test_case("0.1.0")] + #[test_case("=0.1.0"; "exact_version")] + fn failure(req_version: &str) { + async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + + env.fake_release() + .await + .name("foo") + .version("0.1.0") + .build_result_failed() + .create() + .await?; + + let response = env + .web_app() + .await + .get_and_follow_redirects(&format!("/crate/foo/{req_version}/status.json")) + .await?; + response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); + assert_eq!(response.headers()["access-control-allow-origin"], "*"); + assert_eq!(response.status(), StatusCode::OK); + let value: serde_json::Value = serde_json::from_str(&response.text().await?)?; + + assert_eq!( + value, + serde_json::json!({ + "version": "0.1.0", + "doc_status": false, + }) + ); + + Ok(()) + }); + } + + // crate not found + #[test_case("bar", "0.1")] + #[test_case("bar", "0.1.0")] + // version not found + #[test_case("foo", "=0.1.0"; "exact_version")] + #[test_case("foo", "0.2")] + #[test_case("foo", "0.2.0")] + // invalid semver + #[test_case("foo", "0,1")] + #[test_case("foo", "0,1,0")] + fn not_found(krate: &str, req_version: &str) { + async_wrapper(|env| async move { + env.fake_release() + .await + .name("foo") + .version("0.1.1") + .create() + .await?; + + let response = env + .web_app() + .await + .get_and_follow_redirects(&format!("/crate/{krate}/{req_version}/status.json")) + .await?; + response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); + assert_eq!(response.headers()["access-control-allow-origin"], "*"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + Ok(()) + }); + } +} diff --git a/crates/bin/docs_rs_web/src/handlers/mod.rs b/crates/bin/docs_rs_web/src/handlers/mod.rs index c1ccd760e..49dd14afc 100644 --- a/crates/bin/docs_rs_web/src/handlers/mod.rs +++ b/crates/bin/docs_rs_web/src/handlers/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod about; pub(crate) mod build_details; +pub(crate) mod build_status; pub(crate) mod builds; pub(crate) mod crate_details; pub(crate) mod features; @@ -75,7 +76,6 @@ async fn apply_middleware( template_data: Option>, ) -> Result { let has_templates = template_data.is_some(); - let web_metrics = Arc::new(WebMetrics::new(&context.meter_provider)); Ok(router.layer( @@ -262,6 +262,26 @@ mod tests { }); } + #[tokio::test(flavor = "multi_thread")] + async fn test_abnormalities_placeholder_is_rendered() -> Result<()> { + let env = TestEnvironment::new().await?; + + let web = env.web_app().await; + let page = kuchikiki::parse_html().one(web.assert_success("/").await?.text().await?); + let placeholder = page + .select("#abnormalities") + .unwrap() + .next() + .expect("missing abnormalities placeholder"); + + assert_eq!( + placeholder.attributes.borrow().get("data-url"), + Some("/-/partial/abnormalities/") + ); + assert_eq!(page.select("a.pure-menu-link.error").unwrap().count(), 0); + Ok(()) + } + #[test] fn test_doc_coverage_for_crate_pages() { async_wrapper(|env| async move { diff --git a/crates/bin/docs_rs_web/src/handlers/releases.rs b/crates/bin/docs_rs_web/src/handlers/releases.rs index ca19a053c..ce2cf0ef0 100644 --- a/crates/bin/docs_rs_web/src/handlers/releases.rs +++ b/crates/bin/docs_rs_web/src/handlers/releases.rs @@ -733,6 +733,7 @@ struct BuildQueuePage { rebuild_queue: Vec, in_progress_builds: Vec, expand_rebuild_queue: bool, + show_length_warning: bool, } impl_axum_webpage! { BuildQueuePage } @@ -798,6 +799,8 @@ pub(crate) async fn build_queue_handler( }) .collect::>(); + let show_length_warning = build_queue.build_queue_is_too_long(queue.iter()); + queue.retain_mut(|krate| { if krate.priority >= PRIORITY_CONTINUOUS { rebuild_queue.push(krate.clone()); @@ -817,6 +820,7 @@ pub(crate) async fn build_queue_handler( rebuild_queue, in_progress_builds, expand_rebuild_queue: params.expand.is_some(), + show_length_warning, }) } @@ -843,6 +847,7 @@ mod tests { use reqwest::StatusCode; use serde_json::json; use std::collections::HashSet; + use std::str::FromStr; use test_case::test_case; #[test] @@ -1820,6 +1825,7 @@ mod tests { .expect("missing heading") .any(|el| el.text_contents().contains("active CDN deployments")) ); + assert_eq!(empty.select(".warning").unwrap().count(), 0); let queue = env.build_queue()?; queue.add_crate(&FOO, &V1, 0, None).await?; @@ -1855,6 +1861,30 @@ mod tests { }); } + #[tokio::test(flavor = "multi_thread")] + async fn test_releases_queue_shows_length_warning_when_threshold_is_exceeded() -> Result<()> { + let env = TestEnvironment::new().await?; + let web = env.web_app().await; + let queue = env.build_queue()?; + + for idx in 0..1001 { + let name = KrateName::from_str(&format!("queued-crate-{idx}"))?; + queue.add_crate(&name, &V1, 0, None).await?; + } + + let page = kuchikiki::parse_html().one(web.get("/releases/queue").await?.text().await?); + let warning = page + .select(".warning") + .expect("missing warning container") + .next() + .expect("missing queue warning"); + + assert!(warning.text_contents().contains("build queue is too long")); + assert!(warning.text_contents().contains("The team is notified")); + + Ok(()) + } + #[test] fn test_releases_queue_in_progress() { async_wrapper(|env| async move { diff --git a/crates/bin/docs_rs_web/src/handlers/rustdoc.rs b/crates/bin/docs_rs_web/src/handlers/rustdoc.rs index d5262f055..805156bef 100644 --- a/crates/bin/docs_rs_web/src/handlers/rustdoc.rs +++ b/crates/bin/docs_rs_web/src/handlers/rustdoc.rs @@ -18,8 +18,7 @@ use crate::{ TemplateData, templates::{RenderBrands, RenderRegular, RenderSolid, filters}, }, - utils, - utils::licenses, + utils::{self, licenses}, }; use anyhow::{Context as _, anyhow}; use askama::Template; diff --git a/crates/bin/docs_rs_web/src/handlers/status.rs b/crates/bin/docs_rs_web/src/handlers/status.rs index ac560a4c2..f56649015 100644 --- a/crates/bin/docs_rs_web/src/handlers/status.rs +++ b/crates/bin/docs_rs_web/src/handlers/status.rs @@ -1,214 +1,269 @@ use crate::{ cache::CachePolicy, - error::{AxumNope, AxumResult}, - extractors::{DbConnection, rustdoc::RustdocParams}, - match_release::match_version, + error::AxumResult, + extractors::DbConnection, + impl_axum_webpage, + page::{ + templates::{RenderBrands, RenderSolid}, + warnings::{self, ActiveAbnormalities}, + }, }; +use askama::Template; use axum::{ - Json, extract::Extension, http::header::ACCESS_CONTROL_ALLOW_ORIGIN, response::IntoResponse, + extract::Extension, + response::{IntoResponse, Response as AxumResponse}, }; +use docs_rs_build_queue::AsyncBuildQueue; +use docs_rs_database::service_config::Abnormality; +use std::sync::Arc; + +#[derive(Debug, Clone, PartialEq, Template)] +#[template(path = "core/about/status.html")] +struct AboutStatus { + abnormalities: Vec, +} + +impl_axum_webpage!( + AboutStatus, + cache_policy = |_| CachePolicy::ShortInCdnAndBrowser +); + +#[derive(Template)] +#[template(path = "header/abnormalities.html")] +#[derive(Debug, Clone)] +struct Abnormalities { + abnormalities: ActiveAbnormalities, +} + +impl_axum_webpage! { + Abnormalities, + cache_policy = |_| CachePolicy::LongerInCdnAndBrowser +} pub(crate) async fn status_handler( - params: RustdocParams, + Extension(build_queue): Extension>, mut conn: DbConnection, -) -> impl IntoResponse { - ( - Extension(CachePolicy::NoStoreMustRevalidate), - [(ACCESS_CONTROL_ALLOW_ORIGIN, "*")], - // We use an async block to emulate a try block so that we can apply the above CORS header - // and cache policy to both successful and failed responses - async move { - let matched_release = match_version(&mut conn, params.name(), params.req_version()) - .await? - .assume_exact_name()?; - - let rustdoc_status = matched_release.rustdoc_status(); - - let version = matched_release - .into_canonical_req_version_or_else(|confirmed_name, version| { - AxumNope::Redirect( - params - .clone() - .with_name(confirmed_name) - .with_req_version(version) - .build_status_url(), - CachePolicy::NoCaching, - ) - })? - .into_version(); - - let json = Json(serde_json::json!({ - "version": version.to_string(), - "doc_status": rustdoc_status, - })); - - AxumResult::Ok(json.into_response()) - } - .await, - ) +) -> AxumResult { + Ok(AboutStatus { + abnormalities: warnings::load_abnormalities(&mut conn, &build_queue).await?, + }) +} + +pub(crate) async fn abnormalities( + Extension(build_queue): Extension>, + mut conn: DbConnection, +) -> AxumResult { + Ok(Abnormalities { + abnormalities: warnings::load_abnormalities(&mut conn, &build_queue).await?, + } + .into_response()) } #[cfg(test)] mod tests { use crate::{ cache::CachePolicy, - testing::{AxumResponseTestExt, AxumRouterTestExt, TestEnvironmentExt as _, async_wrapper}, + testing::{ + AxumResponseTestExt, AxumRouterTestExt, TestEnvironment, TestEnvironmentExt as _, + }, }; - use docs_rs_types::ReqVersion; - use reqwest::StatusCode; - use test_case::test_case; - - #[test_case("latest")] - #[test_case("0.1")] - #[test_case("0.1.0")] - #[test_case("=0.1.0"; "exact_version")] - fn status(req_version: &str) { - async_wrapper(|env| async move { - let req_version: ReqVersion = req_version.parse()?; - - env.fake_release() - .await - .name("foo") - .version("0.1.0") - .create() - .await?; - - let response = env - .web_app() - .await - .get_and_follow_redirects(&format!("/crate/foo/{req_version}/status.json")) - .await?; - response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); - assert_eq!(response.headers()["access-control-allow-origin"], "*"); - assert_eq!(response.status(), StatusCode::OK); - let value: serde_json::Value = serde_json::from_str(&response.text().await?)?; - - assert_eq!( - value, - serde_json::json!({ - "version": "0.1.0", - "doc_status": true, - }) - ); - - Ok(()) - }); + use anyhow::Result; + use docs_rs_config::AppConfig as _; + use docs_rs_database::service_config::{Abnormality, ConfigName, set_config}; + use docs_rs_types::{KrateName, testing::V1}; + use docs_rs_uri::EscapedURI; + use kuchikiki::traits::TendrilSink; + use std::str::FromStr; + + #[tokio::test(flavor = "multi_thread")] + async fn abnormalities_partial_renders_configured_link() -> Result<()> { + let env = TestEnvironment::new().await?; + + let mut conn = env.async_conn().await?; + set_config( + &mut conn, + ConfigName::Abnormality, + Abnormality { + url: "https://example.com/maintenance" + .parse::() + .unwrap(), + text: "Scheduled maintenance".into(), + explanation: Some("Planned maintenance is in progress.".into()), + }, + ) + .await?; + + let web = env.web_app().await; + let page = kuchikiki::parse_html().one( + web.assert_success_cached( + "/-/partial/abnormalities/", + CachePolicy::LongerInCdnAndBrowser, + env.config(), + ) + .await? + .text() + .await?, + ); + let alert = page + .select("a.pure-menu-link.warn") + .unwrap() + .next() + .expect("missing abnormality"); + + assert_eq!(alert.attributes.borrow().get("href"), Some("/-/status/")); + assert!(alert.text_contents().trim().is_empty()); + Ok(()) } - #[test] - fn redirect_latest() { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("foo") - .version("0.1.0") - .create() - .await?; - - let web = env.web_app().await; - let redirect = web - .assert_redirect("/crate/foo/*/status.json", "/crate/foo/latest/status.json") - .await?; - redirect.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); - assert_eq!(redirect.headers()["access-control-allow-origin"], "*"); - - Ok(()) - }); + #[tokio::test(flavor = "multi_thread")] + async fn abnormalities_partial_renders_queue_alert() -> Result<()> { + let mut queue_config = docs_rs_build_queue::Config::test_config()?; + queue_config.length_warning_threshold = 1; + let env = TestEnvironment::builder() + .build_queue_config(queue_config) + .build() + .await?; + let queue = env.build_queue()?.clone(); + + for idx in 0..2 { + let name = KrateName::from_str(&format!("queued-crate-{idx}"))?; + queue.add_crate(&name, &V1, 0, None).await?; + } + + let web = env.web_app().await; + let page = kuchikiki::parse_html().one( + web.assert_success_cached( + "/-/partial/abnormalities/", + CachePolicy::LongerInCdnAndBrowser, + env.config(), + ) + .await? + .text() + .await?, + ); + let alert = page + .select("a.pure-menu-link.warn") + .unwrap() + .next() + .expect("missing queue alert"); + + assert_eq!(alert.attributes.borrow().get("href"), Some("/-/status/")); + assert!(alert.text_contents().trim().is_empty()); + Ok(()) } - #[test_case("0.1")] - #[test_case("~0.1"; "semver")] - fn redirect(req_version: &str) { - async_wrapper(|env| async move { - let req_version: ReqVersion = req_version.parse()?; - - env.fake_release() - .await - .name("foo") - .version("0.1.0") - .create() - .await?; - - let web = env.web_app().await; - let redirect = web - .assert_redirect( - &format!("/crate/foo/{req_version}/status.json"), - "/crate/foo/0.1.0/status.json", - ) - .await?; - redirect.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); - assert_eq!(redirect.headers()["access-control-allow-origin"], "*"); - - Ok(()) - }); + #[tokio::test(flavor = "multi_thread")] + async fn about_status_page_renders_abnormality_details() -> Result<()> { + let env = TestEnvironment::new().await?; + + let mut conn = env.async_conn().await?; + set_config( + &mut conn, + ConfigName::Abnormality, + Abnormality { + url: "https://example.com/maintenance" + .parse::() + .unwrap(), + text: "Scheduled maintenance".into(), + explanation: Some("Planned maintenance is in progress.".into()), + }, + ) + .await?; + drop(conn); + + let web = env.web_app().await; + let page = kuchikiki::parse_html().one( + web.assert_success_cached( + "/-/status/", + CachePolicy::ShortInCdnAndBrowser, + env.config(), + ) + .await? + .text() + .await?, + ); + + let body_text = page.text_contents(); + assert!(body_text.contains("Scheduled maintenance")); + assert!(body_text.contains("Planned maintenance is in progress.")); + + Ok(()) } - #[test_case("latest")] - #[test_case("0.1")] - #[test_case("0.1.0")] - #[test_case("=0.1.0"; "exact_version")] - fn failure(req_version: &str) { - async_wrapper(|env| async move { - let req_version: ReqVersion = req_version.parse()?; - - env.fake_release() - .await - .name("foo") - .version("0.1.0") - .build_result_failed() - .create() - .await?; - - let response = env - .web_app() - .await - .get_and_follow_redirects(&format!("/crate/foo/{req_version}/status.json")) - .await?; - response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); - assert_eq!(response.headers()["access-control-allow-origin"], "*"); - assert_eq!(response.status(), StatusCode::OK); - let value: serde_json::Value = serde_json::from_str(&response.text().await?)?; - - assert_eq!( - value, - serde_json::json!({ - "version": "0.1.0", - "doc_status": false, - }) - ); - - Ok(()) - }); + #[tokio::test(flavor = "multi_thread")] + async fn about_status_page_shows_no_abnormalities_when_clean() -> Result<()> { + let env = TestEnvironment::new().await?; + let web = env.web_app().await; + + let page = kuchikiki::parse_html().one( + web.assert_success_cached( + "/-/status/", + CachePolicy::ShortInCdnAndBrowser, + env.config(), + ) + .await? + .text() + .await?, + ); + + let body_text = page.text_contents(); + assert!(body_text.contains("No abnormalities detected currently.")); + assert_eq!( + page.select(".about h3").unwrap().count(), + 0, + "should not render any abnormality headings" + ); + + Ok(()) } - // crate not found - #[test_case("bar", "0.1")] - #[test_case("bar", "0.1.0")] - // version not found - #[test_case("foo", "=0.1.0"; "exact_version")] - #[test_case("foo", "0.2")] - #[test_case("foo", "0.2.0")] - // invalid semver - #[test_case("foo", "0,1")] - #[test_case("foo", "0,1,0")] - fn not_found(krate: &str, req_version: &str) { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("foo") - .version("0.1.1") - .create() - .await?; - - let response = env - .web_app() - .await - .get_and_follow_redirects(&format!("/crate/{krate}/{req_version}/status.json")) - .await?; - response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); - assert_eq!(response.headers()["access-control-allow-origin"], "*"); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - Ok(()) - }); + #[tokio::test(flavor = "multi_thread")] + async fn about_status_page_renders_html_explanation() -> Result<()> { + let env = TestEnvironment::new().await?; + + let mut conn = env.async_conn().await?; + set_config( + &mut conn, + ConfigName::Abnormality, + Abnormality { + url: "https://example.com/maintenance" + .parse::() + .unwrap(), + text: "Scheduled maintenance".into(), + explanation: Some( + "Planned maintenance is in progress. See details.".into(), + ), + }, + ) + .await?; + drop(conn); + + let web = env.web_app().await; + let html = web + .assert_success_cached( + "/-/status/", + CachePolicy::ShortInCdnAndBrowser, + env.config(), + ) + .await? + .text() + .await?; + let page = kuchikiki::parse_html().one(html.clone()); + + // The tag should be rendered as an actual HTML element, not escaped. + assert!( + html.contains("in progress"), + "HTML in explanation should be rendered unescaped" + ); + + // The tag should be rendered as an actual link. + let link = page + .select(".about p a[href='/details']") + .unwrap() + .next() + .expect("explanation should contain a rendered link"); + assert!(link.text_contents().contains("details")); + + Ok(()) } } diff --git a/crates/bin/docs_rs_web/src/lib.rs b/crates/bin/docs_rs_web/src/lib.rs index 4ddbb8463..edd28d7c3 100644 --- a/crates/bin/docs_rs_web/src/lib.rs +++ b/crates/bin/docs_rs_web/src/lib.rs @@ -28,16 +28,3 @@ pub use docs_rs_build_limits::DEFAULT_MAX_TARGETS; pub use docs_rs_utils::{APP_USER_AGENT, BUILD_VERSION, RUSTDOC_STATIC_STORAGE_PREFIX}; pub use font_awesome_as_a_crate::icons; pub use handlers::run_web_server; - -use page::GlobalAlert; - -// Warning message shown in the navigation bar of every page. Set to `None` to hide it. -pub(crate) static GLOBAL_ALERT: Option = None; -/* -pub(crate) static GLOBAL_ALERT: Option = Some(GlobalAlert { - url: "https://blog.rust-lang.org/2019/09/18/upcoming-docsrs-changes.html", - text: "Upcoming docs.rs breaking changes!", - css_class: "error", - fa_icon: "exclamation-triangle", -}); -*/ diff --git a/crates/bin/docs_rs_web/src/page/mod.rs b/crates/bin/docs_rs_web/src/page/mod.rs index 8a0146f98..8dc52655a 100644 --- a/crates/bin/docs_rs_web/src/page/mod.rs +++ b/crates/bin/docs_rs_web/src/page/mod.rs @@ -1,12 +1,5 @@ pub(crate) mod templates; +pub(crate) mod warnings; pub(crate) mod web_page; pub(crate) use templates::TemplateData; - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct GlobalAlert { - pub(crate) url: &'static str, - pub(crate) text: &'static str, - pub(crate) css_class: &'static str, - pub(crate) fa_icon: crate::icons::IconTriangleExclamation, -} diff --git a/crates/bin/docs_rs_web/src/page/warnings.rs b/crates/bin/docs_rs_web/src/page/warnings.rs new file mode 100644 index 000000000..b9f8be350 --- /dev/null +++ b/crates/bin/docs_rs_web/src/page/warnings.rs @@ -0,0 +1,98 @@ +use anyhow::{Context as _, Result}; +use docs_rs_build_queue::AsyncBuildQueue; +use docs_rs_database::service_config::{Abnormality, ConfigName, get_config}; + +pub(crate) type ActiveAbnormalities = Vec; + +pub(crate) async fn load_abnormalities( + conn: &mut sqlx::PgConnection, + build_queue: &AsyncBuildQueue, +) -> Result { + let mut active_abnormalities = ActiveAbnormalities::new(); + + if let Some(abnormality) = get_config::(conn, ConfigName::Abnormality) + .await + .context("failed to load manual abnormality from config")? + { + active_abnormalities.push(abnormality); + } + + active_abnormalities.extend( + build_queue + .gather_alerts() + .await + .context("failed to load build queue abnormalities")?, + ); + + Ok(active_abnormalities) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::TestEnvironment; + use anyhow::Result; + use docs_rs_config::AppConfig as _; + use docs_rs_database::service_config::set_config; + use docs_rs_types::{KrateName, Version}; + use docs_rs_uri::EscapedURI; + + #[tokio::test(flavor = "multi_thread")] + async fn load_abnormalities_returns_manual_abnormality() -> Result<()> { + let env = TestEnvironment::new().await?; + + let mut conn = env.async_conn().await?; + let manual_abnormality = Abnormality { + url: "https://example.com/maintenance" + .parse::() + .unwrap(), + text: "Scheduled maintenance".into(), + explanation: Some("Planned maintenance is in progress.".into()), + }; + set_config( + &mut conn, + ConfigName::Abnormality, + manual_abnormality.clone(), + ) + .await?; + + let abnormalities = load_abnormalities(&mut conn, env.build_queue()?).await?; + + assert_eq!(abnormalities, vec![manual_abnormality]); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn load_abnormalities_returns_queue_abnormality() -> Result<()> { + let mut queue_config = docs_rs_build_queue::Config::test_config()?; + queue_config.length_warning_threshold = 1; + let env = crate::testing::TestEnvironment::builder() + .build_queue_config(queue_config) + .build() + .await?; + + let queue = env.build_queue()?.clone(); + for idx in 0..2 { + let name = format!("queued-crate-{idx}").parse::()?; + queue + .add_crate(&name, &Version::parse("1.0.0")?, 0, None) + .await?; + } + + let mut conn = env.async_conn().await?; + let abnormalities = load_abnormalities(&mut conn, &queue).await?; + + assert_eq!( + abnormalities, + vec![Abnormality { + url: EscapedURI::from_path("/releases/queue"), + text: "long build queue".into(), + explanation: Some( + "The build queue currently contains more than 1 crates, so it might take a while before new published crates get documented.".into() + ), + }] + ); + + Ok(()) + } +} diff --git a/crates/bin/docs_rs_web/src/routes.rs b/crates/bin/docs_rs_web/src/routes.rs index 8fef55986..bd5f67144 100644 --- a/crates/bin/docs_rs_web/src/routes.rs +++ b/crates/bin/docs_rs_web/src/routes.rs @@ -2,7 +2,8 @@ use crate::{ cache::CachePolicy, error::AxumNope, handlers::{ - about, build_details, builds, crate_details, features, releases, rustdoc, sitemap, source, + about, build_details, build_status, builds, crate_details, features, releases, rustdoc, + sitemap, source, statics::{build_static_router, static_root_dir}, status, }, @@ -143,6 +144,7 @@ pub(crate) fn build_axum_routes() -> Result { "/-/sitemap/{letter}/sitemap.xml", get_internal(sitemap::sitemap_handler), ) + .route_with_tsr("/-/status/", get_internal(status::status_handler)) .route_with_tsr("/about/builds", get_internal(about::about_builds_handler)) .route_with_tsr("/about", get_internal(about::about_handler)) .route_with_tsr("/about/{subpage}", get_internal(about::about_handler)) @@ -216,7 +218,7 @@ pub(crate) fn build_axum_routes() -> Result { ) .route( "/crate/{name}/{version}/status.json", - get_internal(status::status_handler), + get_internal(build_status::status_handler), ) .route_with_tsr( "/crate/{name}/{version}/builds/{id}", @@ -254,6 +256,10 @@ pub(crate) fn build_axum_routes() -> Result { "/crate/{name}/{version}/menus/releases/{*path}", get_internal(crate_details::get_all_releases), ) + .route( + "/-/partial/abnormalities/", + get_internal(status::abnormalities), + ) .route( "/-/rustdoc.static/{*path}", get_internal(rustdoc::static_asset_handler), diff --git a/crates/bin/docs_rs_web/static/menu.js b/crates/bin/docs_rs_web/static/menu.js index 4d63de4ee..ddc858c6a 100644 --- a/crates/bin/docs_rs_web/static/menu.js +++ b/crates/bin/docs_rs_web/static/menu.js @@ -312,4 +312,19 @@ history.replaceState({}, null, permalink.href); } }); + + (async function loadAbnormalities() { + const abnormalities = document.getElementById("abnormalities"); + if (!abnormalities) { + return; + } + + try { + const response = await fetch(abnormalities.dataset.url); + abnormalities.innerHTML = await response.text(); + } catch (ex) { + console.error(`Failed to load abnormalities: ${ex}`); + abnormalities.innerHTML = ""; + } + })(); })(); diff --git a/crates/bin/docs_rs_web/templates/core/about/status.html b/crates/bin/docs_rs_web/templates/core/about/status.html new file mode 100644 index 000000000..1713eef7f --- /dev/null +++ b/crates/bin/docs_rs_web/templates/core/about/status.html @@ -0,0 +1,28 @@ +{% extends "about-base.html" %} + +{%- block title -%} Docs.rs status {%- endblock title -%} + +{%- block body -%} +

Docs.rs status

+ +
+
+ {%- if abnormalities.is_empty() %} +

No abnormalities detected currently.

+ {%- else %} + {%- for abnormality in abnormalities %} +

+ {{- crate::icons::IconTriangleExclamation.render_solid(false, false, "") }} + {{ abnormality.text }} +

+ {%- if let Some(explanation) = &abnormality.explanation %} +

{{ explanation|safe }}

+ {%- endif %} +

+ More details here. +

+ {%- endfor %} + {%- endif %} +
+
+{%- endblock body %} diff --git a/crates/bin/docs_rs_web/templates/header/abnormalities.html b/crates/bin/docs_rs_web/templates/header/abnormalities.html new file mode 100644 index 000000000..3f239138f --- /dev/null +++ b/crates/bin/docs_rs_web/templates/header/abnormalities.html @@ -0,0 +1,7 @@ +{%- if abnormalities.len() >= 1 -%} +
  • + + {{- crate::icons::IconTriangleExclamation.render_solid(false, false, "") }} + +
    • +{% endif %} diff --git a/crates/bin/docs_rs_web/templates/header/global_alert.html b/crates/bin/docs_rs_web/templates/header/global_alert.html deleted file mode 100644 index fcca2535e..000000000 --- a/crates/bin/docs_rs_web/templates/header/global_alert.html +++ /dev/null @@ -1,11 +0,0 @@ -{# Get the current global alert #} - -{# If there is a global alert, render it #} -{%- if let Some(global_alert) = crate::GLOBAL_ALERT -%} -
  • - - {{- global_alert.fa_icon.render_solid(false, false, "") }} - {{ global_alert.text -}} - -
    • -{% endif %} diff --git a/crates/bin/docs_rs_web/templates/header/topbar_end.html b/crates/bin/docs_rs_web/templates/header/topbar_end.html index 6be3a52b5..3c73a17ee 100644 --- a/crates/bin/docs_rs_web/templates/header/topbar_end.html +++ b/crates/bin/docs_rs_web/templates/header/topbar_end.html @@ -1,8 +1,8 @@ {%- import "macros.html" as macros -%}
    - {# The global alert, if there is one #} - {% include "header/global_alert.html" -%} + {# The current abnormalities, if there are any #} +
      {# diff --git a/crates/bin/docs_rs_web/templates/releases/build_queue.html b/crates/bin/docs_rs_web/templates/releases/build_queue.html index beb11030e..fa44caf25 100644 --- a/crates/bin/docs_rs_web/templates/releases/build_queue.html +++ b/crates/bin/docs_rs_web/templates/releases/build_queue.html @@ -18,6 +18,15 @@ {%- block body -%}
      + + {%- if show_length_warning %} +
      + The docs.rs build queue is too long.
      + Building your crate might take longer, up to a couple of days.
      + The team is notified. +
      + {%- endif %} +
      Currently being built diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/lib/docs_rs_build_limits/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/lib/docs_rs_build_queue/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/lib/docs_rs_build_queue/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/lib/docs_rs_build_queue/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/lib/docs_rs_build_queue/Cargo.toml b/crates/lib/docs_rs_build_queue/Cargo.toml index 2bd5da10c..f1ec57d57 100644 --- a/crates/lib/docs_rs_build_queue/Cargo.toml +++ b/crates/lib/docs_rs_build_queue/Cargo.toml @@ -21,6 +21,7 @@ docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_opentelemetry = { path = "../docs_rs_opentelemetry" } docs_rs_repository_stats = { path = "../docs_rs_repository_stats" } docs_rs_types = { path = "../docs_rs_types" } +docs_rs_uri = { path = "../docs_rs_uri" } docs_rs_utils = { path = "../docs_rs_utils" } futures-util = { workspace = true } opentelemetry = { workspace = true } diff --git a/crates/lib/docs_rs_build_queue/src/config.rs b/crates/lib/docs_rs_build_queue/src/config.rs index 6438589aa..fbbe5855e 100644 --- a/crates/lib/docs_rs_build_queue/src/config.rs +++ b/crates/lib/docs_rs_build_queue/src/config.rs @@ -8,6 +8,7 @@ pub struct Config { pub build_attempts: u16, pub deprioritize_workspace_size: u16, pub delay_between_build_attempts: Duration, + pub length_warning_threshold: usize, } impl Default for Config { @@ -16,6 +17,7 @@ impl Default for Config { build_attempts: 5, deprioritize_workspace_size: 20, delay_between_build_attempts: Duration::from_secs(60), + length_warning_threshold: 1000, } } } @@ -36,6 +38,10 @@ impl AppConfig for Config { config.deprioritize_workspace_size = size; } + if let Some(length) = maybe_env::("DOCSRS_QUEUE_LENGTH_WARNING_THRESHOLD")? { + config.length_warning_threshold = length; + } + Ok(config) } } diff --git a/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs b/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs index 245cf5c53..b0e27da4d 100644 --- a/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs +++ b/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs @@ -1,12 +1,16 @@ -use crate::{Config, PRIORITY_DEFAULT, PRIORITY_DEPRIORITIZED, QueuedCrate, metrics}; -use anyhow::Result; +use crate::{ + Config, PRIORITY_DEFAULT, PRIORITY_DEPRIORITIZED, PRIORITY_MANUAL_FROM_CRATES_IO, QueuedCrate, + metrics, +}; +use anyhow::{Context as _, Result}; use docs_rs_database::{ Pool, - service_config::{ConfigName, get_config, set_config}, + service_config::{Abnormality, ConfigName, get_config, set_config}, }; use docs_rs_opentelemetry::AnyMeterProvider; use docs_rs_repository_stats::workspaces; use docs_rs_types::{KrateName, Version}; +use docs_rs_uri::EscapedURI; use futures_util::TryStreamExt as _; use std::{ collections::{HashMap, HashSet}, @@ -260,6 +264,44 @@ impl AsyncBuildQueue { Ok(()) } + + pub fn build_queue_is_too_long<'a>( + &self, + queued_crates: impl Iterator, + ) -> bool { + queued_crates + .filter(|qc| qc.priority < PRIORITY_MANUAL_FROM_CRATES_IO) + .count() + > self.config.length_warning_threshold + } + + /// fetch the current queue alerts + pub async fn gather_alerts(&self) -> Result> { + let queue_pending_count = self + .pending_count_by_priority() + .await + .context("failed to fetch queue length for alerts")? + .into_iter() + .filter_map(|(prio, amount)| (prio < PRIORITY_MANUAL_FROM_CRATES_IO).then_some(amount)) + .sum::(); + + let mut alerts = Vec::with_capacity(1); + + if queue_pending_count > self.config.length_warning_threshold { + alerts.push(Abnormality { + url: EscapedURI::from_path("/releases/queue"), + text: "long build queue".into(), + explanation: Some( + format!( + "The build queue currently contains more than {} crates, so it might take a while before new published crates get documented.", + self.config.length_warning_threshold, + ) + ), + }); + } + + Ok(alerts) + } } /// Locking functions. @@ -582,4 +624,48 @@ mod tests { Ok(()) } + + #[tokio::test(flavor = "multi_thread")] + async fn test_length_warning_threshold_boundary() -> Result<()> { + let mut config = Config::from_environment()?; + config.length_warning_threshold = 1; + let env = test_queue_with_config(config).await?; + let queue = env.queue; + + queue.add_crate(&FOO, &V1, 0, None).await?; + + assert!(!queue.build_queue_is_too_long(queue.queued_crates().await?.iter())); + assert!(queue.gather_alerts().await?.is_empty()); + + queue.add_crate(&BAR, &V1, 0, None).await?; + + assert!(queue.build_queue_is_too_long(queue.queued_crates().await?.iter())); + assert_eq!( + queue.gather_alerts().await?, + vec![Abnormality { + url: EscapedURI::from_path("/releases/queue"), + text: "long build queue".into(), + explanation: Some("The build queue currently contains more than 1 crates, so it might take a while before new published crates get documented.".into()), + }] + ); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_public_alert_ignores_manual_crates() -> Result<()> { + let mut config = Config::from_environment()?; + config.length_warning_threshold = 0; + let env = test_queue_with_config(config).await?; + let queue = env.queue; + + queue + .add_crate(&FOO, &V1, PRIORITY_MANUAL_FROM_CRATES_IO, None) + .await?; + + assert!(!queue.build_queue_is_too_long(queue.queued_crates().await?.iter())); + assert!(queue.gather_alerts().await?.is_empty()); + + Ok(()) + } } diff --git a/crates/lib/docs_rs_context/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/lib/docs_rs_context/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/lib/docs_rs_context/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs index 4b96d6724..98583d1f5 100644 --- a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs +++ b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs @@ -1,6 +1,7 @@ use crate::Context; use anyhow::Result; use bon::bon; +use docs_rs_build_queue::AsyncBuildQueue; use docs_rs_config::AppConfig; use docs_rs_database::{AsyncPoolClient, Config as DatabaseConfig, testing::TestDatabase}; use docs_rs_fastly::Cdn; @@ -43,6 +44,7 @@ impl TestEnvironment { config: Option, registry_api_config: Option, storage_config: Option, + build_queue_config: Option, ) -> Result { docs_rs_logging::testing::init(); @@ -75,6 +77,18 @@ impl TestEnvironment { let test_storage = TestStorage::from_config(storage_config.clone(), metrics.provider()).await?; + let build_queue_config = Arc::new(if let Some(config) = build_queue_config { + config + } else { + docs_rs_build_queue::Config::from_environment()? + }); + + let build_queue = Arc::new(AsyncBuildQueue::new( + db.pool().clone(), + build_queue_config.clone(), + metrics.provider(), + )); + Ok(Self { config: app_config, context: Context::builder() @@ -83,7 +97,7 @@ impl TestEnvironment { .meter_provider(metrics.provider().clone()) .pool(db_config.into(), db.pool().clone()) .storage(storage_config.clone(), test_storage.storage()) - .with_build_queue()? + .build_queue(build_queue_config, build_queue) .registry_api(registry_api_config, registry_api.into()) .with_repository_stats()? .maybe_cdn( diff --git a/crates/lib/docs_rs_database/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/lib/docs_rs_database/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/lib/docs_rs_database/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/lib/docs_rs_database/Cargo.toml b/crates/lib/docs_rs_database/Cargo.toml index 3076eeab0..2193806dd 100644 --- a/crates/lib/docs_rs_database/Cargo.toml +++ b/crates/lib/docs_rs_database/Cargo.toml @@ -17,6 +17,7 @@ docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_opentelemetry = { path = "../docs_rs_opentelemetry" } docs_rs_registry_api = { path = "../docs_rs_registry_api" } docs_rs_types = { path = "../docs_rs_types" } +docs_rs_uri = { path = "../docs_rs_uri" } docs_rs_utils = { path = "../docs_rs_utils" } futures-util = { workspace = true } hex = "0.4.3" diff --git a/crates/lib/docs_rs_database/src/service_config/abnormalities.rs b/crates/lib/docs_rs_database/src/service_config/abnormalities.rs new file mode 100644 index 000000000..35bf515c5 --- /dev/null +++ b/crates/lib/docs_rs_database/src/service_config/abnormalities.rs @@ -0,0 +1,11 @@ +use docs_rs_uri::EscapedURI; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Abnormality { + pub url: EscapedURI, + pub text: String, + /// explanation to be shown on the status page, can be HTML + #[serde(default)] + pub explanation: Option, +} diff --git a/crates/lib/docs_rs_database/src/service_config.rs b/crates/lib/docs_rs_database/src/service_config/mod.rs similarity index 62% rename from crates/lib/docs_rs_database/src/service_config.rs rename to crates/lib/docs_rs_database/src/service_config/mod.rs index 93ac86c2e..45894246f 100644 --- a/crates/lib/docs_rs_database/src/service_config.rs +++ b/crates/lib/docs_rs_database/src/service_config/mod.rs @@ -1,6 +1,10 @@ +mod abnormalities; + use anyhow::Result; use serde::{Serialize, de::DeserializeOwned}; +pub use abnormalities::Abnormality; + #[derive(strum::IntoStaticStr)] #[strum(serialize_all = "snake_case")] pub enum ConfigName { @@ -8,6 +12,7 @@ pub enum ConfigName { LastSeenIndexReference, QueueLocked, Toolchain, + Abnormality, } pub async fn set_config( @@ -28,6 +33,14 @@ pub async fn set_config( Ok(()) } +pub async fn remove_config(conn: &mut sqlx::PgConnection, name: ConfigName) -> anyhow::Result<()> { + let name: &'static str = name.into(); + sqlx::query!("DELETE FROM config WHERE name = $1;", name) + .execute(conn) + .await?; + Ok(()) +} + pub async fn get_config(conn: &mut sqlx::PgConnection, name: ConfigName) -> Result> where T: DeserializeOwned, @@ -56,6 +69,7 @@ mod tests { #[test_case(ConfigName::RustcVersion, "rustc_version")] #[test_case(ConfigName::QueueLocked, "queue_locked")] #[test_case(ConfigName::LastSeenIndexReference, "last_seen_index_reference")] + #[test_case(ConfigName::Abnormality, "abnormality")] fn test_configname_variants(variant: ConfigName, expected: &'static str) { let name: &'static str = variant.into(); assert_eq!(name, expected); @@ -107,4 +121,52 @@ mod tests { ); Ok(()) } + + #[tokio::test(flavor = "multi_thread")] + async fn test_remove_existing_config() -> anyhow::Result<()> { + let test_metrics = TestMetrics::new(); + let db = TestDatabase::new(&Config::test_config()?, test_metrics.provider()).await?; + + let mut conn = db.async_conn().await?; + set_config( + &mut conn, + ConfigName::RustcVersion, + Value::String("some value".into()), + ) + .await?; + + assert_eq!( + get_config(&mut conn, ConfigName::RustcVersion).await?, + Some("some value".to_string()) + ); + + remove_config(&mut conn, ConfigName::RustcVersion).await?; + + assert!( + get_config::(&mut conn, ConfigName::RustcVersion) + .await? + .is_none() + ); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_remove_missing_config_is_noop() -> anyhow::Result<()> { + let test_metrics = TestMetrics::new(); + let db = TestDatabase::new(&Config::test_config()?, test_metrics.provider()).await?; + + let mut conn = db.async_conn().await?; + sqlx::query!("DELETE FROM config") + .execute(&mut *conn) + .await?; + + remove_config(&mut conn, ConfigName::RustcVersion).await?; + + assert!( + get_config::(&mut conn, ConfigName::RustcVersion) + .await? + .is_none() + ); + Ok(()) + } } diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +} diff --git a/crates/lib/docs_rs_test_fakes/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json b/crates/lib/docs_rs_test_fakes/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json new file mode 100644 index 000000000..c177dd6be --- /dev/null +++ b/crates/lib/docs_rs_test_fakes/.sqlx/query-671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM config WHERE name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "671354f90d42d2d6ce67b952474560588c0187a00f62f3525635182c482eaec3" +}