diff --git a/Cargo.lock b/Cargo.lock index 8c2cf8645..586baa057 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1915,6 +1915,7 @@ dependencies = [ "crates-index-diff", "derive_builder", "derive_more 2.1.0", + "docs_rs_build_limits", "docs_rs_build_queue", "docs_rs_cargo_metadata", "docs_rs_database", @@ -1982,6 +1983,22 @@ dependencies = [ "walkdir", ] +[[package]] +name = "docs_rs_build_limits" +version = "0.1.0" +dependencies = [ + "anyhow", + "docs_rs_database", + "docs_rs_env_vars", + "docs_rs_opentelemetry", + "docs_rs_types", + "futures-util", + "serde", + "sqlx", + "tokio", + "tracing", +] + [[package]] name = "docs_rs_build_queue" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 3ee2298a2..7ed482694 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ url = { version = "2.1.1", features = ["serde"] } walkdir = "2" [dependencies] +docs_rs_build_limits = { path = "crates/lib/docs_rs_build_limits" } docs_rs_build_queue = { path = "crates/lib/docs_rs_build_queue" } docs_rs_cargo_metadata = { path = "crates/lib/docs_rs_cargo_metadata" } docs_rs_database = { path = "crates/lib/docs_rs_database" } diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json b/crates/lib/docs_rs_build_limits/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json new file mode 100644 index 000000000..e1bcf20ab --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE crates\n SET latest_version_id = $2\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json b/crates/lib/docs_rs_build_limits/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json new file mode 100644 index 000000000..ca0a44f4b --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n releases.id as \"id: ReleaseId\",\n releases.version as \"version: Version\",\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.release_time,\n releases.target_name,\n releases.default_target,\n releases.doc_targets\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: ReleaseId", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "version: Version", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "build_status!: BuildStatus", + "type_info": { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "yanked", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "is_library", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "rustdoc_status", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "release_time", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "target_name", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "default_target", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "doc_targets", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json b/crates/lib/docs_rs_build_limits/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json new file mode 100644 index 000000000..530c4d879 --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO config (name, value)\n VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Json" + ] + }, + "nullable": [] + }, + "hash": "4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-4f81678f0d680c4be7215ef0667729e8518063e0ee4ecad5aa7e559e88b8e1cb.json b/crates/lib/docs_rs_build_limits/.sqlx/query-4f81678f0d680c4be7215ef0667729e8518063e0ee4ecad5aa7e559e88b8e1cb.json new file mode 100644 index 000000000..a95b272dd --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-4f81678f0d680c4be7215ef0667729e8518063e0ee4ecad5aa7e559e88b8e1cb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM crates WHERE crates.name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4f81678f0d680c4be7215ef0667729e8518063e0ee4ecad5aa7e559e88b8e1cb" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json b/crates/lib/docs_rs_build_limits/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json new file mode 100644 index 000000000..264e19fd2 --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM config WHERE name = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-718110e13b155eb56cac3f21e1db3b61db1b2d8ba26c160e38e739fd18caa0fc.json b/crates/lib/docs_rs_build_limits/.sqlx/query-718110e13b155eb56cac3f21e1db3b61db1b2d8ba26c160e38e739fd18caa0fc.json new file mode 100644 index 000000000..9cbcbb35c --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-718110e13b155eb56cac3f21e1db3b61db1b2d8ba26c160e38e739fd18caa0fc.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT * FROM sandbox_overrides", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "crate_name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "max_memory_bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "max_targets", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true, + true, + true + ] + }, + "hash": "718110e13b155eb56cac3f21e1db3b61db1b2d8ba26c160e38e739fd18caa0fc" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-73ff86cdb5b9d0ab312493690d4108803ce04531d497d6dd8d67ad05a844eab3.json b/crates/lib/docs_rs_build_limits/.sqlx/query-73ff86cdb5b9d0ab312493690d4108803ce04531d497d6dd8d67ad05a844eab3.json new file mode 100644 index 000000000..6f21daee0 --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-73ff86cdb5b9d0ab312493690d4108803ce04531d497d6dd8d67ad05a844eab3.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO sandbox_overrides (\n crate_name, max_memory_bytes, max_targets, timeout_seconds\n )\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (crate_name) DO UPDATE\n SET\n max_memory_bytes = $2,\n max_targets = $3,\n timeout_seconds = $4\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "73ff86cdb5b9d0ab312493690d4108803ce04531d497d6dd8d67ad05a844eab3" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-96b68919f9016705a1a36ef11a5a659e7fb431beb0017fbcfd21132f105ce722.json b/crates/lib/docs_rs_build_limits/.sqlx/query-96b68919f9016705a1a36ef11a5a659e7fb431beb0017fbcfd21132f105ce722.json new file mode 100644 index 000000000..984eff3ac --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-96b68919f9016705a1a36ef11a5a659e7fb431beb0017fbcfd21132f105ce722.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT relname\n FROM pg_class\n INNER JOIN pg_namespace ON\n pg_class.relnamespace = pg_namespace.oid\n WHERE pg_class.relkind = 'S'\n AND pg_namespace.nspname = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "relname", + "type_info": "Name" + } + ], + "parameters": { + "Left": [ + "Name" + ] + }, + "nullable": [ + false + ] + }, + "hash": "96b68919f9016705a1a36ef11a5a659e7fb431beb0017fbcfd21132f105ce722" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-a3920d6701d1a80f23562ee83682d82ff35a52eeaa93ed45a97adc5e559d3538.json b/crates/lib/docs_rs_build_limits/.sqlx/query-a3920d6701d1a80f23562ee83682d82ff35a52eeaa93ed45a97adc5e559d3538.json new file mode 100644 index 000000000..87fe73890 --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-a3920d6701d1a80f23562ee83682d82ff35a52eeaa93ed45a97adc5e559d3538.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT * FROM sandbox_overrides WHERE crate_name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "crate_name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "max_memory_bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "max_targets", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true + ] + }, + "hash": "a3920d6701d1a80f23562ee83682d82ff35a52eeaa93ed45a97adc5e559d3538" +} diff --git a/crates/lib/docs_rs_build_limits/.sqlx/query-de4ba149a561c4bb467bcea081bdedff233398cddf4996734a64536b6a8c6579.json b/crates/lib/docs_rs_build_limits/.sqlx/query-de4ba149a561c4bb467bcea081bdedff233398cddf4996734a64536b6a8c6579.json new file mode 100644 index 000000000..945d619af --- /dev/null +++ b/crates/lib/docs_rs_build_limits/.sqlx/query-de4ba149a561c4bb467bcea081bdedff233398cddf4996734a64536b6a8c6579.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM sandbox_overrides WHERE crate_name = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "de4ba149a561c4bb467bcea081bdedff233398cddf4996734a64536b6a8c6579" +} diff --git a/crates/lib/docs_rs_build_limits/Cargo.toml b/crates/lib/docs_rs_build_limits/Cargo.toml new file mode 100644 index 000000000..324a36800 --- /dev/null +++ b/crates/lib/docs_rs_build_limits/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "docs_rs_build_limits" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = { workspace = true } +docs_rs_env_vars = { path = "../docs_rs_env_vars" } +docs_rs_types = { path = "../docs_rs_types" } +futures-util = { workspace = true } +serde = { workspace = true} +sqlx = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +docs_rs_database = { path = "../docs_rs_database", features = ["testing"] } +docs_rs_opentelemetry = { path = "../docs_rs_opentelemetry", features = ["testing"] } +docs_rs_types = { path = "../docs_rs_types", features = ["testing"] } +tokio = { workspace = true } + diff --git a/crates/lib/docs_rs_build_limits/src/config.rs b/crates/lib/docs_rs_build_limits/src/config.rs new file mode 100644 index 000000000..3ff6f7857 --- /dev/null +++ b/crates/lib/docs_rs_build_limits/src/config.rs @@ -0,0 +1,14 @@ +use docs_rs_env_vars::maybe_env; + +#[derive(Debug, Default)] +pub struct Config { + pub(crate) build_default_memory_limit: Option, +} + +impl Config { + pub fn from_environment() -> anyhow::Result { + Ok(Self { + build_default_memory_limit: maybe_env("DOCSRS_BUILD_DEFAULT_MEMORY_LIMIT")?, + }) + } +} diff --git a/crates/lib/docs_rs_build_limits/src/lib.rs b/crates/lib/docs_rs_build_limits/src/lib.rs new file mode 100644 index 000000000..7f9a11c1b --- /dev/null +++ b/crates/lib/docs_rs_build_limits/src/lib.rs @@ -0,0 +1,10 @@ +mod config; +mod limits; +mod overrides; + +pub use config::Config; +pub use limits::Limits; +pub use overrides::Overrides; + +/// Maximum number of targets allowed for a crate to be documented on. +pub const DEFAULT_MAX_TARGETS: usize = 10; diff --git a/crates/lib/docs_rs_build_limits/src/limits.rs b/crates/lib/docs_rs_build_limits/src/limits.rs new file mode 100644 index 000000000..bf2e74ef0 --- /dev/null +++ b/crates/lib/docs_rs_build_limits/src/limits.rs @@ -0,0 +1,205 @@ +use crate::{config::Config, overrides::Overrides}; +use anyhow::Result; +use docs_rs_types::KrateName; +use serde::Serialize; +use std::time::Duration; + +const GB: usize = 1024 * 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Limits { + pub memory: usize, + pub targets: usize, + pub timeout: Duration, + pub networking: bool, + pub max_log_size: usize, +} + +impl Limits { + pub fn new(config: &Config) -> Self { + Self { + // 3 GB default default + memory: config.build_default_memory_limit.unwrap_or(3 * GB), + timeout: Duration::from_secs(15 * 60), // 15 minutes + targets: crate::DEFAULT_MAX_TARGETS, + networking: false, + max_log_size: 100 * 1024, // 100 KB + } + } + + pub async fn for_crate( + config: &Config, + conn: &mut sqlx::PgConnection, + name: &KrateName, + ) -> Result { + let default = Self::new(config); + let overrides = Overrides::for_crate(conn, name).await?.unwrap_or_default(); + Ok(Self { + memory: overrides + .memory + .unwrap_or(default.memory) + .max(default.memory), + targets: overrides + .targets + .or(overrides.timeout.map(|_| 1)) + .unwrap_or(default.targets), + timeout: overrides.timeout.unwrap_or(default.timeout), + networking: default.networking, + max_log_size: default.max_log_size, + }) + } + + pub fn memory(&self) -> usize { + self.memory + } + + pub fn timeout(&self) -> Duration { + self.timeout + } + + pub fn networking(&self) -> bool { + self.networking + } + + pub fn max_log_size(&self) -> usize { + self.max_log_size + } + + pub fn targets(&self) -> usize { + self.targets + } +} + +#[cfg(test)] +mod test { + use super::*; + use docs_rs_database::testing::TestDatabase; + use docs_rs_opentelemetry::testing::TestMetrics; + use docs_rs_types::testing::KRATE; + + async fn db() -> anyhow::Result { + let test_metrics = TestMetrics::new(); + TestDatabase::new( + &docs_rs_database::Config::test_config()?, + test_metrics.provider(), + ) + .await + } + + #[tokio::test(flavor = "multi_thread")] + async fn retrieve_limits() -> anyhow::Result<()> { + let db = db().await?; + let mut conn = db.async_conn().await; + + let cfg = Config::default(); + + let defaults = Limits::new(&cfg); + + let krate = KrateName::from_static("hexponent"); + // limits work if no crate has limits set + let hexponent = Limits::for_crate(&cfg, &mut conn, &krate).await?; + assert_eq!(hexponent, defaults); + + Overrides::save( + &mut conn, + &krate, + Overrides { + targets: Some(15), + ..Overrides::default() + }, + ) + .await?; + // limits work if crate has limits set + let hexponent = Limits::for_crate(&cfg, &mut conn, &krate).await?; + assert_eq!( + hexponent, + Limits { + targets: 15, + ..defaults + } + ); + + // all limits work + let krate = KrateName::from_static("regex"); + let limits = Limits { + memory: defaults.memory * 2, + timeout: defaults.timeout * 2, + targets: 1, + ..defaults + }; + Overrides::save( + &mut conn, + &krate, + Overrides { + memory: Some(limits.memory), + targets: Some(limits.targets), + timeout: Some(limits.timeout), + }, + ) + .await?; + assert_eq!(limits, Limits::for_crate(&cfg, &mut conn, &krate).await?); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn targets_default_to_one_with_timeout() -> anyhow::Result<()> { + let db = db().await?; + + let mut conn = db.async_conn().await; + let krate = KrateName::from_static("hexponent"); + Overrides::save( + &mut conn, + &krate, + Overrides { + timeout: Some(Duration::from_secs(20 * 60)), + ..Overrides::default() + }, + ) + .await?; + let limits = Limits::for_crate(&Config::default(), &mut conn, &krate).await?; + assert_eq!(limits.targets, 1); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn config_default_memory_limit() -> Result<()> { + let db = db().await?; + + let cfg = Config { + build_default_memory_limit: Some(6 * GB), + }; + + let mut conn = db.async_conn().await; + + let limits = Limits::for_crate(&cfg, &mut conn, &KRATE).await?; + assert_eq!(limits.memory, 6 * GB); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn overrides_dont_lower_memory_limit() -> Result<()> { + let db = db().await?; + let mut conn = db.async_conn().await; + + let cfg = Config::default(); + + let defaults = Limits::new(&cfg); + + Overrides::save( + &mut conn, + &KRATE, + Overrides { + memory: Some(defaults.memory / 2), + ..Overrides::default() + }, + ) + .await?; + + let limits = Limits::for_crate(&cfg, &mut conn, &KRATE).await?; + assert_eq!(limits, defaults); + + Ok(()) + } +} diff --git a/crates/lib/docs_rs_build_limits/src/overrides.rs b/crates/lib/docs_rs_build_limits/src/overrides.rs new file mode 100644 index 000000000..7f7f6895c --- /dev/null +++ b/crates/lib/docs_rs_build_limits/src/overrides.rs @@ -0,0 +1,159 @@ +use anyhow::Result; +use docs_rs_types::KrateName; +use futures_util::stream::TryStreamExt; +use std::time::Duration; +use tracing::warn; + +#[derive(Default, Debug, Clone, Copy, Eq, PartialEq)] +pub struct Overrides { + pub memory: Option, + pub targets: Option, + pub timeout: Option, +} + +macro_rules! row_to_overrides { + ($row:expr) => {{ + Overrides { + memory: $row.max_memory_bytes.map(|i| i as usize), + targets: $row.max_targets.map(|i| i as usize), + timeout: $row.timeout_seconds.map(|i| Duration::from_secs(i as u64)), + } + }}; +} + +impl Overrides { + pub async fn all(conn: &mut sqlx::PgConnection) -> Result> { + Ok(sqlx::query!("SELECT * FROM sandbox_overrides") + .fetch(conn) + .map_ok(|row| (row.crate_name, row_to_overrides!(row))) + .try_collect() + .await?) + } + + pub async fn for_crate( + conn: &mut sqlx::PgConnection, + krate: &KrateName, + ) -> Result> { + Ok(sqlx::query!( + "SELECT * FROM sandbox_overrides WHERE crate_name = $1", + krate + ) + .fetch_optional(conn) + .await? + .map(|row| row_to_overrides!(row))) + } + + pub async fn save( + conn: &mut sqlx::PgConnection, + krate: &KrateName, + overrides: Self, + ) -> Result<()> { + if overrides.timeout.is_some() && overrides.targets.is_none() { + warn!( + %krate, + ?overrides, + "setting `Overrides::timeout` implies a default `Overrides::targets = 1`, prefer setting this explicitly", + ); + } + + if sqlx::query_scalar!("SELECT id FROM crates WHERE crates.name = $1", krate) + .fetch_optional(&mut *conn) + .await? + .is_none() + { + warn!(%krate, "setting overrides for unknown crate"); + } + + sqlx::query!( + " + INSERT INTO sandbox_overrides ( + crate_name, max_memory_bytes, max_targets, timeout_seconds + ) + VALUES ($1, $2, $3, $4) + ON CONFLICT (crate_name) DO UPDATE + SET + max_memory_bytes = $2, + max_targets = $3, + timeout_seconds = $4 + ", + krate, + overrides.memory.map(|i| i as i64), + overrides.targets.map(|i| i as i32), + overrides.timeout.map(|d| d.as_secs() as i32), + ) + .execute(&mut *conn) + .await?; + Ok(()) + } + + pub async fn remove(conn: &mut sqlx::PgConnection, krate: &str) -> Result<()> { + sqlx::query!("DELETE FROM sandbox_overrides WHERE crate_name = $1", krate) + .execute(conn) + .await?; + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + use docs_rs_database::testing::TestDatabase; + use docs_rs_opentelemetry::testing::TestMetrics; + use std::time::Duration; + + async fn db() -> anyhow::Result { + let test_metrics = TestMetrics::new(); + TestDatabase::new( + &docs_rs_database::Config::test_config()?, + test_metrics.provider(), + ) + .await + } + + #[tokio::test(flavor = "multi_thread")] + async fn retrieve_overrides() -> Result<()> { + let db = db().await?; + let mut conn = db.async_conn().await; + + let krate = KrateName::from_static("hexponent"); + + // no overrides + let actual = Overrides::for_crate(&mut conn, &krate).await?; + assert_eq!(actual, None); + + // add partial overrides + let expected = Overrides { + targets: Some(1), + ..Overrides::default() + }; + Overrides::save(&mut conn, &krate, expected).await?; + let actual = Overrides::for_crate(&mut conn, &krate).await?; + assert_eq!(actual, Some(expected)); + + // overwrite with full overrides + let expected = Overrides { + memory: Some(100_000), + targets: Some(1), + timeout: Some(Duration::from_secs(300)), + }; + Overrides::save(&mut conn, &krate, expected).await?; + let actual = Overrides::for_crate(&mut conn, &krate).await?; + assert_eq!(actual, Some(expected)); + + // overwrite with partial overrides + let expected = Overrides { + memory: Some(1), + ..Overrides::default() + }; + Overrides::save(&mut conn, &krate, expected).await?; + let actual = Overrides::for_crate(&mut conn, &krate).await?; + assert_eq!(actual, Some(expected)); + + // remove overrides + Overrides::remove(&mut conn, &krate).await?; + let actual = Overrides::for_crate(&mut conn, &krate).await?; + assert_eq!(actual, None); + + Ok(()) + } +} diff --git a/crates/lib/docs_rs_utils/src/lib.rs b/crates/lib/docs_rs_utils/src/lib.rs index 0433f87c2..ad0059469 100644 --- a/crates/lib/docs_rs_utils/src/lib.rs +++ b/crates/lib/docs_rs_utils/src/lib.rs @@ -34,9 +34,6 @@ pub const APP_USER_AGENT: &str = concat!( /// `s3://rust-docs-rs//rustdoc-static/something.css` pub const RUSTDOC_STATIC_STORAGE_PREFIX: &str = "/rustdoc-static/"; -/// Maximum number of targets allowed for a crate to be documented on. -pub const DEFAULT_MAX_TARGETS: usize = 10; - /// a wrapper around tokio's `spawn_blocking` that /// enables us to write nicer code when the closure /// returns an `anyhow::Result`. diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 5af5257be..379b302c8 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -4,13 +4,13 @@ use clap::{Parser, Subcommand, ValueEnum}; use docs_rs::{ Config, Context, Index, PackageKind, RustwideBuilder, build_queue::{last_seen_reference, queue_rebuilds_faulty_rustdoc, set_last_seen_reference}, - db::{self, Overrides}, - start_web_server, + db, start_web_server, utils::{ daemon::start_background_service_metric_collector, get_crate_pattern_and_priority, list_crate_priorities, queue_builder, remove_crate_priority, set_crate_priority, }, }; +use docs_rs_build_limits::Overrides; use docs_rs_database::{ crate_details, service_config::{ConfigName, get_config, set_config}, @@ -639,6 +639,7 @@ impl LimitsSubcommand { match self { Self::Get { crate_name } => { + let crate_name: KrateName = crate_name.parse()?; let overrides = Overrides::for_crate(&mut conn, &crate_name).await?; println!("sandbox limit overrides for {crate_name} = {overrides:?}"); } @@ -655,6 +656,7 @@ impl LimitsSubcommand { targets, timeout, } => { + let crate_name: KrateName = crate_name.parse()?; let overrides = Overrides::for_crate(&mut conn, &crate_name).await?; println!("previous sandbox limit overrides for {crate_name} = {overrides:?}"); let overrides = Overrides { @@ -669,6 +671,7 @@ impl LimitsSubcommand { } Self::Remove { crate_name } => { + let crate_name: KrateName = crate_name.parse()?; let overrides = Overrides::for_crate(&mut conn, &crate_name).await?; println!("previous overrides for {crate_name} = {overrides:?}"); Overrides::remove(&mut conn, &crate_name).await?; diff --git a/src/config.rs b/src/config.rs index beedb3d52..81a5bfee7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -62,7 +62,6 @@ pub struct Config { pub(crate) inside_docker: bool, pub(crate) docker_image: Option, pub(crate) build_cpu_limit: Option, - pub(crate) build_default_memory_limit: Option, pub(crate) include_default_targets: bool, pub(crate) disable_memory_limit: bool, @@ -76,6 +75,7 @@ pub struct Config { pub(crate) repository_stats: docs_rs_repository_stats::Config, pub(crate) storage: Arc, pub(crate) build_queue: Arc, + pub(crate) build_limits: Arc, } impl Config { @@ -135,7 +135,6 @@ impl Config { maybe_env("DOCSRS_LOCAL_DOCKER_IMAGE")?.or(maybe_env("DOCSRS_DOCKER_IMAGE")?), ) .build_cpu_limit(maybe_env("DOCSRS_BUILD_CPU_LIMIT")?) - .build_default_memory_limit(maybe_env("DOCSRS_BUILD_DEFAULT_MEMORY_LIMIT")?) .include_default_targets(env("DOCSRS_INCLUDE_DEFAULT_TARGETS", true)?) .disable_memory_limit(env("DOCSRS_DISABLE_MEMORY_LIMIT", false)?) .build_workspace_reinitialization_interval(Duration::from_secs(env( @@ -152,6 +151,7 @@ impl Config { .database(docs_rs_database::Config::from_environment()?) .repository_stats(docs_rs_repository_stats::Config::from_environment()?) .storage(Arc::new(docs_rs_storage::Config::from_environment()?)) - .build_queue(Arc::new(docs_rs_build_queue::Config::from_environment()?))) + .build_queue(Arc::new(docs_rs_build_queue::Config::from_environment()?)) + .build_limits(Arc::new(docs_rs_build_limits::Config::from_environment()?))) } } diff --git a/src/db/mod.rs b/src/db/mod.rs index 715c7ebc9..78f87896f 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -6,10 +6,8 @@ pub(crate) use self::add_package::{ pub use self::{ add_package::{update_build_status, update_crate_data_in_database}, delete::{delete_crate, delete_version}, - overrides::Overrides, }; mod add_package; pub mod blacklist; pub mod delete; -mod overrides; diff --git a/src/db/overrides.rs b/src/db/overrides.rs deleted file mode 100644 index 4329c1c0e..000000000 --- a/src/db/overrides.rs +++ /dev/null @@ -1,139 +0,0 @@ -use crate::error::Result; -use futures_util::stream::TryStreamExt; -use std::time::Duration; - -#[derive(Default, Debug, Clone, Copy, Eq, PartialEq)] -pub struct Overrides { - pub memory: Option, - pub targets: Option, - pub timeout: Option, -} - -macro_rules! row_to_overrides { - ($row:expr) => {{ - Overrides { - memory: $row.max_memory_bytes.map(|i| i as usize), - targets: $row.max_targets.map(|i| i as usize), - timeout: $row.timeout_seconds.map(|i| Duration::from_secs(i as u64)), - } - }}; -} - -impl Overrides { - pub async fn all(conn: &mut sqlx::PgConnection) -> Result> { - Ok(sqlx::query!("SELECT * FROM sandbox_overrides") - .fetch(conn) - .map_ok(|row| (row.crate_name, row_to_overrides!(row))) - .try_collect() - .await?) - } - - pub async fn for_crate(conn: &mut sqlx::PgConnection, krate: &str) -> Result> { - Ok(sqlx::query!( - "SELECT * FROM sandbox_overrides WHERE crate_name = $1", - krate - ) - .fetch_optional(conn) - .await? - .map(|row| row_to_overrides!(row))) - } - - pub async fn save(conn: &mut sqlx::PgConnection, krate: &str, overrides: Self) -> Result<()> { - if overrides.timeout.is_some() && overrides.targets.is_none() { - tracing::warn!( - "setting `Overrides::timeout` implies a default `Overrides::targets = 1`, prefer setting this explicitly" - ); - } - - if sqlx::query_scalar!("SELECT id FROM crates WHERE crates.name = $1", krate) - .fetch_optional(&mut *conn) - .await? - .is_none() - { - tracing::warn!("setting overrides for unknown crate `{krate}`"); - } - - sqlx::query!( - " - INSERT INTO sandbox_overrides ( - crate_name, max_memory_bytes, max_targets, timeout_seconds - ) - VALUES ($1, $2, $3, $4) - ON CONFLICT (crate_name) DO UPDATE - SET - max_memory_bytes = $2, - max_targets = $3, - timeout_seconds = $4 - ", - krate, - overrides.memory.map(|i| i as i64), - overrides.targets.map(|i| i as i32), - overrides.timeout.map(|d| d.as_secs() as i32), - ) - .execute(&mut *conn) - .await?; - Ok(()) - } - - pub async fn remove(conn: &mut sqlx::PgConnection, krate: &str) -> Result<()> { - sqlx::query!("DELETE FROM sandbox_overrides WHERE crate_name = $1", krate) - .execute(conn) - .await?; - Ok(()) - } -} - -#[cfg(test)] -mod test { - use crate::{db::Overrides, test::*}; - use std::time::Duration; - - #[test] - fn retrieve_overrides() { - async_wrapper(|env| async move { - let db = env.async_db(); - let mut conn = db.async_conn().await; - - let krate = "hexponent"; - - // no overrides - let actual = Overrides::for_crate(&mut conn, krate).await?; - assert_eq!(actual, None); - - // add partial overrides - let expected = Overrides { - targets: Some(1), - ..Overrides::default() - }; - Overrides::save(&mut conn, krate, expected).await?; - let actual = Overrides::for_crate(&mut conn, krate).await?; - assert_eq!(actual, Some(expected)); - - // overwrite with full overrides - let expected = Overrides { - memory: Some(100_000), - targets: Some(1), - timeout: Some(Duration::from_secs(300)), - }; - Overrides::save(&mut conn, krate, expected).await?; - let actual = Overrides::for_crate(&mut conn, krate).await?; - assert_eq!(actual, Some(expected)); - - // overwrite with partial overrides - let expected = Overrides { - memory: Some(1), - ..Overrides::default() - }; - Overrides::save(&mut conn, krate, expected).await?; - let actual = Overrides::for_crate(&mut conn, krate).await?; - assert_eq!(actual, Some(expected)); - - // remove overrides - Overrides::remove(&mut conn, krate).await?; - let actual = Overrides::for_crate(&mut conn, krate).await?; - assert_eq!(actual, None); - - Ok(()) - }) - } -} diff --git a/src/docbuilder/limits.rs b/src/docbuilder/limits.rs deleted file mode 100644 index b9571c991..000000000 --- a/src/docbuilder/limits.rs +++ /dev/null @@ -1,197 +0,0 @@ -use crate::{Config, db::Overrides, error::Result}; -use std::time::Duration; - -const GB: usize = 1024 * 1024 * 1024; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct Limits { - pub memory: usize, - pub targets: usize, - pub timeout: Duration, - pub networking: bool, - pub max_log_size: usize, -} - -impl Limits { - pub(crate) fn new(config: &Config) -> Self { - Self { - // 3 GB default default - memory: config.build_default_memory_limit.unwrap_or(3 * GB), - timeout: Duration::from_secs(15 * 60), // 15 minutes - targets: crate::DEFAULT_MAX_TARGETS, - networking: false, - max_log_size: 100 * 1024, // 100 KB - } - } - - pub(crate) async fn for_crate( - config: &Config, - conn: &mut sqlx::PgConnection, - name: &str, - ) -> Result { - let default = Self::new(config); - let overrides = Overrides::for_crate(conn, name).await?.unwrap_or_default(); - Ok(Self { - memory: overrides - .memory - .unwrap_or(default.memory) - .max(default.memory), - targets: overrides - .targets - .or(overrides.timeout.map(|_| 1)) - .unwrap_or(default.targets), - timeout: overrides.timeout.unwrap_or(default.timeout), - networking: default.networking, - max_log_size: default.max_log_size, - }) - } - - pub(crate) fn memory(&self) -> usize { - self.memory - } - - pub(crate) fn timeout(&self) -> Duration { - self.timeout - } - - pub(crate) fn networking(&self) -> bool { - self.networking - } - - pub(crate) fn max_log_size(&self) -> usize { - self.max_log_size - } - - pub(crate) fn targets(&self) -> usize { - self.targets - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::test::*; - - #[test] - fn retrieve_limits() { - async_wrapper(|env| async move { - let db = env.async_db(); - let mut conn = db.async_conn().await; - - let defaults = Limits::new(env.config()); - - let krate = "hexponent"; - // limits work if no crate has limits set - let hexponent = Limits::for_crate(env.config(), &mut conn, krate).await?; - assert_eq!(hexponent, defaults); - - Overrides::save( - &mut conn, - krate, - Overrides { - targets: Some(15), - ..Overrides::default() - }, - ) - .await?; - // limits work if crate has limits set - let hexponent = Limits::for_crate(env.config(), &mut conn, krate).await?; - assert_eq!( - hexponent, - Limits { - targets: 15, - ..defaults - } - ); - - // all limits work - let krate = "regex"; - let limits = Limits { - memory: defaults.memory * 2, - timeout: defaults.timeout * 2, - targets: 1, - ..defaults - }; - Overrides::save( - &mut conn, - krate, - Overrides { - memory: Some(limits.memory), - targets: Some(limits.targets), - timeout: Some(limits.timeout), - }, - ) - .await?; - assert_eq!( - limits, - Limits::for_crate(env.config(), &mut conn, krate).await? - ); - Ok(()) - }) - } - - #[test] - fn targets_default_to_one_with_timeout() { - async_wrapper(|env| async move { - let db = env.async_db(); - let mut conn = db.async_conn().await; - let krate = "hexponent"; - Overrides::save( - &mut conn, - krate, - Overrides { - timeout: Some(Duration::from_secs(20 * 60)), - ..Overrides::default() - }, - ) - .await?; - let limits = Limits::for_crate(env.config(), &mut conn, krate).await?; - assert_eq!(limits.targets, 1); - - Ok(()) - }) - } - - #[tokio::test(flavor = "multi_thread")] - async fn config_default_memory_limit() -> Result<()> { - let env = TestEnvironment::with_config( - TestEnvironment::base_config() - .build_default_memory_limit(Some(6 * GB)) - .build()?, - ) - .await?; - - let db = env.async_db(); - let mut conn = db.async_conn().await; - - let limits = Limits::for_crate(env.config(), &mut conn, "krate").await?; - assert_eq!(limits.memory, 6 * GB); - - Ok(()) - } - - #[test] - fn overrides_dont_lower_memory_limit() { - async_wrapper(|env| async move { - let db = env.async_db(); - let mut conn = db.async_conn().await; - - let defaults = Limits::new(env.config()); - - Overrides::save( - &mut conn, - "krate", - Overrides { - memory: Some(defaults.memory / 2), - ..Overrides::default() - }, - ) - .await?; - - let limits = Limits::for_crate(env.config(), &mut conn, "krate").await?; - assert_eq!(limits, defaults); - - Ok(()) - }) - } -} diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index 6ba423a8d..aabea27ef 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -1,7 +1,5 @@ -mod limits; mod rustwide_builder; -pub(crate) use self::limits::Limits; pub(crate) use self::rustwide_builder::DocCoverage; pub use self::rustwide_builder::{BuilderMetrics, PackageKind, RustwideBuilder}; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index f77f27d74..f9471d84d 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -5,12 +5,12 @@ use crate::{ initialize_build, initialize_crate, initialize_release, update_build_with_error, update_crate_data_in_database, }, - docbuilder::Limits, error::Result, metrics::{BUILD_TIME_HISTOGRAM_BUCKETS, DOCUMENTATION_SIZE_BUCKETS}, utils::{copy_dir_all, report_error}, }; use anyhow::{Context as _, Error, anyhow, bail}; +use docs_rs_build_limits::Limits; use docs_rs_build_queue::BuildPackageSummary; use docs_rs_cargo_metadata::{CargoMetadata, MetadataPackage}; use docs_rs_database::{ @@ -25,7 +25,7 @@ use docs_rs_storage::{ add_path_into_remote_archive, compress, file_list_to_json, get_file_list, rustdoc_archive_path, rustdoc_json_path, source_archive_path, }; -use docs_rs_types::{BuildId, BuildStatus, CrateId, ReleaseId, Version}; +use docs_rs_types::{BuildId, BuildStatus, CrateId, KrateName, ReleaseId, Version}; use docs_rs_utils::{retry, rustc_version::parse_rustc_version}; use docsrs_metadata::{BuildTargets, DEFAULT_TARGETS, HOST_TARGET, Metadata}; use itertools::Itertools as _; @@ -451,12 +451,13 @@ impl RustwideBuilder { #[instrument(skip(self))] fn get_limits(&self, krate: &str) -> Result { + let krate: KrateName = krate.parse()?; self.runtime.block_on({ let db = self.db.clone(); let config = self.config.clone(); async move { let mut conn = db.get_async().await?; - Limits::for_crate(&config, &mut conn, krate).await + Limits::for_crate(&config.build_limits, &mut conn, &krate).await } }) } diff --git a/src/lib.rs b/src/lib.rs index 1f98db8d5..58b3429fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,9 +13,8 @@ pub use self::docbuilder::RustwideBuilder; pub use self::index::Index; pub use self::web::start_web_server; -pub use docs_rs_utils::{ - APP_USER_AGENT, BUILD_VERSION, DEFAULT_MAX_TARGETS, RUSTDOC_STATIC_STORAGE_PREFIX, -}; +pub use docs_rs_build_limits::DEFAULT_MAX_TARGETS; +pub use docs_rs_utils::{APP_USER_AGENT, BUILD_VERSION, RUSTDOC_STATIC_STORAGE_PREFIX}; pub use font_awesome_as_a_crate::icons; pub mod build_queue; diff --git a/src/web/builds.rs b/src/web/builds.rs index 07fb2c021..6cddbf1ed 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -1,7 +1,5 @@ use crate::{ - Config, - docbuilder::Limits, - impl_axum_webpage, + Config, impl_axum_webpage, web::{ MetaData, cache::CachePolicy, @@ -20,6 +18,7 @@ use axum_extra::{ }; use chrono::{DateTime, Utc}; use constant_time_eq::constant_time_eq; +use docs_rs_build_limits::Limits; use docs_rs_build_queue::{AsyncBuildQueue, PRIORITY_MANUAL_FROM_CRATES_IO}; use docs_rs_headers::CanonicalUrl; use docs_rs_types::{BuildId, BuildStatus, KrateName, ReqVersion, Version}; @@ -87,7 +86,7 @@ pub(crate) async fn build_list_handler( Ok(BuildsPage { metadata, builds: get_builds(&mut conn, params.name(), &version).await?, - limits: Limits::for_crate(&config, &mut conn, params.name()).await?, + limits: Limits::for_crate(&config.build_limits, &mut conn, params.name()).await?, canonical_url: CanonicalUrl::from_uri( params .clone() @@ -213,7 +212,6 @@ async fn get_builds( #[cfg(test)] mod tests { use crate::{ - db::Overrides, test::{ AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, V1, V2, async_wrapper, fake_release_that_failed_before_build, @@ -222,6 +220,7 @@ mod tests { }; use anyhow::Result; use axum::{body::Body, http::Request}; + use docs_rs_build_limits::Overrides; use docs_rs_types::{BuildStatus, testing::FOO}; use kuchikiki::traits::TendrilSink; use reqwest::StatusCode; @@ -517,7 +516,7 @@ mod tests { targets: Some(1), timeout: Some(std::time::Duration::from_secs(2 * 60 * 60)), }; - Overrides::save(&mut conn, "foo", limits).await?; + Overrides::save(&mut conn, &FOO, limits).await?; let page = kuchikiki::parse_html().one( env.web_app() diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs index f4fe332a2..2b05156fd 100644 --- a/src/web/sitemap.rs +++ b/src/web/sitemap.rs @@ -1,7 +1,5 @@ use crate::{ - Config, - docbuilder::Limits, - impl_axum_webpage, + Config, impl_axum_webpage, utils::report_error, web::{ AxumErrorPage, @@ -21,6 +19,7 @@ use axum::{ }; use axum_extra::{TypedHeader, headers::ContentType}; use chrono::{TimeZone, Utc}; +use docs_rs_build_limits::Limits; use docs_rs_database::service_config::{ConfigName, get_config}; use docs_rs_mimes as mimes; use futures_util::{StreamExt as _, pin_mut}; @@ -175,7 +174,7 @@ pub(crate) async fn about_builds_handler( ) -> AxumResult { Ok(AboutBuilds { rustc_version: get_config::(&mut conn, ConfigName::RustcVersion).await?, - limits: Limits::new(&config), + limits: Limits::new(&config.build_limits), active_tab: "builds", }) }