From 7db700846efcc8c11ee206b5e197d05518bd2c38 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Sun, 17 May 2026 21:59:33 +0200 Subject: [PATCH 1/5] feat: add `force_remote_build` config param to disable local job handling --- src/compiler/compiler.rs | 22 ++++++++++++++++++++++ src/config.rs | 3 +++ src/dist/http.rs | 6 ++++++ src/dist/mod.rs | 1 + src/server.rs | 4 ++++ tests/harness/mod.rs | 1 + tests/oauth.rs | 1 + 7 files changed, 38 insertions(+) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 1d9ccff751..0a17374cbd 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -864,6 +864,10 @@ where Some(ref client) => client.rewrite_includes_only(), _ => false, }; + let force_remote_build = match dist_client { + Some(ref client) => client.force_remote_build(), + _ => false, + }; let mut path_transformer = dist::PathTransformer::new(); let (compile_cmd, dist_compile_cmd, cacheable) = compilation .generate_compile_commands(&mut path_transformer, rewrite_includes_only) @@ -1056,6 +1060,9 @@ where local_executable2 )) } else { + if force_remote_build { + return Err(e); + } // `{:#}` prints the error and the causes in a single line. let errmsg = format!("{:#}", e); warn!( @@ -3365,6 +3372,9 @@ mod test_dist { fn rewrite_includes_only(&self) -> bool { false } + fn force_remote_build(&self) -> bool { + false + } fn get_custom_toolchain(&self, _exe: &Path) -> Option { None } @@ -3419,6 +3429,9 @@ mod test_dist { fn rewrite_includes_only(&self) -> bool { false } + fn force_remote_build(&self) -> bool { + false + } fn get_custom_toolchain(&self, _exe: &Path) -> Option { None } @@ -3490,6 +3503,9 @@ mod test_dist { fn rewrite_includes_only(&self) -> bool { false } + fn force_remote_build(&self) -> bool { + false + } fn get_custom_toolchain(&self, _exe: &Path) -> Option { None } @@ -3569,6 +3585,9 @@ mod test_dist { fn rewrite_includes_only(&self) -> bool { false } + fn force_remote_build(&self) -> bool { + false + } fn get_custom_toolchain(&self, _exe: &Path) -> Option { None } @@ -3668,6 +3687,9 @@ mod test_dist { fn rewrite_includes_only(&self) -> bool { false } + fn force_remote_build(&self) -> bool { + false + } fn get_custom_toolchain(&self, _exe: &Path) -> Option { None } diff --git a/src/config.rs b/src/config.rs index ecbe98b8d8..3de31f57f0 100644 --- a/src/config.rs +++ b/src/config.rs @@ -744,6 +744,7 @@ pub struct DistConfig { #[serde(deserialize_with = "deserialize_size_from_str")] pub toolchain_cache_size: u64, pub rewrite_includes_only: bool, + pub force_remote_build: bool, } impl Default for DistConfig { @@ -755,6 +756,7 @@ impl Default for DistConfig { toolchains: Default::default(), toolchain_cache_size: default_toolchain_cache_size(), rewrite_includes_only: false, + force_remote_build: false, } } } @@ -2380,6 +2382,7 @@ key_prefix = "cosprefix" toolchains: vec![], toolchain_cache_size: 5368709120, rewrite_includes_only: false, + force_remote_build: false, }, server_startup_timeout_ms: Some(10000), basedirs: vec![], diff --git a/src/dist/http.rs b/src/dist/http.rs index 09aa862ed8..bc6079a19f 100644 --- a/src/dist/http.rs +++ b/src/dist/http.rs @@ -1105,6 +1105,7 @@ mod client { pool: tokio::runtime::Handle, tc_cache: Arc, rewrite_includes_only: bool, + force_remote_build: bool, } impl Client { @@ -1116,6 +1117,7 @@ mod client { toolchain_configs: &[config::DistToolchainConfig], auth_token: String, rewrite_includes_only: bool, + force_remote_build: bool, ) -> Result { let timeout = Duration::new(REQUEST_TIMEOUT_SECS, 0); let connect_timeout = Duration::new(CONNECT_TIMEOUT_SECS, 0); @@ -1138,6 +1140,7 @@ mod client { pool: pool.clone(), tc_cache: Arc::new(client_toolchains), rewrite_includes_only, + force_remote_build, }) } @@ -1328,6 +1331,9 @@ mod client { fn rewrite_includes_only(&self) -> bool { self.rewrite_includes_only } + fn force_remote_build(&self) -> bool { + self.force_remote_build + } fn get_custom_toolchain(&self, exe: &Path) -> Option { match self.tc_cache.get_custom_toolchain(exe) { Some(Ok((_, _, path))) => Some(path), diff --git a/src/dist/mod.rs b/src/dist/mod.rs index 6bc1024aa8..e18345d2c7 100644 --- a/src/dist/mod.rs +++ b/src/dist/mod.rs @@ -747,5 +747,6 @@ pub trait Client: Send + Sync { toolchain_packager: Box, ) -> Result<(Toolchain, Option<(String, PathBuf)>)>; fn rewrite_includes_only(&self) -> bool; + fn force_remote_build(&self) -> bool; fn get_custom_toolchain(&self, exe: &Path) -> Option; } diff --git a/src/server.rs b/src/server.rs index be49c1dfd2..5ec213d3bd 100644 --- a/src/server.rs +++ b/src/server.rs @@ -157,6 +157,7 @@ pub struct DistClientConfig { toolchain_cache_size: u64, toolchains: Vec, rewrite_includes_only: bool, + force_remote_build: bool, } #[cfg(feature = "dist-client")] @@ -213,6 +214,7 @@ impl DistClientContainer { toolchain_cache_size: config.dist.toolchain_cache_size, toolchains: config.dist.toolchains.clone(), rewrite_includes_only: config.dist.rewrite_includes_only, + force_remote_build: config.dist.force_remote_build, }; let state = Self::create_state(config); let state = pool.block_on(state); @@ -378,6 +380,7 @@ impl DistClientContainer { &config.toolchains, auth_token, config.rewrite_includes_only, + config.force_remote_build, ); let dist_client = try_or_retry_later!(dist_client.context("failure during dist client creation")); @@ -978,6 +981,7 @@ where toolchain_cache_size: 0, toolchains: vec![], rewrite_includes_only: false, + force_remote_build: false, }), dist_client, ))), diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index 3ff280845f..453626ac5f 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -190,6 +190,7 @@ pub fn sccache_client_cfg( toolchains: vec![], toolchain_cache_size: TC_CACHE_SIZE, rewrite_includes_only: false, // TODO + force_remote_build: false, }, server_startup_timeout_ms: None, basedirs: vec![], diff --git a/tests/oauth.rs b/tests/oauth.rs index 90c8615059..284a611a6d 100644 --- a/tests/oauth.rs +++ b/tests/oauth.rs @@ -57,6 +57,7 @@ fn config_with_dist_auth( toolchains: vec![], toolchain_cache_size: 0, rewrite_includes_only: true, + force_remote_build: false, }, server_startup_timeout_ms: None, basedirs: vec![], From 7fabc3ca633ffafb7e910f0aaff64b8c5db43755 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Sun, 17 May 2026 22:34:43 +0200 Subject: [PATCH 2/5] chore: use DistConfig::default() when possible --- tests/harness/mod.rs | 4 +--- tests/oauth.rs | 6 +----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index 453626ac5f..e7ef062ebd 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -185,12 +185,10 @@ pub fn sccache_client_cfg( }, dist: sccache::config::DistConfig { auth: Default::default(), // dangerously_insecure - scheduler_url: None, cache_dir: tmpdir.join(dist_cache_relpath), - toolchains: vec![], toolchain_cache_size: TC_CACHE_SIZE, rewrite_includes_only: false, // TODO - force_remote_build: false, + ..sccache::config::DistConfig::default() }, server_startup_timeout_ms: None, basedirs: vec![], diff --git a/tests/oauth.rs b/tests/oauth.rs index 284a611a6d..4c7a9c24c4 100644 --- a/tests/oauth.rs +++ b/tests/oauth.rs @@ -52,12 +52,8 @@ fn config_with_dist_auth( cache: Default::default(), dist: sccache::config::DistConfig { auth: auth_config, - scheduler_url: None, cache_dir: tmpdir.join("unused-cache"), - toolchains: vec![], - toolchain_cache_size: 0, - rewrite_includes_only: true, - force_remote_build: false, + ..sccache::config::DistConfig::default() }, server_startup_timeout_ms: None, basedirs: vec![], From 4915dac82cb98f395d60086428071afbe6147247 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Mon, 18 May 2026 01:13:09 +0200 Subject: [PATCH 3/5] test: add test --- src/compiler/compiler.rs | 125 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 0a17374cbd..54b65b106a 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -3313,6 +3313,82 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } } + + #[cfg(feature = "dist-client")] + #[test] + fn test_compiler_get_cached_or_compile_dist_force_remote() { + drop(env_logger::try_init()); + let creator = new_creator(); + let f = TestFixture::new(); + let gcc = f.mk_bin("gcc").unwrap(); + let runtime = Runtime::new().unwrap(); + let pool = runtime.handle().clone(); + let dist_client = + Arc::new(test_dist::ForceRemoteClient( + test_dist::ErrorRunJobClient::new(), + )); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); + let storage = DiskCache::new( + f.tempdir.path().join("cache"), + u64::MAX, + &pool, + Default::default(), + CacheMode::ReadWrite, + vec![], + ); + let storage = Arc::new(storage); + // Pretend to be GCC. + next_command( + &creator, + Ok(MockChild::new(exit_status(0), "compiler_id=gcc", "")), + ); + let c = get_compiler_info( + creator.clone(), + &gcc, + f.tempdir.path(), + &[], + &[], + &pool, + None, + ) + .wait() + .unwrap() + .0; + // The preprocessor invocation. + next_command( + &creator, + Ok(MockChild::new(exit_status(0), "preprocessor output", "")), + ); + let cwd = f.tempdir.path(); + let arguments = ovec!["-c", "foo.c", "-o", "foo.o"]; + let mut hasher = match c.parse_arguments(&arguments, ".".as_ref(), &[]) { + CompilerArguments::Ok(h) => h, + o => panic!("Bad result from parse_arguments: {:?}", o), + }; + let service = server::SccacheService::mock_with_dist_client( + dist_client.clone(), + storage.clone(), + pool.clone(), + ); + let result = hasher + .get_cached_or_compile( + &service, + Some(dist_client), + creator, + storage, + arguments, + cwd.to_path_buf(), + vec![], + CacheControl::ForceRecache, + pool, + ) + .wait(); + assert!( + result.is_err(), + "Expected dist compilation to fail with force_remote, but it succeeded" + ); + } } #[cfg(test)] @@ -3694,4 +3770,53 @@ mod test_dist { None } } + + pub struct ForceRemoteClient(pub Arc); + + #[async_trait] + impl dist::Client for ForceRemoteClient { + async fn do_alloc_job(&self, tc: Toolchain) -> Result { + self.0.do_alloc_job(tc).await + } + async fn do_get_status(&self) -> Result { + self.0.do_get_status().await + } + async fn do_submit_toolchain( + &self, + job_alloc: JobAlloc, + tc: Toolchain, + ) -> Result { + self.0.do_submit_toolchain(job_alloc, tc).await + } + async fn do_run_job( + &self, + job_alloc: JobAlloc, + command: CompileCommand, + outputs: Vec, + inputs_packager: Box, + ) -> Result<(RunJobResult, PathTransformer)> { + self.0 + .do_run_job(job_alloc, command, outputs, inputs_packager) + .await + } + async fn put_toolchain( + &self, + compiler_path: PathBuf, + weak_key: String, + toolchain_packager: Box, + ) -> Result<(Toolchain, Option<(String, PathBuf)>)> { + self.0 + .put_toolchain(compiler_path, weak_key, toolchain_packager) + .await + } + fn rewrite_includes_only(&self) -> bool { + self.0.rewrite_includes_only() + } + fn force_remote_build(&self) -> bool { + true + } + fn get_custom_toolchain(&self, exe: &Path) -> Option { + self.0.get_custom_toolchain(exe) + } + } } From ded74b44453fde296a5b08b2d6a0e6523edf4eb4 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Mon, 18 May 2026 01:14:33 +0200 Subject: [PATCH 4/5] chore: format --- src/compiler/compiler.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 54b65b106a..2d58b6c818 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -3323,10 +3323,10 @@ LLVM version: 6.0", let gcc = f.mk_bin("gcc").unwrap(); let runtime = Runtime::new().unwrap(); let pool = runtime.handle().clone(); - let dist_client = - Arc::new(test_dist::ForceRemoteClient( - test_dist::ErrorRunJobClient::new(), - )); + let dist_client = Arc::new(test_dist::ForceRemoteClient( + test_dist::ErrorRunJobClient::new(), + )); + // Write a dummy input file so the preprocessor cache mode can work std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let storage = DiskCache::new( @@ -3338,6 +3338,7 @@ LLVM version: 6.0", vec![], ); let storage = Arc::new(storage); + // Pretend to be GCC. next_command( &creator, @@ -3355,6 +3356,7 @@ LLVM version: 6.0", .wait() .unwrap() .0; + // The preprocessor invocation. next_command( &creator, From 481f65d5df14b5f79bf8be11cd36641c1d8b7eac Mon Sep 17 00:00:00 2001 From: iTrooz Date: Mon, 18 May 2026 01:30:12 +0200 Subject: [PATCH 5/5] refactor: pass whole DistClientConfig to Client::new() --- src/dist/http.rs | 24 +++++++++++------------- src/server.rs | 23 +++++++---------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/dist/http.rs b/src/dist/http.rs index bc6079a19f..0caac9fc24 100644 --- a/src/dist/http.rs +++ b/src/dist/http.rs @@ -1067,12 +1067,12 @@ mod server { #[cfg(feature = "dist-client")] mod client { use super::super::cache; - use crate::config; use crate::dist::pkg::{InputsPackager, ToolchainPackager}; use crate::dist::{ self, AllocJobResult, CompileCommand, JobAlloc, PathTransformer, RunJobResult, SchedulerStatusResult, SubmitToolchainResult, Toolchain, }; + use crate::server::DistClientConfig; use async_trait::async_trait; use byteorder::{BigEndian, WriteBytesExt}; @@ -1110,14 +1110,9 @@ mod client { impl Client { pub fn new( - pool: &tokio::runtime::Handle, + config: &DistClientConfig, scheduler_url: reqwest::Url, - cache_dir: &Path, - cache_size: u64, - toolchain_configs: &[config::DistToolchainConfig], auth_token: String, - rewrite_includes_only: bool, - force_remote_build: bool, ) -> Result { let timeout = Duration::new(REQUEST_TIMEOUT_SECS, 0); let connect_timeout = Duration::new(CONNECT_TIMEOUT_SECS, 0); @@ -1129,18 +1124,21 @@ mod client { .pool_max_idle_per_host(0) .build() .context("failed to create an async HTTP client")?; - let client_toolchains = - cache::ClientToolchains::new(cache_dir, cache_size, toolchain_configs) - .context("failed to initialise client toolchains")?; + let client_toolchains = cache::ClientToolchains::new( + &config.cache_dir, + config.toolchain_cache_size, + &config.toolchains, + ) + .context("failed to initialise client toolchains")?; Ok(Self { auth_token, scheduler_url, server_certs: Default::default(), client: Arc::new(Mutex::new(client)), - pool: pool.clone(), + pool: config.pool.clone(), tc_cache: Arc::new(client_toolchains), - rewrite_includes_only, - force_remote_build, + rewrite_includes_only: config.rewrite_includes_only, + force_remote_build: config.force_remote_build, }) } diff --git a/src/server.rs b/src/server.rs index 5ec213d3bd..918123540b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -148,16 +148,16 @@ pub struct DistClientContainer { #[cfg(feature = "dist-client")] pub struct DistClientConfig { // Reusable items tied to an SccacheServer instance - pool: tokio::runtime::Handle, + pub pool: tokio::runtime::Handle, // From the static dist configuration scheduler_url: Option, auth: config::DistAuth, - cache_dir: PathBuf, - toolchain_cache_size: u64, - toolchains: Vec, - rewrite_includes_only: bool, - force_remote_build: bool, + pub cache_dir: PathBuf, + pub toolchain_cache_size: u64, + pub toolchains: Vec, + pub rewrite_includes_only: bool, + pub force_remote_build: bool, } #[cfg(feature = "dist-client")] @@ -372,16 +372,7 @@ impl DistClientContainer { auth_token .context("could not load client auth token, run |sccache --dist-auth|") ); - let dist_client = dist::http::Client::new( - &config.pool, - url, - &config.cache_dir.join("client"), - config.toolchain_cache_size, - &config.toolchains, - auth_token, - config.rewrite_includes_only, - config.force_remote_build, - ); + let dist_client = dist::http::Client::new(&config, url, auth_token); let dist_client = try_or_retry_later!(dist_client.context("failure during dist client creation")); use crate::dist::Client;