feat: add force_remote_build config param to disable local job handling

src/compiler/compiler.rs

@@ -864,6 +864,10 @@ where
         Some(ref client) => client.rewrite_includes_only(),
         _ => false,
     };
+    let force_remote_build = match dist_client {
+        Some(ref client) => client.force_remote_build(),
+        _ => false,
+    };
     let mut path_transformer = dist::PathTransformer::new();
     let (compile_cmd, dist_compile_cmd, cacheable) = compilation
         .generate_compile_commands(&mut path_transformer, rewrite_includes_only)
@@ -1056,6 +1060,9 @@ where
                     local_executable2
                 ))
             } else {
+                if force_remote_build {
+                    return Err(e);
+                }
                 // `{:#}` prints the error and the causes in a single line.
                 let errmsg = format!("{:#}", e);
                 warn!(
@@ -3306,6 +3313,84 @@ LLVM version: 6.0",
             assert_eq!(COMPILER_STDERR, res.stderr.as_slice());
         }
     }
+
+    #[cfg(feature = "dist-client")]
+    #[test]
+    fn test_compiler_get_cached_or_compile_dist_force_remote() {
+        drop(env_logger::try_init());
+        let creator = new_creator();
+        let f = TestFixture::new();
+        let gcc = f.mk_bin("gcc").unwrap();
+        let runtime = Runtime::new().unwrap();
+        let pool = runtime.handle().clone();
+        let dist_client = Arc::new(test_dist::ForceRemoteClient(
+            test_dist::ErrorRunJobClient::new(),
+        ));
+
+        // Write a dummy input file so the preprocessor cache mode can work
+        std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap();
+        let storage = DiskCache::new(
+            f.tempdir.path().join("cache"),
+            u64::MAX,
+            &pool,
+            Default::default(),
+            CacheMode::ReadWrite,
+            vec![],
+        );
+        let storage = Arc::new(storage);
+
+        // Pretend to be GCC.
+        next_command(
+            &creator,
+            Ok(MockChild::new(exit_status(0), "compiler_id=gcc", "")),
+        );
+        let c = get_compiler_info(
+            creator.clone(),
+            &gcc,
+            f.tempdir.path(),
+            &[],
+            &[],
+            &pool,
+            None,
+        )
+        .wait()
+        .unwrap()
+        .0;
+
+        // The preprocessor invocation.
+        next_command(
+            &creator,
+            Ok(MockChild::new(exit_status(0), "preprocessor output", "")),
+        );
+        let cwd = f.tempdir.path();
+        let arguments = ovec!["-c", "foo.c", "-o", "foo.o"];
+        let mut hasher = match c.parse_arguments(&arguments, ".".as_ref(), &[]) {
+            CompilerArguments::Ok(h) => h,
+            o => panic!("Bad result from parse_arguments: {:?}", o),
+        };
+        let service = server::SccacheService::mock_with_dist_client(
+            dist_client.clone(),
+            storage.clone(),
+            pool.clone(),
+        );
+        let result = hasher
+            .get_cached_or_compile(
+                &service,
+                Some(dist_client),
+                creator,
+                storage,
+                arguments,
+                cwd.to_path_buf(),
+                vec![],
+                CacheControl::ForceRecache,
+                pool,
+            )
+            .wait();
+        assert!(
+            result.is_err(),
+            "Expected dist compilation to fail with force_remote, but it succeeded"
+        );
+    }
 }
 
 #[cfg(test)]
@@ -3365,6 +3450,9 @@ mod test_dist {
         fn rewrite_includes_only(&self) -> bool {
             false
         }
+        fn force_remote_build(&self) -> bool {
+            false
+        }
         fn get_custom_toolchain(&self, _exe: &Path) -> Option<PathBuf> {
             None
         }
@@ -3419,6 +3507,9 @@ mod test_dist {
         fn rewrite_includes_only(&self) -> bool {
             false
         }
+        fn force_remote_build(&self) -> bool {
+            false
+        }
         fn get_custom_toolchain(&self, _exe: &Path) -> Option<PathBuf> {
             None
         }
@@ -3490,6 +3581,9 @@ mod test_dist {
         fn rewrite_includes_only(&self) -> bool {
             false
         }
+        fn force_remote_build(&self) -> bool {
+            false
+        }
         fn get_custom_toolchain(&self, _exe: &Path) -> Option<PathBuf> {
             None
         }
@@ -3569,6 +3663,9 @@ mod test_dist {
         fn rewrite_includes_only(&self) -> bool {
             false
         }
+        fn force_remote_build(&self) -> bool {
+            false
+        }
         fn get_custom_toolchain(&self, _exe: &Path) -> Option<PathBuf> {
             None
         }
@@ -3668,8 +3765,60 @@ mod test_dist {
         fn rewrite_includes_only(&self) -> bool {
             false
         }
+        fn force_remote_build(&self) -> bool {
+            false
+        }
         fn get_custom_toolchain(&self, _exe: &Path) -> Option<PathBuf> {
             None
         }
     }
+
+    pub struct ForceRemoteClient(pub Arc<dyn dist::Client>);
+
+    #[async_trait]
+    impl dist::Client for ForceRemoteClient {
+        async fn do_alloc_job(&self, tc: Toolchain) -> Result<AllocJobResult> {
+            self.0.do_alloc_job(tc).await
+        }
+        async fn do_get_status(&self) -> Result<SchedulerStatusResult> {
+            self.0.do_get_status().await
+        }
+        async fn do_submit_toolchain(
+            &self,
+            job_alloc: JobAlloc,
+            tc: Toolchain,
+        ) -> Result<SubmitToolchainResult> {
+            self.0.do_submit_toolchain(job_alloc, tc).await
+        }
+        async fn do_run_job(
+            &self,
+            job_alloc: JobAlloc,
+            command: CompileCommand,
+            outputs: Vec<String>,
+            inputs_packager: Box<dyn pkg::InputsPackager>,
+        ) -> Result<(RunJobResult, PathTransformer)> {
+            self.0
+                .do_run_job(job_alloc, command, outputs, inputs_packager)
+                .await
+        }
+        async fn put_toolchain(
+            &self,
+            compiler_path: PathBuf,
+            weak_key: String,
+            toolchain_packager: Box<dyn pkg::ToolchainPackager>,
+        ) -> Result<(Toolchain, Option<(String, PathBuf)>)> {
+            self.0
+                .put_toolchain(compiler_path, weak_key, toolchain_packager)
+                .await
+        }
+        fn rewrite_includes_only(&self) -> bool {
+            self.0.rewrite_includes_only()
+        }
+        fn force_remote_build(&self) -> bool {
+            true
+        }
+        fn get_custom_toolchain(&self, exe: &Path) -> Option<PathBuf> {
+            self.0.get_custom_toolchain(exe)
+        }
+    }
 }

src/config.rs

@@ -744,6 +744,7 @@ pub struct DistConfig {
     #[serde(deserialize_with = "deserialize_size_from_str")]
     pub toolchain_cache_size: u64,
     pub rewrite_includes_only: bool,
+    pub force_remote_build: bool,
 }
 
 impl Default for DistConfig {
@@ -755,6 +756,7 @@ impl Default for DistConfig {
             toolchains: Default::default(),
             toolchain_cache_size: default_toolchain_cache_size(),
             rewrite_includes_only: false,
+            force_remote_build: false,
         }
     }
 }
@@ -2380,6 +2382,7 @@ key_prefix = "cosprefix"
                 toolchains: vec![],
                 toolchain_cache_size: 5368709120,
                 rewrite_includes_only: false,
+                force_remote_build: false,
             },
             server_startup_timeout_ms: Some(10000),
             basedirs: vec![],

src/dist/http.rs

@@ -1067,12 +1067,12 @@ mod server {
 #[cfg(feature = "dist-client")]
 mod client {
     use super::super::cache;
-    use crate::config;
     use crate::dist::pkg::{InputsPackager, ToolchainPackager};
     use crate::dist::{
         self, AllocJobResult, CompileCommand, JobAlloc, PathTransformer, RunJobResult,
         SchedulerStatusResult, SubmitToolchainResult, Toolchain,
     };
+    use crate::server::DistClientConfig;
 
     use async_trait::async_trait;
     use byteorder::{BigEndian, WriteBytesExt};
@@ -1105,17 +1105,14 @@ mod client {
         pool: tokio::runtime::Handle,
         tc_cache: Arc<cache::ClientToolchains>,
         rewrite_includes_only: bool,
+        force_remote_build: bool,
     }
 
     impl Client {
         pub fn new(
-            pool: &tokio::runtime::Handle,
+            config: &DistClientConfig,
             scheduler_url: reqwest::Url,
-            cache_dir: &Path,
-            cache_size: u64,
-            toolchain_configs: &[config::DistToolchainConfig],
             auth_token: String,
-            rewrite_includes_only: bool,
         ) -> Result<Self> {
             let timeout = Duration::new(REQUEST_TIMEOUT_SECS, 0);
             let connect_timeout = Duration::new(CONNECT_TIMEOUT_SECS, 0);
@@ -1127,17 +1124,21 @@ mod client {
                 .pool_max_idle_per_host(0)
                 .build()
                 .context("failed to create an async HTTP client")?;
-            let client_toolchains =
-                cache::ClientToolchains::new(cache_dir, cache_size, toolchain_configs)
-                    .context("failed to initialise client toolchains")?;
+            let client_toolchains = cache::ClientToolchains::new(
+                &config.cache_dir,
+                config.toolchain_cache_size,
+                &config.toolchains,
+            )
+            .context("failed to initialise client toolchains")?;
             Ok(Self {
                 auth_token,
                 scheduler_url,
                 server_certs: Default::default(),
                 client: Arc::new(Mutex::new(client)),
-                pool: pool.clone(),
+                pool: config.pool.clone(),
                 tc_cache: Arc::new(client_toolchains),
-                rewrite_includes_only,
+                rewrite_includes_only: config.rewrite_includes_only,
+                force_remote_build: config.force_remote_build,
             })
         }
 
@@ -1328,6 +1329,9 @@ mod client {
         fn rewrite_includes_only(&self) -> bool {
             self.rewrite_includes_only
         }
+        fn force_remote_build(&self) -> bool {
+            self.force_remote_build
+        }
         fn get_custom_toolchain(&self, exe: &Path) -> Option<PathBuf> {
             match self.tc_cache.get_custom_toolchain(exe) {
                 Some(Ok((_, _, path))) => Some(path),

src/dist/mod.rs

@@ -747,5 +747,6 @@ pub trait Client: Send + Sync {
         toolchain_packager: Box<dyn pkg::ToolchainPackager>,
     ) -> Result<(Toolchain, Option<(String, PathBuf)>)>;
     fn rewrite_includes_only(&self) -> bool;
+    fn force_remote_build(&self) -> bool;
     fn get_custom_toolchain(&self, exe: &Path) -> Option<PathBuf>;
 }

src/server.rs

@@ -148,15 +148,16 @@ pub struct DistClientContainer {
 #[cfg(feature = "dist-client")]
 pub struct DistClientConfig {
     // Reusable items tied to an SccacheServer instance
-    pool: tokio::runtime::Handle,
+    pub pool: tokio::runtime::Handle,
 
     // From the static dist configuration
     scheduler_url: Option<config::HTTPUrl>,
     auth: config::DistAuth,
-    cache_dir: PathBuf,
-    toolchain_cache_size: u64,
-    toolchains: Vec<config::DistToolchainConfig>,
-    rewrite_includes_only: bool,
+    pub cache_dir: PathBuf,
+    pub toolchain_cache_size: u64,
+    pub toolchains: Vec<config::DistToolchainConfig>,
+    pub rewrite_includes_only: bool,
+    pub force_remote_build: bool,
 }
 
 #[cfg(feature = "dist-client")]
@@ -213,6 +214,7 @@ impl DistClientContainer {
             toolchain_cache_size: config.dist.toolchain_cache_size,
             toolchains: config.dist.toolchains.clone(),
             rewrite_includes_only: config.dist.rewrite_includes_only,
+            force_remote_build: config.dist.force_remote_build,
         };
         let state = Self::create_state(config);
         let state = pool.block_on(state);
@@ -370,15 +372,7 @@ impl DistClientContainer {
                     auth_token
                         .context("could not load client auth token, run |sccache --dist-auth|")
                 );
-                let dist_client = dist::http::Client::new(
-                    &config.pool,
-                    url,
-                    &config.cache_dir.join("client"),
-                    config.toolchain_cache_size,
-                    &config.toolchains,
-                    auth_token,
-                    config.rewrite_includes_only,
-                );
+                let dist_client = dist::http::Client::new(&config, url, auth_token);
                 let dist_client =
                     try_or_retry_later!(dist_client.context("failure during dist client creation"));
                 use crate::dist::Client;
@@ -978,6 +972,7 @@ where
                     toolchain_cache_size: 0,
                     toolchains: vec![],
                     rewrite_includes_only: false,
+                    force_remote_build: false,
                 }),
                 dist_client,
             ))),

tests/harness/mod.rs

@@ -185,11 +185,10 @@ pub fn sccache_client_cfg(
         },
         dist: sccache::config::DistConfig {
             auth: Default::default(), // dangerously_insecure
-            scheduler_url: None,
             cache_dir: tmpdir.join(dist_cache_relpath),
-            toolchains: vec![],
             toolchain_cache_size: TC_CACHE_SIZE,
             rewrite_includes_only: false, // TODO
+            ..sccache::config::DistConfig::default()
         },
         server_startup_timeout_ms: None,
         basedirs: vec![],

tests/oauth.rs

@@ -52,11 +52,8 @@ fn config_with_dist_auth(
         cache: Default::default(),
         dist: sccache::config::DistConfig {
             auth: auth_config,
-            scheduler_url: None,
             cache_dir: tmpdir.join("unused-cache"),
-            toolchains: vec![],
-            toolchain_cache_size: 0,
-            rewrite_includes_only: true,
+            ..sccache::config::DistConfig::default()
         },
         server_startup_timeout_ms: None,
         basedirs: vec![],