ephemeral: Fix run-ssh error handling

If we fail to launch, because we used `--rm` the container
would exit and we couldn't get its logs.

Change to removing the container image via a drop handler,
so if the system fails to launch (e.g. it's missing a kernel or bwrap)
we can get the logs from that.

Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

crates/integration-tests/fixtures/Dockerfile.no-kernel

@@ -0,0 +1,10 @@
+# Test fixture: Bootc image with kernel removed
+# This simulates a broken/malformed bootc image that will fail during ephemeral VM startup
+# Used to test error handling and cleanup in ephemeral run-ssh command
+
+ARG BASE_IMAGE=quay.io/centos-bootc/centos-bootc:stream10
+
+FROM ${BASE_IMAGE}
+
+# Remove kernel and modules to simulate a broken bootc image
+RUN rm -rf /usr/lib/modules

crates/integration-tests/src/tests/run_ephemeral_ssh.rs

@@ -18,11 +18,71 @@ use color_eyre::Result;
 use integration_tests::{integration_test, parameterized_integration_test};
 
 use std::process::Command;
-use std::thread;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use crate::{get_test_image, run_bcvk, INTEGRATION_TEST_LABEL};
 
+/// Poll until a container is removed or timeout is reached
+///
+/// Returns Ok(()) if container is removed within timeout, Err otherwise.
+/// Timeout is set to 60 seconds to account for slow CI runners.
+fn wait_for_container_removal(container_name: &str) -> Result<()> {
+    let timeout = Duration::from_secs(60);
+    let start = Instant::now();
+    let poll_interval = Duration::from_millis(100);
+
+    loop {
+        let output = Command::new("podman")
+            .args(["ps", "-a", "--format", "{{.Names}}"])
+            .output()
+            .expect("Failed to list containers");
+
+        let containers = String::from_utf8_lossy(&output.stdout);
+        if !containers.lines().any(|line| line == container_name) {
+            return Ok(());
+        }
+
+        if start.elapsed() >= timeout {
+            return Err(color_eyre::eyre::eyre!(
+                "Timeout waiting for container {} to be removed. Active containers: {}",
+                container_name,
+                containers
+            ));
+        }
+
+        std::thread::sleep(poll_interval);
+    }
+}
+
+/// Build a test fixture image with the kernel removed
+fn build_broken_image() -> Result<String> {
+    let fixture_path = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/Dockerfile.no-kernel");
+    let image_name = format!("localhost/bcvk-test-no-kernel:{}", std::process::id());
+
+    let output = Command::new("podman")
+        .args([
+            "build",
+            "-f",
+            fixture_path,
+            "-t",
+            &image_name,
+            "--build-arg",
+            &format!("BASE_IMAGE={}", get_test_image()),
+            ".",
+        ])
+        .output()?;
+
+    if !output.status.success() {
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        return Err(color_eyre::eyre::eyre!(
+            "Failed to build broken test image: {}",
+            stderr
+        ));
+    }
+
+    Ok(image_name)
+}
+
 /// Test running a non-interactive command via SSH
 fn test_run_ephemeral_ssh_command() -> Result<()> {
     let output = run_bcvk(&[
@@ -66,20 +126,9 @@ fn test_run_ephemeral_ssh_cleanup() -> Result<()> {
 
     output.assert_success("ephemeral run-ssh");
 
-    thread::sleep(Duration::from_secs(1));
-
-    let check_output = Command::new("podman")
-        .args(["ps", "-a", "--format", "{{.Names}}"])
-        .output()
-        .expect("Failed to list containers");
+    // Poll for container removal with timeout
+    wait_for_container_removal(&container_name)?;
 
-    let containers = String::from_utf8_lossy(&check_output.stdout);
-    assert!(
-        !containers.contains(&container_name),
-        "Container {} was not cleaned up after SSH exit. Active containers: {}",
-        container_name,
-        containers
-    );
     Ok(())
 }
 integration_test!(test_run_ephemeral_ssh_cleanup);
@@ -248,3 +297,64 @@ echo "All checks passed!"
     Ok(())
 }
 integration_test!(test_run_tmpfs);
+
+/// Test that containers are properly cleaned up even when the image is broken
+///
+/// This test verifies that the drop handler for ContainerCleanup works correctly
+/// when ephemeral run-ssh fails early due to a broken image (missing kernel).
+/// Previously this would fail with "setns `mnt`: Bad file descriptor" when using
+/// podman's --rm flag. Now it should fail cleanly and remove the container.
+fn test_run_ephemeral_ssh_broken_image_cleanup() -> Result<()> {
+    // Build a broken test image (bootc image with kernel removed)
+    eprintln!("Building broken test image...");
+    let broken_image = build_broken_image()?;
+    eprintln!("Built broken image: {}", broken_image);
+
+    let container_name = format!("test-broken-cleanup-{}", std::process::id());
+
+    // Try to run ephemeral SSH with the broken image - this should fail
+    let output = run_bcvk(&[
+        "ephemeral",
+        "run-ssh",
+        "--name",
+        &container_name,
+        "--label",
+        INTEGRATION_TEST_LABEL,
+        &broken_image,
+        "--",
+        "echo",
+        "should not reach here",
+    ])?;
+
+    // The command should fail (no kernel found)
+    assert!(
+        !output.success(),
+        "Expected ephemeral run-ssh to fail with broken image, but it succeeded"
+    );
+
+    // Verify the error message indicates the problem
+    assert!(
+        output
+            .stderr
+            .contains("Failed to read kernel modules directory")
+            || output
+                .stderr
+                .contains("Container exited before SSH became available")
+            || output
+                .stderr
+                .contains("Monitor process exited unexpectedly"),
+        "Expected error about missing kernel or container failure, got: {}",
+        output.stderr
+    );
+
+    // Poll for container removal with timeout
+    wait_for_container_removal(&container_name)?;
+
+    // Clean up the test image
+    let _ = Command::new("podman")
+        .args(["rmi", "-f", &broken_image])
+        .output();
+
+    Ok(())
+}
+integration_test!(test_run_ephemeral_ssh_broken_image_cleanup);

crates/kit/src/run_ephemeral.rs

@@ -777,7 +777,10 @@ pub(crate) async fn run_impl(opts: RunEphemeralOpts) -> Result<()> {
     let mut vmlinuz_path: Option<Utf8PathBuf> = None;
     let mut initramfs_path: Option<Utf8PathBuf> = None;
 
-    for entry in fs::read_dir(modules_dir)? {
+    let entries = fs::read_dir(modules_dir)
+        .with_context(|| format!("Failed to read kernel modules directory at {}. This container image may not be a valid bootc image.", modules_dir))?;
+
+    for entry in entries {
         let entry = entry?;
         let path = Utf8PathBuf::from_path_buf(entry.path())
             .map_err(|p| eyre!("Path is not valid UTF-8: {}", p.display()))?;