test(podman): harden preflight retry coverage

crates/openshell-driver-podman/Cargo.toml

@@ -51,6 +51,7 @@ openshell-otel-test-support = { path = "../openshell-otel-test-support" }
 opentelemetry_sdk = { workspace = true, features = ["testing"] }
 prost-types = { workspace = true }
 temp-env = "0.3"
+tempfile = "3"
 tokio = { workspace = true, features = ["test-util"] }
 
 [lints]

crates/openshell-driver-podman/src/driver.rs

@@ -25,13 +25,16 @@ use openshell_core::proto::compute::v1::{
     GpuResourceRequirements, MemoryResourceCapabilities, ResourceCapabilities,
 };
 use std::collections::HashMap;
+use std::future::Future;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 use tracing::{Instrument as _, debug, info, warn};
 
 const STOP_COMPLETION_POLL_INTERVAL: Duration = Duration::from_millis(50);
 const STOP_COMPLETION_TIMEOUT_HEADROOM: Duration = Duration::from_secs(5);
+const MAX_PING_RETRIES: u32 = 5;
+const PING_RETRY_DELAY: Duration = Duration::from_secs(2);
 const POLICY_DNS_RESOLV_CONF: &[u8] = b"nameserver 127.0.0.53\n";
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -413,12 +416,33 @@ fn resolve_socket_path(
     })
 }
 
+async fn ping_with_retry<F, Fut>(mut ping: F) -> Result<(), PodmanApiError>
+where
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<(), PodmanApiError>>,
+{
+    let mut retries = 0;
+    loop {
+        match ping().await {
+            Ok(()) => return Ok(()),
+            Err(error) if retries < MAX_PING_RETRIES => {
+                retries += 1;
+                warn!(
+                    attempt = retries,
+                    max_retries = MAX_PING_RETRIES,
+                    error = %error,
+                    "Podman socket not ready, retrying"
+                );
+                tokio::time::sleep(PING_RETRY_DELAY).await;
+            }
+            Err(error) => return Err(error),
+        }
+    }
+}
+
 impl PodmanComputeDriver {
     /// Create a new driver, verifying the Podman socket is reachable.
     pub async fn new(mut config: PodmanComputeConfig) -> Result<Self, PodmanApiError> {
-        const MAX_PING_RETRIES: u32 = 5;
-        const PING_RETRY_DELAY: Duration = Duration::from_secs(2);
-
         let socket_path = resolve_socket_path(config.socket_path.clone(), detect_socket)?;
         config.socket_path = Some(socket_path.clone());
 
@@ -449,23 +473,7 @@ impl PodmanComputeDriver {
         // unavailability (e.g. podman.socket restarting after a package
         // upgrade). The systemd unit uses Wants=podman.socket (not Requires),
         // so the gateway may start while the socket is briefly re-activating.
-        let mut attempts = 0;
-        loop {
-            match client.ping().await {
-                Ok(()) => break,
-                Err(e) if attempts < MAX_PING_RETRIES => {
-                    attempts += 1;
-                    warn!(
-                        attempt = attempts,
-                        max_retries = MAX_PING_RETRIES,
-                        error = %e,
-                        "Podman socket not ready, retrying"
-                    );
-                    tokio::time::sleep(PING_RETRY_DELAY).await;
-                }
-                Err(e) => return Err(e),
-            }
-        }
+        ping_with_retry(|| client.ping()).await?;
 
         // Verify cgroups v2, detect rootless mode, and log system info.
         let rootless = match client.system_info().await {
@@ -2048,7 +2056,7 @@ mod tests {
     use openshell_core::proto::compute::v1::{
         DriverSandboxSpec, DriverSandboxTemplate, ResourceRequirements,
     };
-    use std::collections::HashMap;
+    use std::collections::{HashMap, VecDeque};
     use std::fs;
     use std::path::{Path, PathBuf};
 
@@ -2109,6 +2117,63 @@ mod tests {
         assert!(err.to_string().contains("no responsive Podman API socket"));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn ping_retries_transient_failures() {
+        let mut outcomes = VecDeque::from([
+            Err(PodmanApiError::Connection("first".to_string())),
+            Err(PodmanApiError::Connection("second".to_string())),
+            Ok(()),
+        ]);
+        let started = tokio::time::Instant::now();
+
+        ping_with_retry(|| std::future::ready(outcomes.pop_front().expect("ping outcome")))
+            .await
+            .expect("a later successful ping should stop retries");
+
+        assert!(outcomes.is_empty());
+        assert_eq!(started.elapsed(), PING_RETRY_DELAY * 2);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn ping_failure_is_bounded_by_retry_policy() {
+        let mut attempts = 0;
+        let started = tokio::time::Instant::now();
+
+        let error = ping_with_retry(|| {
+            attempts += 1;
+            std::future::ready(Err(PodmanApiError::Connection(format!(
+                "attempt {attempts}"
+            ))))
+        })
+        .await
+        .expect_err("persistent connection failures should be returned");
+
+        assert_eq!(attempts, MAX_PING_RETRIES + 1);
+        assert_eq!(started.elapsed(), PING_RETRY_DELAY * MAX_PING_RETRIES);
+        assert_eq!(
+            error.to_string(),
+            format!("connection error: attempt {}", MAX_PING_RETRIES + 1)
+        );
+    }
+
+    #[tokio::test]
+    async fn missing_socket_connection_error_names_configured_path() {
+        let tempdir = tempfile::tempdir().expect("create isolated socket directory");
+        let missing_socket = tempdir.path().join("missing-podman.sock");
+
+        let error = PodmanClient::new(missing_socket.clone())
+            .ping()
+            .await
+            .expect_err("a missing socket should fail to connect");
+
+        assert!(matches!(error, PodmanApiError::Connection(_)));
+        assert!(
+            error
+                .to_string()
+                .contains(&missing_socket.display().to_string())
+        );
+    }
+
     fn cdi_devices_config(device_ids: &[&str]) -> prost_types::Struct {
         prost_types::Struct {
             fields: std::iter::once((

crates/openshell-driver-podman/tests/podman_preflight.rs

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Standalone driver smoke test for the Podman-unavailable diagnostic path.
+//!
+//! Retry timing and connection failures are covered deterministically by unit
+//! tests. This test retains only the executable boundary: argument wiring,
+//! process exit status, and the rendered error shown to operators.
+
+use std::path::PathBuf;
+use std::process::Stdio;
+use std::time::Duration;
+
+#[tokio::test]
+async fn missing_podman_socket_exits_with_actionable_diagnostic() {
+    let tmpdir = tempfile::tempdir().expect("create isolated socket dir");
+    // Use a short relative path so miette cannot insert a line-wrap gutter
+    // inside it on platforms with long temporary-directory paths.
+    let missing_socket = PathBuf::from("missing-podman.sock");
+
+    let mut cmd = tokio::process::Command::new(env!("CARGO_BIN_EXE_openshell-driver-podman"));
+    cmd.arg("--podman-socket")
+        .arg(&missing_socket)
+        .current_dir(tmpdir.path())
+        .kill_on_drop(true)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped());
+
+    let output = tokio::time::timeout(Duration::from_secs(30), cmd.output())
+        .await
+        .expect("driver should stop after its bounded retry window")
+        .expect("spawn openshell-driver-podman");
+
+    assert!(
+        !output.status.success(),
+        "driver should exit non-zero when Podman is unreachable"
+    );
+
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    let combined = format!("{stdout}{stderr}");
+    assert!(
+        combined.contains("connection error"),
+        "driver error should describe a connection failure:\n{combined}"
+    );
+    assert!(
+        combined.contains(missing_socket.to_str().expect("socket path is utf-8")),
+        "driver error should name the unreachable socket path {}:\n{combined}",
+        missing_socket.display()
+    );
+}

e2e/rust/Cargo.toml

@@ -93,11 +93,6 @@ name = "podman_host_gateway"
 path = "tests/podman_host_gateway.rs"
 required-features = ["e2e-podman"]
 
-[[test]]
-name = "podman_preflight"
-path = "tests/podman_preflight.rs"
-required-features = ["e2e-podman"]
-
 [[test]]
 name = "podman_corporate_proxy"
 path = "tests/podman_corporate_proxy.rs"

tests/artifacts.nix

@@ -116,10 +116,6 @@ let
     "podman_corporate_proxy"
     "podman_gateway_start"
     "podman_oci_identity"
-    # This validates the standalone driver binary's daemon-unavailable path and
-    # belongs in the Podman driver crate's integration tests. The E2E archive
-    # does not contain `openshell-driver-podman`.
-    "podman_preflight"
     "provider_auto_create"
     # The provider-refresh feature suite covers revoked Keycloak grants. This
     # binary instead covers stable workload handles across repeated rotations