mana_driver vf reconfig sets hwc failure, fast revokes vtl0 vf

Author: erfrimod

openhcl/underhill_core/src/emuplat/netvsp.rs

@@ -1079,10 +1079,14 @@ impl HclNetworkVFManagerWorker {
                             vtl0_vfid = vtl0_vfid_from_bus_control(&self.vtl0_bus_control),
                             "VTL0 VF being removed as a result of VF Reconfiguration."
                         );
-                        self.try_notify_guest_and_revoke_vtl0_vf(&Vtl0Bus::NotPresent)
-                            .await;
+                        self.remove_vtl0_vf().await;
                     }
 
+                    // Set MAC filters to None.
+                    // Force the restarted device to resend HWC commands to update filters.
+                    // Safety measure for NICs without `cap_filter_state_query` support.
+                    self.save_state.direction_to_vtl0.lock().fill(None);
+
                     // Don't 'keep alive'. VTL2 is reconfigured when in a bad state.
                     let keep_vf_alive = false;
                     self.shutdown_vtl2_device(keep_vf_alive).await;

vm/devices/net/mana_driver/src/gdma_driver.rs

@@ -1021,6 +1021,8 @@ impl<T: DeviceBacking> GdmaDriver<T> {
                     // No data is supplied for VF reconfiguration events.
                     tracing::info!("HWC VF reconfiguration event");
                     self.vf_reconfiguration_pending = true;
+                    // HWC will no longer respond after this event.
+                    self.hwc_failure = true;
                 }
                 ty => tracing::error!(ty, "unknown eq event"),
             }
@@ -1137,8 +1139,16 @@ impl<T: DeviceBacking> GdmaDriver<T> {
                     _ => "response received with delay",
                 }
             );
-            self.report_hwc_timeout(wait_failed, interrupt_loss, eqe_wait_result.elapsed as u32)
+            // Don't report the timeout once VF reconfiguration is pending,
+            // since the SoC will not respond.
+            if !self.vf_reconfiguration_pending {
+                self.report_hwc_timeout(
+                    wait_failed,
+                    interrupt_loss,
+                    eqe_wait_result.elapsed as u32,
+                )
                 .await;
+            }
             if !wait_failed && eqe_wait_result.elapsed > self.hwc_warning_time_in_ms as u128 {
                 // Increase warning threshold after each delay warning occurrence.
                 self.hwc_warning_time_in_ms += HWC_WARNING_INCREASE_IN_MS;
@@ -1162,7 +1172,11 @@ impl<T: DeviceBacking> GdmaDriver<T> {
                 ));
             }
         }
-        self.hwc_failure = false;
+        // If the wait successfully found an EQE, clear hwc_failure caused by prior timeout.
+        // Don't clear hwc_failure during VF reconfiguration (EQE 135), the device is gone.
+        if !self.vf_reconfiguration_pending {
+            self.hwc_failure = false;
+        }
         Ok(())
     }
 
@@ -1478,4 +1492,9 @@ impl<T: DeviceBacking> GdmaDriver<T> {
         )
         .await
     }
+
+    /// Returns true if the HWC has failed and no further requests will succeed.
+    pub(crate) fn hwc_failure(&self) -> bool {
+        self.hwc_failure
+    }
 }

vm/devices/net/mana_driver/src/resources.rs

@@ -63,12 +63,23 @@ impl ResourceArena {
     }
 
     pub(crate) async fn destroy<T: DeviceBacking>(mut self, gdma: &mut GdmaDriver<T>) {
+        let skip_hwc = gdma.hwc_failure();
+        if skip_hwc {
+            tracing::info!(
+                count = self.resources.len(),
+                "skipping HWC resource teardown after hardware failure"
+            );
+        }
         for resource in self.resources.drain(..).rev() {
             let r = match resource {
                 Resource::MemoryBlock(mem) => {
                     drop(ManuallyDrop::into_inner(mem));
                     Ok(())
                 }
+                // When HWC has already failed, skip sending teardown commands for HWC resources:
+                // DmaRegion, Eq, BnicQueue. HWC requests all fail: "Previous hardware failure".
+                // Device should reclaim resources on its own reset.
+                _ if skip_hwc => continue,
                 Resource::DmaRegion {
                     dev_id,
                     gdma_region,

vm/devices/net/mana_driver/src/tests.rs

@@ -231,12 +231,46 @@ async fn test_gdma_reconfig_vf(driver: DefaultDriver) {
         !gdma.get_vf_reconfiguration_pending(),
         "vf_reconfiguration_pending should be false"
     );
+    assert!(
+        !gdma.hwc_failure(),
+        "hwc_failure should be false before EQE 135"
+    );
+
+    // Get the device ID while HWC is still alive (needed for deregister later).
+    let dev_id = gdma
+        .list_devices()
+        .await
+        .unwrap()
+        .iter()
+        .copied()
+        .find(|dev_id| dev_id.ty == GdmaDevType::GDMA_DEVICE_MANA)
+        .unwrap();
 
     // Trigger the reconfig event (EQE 135).
+    // The in-flight HWC request should still complete because the CQE is
+    // delivered in the same batch as the reconfig EQE. Only future requests
+    // should fail.
     gdma.generate_reconfig_vf_event().await.unwrap();
-    gdma.process_all_eqs();
+
     assert!(
         gdma.get_vf_reconfiguration_pending(),
         "vf_reconfiguration_pending should be true after reconfig event"
     );
+    assert!(
+        gdma.hwc_failure(),
+        "hwc_failure should be true after EQE 135"
+    );
+
+    // Deregister should fail immediately because hwc_failure is set.
+    let deregister_result = gdma.deregister_device(dev_id).await;
+    let err = deregister_result.expect_err("deregister_device should fail after EQE 135");
+    let err_msg = format!("{err:#}");
+    assert!(
+        err_msg.contains("Previous hardware failure"),
+        "unexpected error: {err_msg}"
+    );
+    assert!(
+        gdma.hwc_failure(),
+        "hwc_failure should remain true after deregister_device"
+    );
 }