[multicast] M2P forwarding, OPTE port subscription, and sled-agent propagation

This completes the multicast data path by adding per-sled M2P (multicast-to-
physical) mapping, forwarding entry management, and OPTE port subscription
for multicast group members.

## Sled-agent + API update(s)

  - Add multicast endpoints at API v33 (MCAST_M2P_FORWARDING) for M2P,
    forwarding, and per-VMM subscribe/unsubscribe
  - Version v7 join/leave endpoints to v7..v33 with shim conversion
  - Move multicast types from omicron-common to sled-agent-types-versions
    v33 module (mcast_m2p_forwarding) with re-exports through sled-agent-types
  - OPTE port_manager gains set/clear operations for M2P and forwarding
  - Port subscription cleanup on PortTicket release
  - Consolidate per-port mutable state (eip_gateways, mcast) into PortState
  - Seed eip_gateways from global map on port creation to prevent stale
    gateway state on newly created ports
  - Lock ordering documented for ports, routes, eip_gateways

## Nexus

  - New `sled.rs` (MulticastSledClient) encapsulating all sled-agent
    multicast interactions: subscribe/unsubscribe, M2P/forwarding
    propagation and teardown
  - Groups RPW propagates M2P and forwarding entries to all member sleds
    after DPD configuration, with convergent retry on failure
  - Members RPW uses MemberReconcileCtx to thread shared reconciliation
    state. Handles subscribe on join, unsubscribe on leave, and
    re-subscribe on migration
  - `subscribe_vmm` gracefully handles missing propolis (mirrors unsubscribe)
  - `lookup_propolis_id` returns Ok(None) for missing instance
  - `lookup_and_update_member_sled_id` surfaces DB errors instead of
    swallowing them
  - Order-independent forwarding comparison to avoid spurious dataplane churn;
    always create forwarding entries for active groups even with empty next-hops
  - Dataplane client updated for bifurcated replication groups

## illumos-utils

  - Remove CIDR allow rules for multicast (handled by OPTE gateway layer)
  - Reject Reserved replication mode in `list_mcast_fwd` with
    InvalidMcastForwardingState error
  - Consolidate error variants into InvalidMcastUnderlay

## Tests

  - Integration tests for M2P/forwarding/subscribe lifecycle
  - Instance migration multicast re-convergence

Author: zeeshanlakhani

.github/buildomat/jobs/deploy.sh

@@ -182,6 +182,7 @@ ptime -m tar xvzf /input/package/work/package.tar.gz
 source .github/buildomat/ci-env.sh
 
 # Source the OPTE override (if any) from the canonical location and apply it.
+#
 # When set, install the override p5p from buildomat instead of using the
 # version baked into the ramdisk image. The version must be pinned explicitly
 # because IPS version ordering does not match semver.

.github/workflows/check-opte-ver.yml

@@ -1,11 +1,7 @@
 name: check-opte-ver
 on:
   pull_request:
-    paths:
-      - '.github/workflows/check-opte-ver.yml'
-      - 'Cargo.toml'
-      - 'tools/opte_version'
-      - 'tools/opte_version_override'
+    branches: [main]
 jobs:
   check-opte-ver:
     runs-on: ubuntu-22.04
@@ -19,7 +15,12 @@ jobs:
         run: cargo install toml-cli@0.2.3
       - name: Check OPTE version and rev match
         run: ./tools/ci_check_opte_ver.sh
+
+  # Runs on every PR regardless of paths changed, since the override
+  # file could have been set in an earlier commit and slip through on
+  # an unrelated PR otherwise.
   check-opte-override:
+    if: github.base_ref == 'main'
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

dev-tools/releng/src/main.rs

@@ -629,7 +629,7 @@ async fn main() -> Result<()> {
             .arg("-o") // output directory for image
             .arg(args.output_dir.join(format!("os-{}", target)))
             .arg("-F") // pass extra image builder features
-            .arg(format!("optever={}", opte_version))
+            .arg(format!("optever={opte_version}"))
             .arg("-P") // include all files from extra proto area
             .arg(proto_dir.join("root"))
             .arg("-N") // image name
@@ -690,7 +690,7 @@ async fn main() -> Result<()> {
         // When OPTE_COMMIT is set, download the override p5p from buildomat
         // and add it as a package source for the image build.
         if let Some(ov) = &opte_override {
-            let p5p_path = tempdir.path().join(format!("opte-{}.p5p", target));
+            let p5p_path = tempdir.path().join(format!("opte-{target}.p5p"));
             let commit = ov.commit.clone();
             let dest = p5p_path.clone();
             let cl = client.clone();
@@ -702,17 +702,17 @@ async fn main() -> Result<()> {
 
             image_cmd = image_cmd
                 .arg("-p")
-                .arg(format!("helios-dev=file://{}", p5p_path,));
+                .arg(format!("helios-dev=file://{p5p_path}"));
+        }
 
-            jobs.push_command(format!("{target}-image"), image_cmd)
-                .after("helios-setup")
-                .after("helios-incorp")
-                .after(format!("{target}-opte-p5p"));
-        } else {
-            jobs.push_command(format!("{target}-image"), image_cmd)
-                .after("helios-setup")
-                .after("helios-incorp")
-                .after(format!("{target}-proto"));
+        let image_job = jobs
+            .push_command(format!("{target}-image"), image_cmd)
+            .after("helios-setup")
+            .after("helios-incorp")
+            .after(format!("{target}-proto"));
+
+        if opte_override.is_some() {
+            image_job.after(format!("{target}-opte-p5p"));
         }
     }
     // Build the recovery target after we build the host target. Only one

illumos-utils/src/opte/illumos.rs

@@ -13,6 +13,7 @@ use opte_ioctl::OpteHdl;
 use slog::Logger;
 use slog::info;
 use std::net::IpAddr;
+use std::net::Ipv6Addr;
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
@@ -70,6 +71,11 @@ pub enum Error {
         "Tried to update attached subnets on non-existent port ({0}, {1:?})"
     )]
     AttachedSubnetUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind),
+
+    #[error(
+        "address {0} is not within the underlay multicast subnet (ff04::/64)"
+    )]
+    InvalidMcastUnderlay(Ipv6Addr),
 }
 
 /// Delete all xde devices on the system.

illumos-utils/src/opte/mod.rs

@@ -33,10 +33,10 @@ use oxnet::IpNet;
 use oxnet::Ipv4Net;
 use oxnet::Ipv6Net;
 pub use port::Port;
-pub use port_manager::MulticastGroupCfg;
 pub use port_manager::PortCreateParams;
 pub use port_manager::PortManager;
 pub use port_manager::PortTicket;
+pub use sled_agent_types::multicast::MulticastGroupCfg;
 use std::net::IpAddr;
 use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;

illumos-utils/src/opte/non_illumos.rs

@@ -2,26 +2,38 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//! Mock / dummy versions of the OPTE module, for non-illumos platforms
+//! Mock / dummy versions of the OPTE module, for non-illumos platforms.
+//!
+//! Most methods are either `unimplemented!()` or silent no-ops.
+//! Multicast subscribe/unsubscribe is an exception, as it maintains real
+//! in-memory state because port manager tests assert on subscription contents.
 
 use crate::addrobj::AddrObject;
 use omicron_common::api::internal::shared::NetworkInterfaceKind;
 use oxide_vpc::api::AddRouterEntryReq;
+use oxide_vpc::api::ClearMcast2PhysReq;
+use oxide_vpc::api::ClearMcastForwardingReq;
 use oxide_vpc::api::ClearVirt2PhysReq;
 use oxide_vpc::api::DelRouterEntryReq;
 use oxide_vpc::api::DetachSubnetResp;
-use oxide_vpc::api::Direction;
+use oxide_vpc::api::DumpMcast2PhysResp;
+use oxide_vpc::api::DumpMcastForwardingResp;
 use oxide_vpc::api::DumpVirt2PhysResp;
 use oxide_vpc::api::IpCfg;
 use oxide_vpc::api::IpCidr;
 use oxide_vpc::api::ListPortsResp;
+use oxide_vpc::api::McastSubscribeReq;
+use oxide_vpc::api::McastUnsubscribeReq;
 use oxide_vpc::api::NoResp;
 use oxide_vpc::api::PortInfo;
 use oxide_vpc::api::RouterClass;
 use oxide_vpc::api::RouterTarget;
 use oxide_vpc::api::SetExternalIpsReq;
 use oxide_vpc::api::SetFwRulesReq;
+use oxide_vpc::api::SetMcast2PhysReq;
+use oxide_vpc::api::SetMcastForwardingReq;
 use oxide_vpc::api::SetVirt2PhysReq;
+use oxide_vpc::api::SourceFilter;
 use oxide_vpc::api::VpcCfg;
 use slog::Logger;
 use std::collections::HashMap;
@@ -76,6 +88,11 @@ pub enum Error {
         "Tried to update attached subnets on non-existent port ({0}, {1:?})"
     )]
     AttachedSubnetUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind),
+
+    #[error(
+        "address {0} is not within the underlay multicast subnet (ff04::/64)"
+    )]
+    InvalidMcastUnderlay(std::net::Ipv6Addr),
 }
 
 pub fn initialize_xde_driver(
@@ -172,6 +189,8 @@ pub(crate) struct PortData {
     pub port: PortInfo,
     /// The routes for this port. This simulates the router layer.
     pub routes: Vec<RouteInfo>,
+    /// Multicast group subscriptions: group IP → source filter.
+    pub mcast_subscriptions: HashMap<IpAddr, SourceFilter>,
 }
 
 #[derive(Debug)]
@@ -237,7 +256,11 @@ impl Handle {
                 return Err(OpteError::DuplicatePort(entry.key().to_string()));
             }
             Entry::Vacant(entry) => {
-                entry.insert(PortData { port, routes: Vec::new() });
+                entry.insert(PortData {
+                    port,
+                    routes: Vec::new(),
+                    mcast_subscriptions: HashMap::new(),
+                });
             }
         }
         Ok(NO_RESPONSE)
@@ -270,14 +293,46 @@ impl Handle {
         Ok(NO_RESPONSE)
     }
 
-    /// Allow traffic to / from a CIDR block on a port.
-    pub fn allow_cidr(
+    /// Subscribe a port to a multicast group.
+    pub fn mcast_subscribe(
         &self,
-        _: &str,
-        _: IpCidr,
-        _: Direction,
+        req: &McastSubscribeReq,
     ) -> Result<NoResp, OpteError> {
-        unimplemented!("Not yet used in tests")
+        let mut inner = opte_state().lock().unwrap();
+        let Some(port_data) = inner.ports.get_mut(&req.port_name) else {
+            return Err(OpteError::NoPort(req.port_name.clone()));
+        };
+        let group_ip: IpAddr = match req.group {
+            oxide_vpc::api::IpAddr::Ip4(v4) => {
+                std::net::Ipv4Addr::from(v4).into()
+            }
+            oxide_vpc::api::IpAddr::Ip6(v6) => {
+                std::net::Ipv6Addr::from(v6).into()
+            }
+        };
+        port_data.mcast_subscriptions.insert(group_ip, req.filter.clone());
+        Ok(NO_RESPONSE)
+    }
+
+    /// Unsubscribe a port from a multicast group.
+    pub fn mcast_unsubscribe(
+        &self,
+        req: &McastUnsubscribeReq,
+    ) -> Result<NoResp, OpteError> {
+        let mut inner = opte_state().lock().unwrap();
+        let Some(port_data) = inner.ports.get_mut(&req.port_name) else {
+            return Err(OpteError::NoPort(req.port_name.clone()));
+        };
+        let group_ip: IpAddr = match req.group {
+            oxide_vpc::api::IpAddr::Ip4(v4) => {
+                std::net::Ipv4Addr::from(v4).into()
+            }
+            oxide_vpc::api::IpAddr::Ip6(v6) => {
+                std::net::Ipv6Addr::from(v6).into()
+            }
+        };
+        port_data.mcast_subscriptions.remove(&group_ip);
+        Ok(NO_RESPONSE)
     }
 
     /// Delete a router entry from a port.
@@ -323,6 +378,45 @@ impl Handle {
         unimplemented!("Not yet used in tests")
     }
 
+    /// Set a multicast-to-physical mapping.
+    pub fn set_m2p(&self, _: &SetMcast2PhysReq) -> Result<NoResp, OpteError> {
+        Ok(NO_RESPONSE)
+    }
+
+    /// Clear a multicast-to-physical mapping.
+    pub fn clear_m2p(
+        &self,
+        _: &ClearMcast2PhysReq,
+    ) -> Result<NoResp, OpteError> {
+        Ok(NO_RESPONSE)
+    }
+
+    /// Set multicast forwarding for a port.
+    pub fn set_mcast_fwd(
+        &self,
+        _: &SetMcastForwardingReq,
+    ) -> Result<NoResp, OpteError> {
+        Ok(NO_RESPONSE)
+    }
+
+    /// Clear multicast forwarding for a port.
+    pub fn clear_mcast_fwd(
+        &self,
+        _: &ClearMcastForwardingReq,
+    ) -> Result<NoResp, OpteError> {
+        Ok(NO_RESPONSE)
+    }
+
+    /// Dump all multicast-to-physical mappings.
+    pub fn dump_m2p(&self) -> Result<DumpMcast2PhysResp, OpteError> {
+        Ok(DumpMcast2PhysResp { ip4: Vec::new(), ip6: Vec::new() })
+    }
+
+    /// Dump all multicast forwarding entries.
+    pub fn dump_mcast_fwd(&self) -> Result<DumpMcastForwardingResp, OpteError> {
+        Ok(DumpMcastForwardingResp { entries: Vec::new() })
+    }
+
     /// List ports on the current system.
     #[allow(dead_code)]
     pub(crate) fn list_ports(&self) -> Result<ListPortsResp, OpteError> {