Keep adaptor's rings either in normal or netmap mode

After this commit, when an adapter is put in Netmap mode, all of its
rings that are not requested to be in Netmap mode are kept in normal
mode, which is they are still attached to the host stack.

Moreover, if a packet is going to hit a TX ring which is in Netmap mode,
the packets will be copied to the RX host ring.

Author: jibi

sys/dev/netmap/netmap.c

@@ -815,6 +815,8 @@ netmap_krings_create(struct netmap_adapter *na, u_int tailroom)
 			kring->ring_id = i;
 			kring->tx = t;
 			kring->nkr_num_slots = ndesc;
+			kring->nr_mode = NKR_NETMAP_OFF;
+			kring->nr_pending_mode = NKR_NETMAP_OFF;
 			if (i < nma_get_nrings(na, t)) {
 				kring->nm_sync = (t == NR_TX ? na->nm_txsync : na->nm_rxsync);
 			} else {
@@ -909,11 +911,20 @@ static void
 netmap_do_unregif(struct netmap_priv_d *priv)
 {
 	struct netmap_adapter *na = priv->np_na;
+	enum txrx t;
+	int i;
 
 	NMG_LOCK_ASSERT();
 	na->active_fds--;
 	/* release exclusive use if it was requested on regif */
 	netmap_rel_exclusive(priv);
+
+	for_rx_tx(t) {
+		for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
+			kring_rel_netmap_mode(priv, t, i);
+		}
+	}
+
 	if (na->active_fds <= 0) {	/* last instance */
 
 		if (netmap_verbose)
@@ -948,6 +959,10 @@ netmap_do_unregif(struct netmap_priv_d *priv)
 		/* delete rings and buffers */
 		netmap_mem_rings_delete(na);
 		na->nm_krings_delete(na);
+	} else {
+		if (kring_pending(priv)) {
+			na->nm_register(na, 1);
+		}
 	}
 	/* possibily decrement counter of tx_si/rx_si users */
 	netmap_unset_ringid(priv);
@@ -1971,6 +1986,8 @@ netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
 {
 	struct netmap_if *nifp = NULL;
 	int error;
+	enum txrx t;
+	int i;
 
 	NMG_LOCK_ASSERT();
 	/* ring configuration may have changed, fetch from the card */
@@ -2011,6 +2028,12 @@ netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
 	if (error)
 		goto err_del_rings;
 
+	for_rx_tx(t) {
+		for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
+			kring_get_netmap_mode(priv, t, i);
+		}
+	}
+
 	/* in all cases, create a new netmap if */
 	nifp = netmap_mem_if_new(na);
 	if (nifp == NULL) {
@@ -2032,6 +2055,10 @@ netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
 		error = na->nm_register(na, 1); /* mode on */
 		if (error) 
 			goto err_del_if;
+	} else {
+		if (kring_pending(priv)) {
+			na->nm_register(na, 1);
+		}
 	}
 
 	/*
@@ -2050,6 +2077,12 @@ netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
 	netmap_mem_if_delete(na, nifp);
 err_rel_excl:
 	netmap_rel_exclusive(priv);
+
+	for_rx_tx(t) {
+		for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
+			kring_rel_netmap_mode(priv, t, i);
+		}
+	}
 err_del_rings:
 	if (na->active_fds == 0)
 		netmap_mem_rings_delete(na);
@@ -2940,11 +2973,12 @@ int
 netmap_transmit(struct ifnet *ifp, struct mbuf *m)
 {
 	struct netmap_adapter *na = NA(ifp);
-	struct netmap_kring *kring;
+	struct netmap_kring *kring, *tx_kring;
 	u_int len = MBUF_LEN(m);
 	u_int error = ENOBUFS;
 	struct mbq *q;
 	int space;
+	int txr;
 
 	kring = &na->rx_rings[na->num_rx_rings];
 	// XXX [Linux] we do not need this lock
@@ -2957,6 +2991,23 @@ netmap_transmit(struct ifnet *ifp, struct mbuf *m)
 		goto done;
 	}
 
+#if defined (__FreeBSD__)
+	txr = m->m_pkthdr.flowid;
+	tx_kring = &NMR(na, NR_TX)[txr];
+
+	if (tx_kring->nr_mode == NKR_NETMAP_OFF) {
+		return na->if_transmit(ifp, m);
+	}
+
+#elif defined (linux)
+	txr = m->queue_mapping;
+	tx_kring = &NMR(na, NR_TX)[txr];
+
+	if (tx_kring->nr_mode == NKR_NETMAP_OFF) {
+		return ((struct net_device_ops *) na->if_transmit)->ndo_start_xmit(m, ifp);
+	}
+#endif
+
 	q = &kring->rx_queue;
 
 	// XXX reconsider long packets if we handle fragments
@@ -3032,13 +3083,26 @@ netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n,
 	if (tx == NR_TX) {
 		if (n >= na->num_tx_rings)
 			return NULL;
+
 		kring = na->tx_rings + n;
+
+		if (kring->nr_pending_mode == NKR_NETMAP_OFF) {
+			kring->nr_mode = NKR_NETMAP_OFF;
+			return NULL;
+		}
+
 		// XXX check whether we should use hwcur or rcur
 		new_hwofs = kring->nr_hwcur - new_cur;
 	} else {
 		if (n >= na->num_rx_rings)
 			return NULL;
 		kring = na->rx_rings + n;
+
+		if (kring->nr_pending_mode == NKR_NETMAP_OFF) {
+			kring->nr_mode = NKR_NETMAP_OFF;
+			return NULL;
+		}
+
 		new_hwofs = kring->nr_hwtail - new_cur;
 	}
 	lim = kring->nkr_num_slots - 1;
@@ -3075,6 +3139,7 @@ netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n,
 	 * We do the wakeup here, but the ring is not yet reconfigured.
 	 * However, we are under lock so there are no races.
 	 */
+	kring->nr_mode = NKR_NETMAP_ON;
 	kring->nm_notify(kring, 0);
 	return kring->ring->slot;
 }
@@ -3111,10 +3176,15 @@ netmap_common_irq(struct netmap_adapter *na, u_int q, u_int *work_done)
 
 	kring = NMR(na, t) + q;
 
+	if (kring->nr_mode == NKR_NETMAP_OFF) {
+		return 0;
+	}
+
 	if (t == NR_RX) {
 		kring->nr_kflags |= NKR_PENDINTR;	// XXX atomic ?
 		*work_done = 1; /* do not fire napi again */
 	}
+
 	kring->nm_notify(kring, 0);
 	return 1;
 }

sys/dev/netmap/netmap_kern.h

@@ -378,6 +378,12 @@ struct netmap_kring {
 #define NKR_FORWARD	0x4		/* (host ring only) there are
 					   packets to forward
 					 */
+
+	uint32_t	nr_mode;
+	uint32_t	nr_pending_mode;
+#define NKR_NETMAP_OFF	0x0
+#define NKR_NETMAP_ON	0x1
+
 	uint32_t	nkr_num_slots;
 
 	/*
@@ -1153,6 +1159,10 @@ nm_set_native_flags(struct netmap_adapter *na)
 {
 	struct ifnet *ifp = na->ifp;
 
+	if (na->na_refcount > 2) {
+		return;
+	}
+
 	na->na_flags |= NAF_NETMAP_ON;
 #ifdef IFCAP_NETMAP /* or FreeBSD ? */
 	ifp->if_capenable |= IFCAP_NETMAP;
@@ -1708,6 +1718,57 @@ struct netmap_priv_d {
 struct netmap_priv_d *netmap_priv_new(void);
 void netmap_priv_delete(struct netmap_priv_d *);
 
+static inline void kring_get_netmap_mode(struct netmap_priv_d *np, enum txrx ring_txrx, int ring_nr)
+{
+	struct netmap_adapter *na = np->np_na;
+	struct netmap_kring *kring = &NMR(na, ring_txrx)[ring_nr];
+
+	kring->nr_pending_mode = NKR_NETMAP_ON;
+	if (kring->ring_id == nma_get_nrings(na, ring_txrx)) {
+		/*
+		 * If this is a sw ring, just set the mode on (no need to
+		 * wait for netmap_reset())
+		 */
+		kring->nr_mode = NKR_NETMAP_ON;
+	}
+}
+
+static inline void kring_rel_netmap_mode(struct netmap_priv_d *np, enum txrx ring_txrx, int ring_nr)
+{
+	struct netmap_adapter *na = np->np_na;
+	struct netmap_kring *kring = &NMR(na, ring_txrx)[ring_nr];
+
+	/*
+	 * try to release the ring (i.e. put it in normal mode).
+	 * The ring can actually be released only if there are no users
+	 * using it. (users counter is managed by netmap_get_exclusive and
+	 * netmap_rel_exclusive)
+	 */
+	if (kring->users == 0) {
+		kring->nr_pending_mode = NKR_NETMAP_OFF;
+		if (kring->ring_id == nma_get_nrings(na, ring_txrx)) {
+			kring->nr_mode = NKR_NETMAP_OFF;
+		}
+	}
+}
+
+static inline int kring_pending(struct netmap_priv_d *np)
+{
+	struct netmap_adapter *na = np->np_na;
+	enum txrx t;
+	int i;
+
+	for_rx_tx(t) {
+		for (i = np->np_qfirst[t]; i < np->np_qlast[t]; i++) {
+			struct netmap_kring *kring = &NMR(na, t)[i];
+			if (kring->nr_mode != kring->nr_pending_mode) {
+				return 1;
+			}
+		}
+	}
+	return 0;
+}
+
 #ifdef WITH_MONITOR
 
 struct netmap_monitor_adapter {