Make Number of mbufs a Constant Value (#79)

We have been observing performance regression from the last few releases, this PR fixes one of the issues. This PR boosts basic_monitor & pktgen benchmark performance from 7Mpps to 13.1 Mpps. This is done by changing the number of mbufs to a static constant instead of it being a function depending on MAX_NFS number as setting the number of mbufs too high leads to performance degradation.

The best performance is around 16-32 MAX_NFS number. This computes to a num of mbufs value between 27648 (16*1536 + 1536*2) and 52224 (32*1536 + 1536*2). Selected 32767 (2^15 - 1, negative one from DPDK docs)  because it is between those numbers and tends to have the same performance.

Commit log:

* Fix mbuf size to a static constant

* Adjust mbuf num to match dpdk recs

Author: koolzz

onvm/onvm_mgr/onvm_init.c

@@ -265,14 +265,11 @@ init(int argc, char *argv[]) {
  */
 static int
 init_mbuf_pools(void) {
-        const unsigned num_mbufs = (MAX_NFS * MBUFS_PER_NF) \
-                        + (ports->num_ports * MBUFS_PER_PORT);
-
         /* don't pass single-producer/single-consumer flags to mbuf create as it
          * seems faster to use a cache instead */
         printf("Creating mbuf pool '%s' [%u mbufs] ...\n",
-                        PKTMBUF_POOL_NAME, num_mbufs);
-        pktmbuf_pool = rte_mempool_create(PKTMBUF_POOL_NAME, num_mbufs,
+                        PKTMBUF_POOL_NAME, NUM_MBUFS);
+        pktmbuf_pool = rte_mempool_create(PKTMBUF_POOL_NAME, NUM_MBUFS,
                         MBUF_SIZE, MBUF_CACHE_SIZE,
                         sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init,
                         NULL, rte_pktmbuf_init, NULL, rte_socket_id(), NO_FLAGS);

onvm/onvm_mgr/onvm_init.h

@@ -83,8 +83,6 @@
 /***********************************Macros************************************/
 
 
-#define MBUFS_PER_NF 1536
-#define MBUFS_PER_PORT 1536
 #define MBUF_CACHE_SIZE 512
 #define MBUF_OVERHEAD (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
 #define RX_MBUF_DATA_SIZE 2048

onvm/onvm_nflib/onvm_common.h

@@ -50,11 +50,12 @@
 #include "onvm_msg_common.h"
 #include "onvm_config_common.h"
 
-#define ONVM_MAX_CHAIN_LENGTH 4   // the maximum chain length
-#define MAX_NFS 128               // total number of NFs allowed
-#define MAX_SERVICES 32           // total number of unique services allowed
-#define MAX_NFS_PER_SERVICE 32    // max number of NFs per service.
+#define ONVM_MAX_CHAIN_LENGTH 4 // the maximum chain length
+#define MAX_NFS 128             // total number of NFs allowed
+#define MAX_SERVICES 32         // total number of unique services allowed
+#define MAX_NFS_PER_SERVICE 32  // max number of NFs per service.
 
+#define NUM_MBUFS 32767         // total number of mbufs (2^15 - 1)
 #define NF_QUEUE_RINGSIZE 16384 //size of queue for NFs
 
 #define PACKET_READ_SIZE ((uint16_t)32)