Add Latency Measurement to Speed_Tester NF (#256)

This change adds a new, optional, flag to the Speed_Tester NF.  Now, running Speed_Tester with the `-l` flag will generate packets with an additional bit enabled and with a timestamp as data.  The NF will now check for the latency bit when run with the `-l` flag, and then it will extract the timestamp and update the average latency.  The average latency value will be printed with the NF output.

Commit log:

* Initial commit for latency

 - Calculate the latency by timestamping the packets
 - Average the latency over all packets
 - Prints the avg nanosec latency
 * This method is expensive and cuts the throguhput speed *

* working latency in packet

* Clean up latency code

 - Add a flag to enable latency measuring
 - Remove extra testing code

* Readme updates

* Quick README fix

Author: koolzz

examples/speed_tester/README.md

@@ -4,16 +4,16 @@ This program lets you test how fast the manager's TX threads can move packets. I
 
 Optional Libraries
 --
-libpcap library is required for pcap replay functionality.  
-To enable pcap replay set `ENABLE_PCAP=1` in the Makefile.  
+libpcap library is required for pcap replay functionality.
+To enable pcap replay set `ENABLE_PCAP=1` in the Makefile.
 If libpcap is not installed run:
 ```
 sudo apt-get install libpcap-dev
 ```
 Example .pcap files
 --
-`pcap/` directory has a few example pcap files.  
-  - pktgen_big.pcap, pktgen_large.pcap, pktgen_test1.pcap, pktgen_traffic_sample.pcap are taken from the [Pktgen](../../tools/Pktgen/README.md) example pcap files.  
+`pcap/` directory has a few example pcap files.
+  - pktgen_big.pcap, pktgen_large.pcap, pktgen_test1.pcap, pktgen_traffic_sample.pcap are taken from the [Pktgen](../../tools/Pktgen/README.md) example pcap files.
   - 64B_download.pcap, 8K_download.pcap are sample web traffic pcap files.
 
 Compilation and Execution
@@ -22,11 +22,11 @@ Compilation and Execution
 cd examples
 make
 cd speed_tester
-./go.sh CORELIST SERVICE_ID DST_ID [PRINT_DELAY] [ADVANCED_RINGS] [PACKET_SIZE] [DEST_MAC] [PCAP_FILE]
+./go.sh CORELIST SERVICE_ID DST_ID [PRINT_DELAY] [ADVANCED_RINGS] [PACKET_SIZE] [DEST_MAC] [PCAP_FILE] [MEASURE_LATENCY]
 
 OR
 
-sudo ./build/speed_tester -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-a] [-p PRINT_DELAY] [-s PACKET_SIZE] [-m DEST_MAC] [-o PCAP_FILENAME]
+sudo ./build/speed_tester -l CORELIST -n 3 --proc-type=secondary -- -r SERVICE_ID -- -d DST [-a] [-p PRINT_DELAY] [-s PACKET_SIZE] [-m DEST_MAC] [-o PCAP_FILENAME] [-l]
 ```
 
 App Specific Arguments
@@ -35,5 +35,6 @@ App Specific Arguments
   - `-a`: Use advanced rings interface instead of default `packet_handler`
   - `-p PRINT_DELAY`: Number of packets between each print, e.g. `-p 1` prints every packets.
   - `-s PACKET_SIZE`: Size of packet, e.g. `-s 32` allocates 32 bytes for the data segment of `rte_mbuf`.
-  - `-m DEST_MAC`: User specified destination MAC address, e.g. `-m aa:bb:cc:dd:ee:ff` sets the destination address within the ethernet header that is located at the start of the packet data.  
+  - `-m DEST_MAC`: User specified destination MAC address, e.g. `-m aa:bb:cc:dd:ee:ff` sets the destination address within the ethernet header that is located at the start of the packet data.
   - `-o PCAP_FILENAME` : The filename of the pcap file to replay
+  - `-l LATENCY` : Enable latency measurement. This should only be enabled on one Speed Tester NF. Packets must be routed back to the same speed tester NF.

examples/speed_tester/go.sh

@@ -2,12 +2,13 @@
 
 function usage {
         echo "$0 CPU-LIST SERVICE-ID DST [-p PRINT] [-n NF-ID] [-a] [-s PACKET-SIZE] [-m DEST-MAC]
-        [-c PACKET-NUMBER]"
+        [-c PACKET-NUMBER] [-l]"
         echo "$0 3,7,9 1 2 --> cores 3,7, and 9, with Service ID 1, and forwards to service ID 2"
         echo "$0 3,7,9 1 2 1000 --> cores 3,7, and 9, with Service ID 1, forwards to service ID 2,  and Print Rate of 1000"
         echo "$0 3,7,9 1 2 -s 32 --> cores 3,7, and 9, with Service ID 1, forwards to service ID 2, and packet size of 32"
         echo "$0 3,7,9 1 2 -s 32 -m aa:bb:cc:dd:ee:ff --> cores 3,7, and 9, with Service ID 1, forwards to service ID 2, packet size of 32, and destination MAC address of aa:bb:cc:dd:ee:ff"
         echo "$0 5 1 2 -o sample_trafic.pcap --> core 5, with Service ID 1, and forwards to service ID 2, replays sample_trafic.pcap packets (make sure to enable pcap functionality, check README for instructions)"
+        echo "$0 5 1 2 -l --> core 5, with Service ID 1, forwards to service ID 2, measures latency"
         echo "Pass '-a' to signal the NF to use advanced ring manipulation"
         exit 1
 }
@@ -25,7 +26,7 @@ then
     usage
 fi
 
-while getopts ":p:n:as:m:o:c:" opt; do
+while getopts ":p:n:as:m:o:c:l" opt; do
   case $opt in
     p) print="-p $OPTARG";;
     n) instance="-n $OPTARG";;
@@ -34,9 +35,10 @@ while getopts ":p:n:as:m:o:c:" opt; do
     m) dest_mac="-m $OPTARG";;
     o) pcap_filename="-o $OPTARG";;
     c) pkt_num="-c $OPTARG";;
+    l) latency="-l";;
     \?) echo "Unknown option -$OPTARG" && usage
     ;;
   esac
 done
 
-exec sudo $SCRIPTPATH/build/app/speed_tester -l $cpu -n 3 --proc-type=secondary -- -r $service $instance -- -d $dst $print $rings $size $dest_mac $pcap_filename $pkt_num
+exec sudo $SCRIPTPATH/build/app/speed_tester -l $cpu -n 3 --proc-type=secondary -- -r $service $instance -- -d $dst $print $rings $size $dest_mac $pcap_filename $pkt_num $latency

examples/speed_tester/speed_tester.c

@@ -72,8 +72,10 @@
 #define PKTMBUF_POOL_NAME "MProc_pktmbuf_pool"
 #define PKT_READ_SIZE  ((uint16_t)32)
 #define SPEED_TESTER_BIT 7
+#define LATENCY_BIT 6
 #define LOCAL_EXPERIMENTAL_ETHER 0x88B5
 #define DEFAULT_PKT_NUM 128
+#define DEFAULT_LAT_PKT_NUM 16
 #define MAX_PKT_NUM NF_QUEUE_RINGSIZE
 
 /* Struct that contains information about this NF */
@@ -94,6 +96,11 @@ static uint8_t d_addr_bytes[ETHER_ADDR_LEN];
 /* Default number of packets: 128; user can modify it by -c <packet_number> in command line */
 static uint32_t packet_number = 0;
 
+/* Variables for measuring packet latency */
+static uint8_t measure_latency = 0;
+static uint32_t latency_packets = 0;
+static uint64_t total_latency = 0;
+
 /*
  * Variables needed to replay a pcap file
  */
@@ -106,7 +113,7 @@ static void
 usage(const char *progname) {
         printf("Usage: %s [EAL args] -- [NF_LIB args] -- -d <destination> [-p <print_delay>] "
                         "[-a] [-s <packet_length>] [-m <dest_mac_address>] [-o <pcap_filename>] "
-                        "[-c <packet_number>]\n\n", progname);
+                        "[-c <packet_number>] [-l]\n\n", progname);
 }
 
 /*
@@ -117,7 +124,7 @@ parse_app_args(int argc, char *argv[], const char *progname) {
         int c, i, count, dst_flag = 0;
         int values[ETHER_ADDR_LEN];
 
-        while ((c = getopt(argc, argv, "d:p:as:m:o:c:")) != -1) {
+        while ((c = getopt(argc, argv, "d:p:as:m:o:c:l")) != -1) {
                 switch (c) {
                 case 'a':
                         use_direct_rings = 1;
@@ -166,6 +173,9 @@ parse_app_args(int argc, char *argv[], const char *progname) {
                                 return -1;
                         }
                         break;
+                case 'l':
+                        measure_latency = 1;
+                        break;
                 case '?':
                         usage(progname);
                         if (optopt == 'd')
@@ -221,8 +231,14 @@ do_stats_display(struct rte_mbuf* pkt) {
         printf("Total packets: %9"PRIu64" \n", cur_pkts);
         printf("TX pkts per second: %9"PRIu64" \n", (cur_pkts - last_pkts)
                 * rte_get_timer_hz() / (cur_cycles - last_cycles));
+        if (measure_latency && latency_packets > 0)
+                printf("Avg latency nanoseconds: %6"PRIu64" \n", total_latency/(latency_packets)
+                        * 1000000000 / rte_get_timer_hz());
         printf("Initial packets created: %u\n", packet_number);
 
+        total_latency = 0;
+        latency_packets = 0;
+
         last_pkts = cur_pkts;
         last_cycles = cur_cycles;
 
@@ -241,6 +257,15 @@ packet_handler(struct rte_mbuf* pkt, struct onvm_pkt_meta* meta) {
                 /* one of our fake pkts to forward */
                 meta->destination = destination;
                 meta->action = ONVM_NF_ACTION_TONF;
+                if (measure_latency && ONVM_CHECK_BIT(meta->flags, LATENCY_BIT)) {
+                        uint64_t curtime = rte_get_tsc_cycles();
+                        uint64_t* oldtime = (uint64_t *)(rte_pktmbuf_mtod(pkt, uint8_t *) + packet_size);
+                        if (*oldtime != 0) {
+                                total_latency += curtime - *oldtime;
+                                latency_packets++;
+                        }
+                        *oldtime = curtime;
+                }
         } else {
                 /* Drop real incoming packets */
                 meta->action = ONVM_NF_ACTION_DROP;
@@ -391,8 +416,8 @@ int main(int argc, char *argv[]) {
                         ehdr = (struct ether_hdr *) rte_pktmbuf_append(pkt, packet_size);
 
                         /*using manager mac addr for source
-                        *using input string for dest addr 
-                        */ 
+                        *using input string for dest addr
+                        */
                         rte_eth_macaddr_get(0, &ehdr->s_addr);
                         for (j = 0; j < ETHER_ADDR_LEN; ++j) {
                                 ehdr->d_addr.addr_bytes[j] = d_addr_bytes[j];
@@ -405,6 +430,13 @@ int main(int argc, char *argv[]) {
                         pmeta->flags = ONVM_SET_BIT(0, SPEED_TESTER_BIT);
                         pkt->hash.rss = i;
                         pkt->port = 0;
+
+                        if (measure_latency && (packet_number < DEFAULT_LAT_PKT_NUM || i % (packet_number/DEFAULT_LAT_PKT_NUM) == 0)) {
+                                pmeta->flags |= ONVM_SET_BIT(0, LATENCY_BIT);
+                                uint64_t* ts = (uint64_t *) rte_pktmbuf_append(pkt, sizeof(uint64_t));
+                                *ts = 0;
+                        }
+
                         onvm_nflib_return_pkt(pkt);
                 }
 #ifdef LIBPCAP