Merge pull request #75 from CESNET/main

update staging

Author: jaroslavpesek

modules/deduplicator/src/deduplicator.hpp

@@ -46,7 +46,7 @@ class Deduplicator {
 		std::function<bool(const Timestamp&, const Timestamp&)>,
 		std::function<Timestamp(const Timestamp&, uint64_t)>>;
 
-	static inline const int DEFAULT_HASHMAP_TIMEOUT = 5000; ///< Default timeout - 5s
+	static inline const uint64_t DEFAULT_HASHMAP_TIMEOUT = 5000; ///< Default timeout - 5s
 
 	/**
 	 * @brief Deduplicator constructor

modules/deduplicator/src/main.cpp

@@ -112,16 +112,16 @@ int main(int argc, char** argv)
 	try {
 		program.add_argument("-s", "--size")
 			.required()
-			.help("Size of the hash map. Default value is 2^20 for 1'048'576 records.")
+			.help("Exponent N for the hash map size (2^N entries). Default: 20 (~1 048 576).")
 			.default_value(Deduplicator::Deduplicator::DeduplicatorHashMap::
 							   TimeoutHashMapParameters::DEFAULT_HASHMAP_EXPONENT)
-			.scan<'i', int>();
+			.scan<'u', uint32_t>();
 		program.add_argument("-t", "--timeout")
 			.required()
 			.help(
-				"Count of millisecond to consider flows as duplicates. Default value is 5000(5s).")
+				"Count of millisecond to consider flows as duplicates. Default value is 5000 (5s).")
 			.default_value(Deduplicator::Deduplicator::DEFAULT_HASHMAP_TIMEOUT)
-			.scan<'i', int>();
+			.scan<'u', uint64_t>();
 		program.add_argument("-m", "--appfs-mountpoint")
 			.required()
 			.help("path where the appFs directory will be mounted")
@@ -161,7 +161,7 @@ int main(int argc, char** argv)
 			std::cerr << "Table size must be at least 8.\n";
 			return EXIT_FAILURE;
 		}
-		const auto timeout = program.get<uint32_t>("--timeout");
+		const auto timeout = program.get<uint64_t>("--timeout");
 		if (timeout <= 0) {
 			std::cerr << "Timeout must be higher than zero.\n";
 			return EXIT_FAILURE;
@@ -185,6 +185,7 @@ int main(int argc, char** argv)
 		biInterface.setRequieredFormat(
 			"uint16 SRC_PORT, uint16 DST_PORT, ipaddr DST_IP,ipaddr SRC_IP, uint64 LINK_BIT_FIELD, "
 			"uint8 PROTOCOL, time TIME_LAST");
+		deduplicator.updateUnirecIds();
 		processUnirecRecords(biInterface, deduplicator);
 
 	} catch (std::exception& ex) {

modules/deduplicator/src/timeoutHashMap.hpp

@@ -212,7 +212,7 @@ class TimeoutHashMap {
 		/**
 		 * @brief Default size for the hash map.
 		 */
-		static inline const int DEFAULT_HASHMAP_EXPONENT = 20; // 1'048'576 records
+		static inline const uint32_t DEFAULT_HASHMAP_EXPONENT = 20; // 1'048'576 records
 
 		uint32_t bucketCountExponent; ///< Total amount of records in table
 		uint64_t timeout; ///< Time interval to consider flow unique

modules/deduplicator/tests/test.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+function exit_with_error {
+  pkill logger
+  pkill logreplay
+  pkill deduplicator
+  exit 1
+}
+
+function process_started {
+  pid=$1
+  if ! ps -p $pid > /dev/null
+    then
+      echo "Failed to start process"
+      exit_with_error
+  fi
+}
+
+data_path="$(dirname "$0")/testsData/"
+deduplicator=$1
+
+set -e
+trap 'echo "Command \"$BASH_COMMAND\" failed!"; exit_with_error' ERR
+for input_file in $data_path/inputs/*; do
+  index=$(echo "$input_file" | grep -o '[0-9]\+')
+  echo "Running test $index"
+
+  res_file="/tmp/res"
+  logger -i "u:deduplicator" -w $res_file &
+  logger_pid=$!
+  sleep 0.1
+
+  process_started $logger_pid
+
+  $deduplicator \
+    -i "u:din,u:deduplicator" &
+
+  detector_pid=$!
+  sleep 0.1
+  process_started $detector_pid
+
+  logreplay -i "u:din" -f "$data_path/inputs/input$index.csv" 2>/dev/null &
+  sleep 0.1
+  process_started $!
+
+  wait $logger_pid
+  wait $detector_pid
+
+  if [ -f "$res_file" ]; then
+    if ! cmp -s "$data_path/results/res$index.csv" "$res_file"; then
+      echo "Files results/res$index.csv and $res_file are not equal"
+      exit_with_error
+    fi
+  else
+    echo "File $res_file not found"
+    exit_with_error
+  fi
+done
+
+echo "All tests passed"
+exit 0

modules/deduplicator/tests/testsData/inputs/input1.csv

@@ -0,0 +1,9 @@
+ipaddr SRC_IP, ipaddr DST_IP, uint16 SRC_PORT, uint16 DST_PORT, uint8 PROTOCOL, uint64 LINK_BIT_FIELD, time TIME_LAST
+154.175.219.8,155.175.219.8,123,123,6,10,2020-01-01T00:00:01Z
+54.175.29.123,54.175.219.24,123,80,6,10,2020-01-01T00:00:02Z
+54.175.21.66,54.175.219.77,443,22,6,10,2020-01-01T00:00:03Z
+54.175.3.17,54.175.219.99,8080,53,17,10,2020-01-01T00:00:04Z
+154.175.219.8,155.175.219.8,123,123,6,10,2020-01-01T00:00:01Z
+154.175.219.8,155.175.219.8,123,123,6,11,2020-01-01T00:00:01Z
+154.175.219.8,155.175.219.8,123,123,6,10,2020-01-01T00:00:01Z
+154.175.219.8,155.175.219.8,123,123,6,11,2020-01-01T00:00:01Z

modules/deduplicator/tests/testsData/results/res1.csv

@@ -0,0 +1,6 @@
+155.175.219.8,154.175.219.8,10,2020-01-01T00:00:01.000000,123,123,6
+54.175.219.24,54.175.29.123,10,2020-01-01T00:00:02.000000,80,123,6
+54.175.219.77,54.175.21.66,10,2020-01-01T00:00:03.000000,22,443,6
+54.175.219.99,54.175.3.17,10,2020-01-01T00:00:04.000000,53,8080,17
+155.175.219.8,154.175.219.8,10,2020-01-01T00:00:01.000000,123,123,6
+155.175.219.8,154.175.219.8,10,2020-01-01T00:00:01.000000,123,123,6