From d9d366592ce9cbd443c508959a5f43aef6abf7d9 Mon Sep 17 00:00:00 2001 From: stefanDeveloper Date: Tue, 2 Sep 2025 09:19:24 +0200 Subject: [PATCH 1/8] Update docs and fix ignore fields --- .gitignore | 1131 ++++++++++++++++++++-- heidpi-logger/include/Config.hpp | 11 +- heidpi-logger/include/EventProcessor.hpp | 5 +- heidpi-logger/include/GeoIP.hpp | 4 +- heidpi-logger/include/Logger.hpp | 6 +- heidpi-logger/include/NDPIClient.hpp | 7 +- heidpi-logger/src/Config.cpp | 3 - heidpi-logger/src/EventProcessor.cpp | 3 + heidpi-logger/src/GeoIP.cpp | 16 +- heidpi-logger/src/Logger.cpp | 99 +- heidpi-logger/src/main.cpp | 234 +++-- 11 files changed, 1319 insertions(+), 200 deletions(-) diff --git a/.gitignore b/.gitignore index 50ec4d3..5f0fd97 100644 --- a/.gitignore +++ b/.gitignore @@ -1,132 +1,1049 @@ heidpi-data/ heidpi-logs/ .DS_Store -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -# C extensions -*.so +##### Windows +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +##### Linux +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +##### MacOS +# General +.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +##### Android +# Built application files +*.apk +*.ap_ +*.aab + +# Files for the ART/Dalvik VM +*.dex + +# Java class files +*.class -# Distribution / packaging -.Python +# Generated files +bin/ +gen/ +out/ +# Uncomment the following line in case you need and you don't have the release build type files in your app +# release/ + +# Gradle files +.gradle/ build/ -develop-eggs/ + +# Local configuration file (sdk path, etc) +local.properties + +# Proguard folder generated by Eclipse +proguard/ + +# Log Files +*.log + +# Android Studio Navigation editor temp files +.navigation/ + +# Android Studio captures folder +captures/ + +# IntelliJ +*.iml +.idea/workspace.xml +.idea/tasks.xml +.idea/gradle.xml +.idea/assetWizardSettings.xml +.idea/dictionaries +.idea/libraries +# Android Studio 3 in .gitignore file. +.idea/caches +.idea/modules.xml +# Comment next line if keeping position of elements in Navigation Editor is relevant for you +.idea/navEditor.xml + +# Keystore files +# Uncomment the following lines if you do not want to check your keystore files in. +#*.jks +#*.keystore + +# External native build folder generated in Android Studio 2.2 and later +.externalNativeBuild + +# Google Services (e.g. APIs or Firebase) +# google-services.json + +# Freeline +freeline.py +freeline/ +freeline_project_description.json + +# fastlane +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots +fastlane/test_output +fastlane/readme.md + +# Version control +vcs.xml + +# lint +lint/intermediates/ +lint/generated/ +lint/outputs/ +lint/tmp/ +# lint/reports/ + +##### Backup +*.bak +*.gho +*.ori +*.orig +*.tmp + +##### GPG +secring.* + +##### Dropbox +# Dropbox settings and caches +.dropbox +.dropbox.attr +.dropbox.cache + +##### SynopsysVCS +# Waveform formats +*.vcd +*.vpd +*.evcd +*.fsdb + +# Default name of the simulation executable. A different name can be +# specified with this switch (the associated daidir database name is +# also taken from here): -o / +simv + +# Generated for Verilog and VHDL top configs +simv.daidir/ +simv.db.dir/ + +# Infrastructure necessary to co-simulate SystemC models with +# Verilog/VHDL models. An alternate directory may be specified with this +# switch: -Mdir= +csrc/ + +# Log file - the following switch allows to specify the file that will be +# used to write all messages from simulation: -l +*.log + +# Coverage results (generated with urg) and database location. The +# following switch can also be used: urg -dir .vdb +simv.vdb/ +urgReport/ + +# DVE and UCLI related files. +DVEfiles/ +ucli.key + +# When the design is elaborated for DirectC, the following file is created +# with declarations for C/C++ functions. +vc_hdrs.h + +##### SVN +.svn/ + +##### Mercurial +.hg/ +.hgignore +.hgsigs +.hgsub +.hgsubstate +.hgtags + +##### Bazaar +.bzr/ +.bzrignore + +##### CVS +/CVS/* +**/CVS/* +.cvsignore +*/.cvsignore + +##### TortoiseGit +# Project-level settings +/.tgitconfig + +##### PuTTY +# Private key +*.ppk + +##### Vim +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +##### Emacs +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + +##### SublimeText +# Cache files for Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# Workspace files are user-specific +*.sublime-workspace + +# Project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using Sublime Text +# *.sublime-project + +# SFTP configuration file +sftp-config.json +sftp-config-alt*.json + +# Package control specific files +Package Control.last-run +Package Control.ca-list +Package Control.ca-bundle +Package Control.system-ca-bundle +Package Control.cache/ +Package Control.ca-certs/ +Package Control.merged-ca-bundle +Package Control.user-ca-bundle +oscrypto-ca-bundle.crt +bh_unicode_properties.cache + +# Sublime-github package stores a github token in this file +# https://packagecontrol.io/packages/sublime-github +GitHub.sublime-settings + +##### Notepad++ +# Notepad++ backups # +*.bak + +##### TextMate +*.tmproj +*.tmproject +tmtags + +##### VisualStudioCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +##### NetBeans +**/nbproject/private/ +**/nbproject/Makefile-*.mk +**/nbproject/Package-*.bash +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +##### JetBrains +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +##### Eclipse +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +##### Qt +# C++ objects and libs +*.slo +*.lo +*.o +*.a +*.la +*.lai +*.so +*.so.* +*.dll +*.dylib + +# Qt-es +object_script.*.Release +object_script.*.Debug +*_plugin_import.cpp +/.qmake.cache +/.qmake.stash +*.pro.user +*.pro.user.* +*.qbs.user +*.qbs.user.* +*.moc +moc_*.cpp +moc_*.h +qrc_*.cpp +ui_*.h +*.qmlc +*.jsc +Makefile* +*build-* +*.qm +*.prl + +# Qt unit tests +target_wrapper.* + +# QtCreator +*.autosave + +# QtCreator Qml +*.qmlproject.user +*.qmlproject.user.* + +# QtCreator CMake +CMakeLists.txt.user* + +# QtCreator 4.8< compilation database +compile_commands.json + +# QtCreator local machine specific files for imported projects +*creator.user* + +##### VisualStudio +##### VisualStudio +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj *.log -local_settings.py -db.sqlite3 -db.sqlite3-journal +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*[.json, .xml, .info] + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ -# Flask stuff: -instance/ -.webassets-cache +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html -# Scrapy stuff: -.scrapy +# Click-Once directory +publish/ -# Sphinx documentation -docs/_build/ +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj -# PyBuilder -target/ +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ -# Jupyter Notebook -.ipynb_checkpoints +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets -# IPython -profile_default/ -ipython_config.py +# Microsoft Azure Build Output +csx/ +*.build.csdef -# pyenv -.python-version +# Microsoft Azure Emulator +ecf/ +rcf/ -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ -# Celery stuff -celerybeat-schedule -celerybeat.pid +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs -# SageMath parsed files -*.sage.py +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ -# Spyder project settings -.spyderproject -.spyproject +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +##### Gradle +.gradle +**/build/ +!src/**/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties + +##### CMake +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps + +##### C++ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# C/C++ binary extension file +*.bin + +##### C +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib -# Rope project settings -.ropeproject +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex -# mkdocs documentation -/site +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf -# Pyre type checker -.pyre/ +# Raspberry Pi Pico Object file +*.uf2 +# Raspberry Pi Pico disassembler file +*.dis diff --git a/heidpi-logger/include/Config.hpp b/heidpi-logger/include/Config.hpp index 46fa75a..a6d515a 100644 --- a/heidpi-logger/include/Config.hpp +++ b/heidpi-logger/include/Config.hpp @@ -7,14 +7,16 @@ /** * @brief Loads application configuration from a YAML file. */ -struct LoggingConfig { +struct LoggingConfig +{ std::string level{"INFO"}; std::string format{"%Y-%m-%dT%H:%M:%S"}; std::string datefmt{"%Y-%m-%dT%H:%M:%S"}; std::string filename{}; // optional log file }; -struct EventConfig { +struct EventConfig +{ std::vector ignore_fields; std::vector ignore_risks; std::vector event_names; // empty -> allow all event names @@ -26,7 +28,8 @@ struct EventConfig { std::vector geoip_keys; }; -class Config { +class Config +{ public: explicit Config(const std::string &path); const LoggingConfig &logging() const { return logging_cfg; } @@ -34,6 +37,7 @@ class Config { const EventConfig &packetEvent() const { return packet_cfg; } const EventConfig &daemonEvent() const { return daemon_cfg; } const EventConfig &errorEvent() const { return error_cfg; } + private: LoggingConfig logging_cfg; EventConfig flow_cfg; @@ -41,4 +45,3 @@ class Config { EventConfig daemon_cfg; EventConfig error_cfg; }; - diff --git a/heidpi-logger/include/EventProcessor.hpp b/heidpi-logger/include/EventProcessor.hpp index 7138f0d..51ae692 100644 --- a/heidpi-logger/include/EventProcessor.hpp +++ b/heidpi-logger/include/EventProcessor.hpp @@ -8,13 +8,14 @@ /** * @brief Processes events based on configuration and writes them as JSON lines. */ -class EventProcessor { +class EventProcessor +{ public: EventProcessor(const EventConfig &cfg, const std::string &outDir); void process(const nlohmann::json &j); + private: EventConfig config; std::string directory; std::unique_ptr geo; }; - diff --git a/heidpi-logger/include/GeoIP.hpp b/heidpi-logger/include/GeoIP.hpp index 2264c68..8a4ed9a 100644 --- a/heidpi-logger/include/GeoIP.hpp +++ b/heidpi-logger/include/GeoIP.hpp @@ -7,7 +7,8 @@ /** * @brief Performs GeoIP lookups using a MaxMind DB and enriches events. */ -class GeoIP { +class GeoIP +{ public: GeoIP() = default; GeoIP(const std::string &path, const std::vector &keys); @@ -23,4 +24,3 @@ class GeoIP { bool loaded{false}; std::vector keys; }; - diff --git a/heidpi-logger/include/Logger.hpp b/heidpi-logger/include/Logger.hpp index b3b9ad7..3b5c906 100644 --- a/heidpi-logger/include/Logger.hpp +++ b/heidpi-logger/include/Logger.hpp @@ -7,13 +7,15 @@ /** * @brief Very small logger writing to stdout and optional file. */ -class Logger { +class Logger +{ public: static void init(const LoggingConfig &cfg); static void info(const std::string &msg); static void error(const std::string &msg); + ~Logger(); + private: static std::mutex mtx; static std::ofstream file; }; - diff --git a/heidpi-logger/include/NDPIClient.hpp b/heidpi-logger/include/NDPIClient.hpp index 98f9254..05106f8 100644 --- a/heidpi-logger/include/NDPIClient.hpp +++ b/heidpi-logger/include/NDPIClient.hpp @@ -7,14 +7,15 @@ * @brief Simple client for nDPIsrvd server. * Messages are length-prefixed JSON blobs. */ -class NDPIClient { +class NDPIClient +{ public: NDPIClient(); ~NDPIClient(); void connectTcp(const std::string &host, unsigned short port); void connectUnix(const std::string &path); - void loop(const std::function &cb, const std::string &filter=""); + void loop(const std::function &cb, const std::string &filter = ""); + private: int fd{-1}; }; - diff --git a/heidpi-logger/src/Config.cpp b/heidpi-logger/src/Config.cpp index d5be9d0..4a83d63 100644 --- a/heidpi-logger/src/Config.cpp +++ b/heidpi-logger/src/Config.cpp @@ -4,9 +4,6 @@ Config::Config(const std::string &path) { YAML::Node config = YAML::LoadFile(path); auto logNode = config["logging"]; if (logNode) { - logging_cfg.level = logNode["level"].as("INFO"); - logging_cfg.format = logNode["format"].as("%Y-%m-%dT%H:%M:%S"); - logging_cfg.datefmt = logNode["datefmt"].as("%Y-%m-%dT%H:%M:%S"); if (logNode["filename"]) logging_cfg.filename = logNode["filename"].as(); } diff --git a/heidpi-logger/src/EventProcessor.cpp b/heidpi-logger/src/EventProcessor.cpp index 9662b23..52e19b6 100644 --- a/heidpi-logger/src/EventProcessor.cpp +++ b/heidpi-logger/src/EventProcessor.cpp @@ -35,14 +35,17 @@ void EventProcessor::process(const nlohmann::json &j) { std::string dst = j.value("dst_ip", ""); geo->enrich(src, dst, out); } + for (const auto &field : config.ignore_fields) { out.erase(field); } + if (!config.ignore_risks.empty() && out.contains("ndpi") && out["ndpi"].contains("flow_risk")) { for (const auto &risk : config.ignore_risks) { out["ndpi"]["flow_risk"].erase(risk); } } + std::filesystem::create_directories(directory); auto path = std::filesystem::path(directory) / (config.filename + ".json"); std::ofstream ofs(path, std::ios::app); diff --git a/heidpi-logger/src/GeoIP.cpp b/heidpi-logger/src/GeoIP.cpp index 80944a3..1500e07 100644 --- a/heidpi-logger/src/GeoIP.cpp +++ b/heidpi-logger/src/GeoIP.cpp @@ -83,26 +83,38 @@ GeoIP::~GeoIP() { nlohmann::json GeoIP::lookup(const std::string &ip) const { nlohmann::json result; if (!loaded || ip.empty()) return result; + int gai_error = 0, mmdb_error = 0; MMDB_lookup_result_s res = MMDB_lookup_string(&mmdb, ip.c_str(), &gai_error, &mmdb_error); if (gai_error != 0 || mmdb_error != MMDB_SUCCESS || !res.found_entry) { return result; } + for (const auto &key : keys) { + // Split dotted key path into parts std::vector parts; std::stringstream ss(key); std::string part; while (std::getline(ss, part, '.')) parts.push_back(part); + std::vector path; - for (auto &p : parts) path.push_back(p.c_str()); + for (const auto &p : parts) path.push_back(p.c_str()); path.push_back(nullptr); + MMDB_entry_data_s entry{}; int status = MMDB_aget_value(&res.entry, &entry, path.data()); if (status != MMDB_SUCCESS || !entry.has_data) continue; + const std::string &field = parts.back(); + nlohmann::json value = entryToJson(mmdb, entry); + if (!value.is_null() && !(value.is_object() && value.empty())) { - result[field] = value; + if (parts.size() == 1) { + result[parts[0]] = value; + } else { + result[field] = value; + } } } return result; diff --git a/heidpi-logger/src/Logger.cpp b/heidpi-logger/src/Logger.cpp index cf17ebc..f553987 100644 --- a/heidpi-logger/src/Logger.cpp +++ b/heidpi-logger/src/Logger.cpp @@ -3,36 +3,111 @@ #include #include #include +#include std::mutex Logger::mtx; std::ofstream Logger::file; -static std::string timestamp() { +/** + * @file Logger.cpp + * @brief Implements thread-safe logging functionality with console and file output. + * + * This file provides the implementation for the Logger class, which supports logging + * messages with timestamps to both the console and an optional file. It ensures thread + * safety using a mutex and flushes critical error logs immediately to disk. + */ + +/** + * @namespace Logger + * @brief Namespace for logging-related functionality. + * + * Contains the Logger class and associated functions for logging messages. + */ + +/** + * @brief Generates a timestamp in ISO 8601 format. + * + * @return std::string A formatted timestamp (e.g., "2024-03-20T12:34:56"). + */ +static std::string timestamp() +{ auto now = std::chrono::system_clock::now(); std::time_t tt = std::chrono::system_clock::to_time_t(now); - std::tm tm = *std::localtime(&tt); - char buf[64]; - std::strftime(buf, sizeof(buf), "%FT%T", &tm); - return std::string(buf); + std::tm tm = *std::localtime(&tt); // Thread-safe with local variable + std::ostringstream oss; + oss << std::put_time(&tm, "%FT%T"); // ISO 8601 format (2024-03-20T12:34:56) + return oss.str(); } -void Logger::init(const LoggingConfig &cfg) { - if (!cfg.filename.empty()) { - file.open(cfg.filename, std::ios::app); +/** + * @brief Initializes the logger with a configuration. + * + * Opens the log file in append mode if a filename is provided in the configuration. + * If the file cannot be opened, an error is printed to stderr. + * + * @param cfg A LoggingConfig object containing log file settings. + */ +void Logger::init(const LoggingConfig &cfg) +{ + if (!cfg.filename.empty()) + { + file.open(cfg.filename, std::ios::app); // Append mode + if (!file.is_open()) + { + std::cerr << "Failed to open log file: " << cfg.filename << std::endl; + } } } -void Logger::info(const std::string &msg) { +/** + * @brief Logs an informational message. + * + * Writes the message to stdout and the log file (if open), prefixed with a timestamp + * and "INFO:". + * + * @param msg The message to log. + */ +void Logger::info(const std::string &msg) +{ std::lock_guard lock(mtx); std::string line = timestamp() + " INFO: " + msg + "\n"; std::cout << line; - if (file.is_open()) file << line; + if (file.is_open()) + { + file << line; + file.flush(); // Ensure logs are flushed for critical messages + } } -void Logger::error(const std::string &msg) { +/** + * @brief Logs an error message. + * + * Writes the message to stderr and the log file (if open), prefixed with a timestamp + * and "ERROR:". Ensures immediate disk writes to avoid data loss in critical scenarios. + * + * @param msg The error message to log. + */ +void Logger::error(const std::string &msg) +{ std::lock_guard lock(mtx); std::string line = timestamp() + " ERROR: " + msg + "\n"; std::cerr << line; - if (file.is_open()) file << line; + if (file.is_open()) + { + file << line; + file.flush(); // Critical logs should be immediately written + } } +/** + * @brief Closes the log file when the Logger object is destroyed. + * + * Ensures the log file is properly closed to release system resources. + */ +Logger::~Logger() +{ + if (file.is_open()) + { + file.close(); + } +} diff --git a/heidpi-logger/src/main.cpp b/heidpi-logger/src/main.cpp index 6eb63d6..73ebae0 100644 --- a/heidpi-logger/src/main.cpp +++ b/heidpi-logger/src/main.cpp @@ -1,3 +1,19 @@ +/** + * @file main.cpp + * @brief Main entry point for the HeidPi Logger application. + * + * This file implements the primary logic for connecting to an event source (TCP/Unix socket), + * parsing command-line arguments, configuring event processing, and dispatching events to + * appropriate handlers using a thread-safe queue and dispatcher. + * + * Key components: + * - Command-line argument parsing + * - Thread-safe event queue and dispatcher + * - Worker threads for event processing + * - Logging and configuration initialization + * - Graceful shutdown handling + */ + #include "Config.hpp" #include "Logger.hpp" #include "NDPIClient.hpp" @@ -17,25 +33,50 @@ #include #include -struct CLIOptions { - std::string host{"127.0.0.1"}; - std::string unix_path{}; - int port{7000}; - std::string write_path{"/var/log"}; - std::string config_path{"config.yml"}; - std::string filter{}; - bool show_daemon{false}; - bool show_packet{false}; - bool show_error{false}; - bool show_flow{false}; +/** + * @struct CLIOptions + * @brief Stores command-line and environment variable configuration. + * + * Populates default values from environment variables and command-line arguments. + */ +struct CLIOptions +{ + std::string host{"127.0.0.1"}; ///< Default TCP host + std::string unix_path{}; ///< Unix socket path (empty by default) + int port{7000}; ///< Default TCP port + std::string write_path{"/var/log"}; ///< Directory for log/event files + std::string config_path{"config.yml"}; ///< Configuration file path + std::string filter{}; ///< Event filter expression + bool show_daemon{false}; ///< Enable daemon event processing + bool show_packet{false}; ///< Enable packet event processing + bool show_error{false}; ///< Enable error event processing + bool show_flow{false}; ///< Enable flow event processing }; -static std::string envOrDefault(const char *env, const std::string &def) { +/** + * @brief Gets an environment variable or returns a default value. + * + * @param env Environment variable name + * @param def Default value if `env` is not set + * @return std::string The value of `env` or `def` + */ +static std::string envOrDefault(const char *env, const std::string &def) +{ const char *v = std::getenv(env); return v ? std::string(v) : def; } -CLIOptions parse(int argc, char **argv) { +/** + * @brief Parses command-line arguments and environment variables. + * + * Merges environment variables (HOST, UNIX, etc.) with command-line flags. + * + * @param argc Number of command-line arguments + * @param argv Command-line arguments + * @return CLIOptions Parsed configuration + */ +CLIOptions parse(int argc, char **argv) +{ CLIOptions o; o.host = envOrDefault("HOST", o.host); o.unix_path = envOrDefault("UNIX", o.unix_path); @@ -47,20 +88,33 @@ CLIOptions parse(int argc, char **argv) { o.show_packet = envOrDefault("SHOW_PACKET_EVENTS", "0") == "1"; o.show_error = envOrDefault("SHOW_ERROR_EVENTS", "0") == "1"; o.show_flow = envOrDefault("SHOW_FLOW_EVENTS", "0") == "1"; - for (int i = 1; i < argc; ++i) { + for (int i = 1; i < argc; ++i) + { std::string a = argv[i]; - auto next = [&](int &i){ return std::string(argv[++i]); }; - if (a == "--host" && i+1 < argc) o.host = next(i); - else if (a == "--unix" && i+1 < argc) o.unix_path = next(i); - else if (a == "--port" && i+1 < argc) o.port = std::stoi(next(i)); - else if (a == "--write" && i+1 < argc) o.write_path = next(i); - else if (a == "--config" && i+1 < argc) o.config_path = next(i); - else if (a == "--filter" && i+1 < argc) o.filter = next(i); - else if (a == "--show-daemon-events") o.show_daemon = !o.show_daemon; - else if (a == "--show-packet-events") o.show_packet = !o.show_packet; - else if (a == "--show-error-events") o.show_error = !o.show_error; - else if (a == "--show-flow-events") o.show_flow = !o.show_flow; - else if (a == "--help" || a == "-h") { + auto next = [&](int &i) + { return std::string(argv[++i]); }; + if (a == "--host" && i + 1 < argc) + o.host = next(i); + else if (a == "--unix" && i + 1 < argc) + o.unix_path = next(i); + else if (a == "--port" && i + 1 < argc) + o.port = std::stoi(next(i)); + else if (a == "--write" && i + 1 < argc) + o.write_path = next(i); + else if (a == "--config" && i + 1 < argc) + o.config_path = next(i); + else if (a == "--filter" && i + 1 < argc) + o.filter = next(i); + else if (a == "--show-daemon-events") + o.show_daemon = !o.show_daemon; + else if (a == "--show-packet-events") + o.show_packet = !o.show_packet; + else if (a == "--show-error-events") + o.show_error = !o.show_error; + else if (a == "--show-flow-events") + o.show_flow = !o.show_flow; + else if (a == "--help" || a == "-h") + { std::cout << "Usage: " << argv[0] << " [options]\n" << " --host Set host\n" << " --unix Set unix socket path\n" @@ -79,19 +133,47 @@ CLIOptions parse(int argc, char **argv) { return o; } -struct Worker { - std::string eventKey; - EventConfig config; - EventProcessor processor; +/** + * @struct Worker + * @brief Encapsulates event processing logic for a specific event type. + * + * Combines an event configuration and processor to handle events of a single type. + */ +struct Worker +{ + std::string eventKey; ///< Type of event to process (e.g., "flow_event_name") + EventConfig config; ///< Configuration for the event type + EventProcessor processor; ///< Processor for event data + + /** + * @brief Constructs a Worker with configuration and directory. + * + * @param k Event type identifier + * @param c Event-specific configuration + * @param dir Directory for output files + */ Worker(const std::string &k, const EventConfig &c, const std::string &dir) : eventKey(k), config(c), processor(c, dir) {} }; -int main(int argc, char **argv) { - // Help kurz vorher abfangen (wie im Original) - for (int i = 1; i < argc; ++i) { +/** + * @brief Main entry point for the HeidPi Logger application. + * + * Initializes logging, parses command-line arguments, connects to the event source, + * and dispatches events to appropriate workers using a thread-safe queue. + * + * @param argc Number of command-line arguments + * @param argv Command-line arguments + * @return int Exit code (0 for success, 1 for error) + */ +int main(int argc, char **argv) +{ + // Early check for --help to avoid redundant parsing + for (int i = 1; i < argc; ++i) + { std::string a = argv[i]; - if (a == "-h" || a == "--help") { + if (a == "-h" || a == "--help") + { std::string name = std::filesystem::path(argv[0]).filename(); std::cout << "usage: " << name << " [-h] [--host HOST | --unix UNIX] [--port PORT] [--write WRITE]\n" @@ -104,43 +186,64 @@ int main(int argc, char **argv) { } } + // --- [1] Parse Command-Line Options --- CLIOptions opts = parse(argc, argv); + + // --- [2] Initialize Configuration & Logger --- Config cfg(opts.config_path); Logger::init(cfg.logging()); + // --- [3] Create Workers for Enabled Event Types --- std::vector workers; workers.reserve(4); - if (opts.show_flow) workers.emplace_back("flow_event_name", cfg.flowEvent(), opts.write_path); - if (opts.show_packet) workers.emplace_back("packet_event_name", cfg.packetEvent(), opts.write_path); - if (opts.show_daemon) workers.emplace_back("daemon_event_name", cfg.daemonEvent(), opts.write_path); - if (opts.show_error) workers.emplace_back("error_event_name", cfg.errorEvent(), opts.write_path); + if (opts.show_flow) + workers.emplace_back("flow_event_name", cfg.flowEvent(), opts.write_path); + if (opts.show_packet) + workers.emplace_back("packet_event_name", cfg.packetEvent(), opts.write_path); + if (opts.show_daemon) + workers.emplace_back("daemon_event_name", cfg.daemonEvent(), opts.write_path); + if (opts.show_error) + workers.emplace_back("error_event_name", cfg.errorEvent(), opts.write_path); - if (workers.empty()) { + if (workers.empty()) + { Logger::error("No event types enabled. Use --show-*_events flags to enable processing."); return 1; } + // --- [4] Connect to Event Source --- NDPIClient client; - try { + try + { if (!opts.unix_path.empty()) client.connectUnix(opts.unix_path); else - client.connectTcp(opts.host, static_cast(opts.port)); // FIX - } catch (const std::exception &ex) { + client.connectTcp(opts.host, static_cast(opts.port)); + } + catch (const std::exception &ex) + { Logger::error(std::string("Failed to connect: ") + ex.what()); return 1; } - // ------------------------- - // NEU: FIFO-Queue + Dispatcher - // ------------------------- - std::queue eventQueue; // FIX: Typ-Parameter - std::mutex mtx; - std::condition_variable cv; - std::atomic done{false}; + // --- [5] Thread-Safe Event Queue and Dispatcher --- + std::queue eventQueue; // FIFO queue for incoming events + std::mutex mtx; // Mutex for thread-safe access + std::condition_variable cv; // Notifies worker of new events + std::atomic done{false}; // Signals shutdown + + // Dispatcher thread: processes events from the queue + std::thread dispatcher([&] + { + auto getAllowedNames = [&](const std::string &eventKey) -> const std::vector& { + if (eventKey == "flow_event_name") return cfg.flowEvent().event_names; + if (eventKey == "packet_event_name") return cfg.packetEvent().event_names; + if (eventKey == "daemon_event_name") return cfg.daemonEvent().event_names; + if (eventKey == "error_event_name") return cfg.errorEvent().event_names; + static const std::vector empty; + return empty; + }; - // Dispatcher-Thread (arbeitet streng nacheinander ab) - std::thread dispatcher([&]{ while (true) { nlohmann::json event; { @@ -151,26 +254,32 @@ int main(int argc, char **argv) { eventQueue.pop(); } - // Event-Typ ermitteln & Namen lesen std::string key; std::string name; if (event.contains("flow_event_name")) { key = "flow_event_name"; - name = event["flow_event_name"].get(); // FIX: get() + name = event["flow_event_name"].get(); } else if (event.contains("packet_event_name")) { key = "packet_event_name"; - name = event["packet_event_name"].get(); // FIX: get() + name = event["packet_event_name"].get(); } else if (event.contains("daemon_event_name")) { key = "daemon_event_name"; - name = event["daemon_event_name"].get(); // FIX: get() + name = event["daemon_event_name"].get(); } else if (event.contains("error_event_name")) { key = "error_event_name"; - name = event["error_event_name"].get(); // FIX: get() + name = event["error_event_name"].get(); } else { Logger::info("Received unknown event: missing event name"); continue; } + const auto &allowedNames = getAllowedNames(key); + if (!allowedNames.empty() && + std::find(allowedNames.begin(), allowedNames.end(), name) == allowedNames.end()) { + Logger::info("Skipping event '" + name + "' of type " + key); + continue; + } + bool handled = false; for (auto &w : workers) { if (w.eventKey != key) continue; @@ -180,19 +289,18 @@ int main(int argc, char **argv) { if (!handled) { Logger::info("No handler enabled for event '" + name + "' of type " + key); } - } - }); + } }); - // Reader: liest nonstop und füttert nur die Queue - client.loop([&](const nlohmann::json &j) { + // --- [6] Event Reader Loop --- + client.loop([&](const nlohmann::json &j) + { { std::lock_guard lk(mtx); eventQueue.push(j); } - cv.notify_one(); - }, opts.filter); + cv.notify_one(); }, opts.filter); - // Nach Abbruch der Verbindung: Queue leeren lassen und Thread beenden + // --- [7] Graceful Shutdown --- { std::lock_guard lk(mtx); done = true; From 023c3ab2da12f60832ccdefe810372033253a801 Mon Sep 17 00:00:00 2001 From: stefanDeveloper Date: Tue, 2 Sep 2025 10:24:38 +0200 Subject: [PATCH 2/8] Add build pipeline --- .github/workflows/build-linux.yml | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/build-linux.yml diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml new file mode 100644 index 0000000..fd6b7e5 --- /dev/null +++ b/.github/workflows/build-linux.yml @@ -0,0 +1,39 @@ +name: Linux Build + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + + env: + CMAKE_BUILD_TYPE: Release + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up CMake + uses: lukka/get-cmake@v3.29.2 + + - name: Cache APT packages + uses: actions/cache@v4 + with: + path: | + /var/cache/apt/archives + key: ${{ runner.os }}-apt-${{ hashFiles('**/CMakeLists.txt') }} + restore-keys: | + ${{ runner.os }}-apt-${{ hashFiles('**/CMakeLists.txt') }} + + - name: Build + run: | + cd heidpi-logger + cmake . + make \ No newline at end of file From 8aacd3f00c1761f1e0f9da74002a75970fd234d6 Mon Sep 17 00:00:00 2001 From: stefanDeveloper Date: Tue, 2 Sep 2025 13:57:04 +0200 Subject: [PATCH 3/8] Add tests --- config.yml | 21 ++---- heidpi-logger/include/Config.hpp | 5 +- heidpi-logger/include/GeoIP.hpp | 6 +- heidpi-logger/src/EventProcessor.cpp | 69 +++++++++++++----- heidpi-logger/src/GeoIP.cpp | 97 ++++++++++++++++--------- heidpi-logger/src/NDPIClient.cpp | 104 ++++++++++++++++++++++----- heidpi-logger/test/CMakeLists.txt | 25 +++++++ heidpi-logger/test/ConfigTest.cpp | 97 +++++++++++++++++++++++++ heidpi-logger/test/GeoIPTest.cpp | 94 ++++++++++++++++++++++++ 9 files changed, 430 insertions(+), 88 deletions(-) create mode 100644 heidpi-logger/test/CMakeLists.txt create mode 100644 heidpi-logger/test/ConfigTest.cpp create mode 100644 heidpi-logger/test/GeoIPTest.cpp diff --git a/config.yml b/config.yml index ee8cdee..040b040 100644 --- a/config.yml +++ b/config.yml @@ -1,16 +1,13 @@ -appName: heiDPI - logging: - level: ERROR - encoding: utf-8 - format: "%(asctime)s %(levelname)s:%(message)s" - datefmt: "%Y-%m-%dT%I:%M:%S" - # filemode: w # a for append, will not override current file # filename: heiDPI.log flow_event: - ignore_fields: [] - ignore_risks: [] + ignore_fields: [ + "alias" + ] + ignore_risks: [ + "5" + ] flow_event_name: - update - end @@ -24,11 +21,10 @@ flow_event: keys: - country.names.en # Additional configurations - - location + # - location # - city # - traits # - postal -# threads: 4 daemon_event: ignore_fields: [] @@ -36,18 +32,15 @@ daemon_event: - init - status filename: daemon_event -# threads: 4 packet_event: ignore_fields: [] packet_event_name: - packet-flow filename: packet_event -# threads: 4 error_event: ignore_fields: [] error_event_name: - error-flow filename: error_event -# threads: 4 diff --git a/heidpi-logger/include/Config.hpp b/heidpi-logger/include/Config.hpp index a6d515a..711fbc0 100644 --- a/heidpi-logger/include/Config.hpp +++ b/heidpi-logger/include/Config.hpp @@ -9,10 +9,7 @@ */ struct LoggingConfig { - std::string level{"INFO"}; - std::string format{"%Y-%m-%dT%H:%M:%S"}; - std::string datefmt{"%Y-%m-%dT%H:%M:%S"}; - std::string filename{}; // optional log file + std::string filename{}; }; struct EventConfig diff --git a/heidpi-logger/include/GeoIP.hpp b/heidpi-logger/include/GeoIP.hpp index 8a4ed9a..a1d0bf6 100644 --- a/heidpi-logger/include/GeoIP.hpp +++ b/heidpi-logger/include/GeoIP.hpp @@ -16,11 +16,11 @@ class GeoIP void enrich(const std::string &src_ip, const std::string &dst_ip, nlohmann::json &out) const; +protected: + virtual nlohmann::json lookup(const std::string &ip) const; + bool loaded{false}; private: - nlohmann::json lookup(const std::string &ip) const; - MMDB_s mmdb{}; - bool loaded{false}; std::vector keys; }; diff --git a/heidpi-logger/src/EventProcessor.cpp b/heidpi-logger/src/EventProcessor.cpp index 52e19b6..4b8de84 100644 --- a/heidpi-logger/src/EventProcessor.cpp +++ b/heidpi-logger/src/EventProcessor.cpp @@ -1,3 +1,9 @@ +/** + * @file EventProcessor.cpp + * Implementation of event processing logic including timestamping, + * GeoIP enrichment, and output formatting + */ + #include "EventProcessor.hpp" #include #include @@ -5,11 +11,23 @@ #include #include +/** + * @class EventProcessor + * Handles event processing pipeline including: + * - Timestamp injection + * - GeoIP enrichment (if enabled) + * - Field/risk filtering + * - Output file writing + */ EventProcessor::EventProcessor(const EventConfig &cfg, const std::string &outDir) - : config(cfg), directory(outDir) { - if (cfg.geoip_enabled && !cfg.geoip_path.empty()) { + : config(cfg), directory(outDir) +{ + if (cfg.geoip_enabled && !cfg.geoip_path.empty()) + { geo = std::make_unique(cfg.geoip_path, cfg.geoip_keys); - } else { + } + else + { // optional, aber hilfreich zur Diagnose: Logger::info(std::string("GeoIP disabled for '") + cfg.filename + "' (enabled=" + (cfg.geoip_enabled ? "true" : "false") + @@ -17,42 +35,61 @@ EventProcessor::EventProcessor(const EventConfig &cfg, const std::string &outDir } } -static std::string nowTs() { +/** + * @return Current timestamp in ISO 8601 format (YYYY-MM-DDTHH:MM:SS) + */ +static std::string nowTs() +{ auto now = std::chrono::system_clock::now(); std::time_t tt = std::chrono::system_clock::to_time_t(now); - std::tm tm = *std::localtime(&tt); - char buf[64]; - std::strftime(buf, sizeof(buf), "%FT%T", &tm); - return std::string(buf); + std::tm tm = *std::localtime(&tt); // Thread-safe with local variable + std::ostringstream oss; + oss << std::put_time(&tm, "%FT%T"); // ISO 8601 format (2024-03-20T12:34:56) + return oss.str(); } -void EventProcessor::process(const nlohmann::json &j) { +/** + * Processes a raw event JSON object through the full pipeline: + * 1. Injects current timestamp + * 2. Adds GeoIP enrichment (if configured) + * 3. Removes ignored fields + * 4. Filters out ignored risks + * 5. Writes to output file in specified directory + * + * @param j Raw event JSON to process + */ +void EventProcessor::process(const nlohmann::json &j) +{ nlohmann::json out = j; out["timestamp"] = nowTs(); - if (geo) { // statt config.geoip_enabled + if (geo) + { // statt config.geoip_enabled std::string src = j.value("src_ip", ""); std::string dst = j.value("dst_ip", ""); geo->enrich(src, dst, out); } - for (const auto &field : config.ignore_fields) { + for (const auto &field : config.ignore_fields) + { out.erase(field); } - if (!config.ignore_risks.empty() && out.contains("ndpi") && out["ndpi"].contains("flow_risk")) { - for (const auto &risk : config.ignore_risks) { + if (!config.ignore_risks.empty() && out.contains("ndpi") && out["ndpi"].contains("flow_risk")) + { + for (const auto &risk : config.ignore_risks) + { out["ndpi"]["flow_risk"].erase(risk); } } - + std::filesystem::create_directories(directory); auto path = std::filesystem::path(directory) / (config.filename + ".json"); std::ofstream ofs(path, std::ios::app); - if (!ofs.is_open()) { + if (!ofs.is_open()) + { Logger::error("Failed to open output file: " + path.string()); return; } ofs << out.dump() << std::endl; } - diff --git a/heidpi-logger/src/GeoIP.cpp b/heidpi-logger/src/GeoIP.cpp index 1500e07..eec1a09 100644 --- a/heidpi-logger/src/GeoIP.cpp +++ b/heidpi-logger/src/GeoIP.cpp @@ -2,9 +2,12 @@ #include "Logger.hpp" #include -namespace { -nlohmann::json entryToJson(const MMDB_s &db, const MMDB_entry_data_s &entry) { - switch (entry.type) { +namespace +{ + nlohmann::json entryToJson(const MMDB_s &db, const MMDB_entry_data_s &entry) + { + switch (entry.type) + { case MMDB_DATA_TYPE_UTF8_STRING: return std::string(entry.utf8_string, entry.data_size); case MMDB_DATA_TYPE_DOUBLE: @@ -21,18 +24,22 @@ nlohmann::json entryToJson(const MMDB_s &db, const MMDB_entry_data_s &entry) { return entry.uint64; case MMDB_DATA_TYPE_BOOLEAN: return static_cast(entry.boolean); - case MMDB_DATA_TYPE_MAP: { + case MMDB_DATA_TYPE_MAP: + { MMDB_entry_s sub{&db, entry.offset}; MMDB_entry_data_list_s *list = nullptr; - if (MMDB_get_entry_data_list(&sub, &list) == MMDB_SUCCESS && list) { + if (MMDB_get_entry_data_list(&sub, &list) == MMDB_SUCCESS && list) + { nlohmann::json obj = nlohmann::json::object(); MMDB_entry_data_list_s *ptr = list; - while (ptr && ptr->next) { + while (ptr && ptr->next) + { auto key = ptr->entry_data; ptr = ptr->next; auto val = ptr->entry_data; ptr = ptr->next; - if (key.type != MMDB_DATA_TYPE_UTF8_STRING) continue; + if (key.type != MMDB_DATA_TYPE_UTF8_STRING) + continue; std::string k(key.utf8_string, key.data_size); obj[k] = entryToJson(db, val); } @@ -41,13 +48,16 @@ nlohmann::json entryToJson(const MMDB_s &db, const MMDB_entry_data_s &entry) { } break; } - case MMDB_DATA_TYPE_ARRAY: { + case MMDB_DATA_TYPE_ARRAY: + { MMDB_entry_s sub{&db, entry.offset}; MMDB_entry_data_list_s *list = nullptr; - if (MMDB_get_entry_data_list(&sub, &list) == MMDB_SUCCESS && list) { + if (MMDB_get_entry_data_list(&sub, &list) == MMDB_SUCCESS && list) + { nlohmann::json arr = nlohmann::json::array(); MMDB_entry_data_list_s *ptr = list; - while (ptr) { + while (ptr) + { arr.push_back(entryToJson(db, ptr->entry_data)); ptr = ptr->next; } @@ -58,61 +68,78 @@ nlohmann::json entryToJson(const MMDB_s &db, const MMDB_entry_data_s &entry) { } default: break; + } + return {}; } - return {}; -} } // namespace GeoIP::GeoIP(const std::string &path, const std::vector &k) - : keys(k) { + : keys(k) +{ int status = MMDB_open(path.c_str(), MMDB_MODE_MMAP, &mmdb); - if (status != MMDB_SUCCESS) { + if (status != MMDB_SUCCESS) + { Logger::error(std::string("GeoIP open failed: ") + path + " " + MMDB_strerror(status)); loaded = false; - } else { + } + else + { loaded = true; } } -GeoIP::~GeoIP() { - if (loaded) { +GeoIP::~GeoIP() +{ + if (loaded) + { MMDB_close(&mmdb); } } -nlohmann::json GeoIP::lookup(const std::string &ip) const { +nlohmann::json GeoIP::lookup(const std::string &ip) const +{ nlohmann::json result; - if (!loaded || ip.empty()) return result; + if (!loaded || ip.empty()) + return result; int gai_error = 0, mmdb_error = 0; MMDB_lookup_result_s res = MMDB_lookup_string(&mmdb, ip.c_str(), &gai_error, &mmdb_error); - if (gai_error != 0 || mmdb_error != MMDB_SUCCESS || !res.found_entry) { + if (gai_error != 0 || mmdb_error != MMDB_SUCCESS || !res.found_entry) + { return result; } - for (const auto &key : keys) { + for (const auto &key : keys) + { // Split dotted key path into parts std::vector parts; std::stringstream ss(key); std::string part; - while (std::getline(ss, part, '.')) parts.push_back(part); + while (std::getline(ss, part, '.')) + parts.push_back(part); - std::vector path; - for (const auto &p : parts) path.push_back(p.c_str()); + std::vector path; + for (const auto &p : parts) + path.push_back(p.c_str()); path.push_back(nullptr); MMDB_entry_data_s entry{}; int status = MMDB_aget_value(&res.entry, &entry, path.data()); - if (status != MMDB_SUCCESS || !entry.has_data) continue; - + if (status != MMDB_SUCCESS || !entry.has_data) + continue; + const std::string &field = parts.back(); nlohmann::json value = entryToJson(mmdb, entry); - if (!value.is_null() && !(value.is_object() && value.empty())) { - if (parts.size() == 1) { + if (!value.is_null() && !(value.is_object() && value.empty())) + { + if (parts.size() == 1) + { result[parts[0]] = value; - } else { + } + else + { result[field] = value; } } @@ -121,14 +148,18 @@ nlohmann::json GeoIP::lookup(const std::string &ip) const { } void GeoIP::enrich(const std::string &src_ip, const std::string &dst_ip, - nlohmann::json &out) const { - if (!loaded) return; + nlohmann::json &out) const +{ + if (!loaded) + return; auto src = lookup(src_ip); - if (!src.empty()) { + if (!src.empty()) + { out["src_geoip2_city"] = src; } auto dst = lookup(dst_ip); - if (!dst.empty()) { + if (!dst.empty()) + { out["dst_geoip2_city"] = dst; } } \ No newline at end of file diff --git a/heidpi-logger/src/NDPIClient.cpp b/heidpi-logger/src/NDPIClient.cpp index 20cd60b..4dccd0c 100644 --- a/heidpi-logger/src/NDPIClient.cpp +++ b/heidpi-logger/src/NDPIClient.cpp @@ -8,59 +8,127 @@ #include #include #include +#include +/** + * @class NDPIClient + * @brief Client for connecting to the NDPIServer and receiving JSON messages. + * + * This class provides TCP and Unix domain socket connectivity to an NDPIServer, + * with support for optional message filters and automatic JSON parsing. + * Implements safety limits to prevent memory exhaustion from large payloads. + */ NDPIClient::NDPIClient() {} NDPIClient::~NDPIClient() { if (fd >= 0) ::close(fd); } +/** + * @brief Establishes a TCP connection to the specified host and port. + * + * @param host IPv4 address string (e.g., "127.0.0.1") + * @param port TCP port number (0-65535) + * @throws std::runtime_error if socket creation or connection fails + * @throws std::invalid_argument for invalid IPv4 address format + */ void NDPIClient::connectTcp(const std::string &host, unsigned short port) { fd = ::socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) throw std::runtime_error("socket"); + if (fd < 0) + throw std::runtime_error("socket: " + std::string(strerror(errno))); + sockaddr_in addr{}; addr.sin_family = AF_INET; addr.sin_port = htons(port); ::inet_pton(AF_INET, host.c_str(), &addr.sin_addr); - if (::connect(fd, (sockaddr*)&addr, sizeof(addr)) < 0) - throw std::runtime_error("connect"); + + if (::connect(fd, (sockaddr *)&addr, sizeof(addr)) < 0) + throw std::runtime_error("connect: " + std::string(strerror(errno))); } +/** + * @brief Establishes a Unix domain socket connection to the specified path. + * + * @param path Path to Unix socket file (max 107 bytes) + * @throws std::runtime_error if socket creation or connection fails + * @throws std::invalid_argument for invalid path length + */ void NDPIClient::connectUnix(const std::string &path) { fd = ::socket(AF_UNIX, SOCK_STREAM, 0); - if (fd < 0) throw std::runtime_error("socket"); + if (fd < 0) + throw std::runtime_error("socket: " + std::string(strerror(errno))); + sockaddr_un addr{}; addr.sun_family = AF_UNIX; + + if (path.size() >= sizeof(addr.sun_path) - 1) { + throw std::invalid_argument("Unix socket path too long (max " + + std::to_string(sizeof(addr.sun_path) - 1) + + " bytes)"); + } + std::strncpy(addr.sun_path, path.c_str(), sizeof(addr.sun_path)-1); if (::connect(fd, (sockaddr*)&addr, sizeof(addr)) < 0) - throw std::runtime_error("connect"); + throw std::runtime_error("connect: " + std::string(strerror(errno))); } +/** + * @brief Main message loop for receiving and processing JSON messages + * + * @param cb Callback function to process parsed JSON messages + * @param filter Optional message filter (sent to server in 6-digit length format) + * @throws std::runtime_error for send/receive errors or malformed messages + * + * Message format: 6-digit length field followed by JSON payload + * Maximum allowed payload size: 10MB (safety limit) + */ void NDPIClient::loop(const std::function &cb, const std::string &filter) { - // send optional filter expression before starting the receive loop - if (!filter.empty()) { + // Send optional filter expression before starting the receive loop + if (!filter.empty()) + { std::ostringstream ss; ss << std::setw(6) << std::setfill('0') << filter.size() << filter; std::string msg = ss.str(); + ssize_t sent = ::send(fd, msg.c_str(), msg.size(), 0); if (sent < 0 || static_cast(sent) != msg.size()) - throw std::runtime_error("send"); + throw std::runtime_error("send: " + std::string(strerror(errno))); } - while (true) { + while (true) + { char lenbuf[6]; - // Der Generator verwendet immer fünf Ziffern für die Länge + // Read 6 bytes for length (server uses 6-digit format with leading zeros) ssize_t n = ::recv(fd, lenbuf, 5, MSG_WAITALL); if (n <= 0) break; - lenbuf[5] = '\0'; - size_t len = std::stoul(lenbuf); - // anschließend die JSON‑Nutzlast lesen (inklusive '{') + + lenbuf[5] = '\0'; // Properly null-terminate the buffer + size_t len; + + try + { + len = std::stoul(lenbuf); + if (len == 0 || len > 10 * 1024 * 1024) + { // 10MB safety limit + throw std::runtime_error("invalid message length"); + } + } + catch (const std::exception &e) + { + throw std::runtime_error("invalid message length format: " + std::string(e.what())); + } + + // Read JSON payload with safety checks std::string payload(len, '\0'); n = ::recv(fd, payload.data(), len, MSG_WAITALL); - if (n <= 0) break; - try { + if (n <= 0) + break; + + try + { auto j = nlohmann::json::parse(payload); cb(j); - } catch (...) { - // JSON‑Fehler ignorieren + } + catch (...) + { + // JSON-Fehler ignorieren } } } - diff --git a/heidpi-logger/test/CMakeLists.txt b/heidpi-logger/test/CMakeLists.txt new file mode 100644 index 0000000..f41b116 --- /dev/null +++ b/heidpi-logger/test/CMakeLists.txt @@ -0,0 +1,25 @@ +find_package(GTest REQUIRED) + +# Add test executables +add_executable(LoggerTest LoggerTest.cpp ${PROJECT_SOURCE_DIR}/src/Logger.cpp) +target_include_directories(LoggerTest PRIVATE + ${PROJECT_SOURCE_DIR}/include +) +target_link_libraries(LoggerTest PRIVATE ${GTEST_LIBRARIES} nlohmann_json yaml-cpp maxminddb nlohmann_json_schema_validator gtest_main pthread) + +add_executable(ConfigTest ConfigTest.cpp ${PROJECT_SOURCE_DIR}/src/Logger.cpp ${PROJECT_SOURCE_DIR}/src/Config.cpp) +target_include_directories(ConfigTest PRIVATE + ${PROJECT_SOURCE_DIR}/include +) +target_link_libraries(ConfigTest PRIVATE ${GTEST_LIBRARIES} nlohmann_json yaml-cpp maxminddb nlohmann_json_schema_validator gtest_main pthread) + +add_executable(GeoIPTest GeoIPTest.cpp ${PROJECT_SOURCE_DIR}/src/Logger.cpp ${PROJECT_SOURCE_DIR}/src/GeoIP.cpp) +target_include_directories(GeoIPTest PRIVATE + ${PROJECT_SOURCE_DIR}/include +) +target_link_libraries(GeoIPTest PRIVATE ${GTEST_LIBRARIES} nlohmann_json yaml-cpp maxminddb nlohmann_json_schema_validator gtest_main pthread) + +# Register tests with CTest +add_test(NAME LoggerTest COMMAND LoggerTest) +add_test(NAME GeoIPTest COMMAND GeoIPTest) +add_test(NAME ConfigTest COMMAND ConfigTest) \ No newline at end of file diff --git a/heidpi-logger/test/ConfigTest.cpp b/heidpi-logger/test/ConfigTest.cpp new file mode 100644 index 0000000..1c92d7d --- /dev/null +++ b/heidpi-logger/test/ConfigTest.cpp @@ -0,0 +1,97 @@ +// test_config.cpp +#include +#include +#include "Config.hpp" + +namespace +{ + + // Helper to write temporary YAML file + std::string writeTempYAML(const std::string &content) + { + std::string filename = "temp_config.yaml"; + std::ofstream ofs(filename); + ofs << content; + ofs.close(); + return filename; + } + +} // namespace + +TEST(ConfigTest, LoadsLoggingFilename) +{ + std::string yaml = R"( +logging: + filename: "log.txt" +)"; + std::string path = writeTempYAML(yaml); + Config cfg(path); + EXPECT_EQ(cfg.logging().filename, "log.txt"); +} + +TEST(ConfigTest, LoadsBasicEventConfig) +{ + std::string yaml = R"( +flow_event: + ignore_fields: ["field1", "field2"] + ignore_risks: ["risk1"] + flow_event_name: ["flow1", "flow2"] + filename: "flow_file.log" + threads: 4 +)"; + std::string path = writeTempYAML(yaml); + Config cfg(path); + + EXPECT_EQ(cfg.flowEvent().ignore_fields, std::vector({"field1", "field2"})); + EXPECT_EQ(cfg.flowEvent().ignore_risks, std::vector({"risk1"})); + EXPECT_EQ(cfg.flowEvent().event_names, std::vector({"flow1", "flow2"})); + EXPECT_EQ(cfg.flowEvent().filename, "flow_file.log"); + EXPECT_EQ(cfg.flowEvent().threads, 4); +} + +TEST(ConfigTest, OverwritesEventNames) +{ + std::string yaml = R"( +packet_event: + flow_event_name: ["should_be_overwritten"] + packet_event_name: ["packet1"] +)"; + std::string path = writeTempYAML(yaml); + Config cfg(path); + + // Due to multiple overwrites, only the last one should be kept + EXPECT_EQ(cfg.packetEvent().event_names, std::vector({"packet1"})); +} + +TEST(ConfigTest, LoadsGeoIPSettings) +{ + std::string yaml = R"( +error_event: + geoip2_city: + enabled: true + filepath: "/path/to/geoip" + keys: ["country", "city"] +)"; + std::string path = writeTempYAML(yaml); + Config cfg(path); + + EXPECT_TRUE(cfg.errorEvent().geoip_enabled); + EXPECT_EQ(cfg.errorEvent().geoip_path, "/path/to/geoip"); + EXPECT_EQ(cfg.errorEvent().geoip_keys, std::vector({"country", "city"})); +} + +TEST(ConfigTest, HandlesMissingOptionalFieldsGracefully) +{ + std::string yaml = R"( +daemon_event: {} +)"; + std::string path = writeTempYAML(yaml); + Config cfg(path); + + // Defaults + EXPECT_TRUE(cfg.daemonEvent().ignore_fields.empty()); + EXPECT_TRUE(cfg.daemonEvent().ignore_risks.empty()); + EXPECT_TRUE(cfg.daemonEvent().event_names.empty()); + EXPECT_EQ(cfg.daemonEvent().threads, 1); + EXPECT_FALSE(cfg.daemonEvent().geoip_enabled); +} diff --git a/heidpi-logger/test/GeoIPTest.cpp b/heidpi-logger/test/GeoIPTest.cpp new file mode 100644 index 0000000..1366fb2 --- /dev/null +++ b/heidpi-logger/test/GeoIPTest.cpp @@ -0,0 +1,94 @@ +#include +#include "GeoIP.hpp" + +using namespace testing; +using json = nlohmann::json; + +// A helper to simulate MMDB_entry_data_s for UTF8 string +MMDB_entry_data_s makeStringEntry(const std::string &s) { + MMDB_entry_data_s entry{}; + entry.type = MMDB_DATA_TYPE_UTF8_STRING; + entry.utf8_string = s.c_str(); + entry.data_size = static_cast(s.size()); + entry.has_data = 1; + return entry; +} + +class GeoIPTest : public ::testing::Test { +protected: + void SetUp() override { + // Normally, you'd inject or mock MMDB functions. + // For simplicity, assume GeoIP is modifiable for testing. + } + + void TearDown() override { + } +}; + +// A mock/fake class to override behavior +class FakeGeoIP : public GeoIP { +public: + FakeGeoIP(const std::vector& keys) + : GeoIP("/dev/null", keys) { + loaded = true; + } + + // Override lookup to simulate IP data + nlohmann::json lookup(const std::string &ip) const override { + if (ip == "1.2.3.4") { + return { + {"country", "USA"}, + {"city", "San Francisco"}, + {"lat", 37.7749}, + {"lon", -122.4194} + }; + } else if (ip == "8.8.8.8") { + return { + {"country", "USA"}, + {"city", "Mountain View"} + }; + } + return json{}; + } +}; + +TEST_F(GeoIPTest, LookupReturnsExpectedFields) { + FakeGeoIP geoip({"country", "city", "lat", "lon"}); + + json result = geoip.lookup("1.2.3.4"); + + EXPECT_EQ(result["country"], "USA"); + EXPECT_EQ(result["city"], "San Francisco"); + EXPECT_DOUBLE_EQ(result["lat"], 37.7749); + EXPECT_DOUBLE_EQ(result["lon"], -122.4194); +} + +TEST_F(GeoIPTest, LookupReturnsEmptyForUnknownIP) { + FakeGeoIP geoip({"country", "city"}); + json result = geoip.lookup("192.0.2.1"); + + EXPECT_TRUE(result.empty()); +} + +TEST_F(GeoIPTest, EnrichAddsSrcAndDstGeoFields) { + FakeGeoIP geoip({"country", "city"}); + + json out; + geoip.enrich("1.2.3.4", "8.8.8.8", out); + + ASSERT_TRUE(out.contains("src_geoip2_city")); + ASSERT_TRUE(out.contains("dst_geoip2_city")); + + EXPECT_EQ(out["src_geoip2_city"]["city"], "San Francisco"); + EXPECT_EQ(out["dst_geoip2_city"]["city"], "Mountain View"); +} + +TEST_F(GeoIPTest, EnrichHandlesEmptyLookups) { + FakeGeoIP geoip({"country", "city"}); + + json out; + geoip.enrich("192.0.2.1", "203.0.113.5", out); // Non-existent + + EXPECT_FALSE(out.contains("src_geoip2_city")); + EXPECT_FALSE(out.contains("dst_geoip2_city")); +} From e5a3370a75d2d52384ac811b43818e5df2c147e2 Mon Sep 17 00:00:00 2001 From: stefanDeveloper Date: Tue, 2 Sep 2025 14:24:07 +0200 Subject: [PATCH 4/8] Fix logger tests --- heidpi-logger/CMakeLists.txt | 17 +++- heidpi-logger/include/Logger.hpp | 1 + heidpi-logger/src/Logger.cpp | 33 +++++--- heidpi-logger/test/LoggerTest.cpp | 128 ++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 11 deletions(-) create mode 100644 heidpi-logger/test/LoggerTest.cpp diff --git a/heidpi-logger/CMakeLists.txt b/heidpi-logger/CMakeLists.txt index 7d81f51..b8d0704 100644 --- a/heidpi-logger/CMakeLists.txt +++ b/heidpi-logger/CMakeLists.txt @@ -23,7 +23,7 @@ FetchContent_MakeAvailable(json) FetchContent_Declare( json-schema-validator GIT_REPOSITORY https://github.com/pboettch/json-schema-validator.git - GIT_TAG main + GIT_TAG 2.3.0 ) FetchContent_MakeAvailable(json-schema-validator) @@ -39,9 +39,16 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(maxminddb) +FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.17.0 +) +FetchContent_MakeAvailable(googletest) + file(GLOB SOURCES src/*.cpp) add_executable(heidpi_cpp ${SOURCES}) -target_include_directories(heidpi_cpp PRIVATE include) +target_include_directories(heidpi_cpp PRIVATE ${PROJECT_SOURCE_DIR}/include) target_link_libraries(heidpi_cpp PRIVATE yaml-cpp nlohmann_json::nlohmann_json @@ -54,3 +61,9 @@ include(GNUInstallDirs) install(TARGETS heidpi_cpp RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +option(BUILD_TESTING "Build the tests" ON) +if(BUILD_TESTING) + enable_testing() + add_subdirectory(test) +endif() diff --git a/heidpi-logger/include/Logger.hpp b/heidpi-logger/include/Logger.hpp index 3b5c906..0da8fc7 100644 --- a/heidpi-logger/include/Logger.hpp +++ b/heidpi-logger/include/Logger.hpp @@ -13,6 +13,7 @@ class Logger static void init(const LoggingConfig &cfg); static void info(const std::string &msg); static void error(const std::string &msg); + static void destroy(); ~Logger(); private: diff --git a/heidpi-logger/src/Logger.cpp b/heidpi-logger/src/Logger.cpp index f553987..c2da8bc 100644 --- a/heidpi-logger/src/Logger.cpp +++ b/heidpi-logger/src/Logger.cpp @@ -1,3 +1,11 @@ +/** + * @file Logger.cpp + * @brief Implements thread-safe logging functionality with console and file output. + * + * This file provides the implementation for the Logger class, which supports logging + * messages with timestamps to both the console and an optional file. It ensures thread + * safety using a mutex and flushes critical error logs immediately to disk. + */ #include "Logger.hpp" #include #include @@ -8,15 +16,6 @@ std::mutex Logger::mtx; std::ofstream Logger::file; -/** - * @file Logger.cpp - * @brief Implements thread-safe logging functionality with console and file output. - * - * This file provides the implementation for the Logger class, which supports logging - * messages with timestamps to both the console and an optional file. It ensures thread - * safety using a mutex and flushes critical error logs immediately to disk. - */ - /** * @namespace Logger * @brief Namespace for logging-related functionality. @@ -49,6 +48,7 @@ static std::string timestamp() */ void Logger::init(const LoggingConfig &cfg) { + std::lock_guard lock(mtx); if (!cfg.filename.empty()) { file.open(cfg.filename, std::ios::app); // Append mode @@ -99,6 +99,20 @@ void Logger::error(const std::string &msg) } } +/** + * @brief Closes the log file when the Logger object is destroyed. + * + * Ensures the log file is properly closed to release system resources. + */ +void Logger::destroy() { + std::lock_guard lock(mtx); + if (file.is_open()) { + file.close(); + } + file.clear(); // reset flags +} + + /** * @brief Closes the log file when the Logger object is destroyed. * @@ -106,6 +120,7 @@ void Logger::error(const std::string &msg) */ Logger::~Logger() { + std::lock_guard lock(mtx); if (file.is_open()) { file.close(); diff --git a/heidpi-logger/test/LoggerTest.cpp b/heidpi-logger/test/LoggerTest.cpp new file mode 100644 index 0000000..dd8b601 --- /dev/null +++ b/heidpi-logger/test/LoggerTest.cpp @@ -0,0 +1,128 @@ +#include "Logger.hpp" +#include +#include +#include +#include // For remove() +#include +#include + +// Helper to read file contents +std::string readFile(const std::string& filename) { + std::ifstream in(filename); + std::stringstream buffer; + buffer << in.rdbuf(); + return buffer.str(); +} + +// Captures std::cout and std::cerr output +class OutputCapture { +public: + void startCapture() { + oldCout = std::cout.rdbuf(coutStream.rdbuf()); + oldCerr = std::cerr.rdbuf(cerrStream.rdbuf()); + } + + void stopCapture() { + std::cout.rdbuf(oldCout); + std::cerr.rdbuf(oldCerr); + } + + std::string getCapturedStdout() const { + return coutStream.str(); + } + + std::string getCapturedStderr() const { + return cerrStream.str(); + } + +private: + std::ostringstream coutStream; + std::ostringstream cerrStream; + std::streambuf* oldCout = nullptr; + std::streambuf* oldCerr = nullptr; +}; + +TEST(LoggerTest, LogsToFileAndConsole) { + std::string testFile = "test_log.txt"; + std::remove(testFile.c_str()); // Ensure clean start + + LoggingConfig cfg; + cfg.filename = testFile; + Logger::init(cfg); + + OutputCapture capture; + capture.startCapture(); + + Logger::info("This is an info message."); + Logger::error("This is an error message."); + + capture.stopCapture(); + + std::string fileContent = readFile(testFile); + std::string stdoutContent = capture.getCapturedStdout(); + std::string stderrContent = capture.getCapturedStderr(); + + std::regex infoPattern(R"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} INFO: This is an info message\.\n)"); + std::regex errorPattern(R"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} ERROR: This is an error message\.\n)"); + + // Console output checks + EXPECT_TRUE(std::regex_match(stdoutContent, infoPattern)); + EXPECT_TRUE(std::regex_match(stderrContent, errorPattern)); + + // File output checks + EXPECT_NE(fileContent.find("INFO: This is an info message."), std::string::npos); + EXPECT_NE(fileContent.find("ERROR: This is an error message."), std::string::npos); + + std::remove(testFile.c_str()); // Cleanup + Logger::destroy(); +} + +TEST(LoggerTest, LogsWithoutFile) { + LoggingConfig cfg; + cfg.filename = ""; // No file + Logger::init(cfg); + + OutputCapture capture; + capture.startCapture(); + + Logger::info("Console only info"); + Logger::error("Console only error"); + + capture.stopCapture(); + + EXPECT_NE(capture.getCapturedStdout().find("INFO: Console only info"), std::string::npos); + EXPECT_NE(capture.getCapturedStderr().find("ERROR: Console only error"), std::string::npos); + + Logger::destroy(); +} + +TEST(LoggerTest, ThreadSafeLogging) { + std::string testFile = "threaded_log.txt"; + std::remove(testFile.c_str()); + + LoggingConfig cfg; + cfg.filename = testFile; + Logger::init(cfg); + + auto logTask = [](int threadId) { + for (int i = 0; i < 10; ++i) { + Logger::info("Thread " + std::to_string(threadId) + " message " + std::to_string(i)); + } + }; + + std::thread t1(logTask, 1); + std::thread t2(logTask, 2); + + t1.join(); + t2.join(); + + std::string content = readFile(testFile); + + for (int i = 0; i < 10; ++i) { + EXPECT_NE(content.find("Thread 1 message " + std::to_string(i)), std::string::npos); + EXPECT_NE(content.find("Thread 2 message " + std::to_string(i)), std::string::npos); + } + + std::remove(testFile.c_str()); + Logger::destroy(); +} From 8226a348b7c5d8bfe4b1b733050c9d3abdd99fb7 Mon Sep 17 00:00:00 2001 From: stefanDeveloper Date: Tue, 2 Sep 2025 14:27:39 +0200 Subject: [PATCH 5/8] Add tests --- .github/workflows/build-linux.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index fd6b7e5..647fb1b 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -32,8 +32,19 @@ jobs: restore-keys: | ${{ runner.os }}-apt-${{ hashFiles('**/CMakeLists.txt') }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + libgtest-dev + - name: Build run: | cd heidpi-logger cmake . - make \ No newline at end of file + make + + - name: Test + run: | + cd heidpi-logger + ctest \ No newline at end of file From fd6fdea6fa83ed1b3f7105f918e653c1cbf5945b Mon Sep 17 00:00:00 2001 From: stefanDeveloper Date: Tue, 2 Sep 2025 14:54:37 +0200 Subject: [PATCH 6/8] Update docker compose --- docker-compose.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e64622..4d894bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ services: image: stefan96/heidpi-producer:latest container_name: heidpi_producer network_mode: host + # platform: linux/amd64 security_opt: - no-new-privileges # pids_limit: 8192 @@ -18,6 +19,13 @@ services: - MAX_BUFFERED_LINES=32768 - JA3_URL=https://sslbl.abuse.ch/blacklist/ja3_fingerprints.csv - SSL_SHA1_URL=https://sslbl.abuse.ch/blacklist/sslblacklist.csv + healthcheck: + test: ["CMD-SHELL", "pgrep -f /root/nDPId || exit 1"] + interval: 1m + timeout: 10s + retries: 3 + start_period: 30s + consumer: build: @@ -25,9 +33,11 @@ services: dockerfile: Dockerfile.consumer #image: stefan96/heidpi-consumer:latest container_name: heidpi_consumer + # platform: linux/amd64 volumes: - ./heidpi-logs:/var/log/:rw - ./config.yml:/usr/src/app/config.yml:ro + # - .../GeoLite2-City.mmdb:/tmp/city.mmdb:ro network_mode: host security_opt: - no-new-privileges @@ -39,7 +49,8 @@ services: cpus: '1' memory: 2G depends_on: - - producer + producer: + condition: service_healthy environment: - SHOW_DAEMON_EVENTS=1 - SHOW_PACKET_EVENTS=1 From 3e6708d72bca2c2ab7aa1364192f5fd9a4eb0909 Mon Sep 17 00:00:00 2001 From: stefanDeveloper Date: Tue, 2 Sep 2025 14:58:43 +0200 Subject: [PATCH 7/8] Change to alpine --- Dockerfile.consumer | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/Dockerfile.consumer b/Dockerfile.consumer index 787860a..0a9b147 100644 --- a/Dockerfile.consumer +++ b/Dockerfile.consumer @@ -1,10 +1,14 @@ # ---------- Build stage ---------- -FROM debian:bookworm AS build +FROM alpine:3.22.1 AS build ARG CMAKE_BUILD_TYPE=Release -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential cmake pkg-config git ca-certificates \ - && rm -rf /var/lib/apt/lists/* +# Install dependencies +RUN apk add --no-cache \ + build-base \ + cmake \ + git \ + pkgconfig \ + ca-certificates WORKDIR /src COPY heidpi-logger/ ./heidpi-logger/ @@ -19,21 +23,22 @@ RUN cmake -S heidpi-logger -B /build \ && strip /build/heidpi_cpp || true # ---------- Runtime stage ---------- -FROM debian:bookworm-slim AS runtime +FROM alpine:3.22.1 AS runtime -RUN apt-get update && apt-get install -y --no-install-recommends \ - libstdc++6 ca-certificates \ - && rm -rf /var/lib/apt/lists/* +# Install runtime dependencies +RUN apk add --no-cache \ + libstdc++ \ + ca-certificates -# non-root -RUN useradd -r -u 10001 appuser +# Create non-root user +RUN adduser -D -u 10001 appuser WORKDIR /app -# Configs (aus Repo-Root) +# Copy configuration and built binary COPY config.yml /app/config.yml - COPY --from=build /build/heidpi_cpp /usr/local/bin/app +# Environment variables ENV WRITE="/var/log" \ SHOW_FLOW_EVENTS=1 \ SHOW_PACKET_EVENTS=1 \ From 4b97ab0b302e7b764e7e56a1768ecf0c09c95a58 Mon Sep 17 00:00:00 2001 From: stefanDeveloper Date: Tue, 2 Sep 2025 15:07:48 +0200 Subject: [PATCH 8/8] Improve image size --- Dockerfile.producer | 51 ++++++++++++------------ docker-entrypoint.sh | 93 ++++++++++++++++++++++++++++---------------- 2 files changed, 85 insertions(+), 59 deletions(-) diff --git a/Dockerfile.producer b/Dockerfile.producer index 3b524a3..693bbed 100644 --- a/Dockerfile.producer +++ b/Dockerfile.producer @@ -1,24 +1,24 @@ # ---------- Build stage ---------- -FROM alpine:3.22.1 as builder +FROM alpine:3.22.1 AS builder WORKDIR /root -RUN apk update && \ - apk add --no-cache git \ - make \ - cmake \ - clang \ - clang-dev \ - make \ - automake \ - gcc \ - g++ \ - libc-dev \ - linux-headers \ - pkgconfig \ - libpcap-dev \ - autoconf \ - bash \ - libtool + +RUN apk add --no-cache \ + git \ + cmake \ + clang \ + clang-dev \ + gcc \ + g++ \ + libc-dev \ + linux-headers \ + pkgconfig \ + libpcap-dev \ + autoconf \ + automake \ + libtool \ + bash \ + make RUN git clone --branch 1.6 https://github.com/utoni/nDPId.git @@ -27,7 +27,7 @@ COPY nDPId/config.h nDPId RUN cd nDPId && mkdir build && cd build && cmake .. -DBUILD_NDPI=ON && make # ---------- Runtime stage ---------- -FROM alpine:3.22.1 as runtime +FROM alpine:3.22.1 AS runtime ENV MAX_BUFFERED_LINES=1024 \ MAX_THREADS=4 \ @@ -43,14 +43,15 @@ ENV MAX_BUFFERED_LINES=1024 \ HOSTNAME="" WORKDIR /root -RUN apk update && \ - apk add --no-cache curl libpcap-dev bash -COPY --from=builder /root/nDPId/libnDPI/ /root/ -COPY --from=builder /root/nDPId/build/nDPIsrvd /root/nDPId/build/nDPId /root/ +RUN apk add --no-cache libpcap curl + +COPY --from=builder /root/nDPId/libnDPI/libndpi.so* /root/ +COPY --from=builder /root/nDPId/build/nDPIsrvd /root/ +COPY --from=builder /root/nDPId/build/nDPId /root/ COPY docker-entrypoint.sh /root/ RUN chmod +x /root/docker-entrypoint.sh -ENTRYPOINT ["/bin/bash", "/root/docker-entrypoint.sh"] -CMD ["nDPId"] \ No newline at end of file +ENTRYPOINT ["/bin/sh", "/root/docker-entrypoint.sh"] +CMD ["nDPId"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 185e0c5..dff3b6a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh MY_INFO="\ ########################################### @@ -11,50 +11,74 @@ Starting services echo "$MY_INFO" -# exit script if any command fails (non-zero value) +# Exit on any command failure set -e -params_ndpisrvd=() -params_ndpid=() + +params_ndpisrvd="" +params_ndpid="" ########################################### ### Create params for nDPIsrvd ### ########################################### -[[ $MAX_BUFFERED_LINES -gt 0 ]] && params_ndpisrvd+=(-C $MAX_BUFFERED_LINES) +if [ "$MAX_BUFFERED_LINES" -gt 0 ] 2>/dev/null; then + params_ndpisrvd="$params_ndpisrvd -C $MAX_BUFFERED_LINES" +fi ########################################### ### Create params for nDPId ### ########################################### -regex='(https?|ftp|file)://[-[:alnum:]\+&@#/%?=~_|!:,.;]*[-[:alnum:]\+&@#/%=~_|]' +url_regex="^\(https\?\|ftp\|file\)://" -if [[ $JA3_URL =~ $regex ]]; then - curl "$JA3_URL" > /root/ja3_fingerprints.csv - params_ndpid+=(-J /root/ja3_fingerprints.csv) -fi +case "$JA3_URL" in + http://* | https://* | ftp://* | file://*) + curl "$JA3_URL" > /root/ja3_fingerprints.csv + params_ndpid="$params_ndpid -J /root/ja3_fingerprints.csv" + ;; +esac -if [[ $SSL_SHA1_URL =~ $regex ]]; then - curl "$SSL_SHA1_URL" > /root/sslblacklist.csv - params_ndpid+=(-S /root/sslblacklist.csv) +case "$SSL_SHA1_URL" in + http://* | https://* | ftp://* | file://*) + curl "$SSL_SHA1_URL" > /root/sslblacklist.csv + params_ndpid="$params_ndpid -S /root/sslblacklist.csv" + ;; +esac + +if [ -n "$INTERFACE" ]; then + params_ndpid="$params_ndpid -i $INTERFACE" fi -[[ -n $INTERFACE ]] && params_ndpid+=(-i $INTERFACE) +if [ "$FLOW_ANALYSIS" = "1" ]; then + params_ndpid="$params_ndpid -A" +fi -[[ $FLOW_ANALYSIS = 1 ]] && params_ndpid+=(-A) +if [ -n "$TUNE_PARAM" ]; then + OLD_IFS="$IFS" + IFS=',' -if [[ -n $TUNE_PARAM ]]; then - for word in $(echo $TUNE_PARAM | tr "," "\n"); do - params_ndpid+=(-o $word) + for word in $TUNE_PARAM; do + params_ndpid="$params_ndpid -o $word" done + + IFS="$OLD_IFS" fi -[[ -n $PCAP_FILTER ]] && params_ndpid+=(-B $PCAP_FILTER) +if [ -n "$PCAP_FILTER" ]; then + params_ndpid="$params_ndpid -B $PCAP_FILTER" +fi -[[ -n $NDPI_CUSTOM_PROTOCOLS ]] && params_ndpid+=(-P $NDPI_CUSTOM_PROTOCOLS) +if [ -n "$NDPI_CUSTOM_PROTOCOLS" ]; then + params_ndpid="$params_ndpid -P $NDPI_CUSTOM_PROTOCOLS" +fi -[[ -n $NDPI_CUSTOM_CATEGORIES ]] && params_ndpid+=(-C $NDPI_CUSTOM_CATEGORIES) +if [ -n "$NDPI_CUSTOM_CATEGORIES" ]; then + params_ndpid="$params_ndpid -C $NDPI_CUSTOM_CATEGORIES" +fi -[[ -n $HOSTNAME ]] && params_ndpid+=(-a $HOSTNAME) +if [ -n "$HOSTNAME" ]; then + params_ndpid="$params_ndpid -a $HOSTNAME" +fi ########################################### ### Start nDPIsrvd ### @@ -62,14 +86,15 @@ fi echo "Start nDPIsrvd..." -/root/nDPIsrvd -p /tmp/nDPIsrvd-daemon.pid \ - -c /tmp/nDPIsrvd-daemon-collector.sock \ - -s /tmp/nDPIsrvd-daemon-distributor.sock \ - -S 0.0.0.0:$PORT \ - -u root \ - -d \ - -L /tmp/nDPIsrvd.log \ - "${params_ndpisrvd[@]}" +# Use eval to expand parameter string correctly +eval /root/nDPIsrvd -p /tmp/nDPIsrvd-daemon.pid \ + -c /tmp/nDPIsrvd-daemon-collector.sock \ + -s /tmp/nDPIsrvd-daemon-distributor.sock \ + -S 0.0.0.0:$PORT \ + -u root \ + -d \ + -L /tmp/nDPIsrvd.log \ + $params_ndpisrvd ########################################### ### Start nDPId ### @@ -78,7 +103,7 @@ echo "Start nDPIsrvd..." echo "Start nDPId..." exec /root/nDPId -p /tmp/nDPId-daemon.pid \ - -c /tmp/nDPIsrvd-daemon-collector.sock \ - -u root \ - -L /tmp/nDPId.log \ - "${params_ndpid[@]}" + -c /tmp/nDPIsrvd-daemon-collector.sock \ + -u root \ + -L /tmp/nDPId.log \ + $params_ndpid