configuration of memory banks realtime display

Author: sy-c

doc/releaseNotes.md

@@ -529,3 +529,4 @@ This file describes the main feature changes for each readout.exe released versi
 - Updated configuration parameters:
   - added equipment-*.dropPagesWithError: if set, the pages with RDH errors are discarded (requires rdhCheckEnabled or rdhUseFirstInPage). This may be used if downstream software is not robust to RDH errors.
 - Disabled unused RDMA features (still available by switch in CMake)
+- Added feature for memory banks real time monitoring. Enabled by setting membanksMonitorRate in /etc/o2.d/readout-defaults.cfg. Output (high-rate text) can be seen locally in real time with `tail -f /tmp/readout-monitor-mempool-(id)`. The status of each page in the bank is displayed.

src/MemoryBankManager.cxx

@@ -22,17 +22,10 @@
 #include "readoutInfoLogger.h"
 
 MemoryBankManager::MemoryBankManager() {
-  std::function<void(void)> f = std::bind(&MemoryBankManager::monitorThLoop, this);
-  monitorThShutdown = 0;
-  monitorTh=std::make_unique<std::thread>(f);
 }
 
 MemoryBankManager::~MemoryBankManager() {
-  if (monitorTh!=nullptr) {
-    monitorThShutdown = 	1;
-    monitorTh->join();
-    monitorTh = nullptr;
-  }
+  stopMonitoring();
 }
 
 int MemoryBankManager::addBank(std::shared_ptr<MemoryBank> bankPtr, std::string name)
@@ -53,9 +46,12 @@ int MemoryBankManager::addBank(std::shared_ptr<MemoryBank> bankPtr, std::string
   return 0;
 }
 
-std::string getMonitorFifoPath(int id) {
+std::string MemoryBankManager::getMonitorFifoPath(int id) {
+  if (id < 0) {
+    return monitorPath;
+  }
   char fn[128];
-  snprintf(fn,sizeof(fn),"/tmp/readout-monitor-pool-%d",id);
+  snprintf(fn,sizeof(fn),"%s-%d", monitorPath.c_str(), id);
   return fn;
 }
 
@@ -256,11 +252,12 @@ void MemoryBankManager::reset()
   }
   banks.clear();
   poolIndex = -1;
+  stopMonitoring();
 }
 
 void MemoryBankManager::monitorThLoop() {
   AliceO2::Common::Timer t;
-  t.reset(200000);
+  t.reset(1000000.0 / monitorUpdateRate);
   for(;!monitorThShutdown.load();) {
     if (t.isTimeout()) {
       std::unique_lock<std::mutex> lock(bankMutex);
@@ -280,3 +277,30 @@ void MemoryBankManager::monitorThLoop() {
     }
   }
 }
+
+
+void MemoryBankManager::startMonitoring(double v_updateRate, const char* v_monitorPath) {
+  if (monitorTh != nullptr) {
+    stopMonitoring();
+  }
+  if (v_updateRate <= 0) {
+    return;
+  }
+  monitorUpdateRate = v_updateRate;
+  if ((v_monitorPath != nullptr) && (strlen(v_monitorPath) != 0)) {
+    monitorPath = v_monitorPath;
+  } else {
+    monitorPath = monitorPathDefault;
+  }
+  std::function<void(void)> f = std::bind(&MemoryBankManager::monitorThLoop, this);
+  monitorThShutdown = 0;
+  monitorTh=std::make_unique<std::thread>(f);
+}
+
+void MemoryBankManager::stopMonitoring() {
+  if (monitorTh != nullptr) {
+    monitorThShutdown = 1;
+    monitorTh->join();
+    monitorTh = nullptr;
+  }
+}

src/MemoryBankManager.h

@@ -67,6 +67,10 @@ class MemoryBankManager
   // reset bank manager in fresh state, in particular: clear all banks
   void reset();
 
+  void startMonitoring(double updateRate, const char* monitorPath = ""); // starts monitoring at given updateRate and output path
+  void stopMonitoring(); // stop monitoring
+  std::string getMonitorFifoPath(int id); // return full path to monitor pipe for given pool index
+
  private:
   std::vector<bankDescriptor> banks; // list of registered memory banks
   std::mutex bankMutex;              // instance mutex to handle concurrent access to public methods
@@ -76,6 +80,9 @@ class MemoryBankManager
   std::atomic<int> monitorThShutdown; // flag to terminate monitor thread
   void monitorThLoop(); // function launched in monitor thread
   std::vector<std::shared_ptr<MemoryPagesPool>> pools; // reference to existing page pools, for monitoring purpose
+  double monitorUpdateRate = 0; // monitoring period, in Hz
+  const char *monitorPathDefault = "/tmp/readout-monitor-mempool";
+  std::string monitorPath = monitorPathDefault; // prefix path to output monitoring pipes. "-%d" (bank number) added to it and fifo created.
 };
 
 // a global MemoryBankManager instance

src/mainReadout.cxx

@@ -346,6 +346,8 @@ int Readout::init(int argc, char* argv[])
   const std::string cfgDefaultsEntryPoint = "readout"; // entry point for default configuration variables (e.g. section named [readout])
   std::string cfgStatsPublishAddress; // address where to publish readout stats, eg "tcp://127.0.0.1:6008"
   double cfgStatsPublishInterval = 5.0; // interval for readout stats publish, in seconds
+  std::string cfgMembanksMonitorPath = ""; // path to memory banks monitor output
+  double cfgMembanksMonitorRate = 0.0; // rate of memory banks monitor updates
   try {
     cfgDefaults.load(cfgDefaultsPath.c_str());
     initLogs.push_back({LogInfoDevel, "Defaults loaded from " + cfgDefaultsPath});
@@ -357,6 +359,8 @@ int Readout::init(int argc, char* argv[])
     cfgDefaults.getOptionalValue<double>(cfgDefaultsEntryPoint + ".statsPublishInterval", cfgStatsPublishInterval, cfgStatsPublishInterval);
     cfgDefaults.getOptionalValue<std::string>(cfgDefaultsEntryPoint + ".db", cfgDatabaseCxParams);
     cfgDefaults.getOptionalValue<int>(cfgDefaultsEntryPoint + ".customCommandsEnabled", cfgCustomCommandsEnabled);
+    cfgDefaults.getOptionalValue<std::string>(cfgDefaultsEntryPoint + ".membanksMonitorPath", cfgMembanksMonitorPath, cfgMembanksMonitorPath);
+    cfgDefaults.getOptionalValue<double>(cfgDefaultsEntryPoint + ".membanksMonitorRate", cfgMembanksMonitorRate, cfgMembanksMonitorRate);
   }
   catch(...) {
     //initLogs.push_back({LogWarningSupport_(3100), std::string("Error loading defaults")});
@@ -412,6 +416,11 @@ int Readout::init(int argc, char* argv[])
     initLogs.push_back({LogWarningDevel_(3236), "Failed to start Stats publish"});
   } //otherwise: disabled
 
+  if (cfgMembanksMonitorRate > 0) {
+    theMemoryBankManager.startMonitoring(cfgMembanksMonitorRate, cfgMembanksMonitorPath.c_str());
+    initLogs.push_back({LogInfoDevel, "Started MemoryBanks monitor @ " + std::to_string(cfgMembanksMonitorRate) + " Hz : " + theMemoryBankManager.getMonitorFifoPath(-1) + "..."});
+  }
+
   // configure signal handlers for clean exit
   struct sigaction signalSettings;
   bzero(&signalSettings, sizeof(signalSettings));