Merge pull request #168 from NodeFactoryIo/mmuftic/fix-data-collection-interval

Custom stats collecting intervals

Author: mpetrunic

.version

@@ -1 +1 @@
-version=0.4.1
+version=0.4.2

CHANGELOG.md

@@ -1,24 +1,34 @@
 # Changelog
 
+## [v0.4.2]((https://github.com/NodeFactoryIo/vedran/tree/v0.4.2))
+[Full Changelog](https://github.com/NodeFactoryIo/vedran/compare/v0.4.1...v0.4.2\)
+
+### Added
+
+### Fix
+
+### Changed
+- Custom stats collecting intervals [\#168](https://github.com/NodeFactoryIo/vedran/pull/168) ([MakMuftic](https://github.com/MakMuftic))
+
 ## [v0.4.1]((https://github.com/NodeFactoryIo/vedran/tree/v0.4.1))
-[Full Changelog](https://github.com/NodeFactoryIo/vedran/compare/v0.4.0...v0.4.1)
+[Full Changelog](https://github.com/NodeFactoryIo/vedran/compare/v0.4.0...v0.4.1\)
 
 ### Added
 
 ### Fix
-- Fix manual payout #165 [#\165](https://github.com/NodeFactoryIo/vedran/pull/165) ([MakMuftic](https://github.com/MakMuftic))
+- Fix manual payout [\#165](https://github.com/NodeFactoryIo/vedran/pull/165) ([MakMuftic](https://github.com/MakMuftic))
 
 ### Changed
 
 ## [v0.4.0]((https://github.com/NodeFactoryIo/vedran/tree/v0.4.0))
 [Full Changelog](https://github.com/NodeFactoryIo/vedran/compare/v0.3.1...v0.4.0)
 
 ### Added
-- Provide LB fee information [#\158](https://github.com/NodeFactoryIo/vedran/pull/158) ([MakMuftic](https://github.com/MakMuftic))
+- Provide LB fee information [\#158](https://github.com/NodeFactoryIo/vedran/pull/158) ([MakMuftic](https://github.com/MakMuftic))
 
 ### Fix
-- Metrics endpoint and grafana dashboard example [#\138](https://github.com/NodeFactoryIo/vedran/pull/138) ([mpetrun5](https://github.com/mpetrun5))
-- Fix next payout date prometheus value [#\161](https://github.com/NodeFactoryIo/vedran/pull/161) ([mpetrun5](https://github.com/mpetrun5))
+- Metrics endpoint and grafana dashboard example [\#138](https://github.com/NodeFactoryIo/vedran/pull/138) ([mpetrun5](https://github.com/mpetrun5))
+- Fix next payout date prometheus value [\#161](https://github.com/NodeFactoryIo/vedran/pull/161) ([mpetrun5](https://github.com/mpetrun5))
 
 ### Changed
 

README.md

@@ -41,13 +41,13 @@ This demo starts five separate dockerized components:
 
 ### Trigger Manual Payout
 
-Our compose setup runs dev chain and our load balancer uses Allice account to do payout
+Our compose setup runs dev chain, and our load balancer uses Allice account to do payout, 
 so you don't have to obtain dev DOTs. Polkadot node operator is Bob (he received payout from Allice).
-Load balancer in this setup runs payout daily, if you don't wan't to wait,
-you can run following command which will create additional container (in compose network) and trigger payout from Allice account:
+Load balancer in this setup runs payout daily, if you don't want to wait,
+you can run following command which will create an additional container (in compose network) and trigger payout from Allice account:
 
 ```
-docker run --network vedran_default nodefactory/vedran:v0.3.0 payout --private-key 0xe5be9a5092b81bca64be81d212e7f2f9eba183bb7a90954f7b76361f6edb5c0a --payout-reward 100 --load-balancer-url "http://vedran:4000/ws"
+docker run --network vedran_default nodefactory/vedran:latest payout --private-key 0xe5be9a5092b81bca64be81d212e7f2f9eba183bb7a90954f7b76361f6edb5c0a --payout-reward 100 --load-balancer-url "http://vedran:4000/ws"
 ```
 
 ## Get **vedran** binary releases
@@ -194,6 +194,20 @@ Monitoring is done via grafana and prometheus which are expected to be installed
 -  ### Prometheus
     Prometheus should be configured to scrape metrics from vedran's `/metrics` endpoint
     via prometheus .yml configuration. Example of which can be found [here](./infra/prometheus/prometheus.yml)
+   
+### Stats collection intervals
+
+It is possible to change default calculation intervals for specific statistic categories. These intervals define how
+often will these statistics be recalculated.
+
+Valid values are time intervals such as "5s", "1.5h" or "2h45m". Valid time units are "ms", "s", "m", "h"
+
+| ENV | Description | Default value |
+|----|-----------|:--------:|
+|`PROM_FEE_STATS_INTERVAL`|payout fees for nodes and load balancer|12 hours|
+|`PROM_NODE_STATS_INTERVAL`|active and penalized nodes|15 seconds|
+|`PROM_REQUEST_STATS_INTERVAL`|successful and failed requests|15 seconds|
+|`PROM_PAYOUT_STATS_INTERVAL`|payout distribution|1 minute|
 
 
 ## Vedran loadbalancer API

docker-compose.yml

@@ -17,7 +17,10 @@ services:
 
   vedran:
     command: start --auth-secret=test-secret --log-level debug --public-ip vedran --server-port 4000  --private-key ${VEDRAN_LB_PK:-0xe5be9a5092b81bca64be81d212e7f2f9eba183bb7a90954f7b76361f6edb5c0a} --payout-interval 1 --payout-reward ${VEDRAN_LB_REWARD_POOL:-10}
-    image: nodefactory/vedran:v0.4.1
+    image: nodefactory/vedran:v0.4.2
+    environment:
+      - PROM_FEE_STATS_INTERVAL=10s
+      - PROM_PAYOUT_STATS_INTERVAL=5s
     ports:
       - "4000:4000"
     container_name: "vedran"

internal/prometheus/prometheus.go

@@ -3,6 +3,7 @@ package prometheus
 import (
 	"fmt"
 	"github.com/NodeFactoryIo/vedran/internal/stats"
+	"os"
 	"runtime"
 	"strconv"
 	"time"
@@ -19,8 +20,22 @@ import (
 
 const (
 	nextPayoutDateLayout = "Mon, Jan 2 2006."
+
+	FeeStatsIntervalEnv                  = "PROM_FEE_STATS_INTERVAL"
+	DefaultFeeStatsCollectionInterval    = 12 * time.Hour
+	NodeStatsIntervalEnv                 = "PROM_NODE_STATS_INTERVAL"
+	DefaultNodeStatsCollectionInterval   = 15 * time.Second
+	RequestStatsIntervalEnv              = "PROM_REQUEST_STATS_INTERVAL"
+	DefaultRecordStatsCollectionInterval = 15 * time.Second
+	PayoutStatsIntervalEnv               = "PROM_PAYOUT_STATS_INTERVAL"
+	DefaultPayoutStatsCollectionInterval = 1 * time.Minute
 )
 
+var feeStatsCollectionInterval time.Duration
+var nodeStatsCollectionInterval time.Duration
+var requestStatsCollectionInterval time.Duration
+var payoutStatsCollectionInterval time.Duration
+
 var (
 	activeNodes = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "vedran_number_of_active_nodes",
@@ -101,6 +116,8 @@ func RecordMetrics(repos repositories.Repos) {
 		})
 	fee.Set(float64(configuration.Config.Fee))
 
+	setUpCollectionIntervals()
+
 	go recordPayoutDistribution(repos)
 	go recordActiveNodeCount(repos.NodeRepo)
 	go recordPenalizedNodeCount(repos.NodeRepo)
@@ -122,7 +139,7 @@ func recordNodeFees(repos repositories.FeeRepository) {
 		for _, fee := range *fees {
 			nodeFees.With(prometheus.Labels{"node": fee.NodeId}).Set(float64(fee.TotalFee))
 		}
-		time.Sleep(12 * time.Hour)
+		time.Sleep(feeStatsCollectionInterval)
 	}
 }
 
@@ -144,7 +161,7 @@ func recordLbFeeAmount(payoutRepo repositories.PayoutRepository) {
 			totalFeeCollected += p.LbFee
 		}
 		totalFee.Set(totalFeeCollected)
-		time.Sleep(12 * time.Hour)
+		time.Sleep(feeStatsCollectionInterval)
 	}
 }
 
@@ -176,7 +193,7 @@ func recordPayoutDistribution(repos repositories.Repos) {
 			)
 		}
 
-		time.Sleep(1 * time.Minute)
+		time.Sleep(payoutStatsCollectionInterval)
 	}
 }
 
@@ -188,37 +205,68 @@ func recordPayoutDate(repos repositories.Repos) {
 		} else {
 			payoutDate.With(prometheus.Labels{"date": date.Format(nextPayoutDateLayout)}).Set(1)
 		}
-		time.Sleep(12 * time.Hour)
+		time.Sleep(feeStatsCollectionInterval)
 	}
 }
 
 func recordActiveNodeCount(nodeRepo repositories.NodeRepository) {
 	for {
 		activeNodes.Set(float64(len(*nodeRepo.GetAllActiveNodes())))
-		time.Sleep(15 * time.Second)
+		time.Sleep(nodeStatsCollectionInterval)
 	}
 }
 
 func recordPenalizedNodeCount(nodeRepo repositories.NodeRepository) {
 	for {
 		nodes, _ := nodeRepo.GetPenalizedNodes()
 		penalizedNodes.Set(float64(len(*nodes)))
-		time.Sleep(15 * time.Second)
+		time.Sleep(nodeStatsCollectionInterval)
 	}
 }
 
 func recordSuccessfulRequestCount(recordRepo repositories.RecordRepository) {
 	for {
 		count, _ := recordRepo.CountSuccessfulRequests()
 		successfulRequests.Set(float64(count))
-		time.Sleep(15 * time.Second)
+		time.Sleep(requestStatsCollectionInterval)
 	}
 }
 
 func recordFailedRequestCount(recordRepo repositories.RecordRepository) {
 	for {
 		count, _ := recordRepo.CountFailedRequests()
 		failedRequests.Set(float64(count))
-		time.Sleep(15 * time.Second)
+		time.Sleep(requestStatsCollectionInterval)
+	}
+}
+
+func setUpCollectionIntervals() {
+	fsi := os.Getenv(FeeStatsIntervalEnv)
+	if fsi != "" {
+		feeStatsCollectionInterval, _ = time.ParseDuration(fsi)
+	}
+	if feeStatsCollectionInterval == 0 {
+		feeStatsCollectionInterval = DefaultFeeStatsCollectionInterval
+	}
+	nsi := os.Getenv(NodeStatsIntervalEnv)
+	if nsi != "" {
+		nodeStatsCollectionInterval, _ = time.ParseDuration(nsi)
+	}
+	if nodeStatsCollectionInterval == 0 {
+		nodeStatsCollectionInterval = DefaultNodeStatsCollectionInterval
+	}
+	rsi := os.Getenv(RequestStatsIntervalEnv)
+	if rsi != "" {
+		requestStatsCollectionInterval, _ = time.ParseDuration(rsi)
+	}
+	if requestStatsCollectionInterval == 0 {
+		requestStatsCollectionInterval = DefaultRecordStatsCollectionInterval
+	}
+	psi := os.Getenv(PayoutStatsIntervalEnv)
+	if psi != "" {
+		payoutStatsCollectionInterval, _ = time.ParseDuration(psi)
+	}
+	if payoutStatsCollectionInterval == 0 {
+		payoutStatsCollectionInterval = DefaultPayoutStatsCollectionInterval
 	}
 }