diff --git a/apis/fluentbit/v1alpha2/plugins/filter/log_to_metrics_types.go b/apis/fluentbit/v1alpha2/plugins/filter/log_to_metrics_types.go index ff35e48f7..91165ce29 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/log_to_metrics_types.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/log_to_metrics_types.go @@ -49,6 +49,15 @@ type LogToMetrics struct { // Flag that defines if logs should be discarded after processing. This applies // for all logs, no matter if they have emitted metrics or not. DiscardLogs *bool `json:"discardLogs,omitempty"` + // The interval for metrics emission, in seconds. If flushIntervalSec and flushIntervalNsec are both + // unset or 0, metrics are emitted right after each filter match. Otherwise they are emitted at + // this interval, which lowers resource usage under high load. + // +kubebuilder:validation:Minimum:=0 + FlushIntervalSec *int32 `json:"flushIntervalSec,omitempty"` + // The interval for metrics emission, in nanoseconds. Works together with flushIntervalSec. + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=999999999 + FlushIntervalNsec *int32 `json:"flushIntervalNsec,omitempty"` } func (*LogToMetrics) Name() string { @@ -109,5 +118,11 @@ func (l *LogToMetrics) Params(_ plugins.SecretLoader) (*params.KVs, error) { if l.DiscardLogs != nil { kvs.Insert("Discard_logs", fmt.Sprintf("%t", *l.DiscardLogs)) } + if l.FlushIntervalSec != nil { + kvs.Insert("Flush_Interval_Sec", fmt.Sprintf("%d", *l.FlushIntervalSec)) + } + if l.FlushIntervalNsec != nil { + kvs.Insert("Flush_Interval_Nsec", fmt.Sprintf("%d", *l.FlushIntervalNsec)) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/filter/log_to_metrics_types_test.go b/apis/fluentbit/v1alpha2/plugins/filter/log_to_metrics_types_test.go new file mode 100644 index 000000000..5f42aba6f --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/filter/log_to_metrics_types_test.go @@ -0,0 +1,49 @@ +package filter + +import ( + "testing" + + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params" + . "github.com/onsi/gomega" +) + +func TestFilter_LogToMetrics_FlushInterval(t *testing.T) { + g := NewGomegaWithT(t) + + sl := plugins.NewSecretLoader(nil, "test namespace") + + l := LogToMetrics{ + Tag: "metrics", + MetricMode: "counter", + MetricName: "count_all", + FlushIntervalSec: new(int32(30)), + FlushIntervalNsec: new(int32(500000000)), + } + + expected := params.NewKVs() + expected.Insert("Tag", "metrics") + expected.Insert("Metric_mode", "counter") + expected.Insert("Metric_name", "count_all") + expected.Insert("Flush_Interval_Sec", "30") + expected.Insert("Flush_Interval_Nsec", "500000000") + + kvs, err := l.Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kvs).To(Equal(expected)) +} + +func TestFilter_LogToMetrics_NoFlushInterval(t *testing.T) { + g := NewGomegaWithT(t) + + sl := plugins.NewSecretLoader(nil, "test namespace") + + l := LogToMetrics{Tag: "metrics"} + + expected := params.NewKVs() + expected.Insert("Tag", "metrics") + + kvs, err := l.Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kvs).To(Equal(expected)) +} diff --git a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go index bc138d563..c73d56803 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go @@ -323,6 +323,16 @@ func (in *LogToMetrics) DeepCopyInto(out *LogToMetrics) { *out = new(bool) **out = **in } + if in.FlushIntervalSec != nil { + in, out := &in.FlushIntervalSec, &out.FlushIntervalSec + *out = new(int32) + **out = **in + } + if in.FlushIntervalNsec != nil { + in, out := &in.FlushIntervalNsec, &out.FlushIntervalNsec + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogToMetrics. diff --git a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterfilters.yaml index 175ed596d..189cac00d 100644 --- a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterfilters.yaml @@ -367,6 +367,21 @@ spec: items: type: string type: array + flushIntervalNsec: + description: The interval for metrics emission, in nanoseconds. + Works together with flushIntervalSec. + format: int32 + maximum: 999999999 + minimum: 0 + type: integer + flushIntervalSec: + description: |- + The interval for metrics emission, in seconds. If flushIntervalSec and flushIntervalNsec are both + unset or 0, metrics are emitted right after each filter match. Otherwise they are emitted at + this interval, which lowers resource usage under high load. + format: int32 + minimum: 0 + type: integer kubernetesMode: description: |- If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name diff --git a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_filters.yaml index 85cedbbdb..cbb01dfbc 100644 --- a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_filters.yaml @@ -367,6 +367,21 @@ spec: items: type: string type: array + flushIntervalNsec: + description: The interval for metrics emission, in nanoseconds. + Works together with flushIntervalSec. + format: int32 + maximum: 999999999 + minimum: 0 + type: integer + flushIntervalSec: + description: |- + The interval for metrics emission, in seconds. If flushIntervalSec and flushIntervalNsec are both + unset or 0, metrics are emitted right after each filter match. Otherwise they are emitted at + this interval, which lowers resource usage under high load. + format: int32 + minimum: 0 + type: integer kubernetesMode: description: |- If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfilters.yaml index f0d37ece6..6f6c908b1 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -365,6 +365,21 @@ spec: items: type: string type: array + flushIntervalNsec: + description: The interval for metrics emission, in nanoseconds. + Works together with flushIntervalSec. + format: int32 + maximum: 999999999 + minimum: 0 + type: integer + flushIntervalSec: + description: |- + The interval for metrics emission, in seconds. If flushIntervalSec and flushIntervalNsec are both + unset or 0, metrics are emitted right after each filter match. Otherwise they are emitted at + this interval, which lowers resource usage under high load. + format: int32 + minimum: 0 + type: integer kubernetesMode: description: |- If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_filters.yaml index 5a9d2993a..80c0f4c69 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_filters.yaml @@ -365,6 +365,21 @@ spec: items: type: string type: array + flushIntervalNsec: + description: The interval for metrics emission, in nanoseconds. + Works together with flushIntervalSec. + format: int32 + maximum: 999999999 + minimum: 0 + type: integer + flushIntervalSec: + description: |- + The interval for metrics emission, in seconds. If flushIntervalSec and flushIntervalNsec are both + unset or 0, metrics are emitted right after each filter match. Otherwise they are emitted at + this interval, which lowers resource usage under high load. + format: int32 + minimum: 0 + type: integer kubernetesMode: description: |- If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml index 6af7e8cc4..9d4bf849b 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -366,6 +366,21 @@ spec: items: type: string type: array + flushIntervalNsec: + description: The interval for metrics emission, in nanoseconds. + Works together with flushIntervalSec. + format: int32 + maximum: 999999999 + minimum: 0 + type: integer + flushIntervalSec: + description: |- + The interval for metrics emission, in seconds. If flushIntervalSec and flushIntervalNsec are both + unset or 0, metrics are emitted right after each filter match. Otherwise they are emitted at + this interval, which lowers resource usage under high load. + format: int32 + minimum: 0 + type: integer kubernetesMode: description: |- If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index 78947f4b4..eddc61552 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -366,6 +366,21 @@ spec: items: type: string type: array + flushIntervalNsec: + description: The interval for metrics emission, in nanoseconds. + Works together with flushIntervalSec. + format: int32 + maximum: 999999999 + minimum: 0 + type: integer + flushIntervalSec: + description: |- + The interval for metrics emission, in seconds. If flushIntervalSec and flushIntervalNsec are both + unset or 0, metrics are emitted right after each filter match. Otherwise they are emitted at + this interval, which lowers resource usage under high load. + format: int32 + minimum: 0 + type: integer kubernetesMode: description: |- If enabled, it will automatically put pod_id, pod_name, namespace_name, docker_id and container_name diff --git a/docs/plugins/fluentbit/filter/log_to_metrics.md b/docs/plugins/fluentbit/filter/log_to_metrics.md index 00ef21537..abcb60a06 100644 --- a/docs/plugins/fluentbit/filter/log_to_metrics.md +++ b/docs/plugins/fluentbit/filter/log_to_metrics.md @@ -21,3 +21,5 @@ The Log To Metrics Filter plugin allows you to generate log-derived metrics.