From e5e8b638f76d0d5b10be11619f01b41a7877e4ac Mon Sep 17 00:00:00 2001 From: Srinivas Chippagiri Date: Thu, 22 Oct 2026 16:19:04 -0700 Subject: [PATCH] feat(fluentbit): add timeStrict option to parsers Expose Fluent Bit's Time_Strict setting on the json, logfmt, ltsv and regex parsers so users can turn off strict time parsing, e.g. when a timestamp has no fractional seconds but the format expects them. Fixes #1433 Signed-off-by: Srinivas Chippagiri --- .../v1alpha2/plugins/parser/json_types.go | 5 ++ .../v1alpha2/plugins/parser/logfmt_types.go | 5 ++ .../v1alpha2/plugins/parser/lstv_types.go | 9 +++- .../v1alpha2/plugins/parser/regex_types.go | 5 ++ .../plugins/parser/time_strict_test.go | 46 +++++++++++++++++++ .../plugins/parser/zz_generated.deepcopy.go | 20 ++++++++ .../fluentbit.fluent.io_clusterparsers.yaml | 12 +++++ .../fluentbit.fluent.io_parsers.yaml | 12 +++++ .../fluentbit.fluent.io_clusterparsers.yaml | 12 +++++ .../crds/fluentbit.fluent.io_parsers.yaml | 12 +++++ .../fluentbit.fluent.io_clusterparsers.yaml | 12 +++++ .../bases/fluentbit.fluent.io_parsers.yaml | 12 +++++ docs/plugins/fluentbit/parser/json.md | 1 + docs/plugins/fluentbit/parser/logfmt.md | 1 + docs/plugins/fluentbit/parser/lstv.md | 1 + docs/plugins/fluentbit/parser/regex.md | 1 + manifests/setup/setup.yaml | 24 ++++++++++ 17 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 apis/fluentbit/v1alpha2/plugins/parser/time_strict_test.go diff --git a/apis/fluentbit/v1alpha2/plugins/parser/json_types.go b/apis/fluentbit/v1alpha2/plugins/parser/json_types.go index de819befc..b6d09b524 100644 --- a/apis/fluentbit/v1alpha2/plugins/parser/json_types.go +++ b/apis/fluentbit/v1alpha2/plugins/parser/json_types.go @@ -18,6 +18,8 @@ type JSON struct { TimeFormat string `json:"timeFormat,omitempty"` // Time_Keep TimeKeep *bool `json:"timeKeep,omitempty"` + // Time_Strict + TimeStrict *bool `json:"timeStrict,omitempty"` } func (*JSON) Name() string { @@ -35,5 +37,8 @@ func (j *JSON) Params(_ plugins.SecretLoader) (*params.KVs, error) { if j.TimeKeep != nil { kvs.Insert("Time_Keep", fmt.Sprint(*j.TimeKeep)) } + if j.TimeStrict != nil { + kvs.Insert("Time_Strict", fmt.Sprint(*j.TimeStrict)) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/parser/logfmt_types.go b/apis/fluentbit/v1alpha2/plugins/parser/logfmt_types.go index 98af4fa8f..cb8640f0d 100644 --- a/apis/fluentbit/v1alpha2/plugins/parser/logfmt_types.go +++ b/apis/fluentbit/v1alpha2/plugins/parser/logfmt_types.go @@ -18,6 +18,8 @@ type Logfmt struct { TimeFormat string `json:"timeFormat,omitempty"` // Time_Keep TimeKeep *bool `json:"timeKeep,omitempty"` + // Time_Strict + TimeStrict *bool `json:"timeStrict,omitempty"` } func (*Logfmt) Name() string { @@ -35,5 +37,8 @@ func (l *Logfmt) Params(_ plugins.SecretLoader) (*params.KVs, error) { if l.TimeKeep != nil { kvs.Insert("Time_Keep", fmt.Sprint(*l.TimeKeep)) } + if l.TimeStrict != nil { + kvs.Insert("Time_Strict", fmt.Sprint(*l.TimeStrict)) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/parser/lstv_types.go b/apis/fluentbit/v1alpha2/plugins/parser/lstv_types.go index f33e5b297..34e309999 100644 --- a/apis/fluentbit/v1alpha2/plugins/parser/lstv_types.go +++ b/apis/fluentbit/v1alpha2/plugins/parser/lstv_types.go @@ -17,8 +17,10 @@ type LSTV struct { // Time_Format, eg. %Y-%m-%dT%H:%M:%S %z TimeFormat string `json:"timeFormat,omitempty"` // Time_Keep - TimeKeep *bool `json:"timeKeep,omitempty"` - Types string `json:"types,omitempty"` + TimeKeep *bool `json:"timeKeep,omitempty"` + // Time_Strict + TimeStrict *bool `json:"timeStrict,omitempty"` + Types string `json:"types,omitempty"` } func (*LSTV) Name() string { @@ -36,6 +38,9 @@ func (l *LSTV) Params(_ plugins.SecretLoader) (*params.KVs, error) { if l.TimeKeep != nil { kvs.Insert("Time_Format", fmt.Sprint(*l.TimeKeep)) } + if l.TimeStrict != nil { + kvs.Insert("Time_Strict", fmt.Sprint(*l.TimeStrict)) + } if l.Types != "" { kvs.Insert("Types", l.Types) } diff --git a/apis/fluentbit/v1alpha2/plugins/parser/regex_types.go b/apis/fluentbit/v1alpha2/plugins/parser/regex_types.go index f08113328..bf8ecc6aa 100644 --- a/apis/fluentbit/v1alpha2/plugins/parser/regex_types.go +++ b/apis/fluentbit/v1alpha2/plugins/parser/regex_types.go @@ -19,6 +19,8 @@ type Regex struct { TimeFormat string `json:"timeFormat,omitempty"` // Time_Keep TimeKeep *bool `json:"timeKeep,omitempty"` + // Time_Strict + TimeStrict *bool `json:"timeStrict,omitempty"` // Time_Offset, eg. +0200 TimeOffset string `json:"timeOffset,omitempty"` Types string `json:"types,omitempty"` @@ -42,6 +44,9 @@ func (re *Regex) Params(_ plugins.SecretLoader) (*params.KVs, error) { if re.TimeKeep != nil { kvs.Insert("Time_Keep", fmt.Sprint(*re.TimeKeep)) } + if re.TimeStrict != nil { + kvs.Insert("Time_Strict", fmt.Sprint(*re.TimeStrict)) + } if re.TimeOffset != "" { kvs.Insert("Time_Offset", re.TimeOffset) } diff --git a/apis/fluentbit/v1alpha2/plugins/parser/time_strict_test.go b/apis/fluentbit/v1alpha2/plugins/parser/time_strict_test.go new file mode 100644 index 000000000..d9ab6af8b --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/parser/time_strict_test.go @@ -0,0 +1,46 @@ +package parser + +import ( + "testing" + + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" + . "github.com/onsi/gomega" +) + +func TestParsers_TimeStrict(t *testing.T) { + strict := false + + tests := []struct { + name string + plugin plugins.Plugin + }{ + {"json", &JSON{TimeStrict: &strict}}, + {"logfmt", &Logfmt{TimeStrict: &strict}}, + {"ltsv", &LSTV{TimeStrict: &strict}}, + {"regex", &Regex{Regex: "^(?.*)$", TimeStrict: &strict}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + kvs, err := tt.plugin.Params(plugins.NewSecretLoader(nil, "test")) + g.Expect(err).NotTo(HaveOccurred()) + out, err := kvs.String() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).To(MatchRegexp(`Time_Strict\s+false\n`)) + }) + } +} + +func TestParsers_TimeStrictUnset(t *testing.T) { + g := NewGomegaWithT(t) + + for _, p := range []plugins.Plugin{&JSON{}, &Logfmt{}, &LSTV{}, &Regex{}} { + kvs, err := p.Params(plugins.NewSecretLoader(nil, "test")) + g.Expect(err).NotTo(HaveOccurred()) + out, err := kvs.String() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).NotTo(ContainSubstring("Time_Strict")) + } +} diff --git a/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go index ffaf6c89a..d0cb0c9d5 100644 --- a/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/parser/zz_generated.deepcopy.go @@ -30,6 +30,11 @@ func (in *JSON) DeepCopyInto(out *JSON) { *out = new(bool) **out = **in } + if in.TimeStrict != nil { + in, out := &in.TimeStrict, &out.TimeStrict + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JSON. @@ -50,6 +55,11 @@ func (in *LSTV) DeepCopyInto(out *LSTV) { *out = new(bool) **out = **in } + if in.TimeStrict != nil { + in, out := &in.TimeStrict, &out.TimeStrict + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LSTV. @@ -70,6 +80,11 @@ func (in *Logfmt) DeepCopyInto(out *Logfmt) { *out = new(bool) **out = **in } + if in.TimeStrict != nil { + in, out := &in.TimeStrict, &out.TimeStrict + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Logfmt. @@ -90,6 +105,11 @@ func (in *Regex) DeepCopyInto(out *Regex) { *out = new(bool) **out = **in } + if in.TimeStrict != nil { + in, out := &in.TimeStrict, &out.TimeStrict + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Regex. diff --git a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterparsers.yaml b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterparsers.yaml index ef16da822..4073b010b 100644 --- a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterparsers.yaml +++ b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterparsers.yaml @@ -72,6 +72,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object logfmt: description: Logfmt defines logfmt parser configuration. @@ -85,6 +88,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object x-kubernetes-validations: - message: logfmt must not be null @@ -101,6 +107,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -121,6 +130,9 @@ spec: timeOffset: description: Time_Offset, eg. +0200 type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object diff --git a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_parsers.yaml b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_parsers.yaml index ae30840cb..a705da3ee 100644 --- a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_parsers.yaml +++ b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_parsers.yaml @@ -72,6 +72,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object logfmt: description: Logfmt defines logfmt parser configuration. @@ -85,6 +88,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object x-kubernetes-validations: - message: logfmt must not be null @@ -101,6 +107,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -121,6 +130,9 @@ spec: timeOffset: description: Time_Offset, eg. +0200 type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_clusterparsers.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_clusterparsers.yaml index eb01a86b3..ab213f74d 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_clusterparsers.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_clusterparsers.yaml @@ -70,6 +70,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object logfmt: description: Logfmt defines logfmt parser configuration. @@ -83,6 +86,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object x-kubernetes-validations: - message: logfmt must not be null @@ -99,6 +105,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -119,6 +128,9 @@ spec: timeOffset: description: Time_Offset, eg. +0200 type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_parsers.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_parsers.yaml index cd4027856..3fa5742a9 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_parsers.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_parsers.yaml @@ -70,6 +70,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object logfmt: description: Logfmt defines logfmt parser configuration. @@ -83,6 +86,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object x-kubernetes-validations: - message: logfmt must not be null @@ -99,6 +105,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -119,6 +128,9 @@ spec: timeOffset: description: Time_Offset, eg. +0200 type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object diff --git a/config/crd/bases/fluentbit.fluent.io_clusterparsers.yaml b/config/crd/bases/fluentbit.fluent.io_clusterparsers.yaml index 5d0e41164..640ccc350 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterparsers.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterparsers.yaml @@ -71,6 +71,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object logfmt: description: Logfmt defines logfmt parser configuration. @@ -84,6 +87,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object x-kubernetes-validations: - message: logfmt must not be null @@ -100,6 +106,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -120,6 +129,9 @@ spec: timeOffset: description: Time_Offset, eg. +0200 type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object diff --git a/config/crd/bases/fluentbit.fluent.io_parsers.yaml b/config/crd/bases/fluentbit.fluent.io_parsers.yaml index 05beebf22..4e83c7010 100644 --- a/config/crd/bases/fluentbit.fluent.io_parsers.yaml +++ b/config/crd/bases/fluentbit.fluent.io_parsers.yaml @@ -71,6 +71,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object logfmt: description: Logfmt defines logfmt parser configuration. @@ -84,6 +87,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object x-kubernetes-validations: - message: logfmt must not be null @@ -100,6 +106,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -120,6 +129,9 @@ spec: timeOffset: description: Time_Offset, eg. +0200 type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object diff --git a/docs/plugins/fluentbit/parser/json.md b/docs/plugins/fluentbit/parser/json.md index 18ac80bb4..d7d9ec193 100644 --- a/docs/plugins/fluentbit/parser/json.md +++ b/docs/plugins/fluentbit/parser/json.md @@ -8,3 +8,4 @@ The JSON parser plugin.
**For full documentation, refer to https://docs.f | timeKey | Time_Key | string | | timeFormat | Time_Format, eg. %Y-%m-%dT%H:%M:%S %z | string | | timeKeep | Time_Keep | *bool | +| timeStrict | Time_Strict | *bool | diff --git a/docs/plugins/fluentbit/parser/logfmt.md b/docs/plugins/fluentbit/parser/logfmt.md index b58493c95..c48086994 100644 --- a/docs/plugins/fluentbit/parser/logfmt.md +++ b/docs/plugins/fluentbit/parser/logfmt.md @@ -8,3 +8,4 @@ The logfmt parser allows to parse the logfmt format described in https://brandur | timeKey | Time_Key | string | | timeFormat | Time_Format, eg. %Y-%m-%dT%H:%M:%S %z | string | | timeKeep | Time_Keep | *bool | +| timeStrict | Time_Strict | *bool | diff --git a/docs/plugins/fluentbit/parser/lstv.md b/docs/plugins/fluentbit/parser/lstv.md index efaab907d..4e6437e82 100644 --- a/docs/plugins/fluentbit/parser/lstv.md +++ b/docs/plugins/fluentbit/parser/lstv.md @@ -8,4 +8,5 @@ The LSTV parser allows to parse LTSV formatted texts.
**For full document | timeKey | Time_Key | string | | timeFormat | Time_Format, eg. %Y-%m-%dT%H:%M:%S %z | string | | timeKeep | Time_Keep | *bool | +| timeStrict | Time_Strict | *bool | | types | | string | diff --git a/docs/plugins/fluentbit/parser/regex.md b/docs/plugins/fluentbit/parser/regex.md index ce7ebda07..865e19887 100644 --- a/docs/plugins/fluentbit/parser/regex.md +++ b/docs/plugins/fluentbit/parser/regex.md @@ -10,4 +10,5 @@ The regex parser allows to define a custom Ruby Regular Expression that will use | timeFormat | Time_Format, eg. %Y-%m-%dT%H:%M:%S %z | string | | timeKeep | Time_Keep | *bool | | timeOffset | Time_Offset, eg. +0200 | string | +| timeStrict | Time_Strict | *bool | | types | | string | diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index b0ca45551..61fa9328c 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -11972,6 +11972,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object logfmt: description: Logfmt defines logfmt parser configuration. @@ -11985,6 +11988,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object x-kubernetes-validations: - message: logfmt must not be null @@ -12001,6 +12007,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -12021,6 +12030,9 @@ spec: timeOffset: description: Time_Offset, eg. +0200 type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -43465,6 +43477,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object logfmt: description: Logfmt defines logfmt parser configuration. @@ -43478,6 +43493,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean type: object x-kubernetes-validations: - message: logfmt must not be null @@ -43494,6 +43512,9 @@ spec: timeKey: description: Time_Key type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object @@ -43514,6 +43535,9 @@ spec: timeOffset: description: Time_Offset, eg. +0200 type: string + timeStrict: + description: Time_Strict + type: boolean types: type: string type: object