From eb1ac057299e6206c3f5f5b1e7b471e61ec8ce5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?George=20Ga=C3=A1l?= Date: Wed, 1 Jul 2026 10:35:38 +0200 Subject: [PATCH] Reduce misleading log noise from benign/unavailable integrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On hosts where some integrations are simply not present (e.g. Bottlerocket: no Docker, memory-only journald, SELinux-protected systemd bus, VPC CNI instead of Cilium) the node-agent logs a lot of errors and warnings that do not indicate any real failure. This sends operators down the wrong path when debugging unrelated problems, and the repeated systemd/SELinux warnings in particular consume the global log rate-limiter budget, hiding genuinely useful messages. Adjust these to reflect that they are expected, benign conditions: - cilium: log the missing eBPF map name and that Cilium is assumed unused, instead of a bare "no such file or directory". - container runtime / journald probes: an unavailable integration is normal (only one runtime is present on a node), so log at info level. - systemd: reading unit properties can be permanently denied by the host SELinux policy. Cache the failure per unit so the private bus is not re-queried and the identical warning is logged only once instead of on every scan. - journald: a missing or empty journal path is an expected probe outcome, log at info level. - aws metadata: a missing optional field (e.g. public-ipv4 on a private instance) returns a non-200 response; that is expected, not an error. No behavior changes other than logging; metric collection is unaffected. Signed-off-by: George Gaál --- containers/cilium.go | 8 ++++---- containers/registry.go | 15 ++++++++++----- containers/systemd.go | 23 ++++++++++++++++++++--- logs/journald_reader.go | 4 ++-- node/metadata/aws.go | 4 +++- 5 files changed, 39 insertions(+), 15 deletions(-) diff --git a/containers/cilium.go b/containers/cilium.go index 2494ff9..388a799 100644 --- a/containers/cilium.go +++ b/containers/cilium.go @@ -43,7 +43,7 @@ func init() { &ctmap.CtEntry{}, ) if err != nil { - klog.Infoln(err) + klog.Infof("cilium ebpf-map %s is not available, assuming Cilium is not used: %s", ctmap.MapNameTCP4Global, err) } else { klog.Infoln("found cilium ebpf-map:", ctmap.MapNameTCP4Global) } @@ -53,7 +53,7 @@ func init() { &ctmap.CtEntry{}, ) if err != nil { - klog.Infoln(err) + klog.Infof("cilium ebpf-map %s is not available, assuming Cilium is not used: %s", ctmap.MapNameTCP6Global, err) } else { klog.Infoln("found cilium ebpf-map:", ctmap.MapNameTCP6Global) } @@ -61,7 +61,7 @@ func init() { def := ciliumMaps[n] backends4Map, err = bpf.OpenMap(proc.HostPath(filepath.Join(defaults.BPFFSRoot, defaults.TCGlobalsPath, n)), def.key, def.value) if err != nil { - klog.Infoln(err) + klog.Infof("cilium ebpf-map %s is not available, assuming Cilium is not used: %s", n, err) } else { klog.Infoln("found cilium ebpf-map:", n) break @@ -71,7 +71,7 @@ func init() { def := ciliumMaps[n] backends6Map, err = bpf.OpenMap(proc.HostPath(filepath.Join(defaults.BPFFSRoot, defaults.TCGlobalsPath, n)), def.key, def.value) if err != nil { - klog.Infoln(err) + klog.Infof("cilium ebpf-map %s is not available, assuming Cilium is not used: %s", n, err) } else { klog.Infoln("found cilium ebpf-map:", n) break diff --git a/containers/registry.go b/containers/registry.go index f939c04..e93edc8 100644 --- a/containers/registry.go +++ b/containers/registry.go @@ -97,16 +97,16 @@ func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo, pr return nil, err } if err = DockerdInit(); err != nil { - klog.Warningln(err) + klog.Infoln("dockerd integration is not available:", err) } if err = ContainerdInit(); err != nil { - klog.Warningln(err) + klog.Infoln("containerd integration is not available:", err) } if err = CrioInit(); err != nil { - klog.Warningln(err) + klog.Infoln("cri-o integration is not available:", err) } if err = JournaldInit(); err != nil { - klog.Warningln(err) + klog.Infoln("journald integration is not available:", err) } r := &Registry{ @@ -371,7 +371,12 @@ func (r *Registry) getOrCreateContainer(pid uint32) *Container { } md, err := getContainerMetadata(cg) if err != nil { - klog.Warningf("failed to get container metadata for pid %d -> %s: %s", pid, cg.Id, err) + // Some hosts (e.g. Bottlerocket) deny reading systemd unit properties + // via SELinux. Such failures are permanent and identical for a given + // unit, so log them only once instead of on every scan. + if !errors.Is(err, ErrUnitPropertiesUnavailable) { + klog.Warningf("failed to get container metadata for pid %d -> %s: %s", pid, cg.Id, err) + } return nil } id := calcId(cg, md) diff --git a/containers/systemd.go b/containers/systemd.go index 7e865d4..c82eb95 100644 --- a/containers/systemd.go +++ b/containers/systemd.go @@ -2,6 +2,7 @@ package containers import ( "context" + "errors" "fmt" "os" "strconv" @@ -14,6 +15,12 @@ import ( gdbus "github.com/godbus/dbus/v5" ) +// ErrUnitPropertiesUnavailable is returned when a systemd unit's properties +// have already failed to be read once (e.g. the host's SELinux policy denies +// access to the private systemd bus). It is used to suppress repeated, +// identical log lines for the same unit. +var ErrUnitPropertiesUnavailable = errors.New("systemd unit properties are unavailable") + var ( dbusTimeout = time.Second dbusClient = NewDbusClient() @@ -37,13 +44,15 @@ var ( ) type DbusClient struct { - conn *dbus.Conn - cache map[string]map[string]any + conn *dbus.Conn + cache map[string]map[string]any + failed map[string]bool } func NewDbusClient() *DbusClient { return &DbusClient{ - cache: map[string]map[string]any{}, + cache: map[string]map[string]any{}, + failed: map[string]bool{}, } } @@ -78,6 +87,13 @@ func (c *DbusClient) GetAllPropertiesContext(ctx context.Context, unit string, r if res, ok := c.cache[unit]; ok { return res, nil } + // A unit whose properties could not be read once (most often because the + // host's SELinux policy denies access to the private systemd bus) will keep + // failing. Remember it and return a sentinel error so callers can avoid both + // re-querying the bus and re-logging the same failure on every scan. + if c.failed[unit] { + return nil, ErrUnitPropertiesUnavailable + } if c.conn == nil { if err := c.connect(); err != nil { return nil, err @@ -92,6 +108,7 @@ func (c *DbusClient) GetAllPropertiesContext(ctx context.Context, unit string, r c.close() return c.GetAllPropertiesContext(ctx, unit, false) default: + c.failed[unit] = true return nil, err } } diff --git a/logs/journald_reader.go b/logs/journald_reader.go index 803286a..ee01e9c 100644 --- a/logs/journald_reader.go +++ b/logs/journald_reader.go @@ -32,7 +32,7 @@ func NewJournaldReader(journalPaths ...string) (*JournaldReader, error) { var err error for _, journalPath := range journalPaths { if r.journal, err = sdjournal.NewJournalFromDir(journalPath); err != nil { - klog.Errorf("failed to get journal at %s: %s", journalPath, err) + klog.Infof("journal at %s is not available: %s", journalPath, err) continue } usage, err := r.journal.GetUsage() @@ -41,7 +41,7 @@ func NewJournaldReader(journalPaths ...string) (*JournaldReader, error) { continue } if usage == 0 { - klog.Errorf("journal at %s is empty", journalPath) + klog.Infof("journal at %s is empty", journalPath) r.journal = nil continue } diff --git a/node/metadata/aws.go b/node/metadata/aws.go index 0441818..9221099 100644 --- a/node/metadata/aws.go +++ b/node/metadata/aws.go @@ -64,7 +64,9 @@ func getAwsMetadataVariable(token string, path string) string { r.Header.Set("X-aws-ec2-metadata-token", string(token)) resp, err := httpCallWithTimeout(r) if err != nil { - klog.Errorln(err) + // A missing optional field (e.g. public-ipv4 on a private instance) + // returns a non-200 response; that is expected and not an error. + klog.Infof("aws metadata %q is not available: %s", path, err) return "" } defer resp.Body.Close()