Files
archived-talos/pkg/logging/error_suppress.go
Andrey Smirnov 0454130ad9 feat: suppress controller runtime first N failures on the console
As the controllers might fail with transient errors on machine startup,
but errors are always retried, persisten errors will anyway show up in
the console.

The full `talosctl logs controller-runtime` are not suppressed.

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
2024-07-05 15:36:54 +04:00

80 lines
1.9 KiB
Go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package logging
import (
"sync/atomic"
"go.uber.org/zap/zapcore"
)
// NewControllerErrorSuppressor creates a new controller error suppressor.
//
// It suppresses error logs for a given controller unless it logs >= threshold errors.
// The idea is that all controllers reconcile errors, so if the error is not transient,
// it will be reported enough time to hit the threshold, but transient errors will be
// suppressed.
//
// The suppressor records the controller name by inspecting a log field named "controller"
// passed via `logger.With()` call.
func NewControllerErrorSuppressor(core zapcore.Core, threshold int64) zapcore.Core {
return &consoleSampler{
Core: core,
threshold: threshold,
}
}
type consoleSampler struct {
zapcore.Core
hits *atomic.Int64
threshold int64
controller string
}
var _ zapcore.Core = (*consoleSampler)(nil)
func (s *consoleSampler) Level() zapcore.Level {
return zapcore.LevelOf(s.Core)
}
func (s *consoleSampler) With(fields []zapcore.Field) zapcore.Core {
controller := s.controller
num := s.hits
for _, field := range fields {
if field.Key == "controller" {
if field.String != controller {
controller = field.String
num = new(atomic.Int64)
}
break
}
}
return &consoleSampler{
threshold: s.threshold,
controller: controller,
hits: num,
Core: s.Core.With(fields),
}
}
func (s *consoleSampler) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry {
if !s.Enabled(ent.Level) {
return ce
}
if ent.Level == zapcore.ErrorLevel && s.controller != "" {
if s.hits.Add(1) <= s.threshold {
// suppress the log
return ce
}
}
return s.Core.Check(ent, ce)
}