mirror of
https://github.com/outbackdingo/cozystack.git
synced 2026-01-27 18:18:41 +00:00
[cozystack-controller] Implement cache for CozystackResourceDefinitions (#1427)
Signed-off-by: Andrei Kvapil <kvapss@gmail.com> <!-- Thank you for making a contribution! Here are some tips for you: - Start the PR title with the [label] of Cozystack component: - For system components: [platform], [system], [linstor], [cilium], [kube-ovn], [dashboard], [cluster-api], etc. - For managed apps: [apps], [tenant], [kubernetes], [postgres], [virtual-machine] etc. - For development and maintenance: [tests], [ci], [docs], [maintenance]. - If it's a work in progress, consider creating this PR as a draft. - Don't hesistate to ask for opinion and review in the community chats, even if it's still a draft. - Add the label `backport` if it's a bugfix that needs to be backported to a previous version. --> ## What this PR does This PR introduces shared cache for CozystackResourceDefinitions and warbs it up before making decidion on restart cozystack-api server. Reastart logic was also updated to trigger restart only if consistent hash from the configuration has been changed. ### Release note <!-- Write a release note: - Explain what has changed internally and for users. - Start with the same [label] as in the PR title - Follow the guidelines at https://github.com/kubernetes/community/blob/master/contributors/guide/release-notes.md. --> ```release-note [cozystack-controller] Implement cache for CozystackResourceDefinitions ``` <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - New Features - Smarter, hash-based restarts for the API component, triggered only when configuration truly changes. - Debounced restart behavior to avoid rapid, repeated restarts during bursts of updates. - Performance - Introduces an internal in-memory configuration cache to speed up evaluations and reduce API calls. - Cache is primed at startup for faster, more responsive operations. - Bug Fixes - Eliminates unnecessary restarts when there are no effective config changes, improving stability and reducing disruption. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -2,14 +2,25 @@ package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cozystack/cozystack/internal/shared/crdmem"
|
||||
|
||||
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/builder"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
@@ -20,85 +31,55 @@ type CozystackResourceDefinitionReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
|
||||
// Configurable debounce duration
|
||||
Debounce time.Duration
|
||||
|
||||
// Internal state for debouncing
|
||||
mu sync.Mutex
|
||||
lastEvent time.Time // Time of last CRUD event on CozystackResourceDefinition
|
||||
lastHandled time.Time // Last time the Deployment was actually restarted
|
||||
lastEvent time.Time
|
||||
lastHandled time.Time
|
||||
|
||||
mem *crdmem.Memory
|
||||
}
|
||||
|
||||
// Reconcile handles the logic to restart the target Deployment only once,
|
||||
// even if multiple events occur close together
|
||||
func (r *CozystackResourceDefinitionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
log := log.FromContext(ctx)
|
||||
logger := log.FromContext(ctx)
|
||||
|
||||
// Only respond to our target deployment
|
||||
if req.Namespace != "cozy-system" || req.Name != "cozystack-api" {
|
||||
crd := &cozyv1alpha1.CozystackResourceDefinition{}
|
||||
err := r.Get(ctx, types.NamespacedName{Name: req.Name}, crd)
|
||||
if err == nil {
|
||||
if r.mem != nil {
|
||||
r.mem.Upsert(crd)
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
r.lastEvent = time.Now()
|
||||
r.mu.Unlock()
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
le := r.lastEvent
|
||||
lh := r.lastHandled
|
||||
debounce := r.Debounce
|
||||
r.mu.Unlock()
|
||||
|
||||
if debounce <= 0 {
|
||||
debounce = 5 * time.Second
|
||||
}
|
||||
|
||||
// No events received yet — nothing to do
|
||||
if le.IsZero() {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// Wait until the debounce duration has passed since the last event
|
||||
if d := time.Since(le); d < debounce {
|
||||
return ctrl.Result{RequeueAfter: debounce - d}, nil
|
||||
}
|
||||
|
||||
// Already handled this event — skip restart
|
||||
if !lh.Before(le) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// Perform the restart by patching the deployment annotation
|
||||
deploy := &appsv1.Deployment{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: "cozy-system", Name: "cozystack-api"}, deploy); err != nil {
|
||||
log.Error(err, "Failed to get Deployment cozy-system/cozystack-api")
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
patch := client.MergeFrom(deploy.DeepCopy())
|
||||
if deploy.Spec.Template.Annotations == nil {
|
||||
deploy.Spec.Template.Annotations = make(map[string]string)
|
||||
}
|
||||
deploy.Spec.Template.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339)
|
||||
|
||||
if err := r.Patch(ctx, deploy, patch); err != nil {
|
||||
log.Error(err, "Failed to patch Deployment annotation")
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
// Mark this event as handled
|
||||
r.mu.Lock()
|
||||
r.lastHandled = le
|
||||
r.mu.Unlock()
|
||||
|
||||
log.Info("Deployment cozy-system/cozystack-api successfully restarted")
|
||||
if apierrors.IsNotFound(err) && r.mem != nil {
|
||||
r.mem.Delete(req.Name)
|
||||
}
|
||||
if req.Namespace == "cozy-system" && req.Name == "cozystack-api" {
|
||||
return r.debouncedRestart(ctx, logger)
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// SetupWithManager configures how the controller listens to events
|
||||
func (r *CozystackResourceDefinitionReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
if r.Debounce == 0 {
|
||||
r.Debounce = 5 * time.Second
|
||||
}
|
||||
|
||||
if r.mem == nil {
|
||||
r.mem = crdmem.Global()
|
||||
}
|
||||
if err := r.mem.EnsurePrimingWithManager(mgr); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
Named("cozystack-restart-controller").
|
||||
Named("cozystackresource-controller").
|
||||
For(&cozyv1alpha1.CozystackResourceDefinition{}, builder.WithPredicates()).
|
||||
Watches(
|
||||
&cozyv1alpha1.CozystackResourceDefinition{},
|
||||
handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request {
|
||||
@@ -115,3 +96,88 @@ func (r *CozystackResourceDefinitionReconciler) SetupWithManager(mgr ctrl.Manage
|
||||
).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
type crdHashView struct {
|
||||
Name string `json:"name"`
|
||||
Spec cozyv1alpha1.CozystackResourceDefinitionSpec `json:"spec"`
|
||||
}
|
||||
|
||||
func (r *CozystackResourceDefinitionReconciler) computeConfigHash() (string, error) {
|
||||
if r.mem == nil {
|
||||
return "", nil
|
||||
}
|
||||
snapshot := r.mem.Snapshot()
|
||||
sort.Slice(snapshot, func(i, j int) bool { return snapshot[i].Name < snapshot[j].Name })
|
||||
views := make([]crdHashView, 0, len(snapshot))
|
||||
for i := range snapshot {
|
||||
views = append(views, crdHashView{
|
||||
Name: snapshot[i].Name,
|
||||
Spec: snapshot[i].Spec,
|
||||
})
|
||||
}
|
||||
b, err := json.Marshal(views)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func (r *CozystackResourceDefinitionReconciler) debouncedRestart(ctx context.Context, logger logr.Logger) (ctrl.Result, error) {
|
||||
r.mu.Lock()
|
||||
le := r.lastEvent
|
||||
lh := r.lastHandled
|
||||
debounce := r.Debounce
|
||||
r.mu.Unlock()
|
||||
|
||||
if debounce <= 0 {
|
||||
debounce = 5 * time.Second
|
||||
}
|
||||
if le.IsZero() {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
if d := time.Since(le); d < debounce {
|
||||
return ctrl.Result{RequeueAfter: debounce - d}, nil
|
||||
}
|
||||
if !lh.Before(le) {
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
newHash, err := r.computeConfigHash()
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
deploy := &appsv1.Deployment{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: "cozy-system", Name: "cozystack-api"}, deploy); err != nil {
|
||||
return ctrl.Result{}, client.IgnoreNotFound(err)
|
||||
}
|
||||
|
||||
if deploy.Spec.Template.Annotations == nil {
|
||||
deploy.Spec.Template.Annotations = map[string]string{}
|
||||
}
|
||||
oldHash := deploy.Spec.Template.Annotations["cozystack.io/config-hash"]
|
||||
|
||||
if oldHash == newHash && oldHash != "" {
|
||||
r.mu.Lock()
|
||||
r.lastHandled = le
|
||||
r.mu.Unlock()
|
||||
logger.Info("No changes in CRD config; skipping restart", "hash", newHash)
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
patch := client.MergeFrom(deploy.DeepCopy())
|
||||
deploy.Spec.Template.Annotations["cozystack.io/config-hash"] = newHash
|
||||
|
||||
if err := r.Patch(ctx, deploy, patch); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
r.lastHandled = le
|
||||
r.mu.Unlock()
|
||||
|
||||
logger.Info("Updated cozystack-api podTemplate config-hash; rollout triggered",
|
||||
"old", oldHash, "new", newHash)
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
99
internal/shared/crdmem/memory.go
Normal file
99
internal/shared/crdmem/memory.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package crdmem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
type Memory struct {
|
||||
mu sync.RWMutex
|
||||
data map[string]cozyv1alpha1.CozystackResourceDefinition
|
||||
primed bool
|
||||
primeOnce sync.Once
|
||||
}
|
||||
|
||||
func New() *Memory {
|
||||
return &Memory{data: make(map[string]cozyv1alpha1.CozystackResourceDefinition)}
|
||||
}
|
||||
|
||||
var (
|
||||
global *Memory
|
||||
globalOnce sync.Once
|
||||
)
|
||||
|
||||
func Global() *Memory {
|
||||
globalOnce.Do(func() { global = New() })
|
||||
return global
|
||||
}
|
||||
|
||||
func (m *Memory) Upsert(obj *cozyv1alpha1.CozystackResourceDefinition) {
|
||||
if obj == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.data[obj.Name] = *obj.DeepCopy()
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *Memory) Delete(name string) {
|
||||
m.mu.Lock()
|
||||
delete(m.data, name)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *Memory) Snapshot() []cozyv1alpha1.CozystackResourceDefinition {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
out := make([]cozyv1alpha1.CozystackResourceDefinition, 0, len(m.data))
|
||||
for _, v := range m.data {
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Memory) IsPrimed() bool {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.primed
|
||||
}
|
||||
|
||||
type runnable func(context.Context) error
|
||||
|
||||
func (r runnable) Start(ctx context.Context) error { return r(ctx) }
|
||||
|
||||
func (m *Memory) EnsurePrimingWithManager(mgr ctrl.Manager) error {
|
||||
var errOut error
|
||||
m.primeOnce.Do(func() {
|
||||
errOut = mgr.Add(runnable(func(ctx context.Context) error {
|
||||
if ok := mgr.GetCache().WaitForCacheSync(ctx); !ok {
|
||||
return nil
|
||||
}
|
||||
var list cozyv1alpha1.CozystackResourceDefinitionList
|
||||
if err := mgr.GetClient().List(ctx, &list); err == nil {
|
||||
for i := range list.Items {
|
||||
m.Upsert(&list.Items[i])
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.primed = true
|
||||
m.mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
})
|
||||
return errOut
|
||||
}
|
||||
|
||||
func (m *Memory) ListFromCacheOrAPI(ctx context.Context, c client.Client) ([]cozyv1alpha1.CozystackResourceDefinition, error) {
|
||||
if m.IsPrimed() {
|
||||
return m.Snapshot(), nil
|
||||
}
|
||||
var list cozyv1alpha1.CozystackResourceDefinitionList
|
||||
if err := c.List(ctx, &list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return list.Items, nil
|
||||
}
|
||||
Reference in New Issue
Block a user