[cozystack-controller] Ancestor tracking webhook

Many resources created as part of managed apps in cozystack (pods,
secrets, etc) do not carry predictable labels that unambiguously
indicate which app originally triggered their creation. Some resources
are managed by controllers and other custom resources and this
indirection can lead to loss of information. Other controllers sometimes
simply do not allow setting labels on controlled resources and the
latter do not inherit labels from the owner. This patch implements a
webhook that sidesteps this problem with a universal solution. On
creation of a pod/secret/PVC etc it walks through the owner references
until a HelmRelease is found that can be matched with a managed app
dynamically registered in the Cozystack API server. The pod is mutated
with labels identifying the managed app.

```release-note
[cozystack-controller] Add a mutating webhook to identify the Cozystack
managed app that ultimately owns low-level resources created in the
cluster and label these resources with a reference to said app.
```

Signed-off-by: Timofei Larkin <lllamnyp@gmail.com>

internal/lineagecontrollerwebhook/config.go

@@ -0,0 +1,44 @@
+package lineagecontrollerwebhook
+
+import (
+	"fmt"
+
+	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
+	helmv2 "github.com/fluxcd/helm-controller/api/v2"
+)
+
+type chartRef struct {
+	repo  string
+	chart string
+}
+
+type appRef struct {
+	group string
+	kind  string
+}
+
+type runtimeConfig struct {
+	chartAppMap map[chartRef]*cozyv1alpha1.CozystackResourceDefinition
+	appCRDMap   map[appRef]*cozyv1alpha1.CozystackResourceDefinition
+}
+
+func (l *LineageControllerWebhook) initConfig() {
+	l.initOnce.Do(func() {
+		if l.config.Load() == nil {
+			l.config.Store(&runtimeConfig{chartAppMap: make(map[chartRef]*cozyv1alpha1.CozystackResourceDefinition)})
+		}
+	})
+}
+
+func (l *LineageControllerWebhook) Map(hr *helmv2.HelmRelease) (string, string, string, error) {
+	cfg, ok := l.config.Load().(*runtimeConfig)
+	if !ok {
+		return "", "", "", fmt.Errorf("failed to load chart-app mapping from config")
+	}
+	s := hr.Spec.Chart.Spec
+	val, ok := cfg.chartAppMap[chartRef{s.SourceRef.Name, s.Chart}]
+	if !ok {
+		return "", "", "", fmt.Errorf("cannot map helm release %s/%s to dynamic app", hr.Namespace, hr.Name)
+	}
+	return "apps.cozystack.io/v1alpha1", val.Spec.Application.Kind, val.Spec.Release.Prefix, nil
+}

internal/lineagecontrollerwebhook/controller.go

@@ -0,0 +1,55 @@
+package lineagecontrollerwebhook
+
+import (
+	"context"
+
+	cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+// +kubebuilder:rbac:groups=cozystack.io,resources=cozystackresourcedefinitions,verbs=list;watch
+
+func (c *LineageControllerWebhook) SetupWithManagerAsController(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&cozyv1alpha1.CozystackResourceDefinition{}).
+		Complete(c)
+}
+
+func (c *LineageControllerWebhook) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	l := log.FromContext(ctx)
+	crds := &cozyv1alpha1.CozystackResourceDefinitionList{}
+	if err := c.List(ctx, crds, &client.ListOptions{Namespace: "cozy-system"}); err != nil {
+		l.Error(err, "failed reading CozystackResourceDefinitions")
+		return ctrl.Result{}, err
+	}
+	cfg := &runtimeConfig{
+		chartAppMap: make(map[chartRef]*cozyv1alpha1.CozystackResourceDefinition),
+		appCRDMap:   make(map[appRef]*cozyv1alpha1.CozystackResourceDefinition),
+	}
+	for _, crd := range crds.Items {
+		chRef := chartRef{
+			crd.Spec.Release.Chart.SourceRef.Name,
+			crd.Spec.Release.Chart.Name,
+		}
+		appRef := appRef{
+			"apps.cozystack.io",
+			crd.Spec.Application.Kind,
+		}
+
+		newRef := crd
+		if _, exists := cfg.chartAppMap[chRef]; exists {
+			l.Info("duplicate chart mapping detected; ignoring subsequent entry", "key", chRef)
+		} else {
+			cfg.chartAppMap[chRef] = &newRef
+		}
+		if _, exists := cfg.appCRDMap[appRef]; exists {
+			l.Info("duplicate app mapping detected; ignoring subsequent entry", "key", appRef)
+		} else {
+			cfg.appCRDMap[appRef] = &newRef
+		}
+	}
+	c.config.Store(cfg)
+	return ctrl.Result{}, nil
+}

internal/lineagecontrollerwebhook/matcher.go

@@ -0,0 +1,34 @@
+package lineagecontrollerwebhook
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+func matchLabelsToSelector(l map[string]string, s *metav1.LabelSelector) bool {
+	// TODO: emit warning if error
+	sel, err := metav1.LabelSelectorAsSelector(s)
+	if err != nil {
+		return false
+	}
+	return sel.Matches(labels.Set(l))
+}
+
+func matchLabelsToSelectorArray(l map[string]string, ss []*metav1.LabelSelector) bool {
+	for _, s := range ss {
+		if matchLabelsToSelector(l, s) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchLabelsToExcludeInclude(l map[string]string, ex, in []*metav1.LabelSelector) bool {
+	if matchLabelsToSelectorArray(l, ex) {
+		return false
+	}
+	if matchLabelsToSelectorArray(l, in) {
+		return true
+	}
+	return false
+}

internal/lineagecontrollerwebhook/types.go

@@ -0,0 +1,23 @@
+package lineagecontrollerwebhook
+
+import (
+	"sync"
+	"sync/atomic"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/dynamic"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// +kubebuilder:webhook:path=/mutate-lineage,mutating=true,failurePolicy=Fail,sideEffects=None,groups="",resources=pods,secrets,services,persistentvolumeclaims,verbs=create;update,versions=v1,name=mlineage.cozystack.io,admissionReviewVersions={v1}
+type LineageControllerWebhook struct {
+	client.Client
+	Scheme    *runtime.Scheme
+	decoder   admission.Decoder
+	dynClient dynamic.Interface
+	mapper    meta.RESTMapper
+	config    atomic.Value
+	initOnce  sync.Once
+}

internal/lineagecontrollerwebhook/webhook.go

@@ -0,0 +1,180 @@
+package lineagecontrollerwebhook
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/cozystack/cozystack/pkg/lineage"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/discovery/cached/memory"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+var (
+	NoAncestors       = fmt.Errorf("no managed apps found in lineage")
+	AncestryAmbiguous = fmt.Errorf("object ancestry is ambiguous")
+)
+
+// SetupWithManager registers the handler with the webhook server.
+func (h *LineageControllerWebhook) SetupWithManagerAsWebhook(mgr ctrl.Manager) error {
+	cfg := rest.CopyConfig(mgr.GetConfig())
+
+	var err error
+	h.dynClient, err = dynamic.NewForConfig(cfg)
+	if err != nil {
+		return err
+	}
+
+	discoClient, err := discovery.NewDiscoveryClientForConfig(cfg)
+	if err != nil {
+		return err
+	}
+
+	cachedDisco := memory.NewMemCacheClient(discoClient)
+	h.mapper = restmapper.NewDeferredDiscoveryRESTMapper(cachedDisco)
+
+	h.initConfig()
+	// Register HTTP path -> handler.
+	mgr.GetWebhookServer().Register("/mutate-lineage", &admission.Webhook{Handler: h})
+
+	return nil
+}
+
+// InjectDecoder lets controller-runtime give us a decoder for AdmissionReview requests.
+func (h *LineageControllerWebhook) InjectDecoder(d admission.Decoder) error {
+	h.decoder = d
+	return nil
+}
+
+// Handle is called for each AdmissionReview that matches the webhook config.
+func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Request) admission.Response {
+	logger := log.FromContext(ctx).WithValues(
+		"gvk", req.Kind.String(),
+		"namespace", req.Namespace,
+		"name", req.Name,
+		"operation", req.Operation,
+	)
+	warn := make(admission.Warnings, 0)
+
+	obj := &unstructured.Unstructured{}
+	if err := h.decodeUnstructured(req, obj); err != nil {
+		return admission.Errored(400, fmt.Errorf("decode object: %w", err))
+	}
+
+	labels, err := h.computeLabels(ctx, obj)
+	for {
+		if err != nil && errors.Is(err, NoAncestors) {
+			return admission.Allowed("object not managed by app")
+		}
+		if err != nil && errors.Is(err, AncestryAmbiguous) {
+			warn = append(warn, "object ancestry ambiguous, using first ancestor found")
+			break
+		}
+		if err != nil {
+			logger.Error(err, "error computing lineage labels")
+			return admission.Errored(500, fmt.Errorf("error computing lineage labels: %w", err))
+		}
+		if err == nil {
+			break
+		}
+	}
+
+	h.applyLabels(obj, labels)
+
+	mutated, err := json.Marshal(obj)
+	if err != nil {
+		return admission.Errored(500, fmt.Errorf("marshal mutated pod: %w", err))
+	}
+	logger.V(1).Info("mutated pod", "namespace", obj.GetNamespace(), "name", obj.GetName())
+	return admission.PatchResponseFromRaw(req.Object.Raw, mutated).WithWarnings(warn...)
+}
+
+func (h *LineageControllerWebhook) computeLabels(ctx context.Context, o *unstructured.Unstructured) (map[string]string, error) {
+	owners := lineage.WalkOwnershipGraph(ctx, h.dynClient, h.mapper, h, o)
+	if len(owners) == 0 {
+		return nil, NoAncestors
+	}
+	obj, err := owners[0].GetUnstructured(ctx, h.dynClient, h.mapper)
+	if err != nil {
+		return nil, err
+	}
+	gv, err := schema.ParseGroupVersion(obj.GetAPIVersion())
+	if err != nil {
+		// should never happen, we got an APIVersion right from the API
+		return nil, fmt.Errorf("could not parse APIVersion %s to a group and version: %w", obj.GetAPIVersion(), err)
+	}
+	if len(owners) > 1 {
+		err = AncestryAmbiguous
+	}
+	labels := map[string]string{
+		// truncate apigroup to first 63 chars
+		"apps.cozystack.io/application.group": func(s string) string {
+			if len(s) < 63 {
+				return s
+			}
+			s = s[:63]
+			for b := s[62]; !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')); s = s[:len(s)-1] {
+				b = s[len(s)-1]
+			}
+			return s
+		}(gv.Group),
+		"apps.cozystack.io/application.kind": obj.GetKind(),
+		"apps.cozystack.io/application.name": obj.GetName(),
+	}
+	if o.GetAPIVersion() != "v1" || o.GetKind() != "Secret" {
+		return labels, err
+	}
+	cfg := h.config.Load().(*runtimeConfig)
+	crd := cfg.appCRDMap[appRef{gv.Group, obj.GetKind()}]
+	if matchLabelsToExcludeInclude(o.GetLabels(), crd.Spec.Secrets.Exclude, crd.Spec.Secrets.Include) {
+		labels["internal.cozystack.io/tenantsecret"] = ""
+	}
+	return labels, err
+}
+
+func (h *LineageControllerWebhook) applyLabels(o *unstructured.Unstructured, labels map[string]string) {
+	if o.GetAPIVersion() == "operator.victoriametrics.com/v1beta1" && o.GetKind() == "VMCluster" {
+		unstructured.SetNestedStringMap(o.Object, labels, "spec", "managedMetadata", "labels")
+		return
+	}
+
+	existing := o.GetLabels()
+	if existing == nil {
+		existing = make(map[string]string)
+	}
+	for k, v := range labels {
+		existing[k] = v
+	}
+	o.SetLabels(existing)
+}
+
+func (h *LineageControllerWebhook) decodeUnstructured(req admission.Request, out *unstructured.Unstructured) error {
+	if h.decoder != nil {
+		if err := h.decoder.Decode(req, out); err == nil {
+			return nil
+		}
+		if req.Kind.Group != "" || req.Kind.Kind != "" || req.Kind.Version != "" {
+			out.SetGroupVersionKind(schema.GroupVersionKind{
+				Group:   req.Kind.Group,
+				Version: req.Kind.Version,
+				Kind:    req.Kind.Kind,
+			})
+			if err := h.decoder.Decode(req, out); err == nil {
+				return nil
+			}
+		}
+	}
+	if len(req.Object.Raw) == 0 {
+		return errors.New("empty admission object")
+	}
+	return json.Unmarshal(req.Object.Raw, &out.Object)
+}