Revert "[cozystack-controller] Ancestor tracking webhook"

This commit is contained in:
Andrei Kvapil
2025-09-17 12:46:00 +02:00
committed by GitHub
parent 65a734bb65
commit ce522284c4
12 changed files with 1 additions and 686 deletions

View File

@@ -38,7 +38,6 @@ import (
cozystackiov1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
"github.com/cozystack/cozystack/internal/controller"
lcw "github.com/cozystack/cozystack/internal/lineagecontrollerwebhook"
"github.com/cozystack/cozystack/internal/telemetry"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
@@ -215,20 +214,6 @@ func main() {
os.Exit(1)
}
// special one that's both a webhook and a reconciler
lineageControllerWebhook := &lcw.LineageControllerWebhook{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}
if err := lineageControllerWebhook.SetupWithManagerAsController(mgr); err != nil {
setupLog.Error(err, "unable to setup controller", "controller", "LineageController")
os.Exit(1)
}
if err := lineageControllerWebhook.SetupWithManagerAsWebhook(mgr); err != nil {
setupLog.Error(err, "unable to setup webhook", "webhook", "LineageWebhook")
os.Exit(1)
}
// +kubebuilder:scaffold:builder
if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

View File

@@ -1,40 +0,0 @@
package lineagecontrollerwebhook
import (
"fmt"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
)
type chartRef struct {
repo string
chart string
}
type appRef struct {
groupVersion string
kind string
prefix string
}
type runtimeConfig struct {
chartAppMap map[chartRef]appRef
}
func (l *LineageControllerWebhook) initConfig() {
l.initOnce.Do(func() {
if l.config.Load() == nil {
l.config.Store(&runtimeConfig{chartAppMap: make(map[chartRef]appRef)})
}
})
}
func (l *LineageControllerWebhook) Map(hr *helmv2.HelmRelease) (string, string, string, error) {
cfg := l.config.Load().(*runtimeConfig).chartAppMap
s := &hr.Spec.Chart.Spec
val, ok := cfg[chartRef{s.SourceRef.Name, s.Chart}]
if !ok {
return "", "", "", fmt.Errorf("cannot map helm release %s/%s to dynamic app", hr.Namespace, hr.Name)
}
return val.groupVersion, val.kind, val.prefix, nil
}

View File

@@ -1,42 +0,0 @@
package lineagecontrollerwebhook
import (
"context"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)
// +kubebuilder:rbac:groups=cozystack.io,resources=cozystackresourcedefinitions,verbs=list;watch
func (c *LineageControllerWebhook) SetupWithManagerAsController(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&cozyv1alpha1.CozystackResourceDefinition{}).
Complete(c)
}
func (c *LineageControllerWebhook) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
l := log.FromContext(ctx)
crds := &cozyv1alpha1.CozystackResourceDefinitionList{}
if err := c.List(ctx, crds, &client.ListOptions{Namespace: "cozy-system"}); err != nil {
l.Error(err, "failed reading CozystackResourceDefinitions")
return ctrl.Result{}, err
}
newConfig := make(map[chartRef]appRef)
for _, crd := range crds.Items {
k := chartRef{
crd.Spec.Release.Chart.SourceRef.Name,
crd.Spec.Release.Chart.Name,
}
newRef := appRef{"apps.cozystack.io/v1alpha1", crd.Spec.Application.Kind, crd.Spec.Release.Prefix}
if oldRef, exists := newConfig[k]; exists {
l.Info("duplicate chart mapping detected; ignoring subsequent entry", "key", k, "retained value", oldRef, "ignored value", newRef)
continue
}
newConfig[k] = newRef
}
c.config.Store(&runtimeConfig{newConfig})
return ctrl.Result{}, nil
}

View File

@@ -1,23 +0,0 @@
package lineagecontrollerwebhook
import (
"sync"
"sync/atomic"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/dynamic"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)
// +kubebuilder:webhook:path=/mutate-lineage,mutating=true,failurePolicy=Fail,sideEffects=None,groups="",resources=pods,secrets,services,persistentvolumeclaims,verbs=create;update,versions=v1,name=mlineage.cozystack.io,admissionReviewVersions={v1}
type LineageControllerWebhook struct {
client.Client
Scheme *runtime.Scheme
decoder admission.Decoder
dynClient dynamic.Interface
mapper meta.RESTMapper
config atomic.Value
initOnce sync.Once
}

View File

@@ -1,166 +0,0 @@
package lineagecontrollerwebhook
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/cozystack/cozystack/pkg/lineage"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
"k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)
var (
NoAncestors = fmt.Errorf("no managed apps found in lineage")
AncestryAmbiguous = fmt.Errorf("object ancestry is ambiguous")
)
// SetupWithManager registers the handler with the webhook server.
func (h *LineageControllerWebhook) SetupWithManagerAsWebhook(mgr ctrl.Manager) error {
cfg := rest.CopyConfig(mgr.GetConfig())
var err error
h.dynClient, err = dynamic.NewForConfig(cfg)
if err != nil {
return err
}
discoClient, err := discovery.NewDiscoveryClientForConfig(cfg)
if err != nil {
return err
}
cachedDisco := memory.NewMemCacheClient(discoClient)
h.mapper = restmapper.NewDeferredDiscoveryRESTMapper(cachedDisco)
h.initConfig()
// Register HTTP path -> handler.
mgr.GetWebhookServer().Register("/mutate-lineage", &admission.Webhook{Handler: h})
return nil
}
// InjectDecoder lets controller-runtime give us a decoder for AdmissionReview requests.
func (h *LineageControllerWebhook) InjectDecoder(d admission.Decoder) error {
h.decoder = d
return nil
}
// Handle is called for each AdmissionReview that matches the webhook config.
func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Request) admission.Response {
logger := log.FromContext(ctx).WithValues(
"gvk", req.Kind.String(),
"namespace", req.Namespace,
"name", req.Name,
"operation", req.Operation,
)
warn := make(admission.Warnings, 0)
obj := &unstructured.Unstructured{}
if err := h.decodeUnstructured(req, obj); err != nil {
return admission.Errored(400, fmt.Errorf("decode object: %w", err))
}
labels, err := h.computeLabels(ctx, obj)
for {
if err != nil && errors.Is(err, NoAncestors) {
return admission.Allowed("object not managed by app")
}
if err != nil && errors.Is(err, AncestryAmbiguous) {
warn = append(warn, "object ancestry ambiguous, using first ancestor found")
break
}
if err != nil {
return admission.Errored(500, fmt.Errorf("error computing lineage labels: %w", err))
}
if err == nil {
break
}
}
h.applyLabels(obj, labels)
mutated, err := json.Marshal(obj)
if err != nil {
return admission.Errored(500, fmt.Errorf("marshal mutated pod: %w", err))
}
logger.V(1).Info("mutated pod", "namespace", obj.GetNamespace(), "name", obj.GetName())
return admission.PatchResponseFromRaw(req.Object.Raw, mutated).WithWarnings(warn...)
}
func (h *LineageControllerWebhook) computeLabels(ctx context.Context, o *unstructured.Unstructured) (map[string]string, error) {
owners := lineage.WalkOwnershipGraph(ctx, h.dynClient, h.mapper, h, o)
if len(owners) == 0 {
return nil, NoAncestors
}
obj, err := owners[0].GetUnstructured(ctx, h.dynClient, h.mapper)
if err != nil {
return nil, err
}
gv, err := schema.ParseGroupVersion(obj.GetAPIVersion())
if err != nil {
// should never happen, we got an APIVersion right from the API
return nil, fmt.Errorf("could not parse APIVersion %s to a group and version: %w", obj.GetAPIVersion(), err)
}
if len(owners) > 1 {
err = AncestryAmbiguous
}
return map[string]string{
// truncate apigroup to first 63 chars
"apps.cozystack.io/application.group": func(s string) string {
if len(s) < 63 {
return s
}
s = s[:63]
for b := s[62]; !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')); s = s[:len(s)-1] {
b = s[len(s)-1]
}
return s
}(gv.Group),
"apps.cozystack.io/application.kind": obj.GetKind(),
"apps.cozystack.io/application.name": obj.GetName(),
}, err
}
func (h *LineageControllerWebhook) applyLabels(o client.Object, labels map[string]string) {
existing := o.GetLabels()
if existing == nil {
existing = make(map[string]string)
}
for k, v := range labels {
existing[k] = v
}
o.SetLabels(existing)
}
func (h *LineageControllerWebhook) decodeUnstructured(req admission.Request, out *unstructured.Unstructured) error {
if h.decoder != nil {
if err := h.decoder.Decode(req, out); err == nil {
return nil
}
if req.Kind.Group != "" || req.Kind.Kind != "" || req.Kind.Version != "" {
out.SetGroupVersionKind(schema.GroupVersionKind{
Group: req.Kind.Group,
Version: req.Kind.Version,
Kind: req.Kind.Kind,
})
if err := h.decoder.Decode(req, out); err == nil {
return nil
}
}
}
if len(req.Object.Raw) == 0 {
return errors.New("empty admission object")
}
return json.Unmarshal(req.Object.Raw, &out.Object)
}

View File

@@ -1,45 +0,0 @@
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: cozystack-controller-webhook-selfsigned
namespace: {{ .Release.Namespace }}
spec:
selfSigned: {}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: cozystack-controller-webhook-ca
namespace: {{ .Release.Namespace }}
spec:
secretName: cozystack-controller-webhook-ca
duration: 43800h # 5 years
commonName: cozystack-controller-webhook-ca
issuerRef:
name: cozystack-controller-webhook-selfsigned
isCA: true
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: cozystack-controller-webhook-ca
namespace: {{ .Release.Namespace }}
spec:
ca:
secretName: cozystack-controller-webhook-ca
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: cozystack-controller-webhook
namespace: {{ .Release.Namespace }}
spec:
secretName: cozystack-controller-webhook-cert
duration: 8760h
renewBefore: 720h
issuerRef:
name: cozystack-controller-webhook-ca
commonName: cozystack-controller
dnsNames:
- cozystack-controller
- cozystack-controller.{{ .Release.Namespace }}.svc

View File

@@ -2,6 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: cozystack-controller
namespace: cozy-system
labels:
app: cozystack-controller
spec:
@@ -28,15 +29,3 @@ spec:
{{- if .Values.cozystackController.disableTelemetry }}
- --disable-telemetry
{{- end }}
ports:
- name: webhook
containerPort: 9443
volumeMounts:
- name: webhook-certs
mountPath: /tmp/k8s-webhook-server/serving-certs
readOnly: true
volumes:
- name: webhook-certs
secret:
secretName: cozystack-controller-webhook-cert
defaultMode: 0400

View File

@@ -1,33 +0,0 @@
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
name: lineage
annotations:
cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/cozystack-controller-webhook
labels:
app: cozystack-controller
webhooks:
- name: lineage.cozystack.io
admissionReviewVersions: ["v1"]
sideEffects: None
clientConfig:
service:
name: cozystack-controller
namespace: {{ .Release.Namespace }}
path: /mutate-lineage
rules:
- operations: ["CREATE"]
apiGroups: [""]
apiVersions: ["v1"]
resources: ["pods","secrets", "services", "persistentvolumeclaims"]
failurePolicy: Fail
namespaceSelector:
matchExpressions:
- key: cozystack.io/system
operator: NotIn
values:
- "true"
- key: kubernetes.io/metadata.name
operator: NotIn
values:
- kube-system

View File

@@ -1,15 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: cozystack-controller
labels:
app: cozystack-controller
spec:
type: ClusterIP
ports:
- port: 443
targetPort: 9443
protocol: TCP
name: webhook
selector:
app: cozystack-controller

View File

@@ -1,193 +0,0 @@
package lineage
import (
"context"
"fmt"
"os"
"strings"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"sigs.k8s.io/controller-runtime/pkg/log"
)
const (
HRAPIVersion = "helm.toolkit.fluxcd.io/v2"
HRKind = "HelmRelease"
HRLabel = "helm.toolkit.fluxcd.io/name"
)
type ObjectID struct {
APIVersion string
Kind string
Namespace string
Name string
}
func (o ObjectID) GetUnstructured(ctx context.Context, client dynamic.Interface, mapper meta.RESTMapper) (*unstructured.Unstructured, error) {
u, err := getUnstructuredObject(ctx, client, mapper, o.APIVersion, o.Kind, o.Namespace, o.Name)
if err != nil {
return nil, err
}
return u, nil
}
func WalkOwnershipGraph(
ctx context.Context,
client dynamic.Interface,
mapper meta.RESTMapper,
appMapper AppMapper,
obj *unstructured.Unstructured,
memory ...interface{},
) (out []ObjectID) {
id := ObjectID{APIVersion: obj.GetAPIVersion(), Kind: obj.GetKind(), Namespace: obj.GetNamespace(), Name: obj.GetName()}
out = []ObjectID{}
l := log.FromContext(ctx)
l.Info("processing object", "apiVersion", obj.GetAPIVersion(), "kind", obj.GetKind(), "name", obj.GetName())
var visited map[ObjectID]bool
var ok bool
if len(memory) == 1 {
visited, ok = memory[0].(map[ObjectID]bool)
if !ok {
l.Error(
fmt.Errorf("invalid argument"), "could not parse visited map in WalkOwnershipGraph call",
"received", memory[0], "expected", "map[ObjectID]bool",
)
return out
}
}
if len(memory) == 0 {
visited = make(map[ObjectID]bool)
}
if len(memory) != 0 && len(memory) != 1 {
l.Error(
fmt.Errorf("invalid argument count"), "could not parse variadic arguments to WalkOwnershipGraph",
"args passed", len(memory)+5, "expected args", "4|5",
)
return out
}
if visited[id] {
return out
}
visited[id] = true
ownerRefs := obj.GetOwnerReferences()
for _, owner := range ownerRefs {
ownerObj, err := getUnstructuredObject(ctx, client, mapper, owner.APIVersion, owner.Kind, obj.GetNamespace(), owner.Name)
if err != nil {
fmt.Fprintf(os.Stderr, "Could not fetch owner %s/%s (%s): %v\n", obj.GetNamespace(), owner.Name, owner.Kind, err)
continue
}
out = append(out, WalkOwnershipGraph(ctx, client, mapper, appMapper, ownerObj, visited)...)
}
// if object has owners, it couldn't be owned directly by the custom app
if len(ownerRefs) > 0 {
return
}
// I want "if err1 != nil go to next block, if err2 != nil, go to next block, etc semantics",
// like an early return from a function, but if all checks succeed, I don't want to do the rest
// of the function, so it's a `for { if err { break } if othererr { break } if allgood { return }
for {
if obj.GetAPIVersion() != HRAPIVersion || obj.GetKind() != HRKind {
break
}
hr := helmReleaseFromUnstructured(obj)
if hr == nil {
break
}
a, k, p, err := appMapper.Map(hr)
if err != nil {
break
}
ownerObj, err := getUnstructuredObject(ctx, client, mapper, a, k, obj.GetNamespace(), strings.TrimPrefix(obj.GetName(), p))
if err != nil {
break
}
// successfully mapped a HelmRelease to a custom app, no need to continue
out = append(out,
ObjectID{
APIVersion: ownerObj.GetAPIVersion(),
Kind: ownerObj.GetKind(),
Namespace: ownerObj.GetNamespace(),
Name: ownerObj.GetName(),
},
)
return
}
labels := obj.GetLabels()
name, ok := labels[HRLabel]
if !ok {
return
}
ownerObj, err := getUnstructuredObject(ctx, client, mapper, HRAPIVersion, HRKind, obj.GetNamespace(), name)
if err != nil {
return
}
out = append(out, WalkOwnershipGraph(ctx, client, mapper, appMapper, ownerObj, visited)...)
return
}
func getUnstructuredObject(
ctx context.Context,
client dynamic.Interface,
mapper meta.RESTMapper,
apiVersion, kind, namespace, name string,
) (*unstructured.Unstructured, error) {
l := log.FromContext(ctx)
gv, err := schema.ParseGroupVersion(apiVersion)
if err != nil {
l.Error(
err, "failed to parse groupversion",
"apiVersion", apiVersion,
)
return nil, err
}
gvk := schema.GroupVersionKind{
Group: gv.Group,
Version: gv.Version,
Kind: kind,
}
mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
l.Error(err, "Could not map GVK "+gvk.String())
return nil, err
}
ns := namespace
if mapping.Scope.Name() != meta.RESTScopeNameNamespace {
ns = ""
}
ownerObj, err := client.Resource(mapping.Resource).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return nil, err
}
return ownerObj, nil
}
func helmReleaseFromUnstructured(obj *unstructured.Unstructured) *helmv2.HelmRelease {
if obj.GetAPIVersion() == HRAPIVersion && obj.GetKind() == HRKind {
hr := &helmv2.HelmRelease{}
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, hr); err == nil {
return hr
}
}
return nil
}

View File

@@ -1,53 +0,0 @@
package lineage
import (
"context"
"fmt"
"os"
"testing"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
"k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/restmapper"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)
var (
dynClient dynamic.Interface
mapper meta.RESTMapper
l logr.Logger
ctx context.Context
)
func init() {
cfg := config.GetConfigOrDie()
dynClient, _ = dynamic.NewForConfig(cfg)
discoClient, _ := discovery.NewDiscoveryClientForConfig(cfg)
cachedDisco := memory.NewMemCacheClient(discoClient)
mapper = restmapper.NewDeferredDiscoveryRESTMapper(cachedDisco)
zapLogger, _ := zap.NewDevelopment()
l = zapr.NewLogger(zapLogger)
ctx = logr.NewContext(context.Background(), l)
}
func TestWalkingOwnershipGraph(t *testing.T) {
obj, err := dynClient.Resource(schema.GroupVersionResource{"", "v1", "pods"}).Namespace(os.Args[1]).Get(ctx, os.Args[2], metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
nodes := WalkOwnershipGraph(ctx, dynClient, mapper, obj)
for _, node := range nodes {
fmt.Printf("%#v\n", node)
}
}

View File

@@ -1,49 +0,0 @@
package lineage
import (
"fmt"
"strings"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
)
type AppMapper interface {
Map(*helmv2.HelmRelease) (apiVersion, kind, prefix string, err error)
}
type stubMapper struct{}
var stubMapperMap = map[string]string{
"cozystack-extra/bootbox": "apps.cozystack.io/v1alpha1/BootBox/",
"cozystack-apps/bucket": "apps.cozystack.io/v1alpha1/Bucket/bucket-",
"cozystack-apps/clickhouse": "apps.cozystack.io/v1alpha1/ClickHouse/clickhouse-",
"cozystack-extra/etcd": "apps.cozystack.io/v1alpha1/Etcd/",
"cozystack-apps/ferretdb": "apps.cozystack.io/v1alpha1/FerretDB/ferretdb-",
"cozystack-apps/http-cache": "apps.cozystack.io/v1alpha1/HTTPCache/http-cache-",
"cozystack-extra/info": "apps.cozystack.io/v1alpha1/Info/",
"cozystack-extra/ingress": "apps.cozystack.io/v1alpha1/Ingress/",
"cozystack-apps/kafka": "apps.cozystack.io/v1alpha1/Kafka/kafka-",
"cozystack-apps/kubernetes": "apps.cozystack.io/v1alpha1/Kubernetes/kubernetes-",
"cozystack-extra/monitoring": "apps.cozystack.io/v1alpha1/Monitoring/",
"cozystack-apps/mysql": "apps.cozystack.io/v1alpha1/MySQL/mysql-",
"cozystack-apps/nats": "apps.cozystack.io/v1alpha1/NATS/nats-",
"cozystack-apps/postgres": "apps.cozystack.io/v1alpha1/Postgres/postgres-",
"cozystack-apps/rabbitmq": "apps.cozystack.io/v1alpha1/RabbitMQ/rabbitmq-",
"cozystack-apps/redis": "apps.cozystack.io/v1alpha1/Redis/redis-",
"cozystack-extra/seaweedfs": "apps.cozystack.io/v1alpha1/SeaweedFS/",
"cozystack-apps/tcp-balancer": "apps.cozystack.io/v1alpha1/TCPBalancer/tcp-balancer-",
"cozystack-apps/tenant": "apps.cozystack.io/v1alpha1/Tenant/tenant-",
"cozystack-apps/virtual-machine": "apps.cozystack.io/v1alpha1/VirtualMachine/virtual-machine-",
"cozystack-apps/vm-disk": "apps.cozystack.io/v1alpha1/VMDisk/vm-disk-",
"cozystack-apps/vm-instance": "apps.cozystack.io/v1alpha1/VMInstance/vm-instance-",
"cozystack-apps/vpn": "apps.cozystack.io/v1alpha1/VPN/vpn-",
}
func (s *stubMapper) Map(hr *helmv2.HelmRelease) (string, string, string, error) {
val, ok := stubMapperMap[hr.Spec.Chart.Spec.SourceRef.Name+"/"+hr.Spec.Chart.Spec.Chart]
if !ok {
return "", "", "", fmt.Errorf("cannot map helm release %s/%s to dynamic app", hr.Namespace, hr.Name)
}
split := strings.Split(val, "/")
return strings.Join(split[:2], "/"), split[2], split[3], nil
}