(#112) Refactor config flag handling

Remove the server.Config struct, not needed.  Remove the app struct and
move the configuration to the main holos.Config.ServerConfig.

Add flags specific to server configuration.

With this patch logging is simplified.  Subcommands have a handle on the
top level holos.Config and can get a fully configured logger from
cfg.Logger() after flag parsing happens.

api/v1alpha1/helm.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 
 	"github.com/holos-run/holos"
+	"github.com/holos-run/holos/pkg/errors"
 	"github.com/holos-run/holos/pkg/logger"
 	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
 )
 
 // A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
@@ -42,7 +42,7 @@ func (hc *HelmChart) Render(ctx context.Context, path holos.InstancePath) (*Resu
 	}
 	result.addObjectMap(ctx, hc.APIObjectMap)
 	if err := result.kustomize(ctx); err != nil {
-		return nil, wrapper.Wrap(fmt.Errorf("could not kustomize: %w", err))
+		return nil, errors.Wrap(fmt.Errorf("could not kustomize: %w", err))
 	}
 	return &result, nil
 }
@@ -64,13 +64,13 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.InstancePat
 			out, err := util.RunCmd(ctx, "helm", "repo", "add", repo.Name, repo.URL)
 			if err != nil {
 				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-				return wrapper.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
+				return errors.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
 			}
 			// Update repository
 			out, err = util.RunCmd(ctx, "helm", "repo", "update", repo.Name)
 			if err != nil {
 				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-				return wrapper.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
+				return errors.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
 			}
 		} else {
 			log.DebugContext(ctx, "no chart repository url proceeding assuming oci chart")
@@ -85,13 +85,13 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.InstancePat
 	// Write values file
 	tempDir, err := os.MkdirTemp("", "holos")
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+		return errors.Wrap(fmt.Errorf("could not make temp dir: %w", err))
 	}
 	defer util.Remove(ctx, tempDir)
 
 	valuesPath := filepath.Join(tempDir, "values.yaml")
 	if err := os.WriteFile(valuesPath, []byte(hc.ValuesContent), 0644); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not write values: %w", err))
+		return errors.Wrap(fmt.Errorf("could not write values: %w", err))
 	}
 	log.DebugContext(ctx, "helm: wrote values", "path", valuesPath, "bytes", len(hc.ValuesContent))
 
@@ -112,7 +112,7 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.InstancePat
 				err = fmt.Errorf("%s: %w", line, err)
 			}
 		}
-		return wrapper.Wrap(fmt.Errorf("could not run helm template: %w", err))
+		return errors.Wrap(fmt.Errorf("could not run helm template: %w", err))
 	}
 
 	r.accumulatedOutput = helmOut.Stdout.String()
@@ -126,7 +126,7 @@ func cacheChart(ctx context.Context, path holos.InstancePath, chartDir string, c
 
 	cacheTemp, err := os.MkdirTemp(string(path), chartDir)
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+		return errors.Wrap(fmt.Errorf("could not make temp dir: %w", err))
 	}
 	defer util.Remove(ctx, cacheTemp)
 
@@ -136,13 +136,13 @@ func cacheChart(ctx context.Context, path holos.InstancePath, chartDir string, c
 	}
 	helmOut, err := util.RunCmd(ctx, "helm", "pull", "--destination", cacheTemp, "--untar=true", "--version", chart.Version, chartName)
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not run helm pull: %w", err))
+		return errors.Wrap(fmt.Errorf("could not run helm pull: %w", err))
 	}
 	log.Debug("helm pull", "stdout", helmOut.Stdout, "stderr", helmOut.Stderr)
 
 	cachePath := filepath.Join(string(path), chartDir)
 	if err := os.Rename(cacheTemp, cachePath); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not rename: %w", err))
+		return errors.Wrap(fmt.Errorf("could not rename: %w", err))
 	}
 	log.InfoContext(ctx, "cached", "chart", chart.Name, "version", chart.Version, "path", cachePath)
 

api/v1alpha1/kustomize.go

@@ -4,9 +4,9 @@ import (
 	"context"
 
 	"github.com/holos-run/holos"
+	"github.com/holos-run/holos/pkg/errors"
 	"github.com/holos-run/holos/pkg/logger"
 	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
 )
 
 const KustomizeBuildKind = "KustomizeBuild"
@@ -37,7 +37,7 @@ func (kb *KustomizeBuild) Render(ctx context.Context, path holos.InstancePath) (
 	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", string(path))
 	if err != nil {
 		log.ErrorContext(ctx, kOut.Stderr.String())
-		return nil, wrapper.Wrap(err)
+		return nil, errors.Wrap(err)
 	}
 	// Replace the accumulated output
 	result.accumulatedOutput = kOut.Stdout.String()

api/v1alpha1/result.go

@@ -7,9 +7,9 @@ import (
 	"path/filepath"
 	"slices"
 
+	"github.com/holos-run/holos/pkg/errors"
 	"github.com/holos-run/holos/pkg/logger"
 	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
 )
 
 // Result is the build result for display or writing.  Holos components Render the Result as a data pipeline.
@@ -82,7 +82,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	}
 	tempDir, err := os.MkdirTemp("", "holos.kustomize")
 	if err != nil {
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	defer util.Remove(ctx, tempDir)
 
@@ -91,7 +91,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	b := []byte(r.AccumulatedOutput())
 	b = util.EnsureNewline(b)
 	if err := os.WriteFile(target, b, 0644); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not write resources: %w", err))
+		return errors.Wrap(fmt.Errorf("could not write resources: %w", err))
 	}
 	log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
 
@@ -99,12 +99,12 @@ func (r *Result) kustomize(ctx context.Context) error {
 	for file, content := range r.KustomizeFiles {
 		target := filepath.Join(tempDir, file)
 		if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
-			return wrapper.Wrap(err)
+			return errors.Wrap(err)
 		}
 		b := []byte(content)
 		b = util.EnsureNewline(b)
 		if err := os.WriteFile(target, b, 0644); err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not write: %w", err))
+			return errors.Wrap(fmt.Errorf("could not write: %w", err))
 		}
 		log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
 	}
@@ -113,7 +113,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", tempDir)
 	if err != nil {
 		log.ErrorContext(ctx, kOut.Stderr.String())
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	// Replace the accumulated output
 	r.accumulatedOutput = kOut.Stdout.String()
@@ -126,12 +126,12 @@ func (r *Result) Save(ctx context.Context, path string, content string) error {
 	dir := filepath.Dir(path)
 	if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
 		log.WarnContext(ctx, "could not mkdir", "path", dir, "err", err)
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	// Write the kube api objects
 	if err := os.WriteFile(path, []byte(content), os.FileMode(0644)); err != nil {
 		log.WarnContext(ctx, "could not write", "path", path, "err", err)
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	log.DebugContext(ctx, "out: wrote "+path, "action", "write", "path", path, "status", "ok")
 	return nil

docs/examples/authpolicy.cue

@@ -0,0 +1,37 @@
+package holos
+
+import ap "security.istio.io/authorizationpolicy/v1"
+
+// #AuthPolicyRules represents AuthorizationPolicy rules for hosts that need specialized treatment.  Entries in this struct are exclused from the blank ingressauth AuthorizationPolicy governing the ingressgateway and included in a spcialized policy
+#AuthPolicyRules: {
+	// AuthProxySpec represents the identity provider configuration
+	AuthProxySpec: #AuthProxySpec & #Platform.authproxy
+
+	// Hosts are hosts that need specialized treatment
+	hosts: {
+		[Name=_]: {
+			// name is the fully qualifed hostname, a Host: header value.
+			name: Name
+			// slug is the resource name prefix
+			slug: string
+			// Refer to https://istio.io/latest/docs/reference/config/security/authorization-policy/#Rule
+			spec: ap.#AuthorizationPolicySpec & {
+				action: "CUSTOM"
+				provider: name: AuthProxySpec.provider
+				selector: matchLabels: istio: "ingressgateway"
+			}
+		}
+	}
+
+	objects: #APIObjects & {
+		for Host in hosts {
+			apiObjects: {
+				AuthorizationPolicy: "\(Host.slug)-custom": {
+					metadata: namespace: "istio-ingress"
+					metadata: name:      "\(Host.slug)-custom"
+					spec: Host.spec
+				}
+			}
+		}
+	}
+}

docs/examples/cue.mod/gen/argoproj.io/appproject/v1alpha1/types_gen.cue

@@ -0,0 +1,189 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-argocd/prod-platform-argocd.gen.yaml
+
+package v1alpha1
+
+import "strings"
+
+// AppProject provides a logical grouping of applications,
+// providing controls for: * where the apps may deploy to
+// (cluster whitelist) * what may be deployed (repository
+// whitelist, resource whitelist/blacklist) * who can access
+// these applications (roles, OIDC group claims bindings) * and
+// what they can do (RBAC policies) * automation access to these
+// roles (JWT tokens)
+#AppProject: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "argoproj.io/v1alpha1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "AppProject"
+	metadata: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// AppProjectSpec is the specification of an AppProject
+	spec!: #AppProjectSpec
+}
+
+// AppProjectSpec is the specification of an AppProject
+#AppProjectSpec: {
+	// ClusterResourceBlacklist contains list of blacklisted cluster
+	// level resources
+	clusterResourceBlacklist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// ClusterResourceWhitelist contains list of whitelisted cluster
+	// level resources
+	clusterResourceWhitelist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// Description contains optional project description
+	description?: string
+
+	// Destinations contains list of destinations available for
+	// deployment
+	destinations?: [...{
+		// Name is an alternate way of specifying the target cluster by
+		// its symbolic name. This must be set if Server is not set.
+		name?: string
+
+		// Namespace specifies the target namespace for the application's
+		// resources. The namespace will only be set for namespace-scoped
+		// resources that have not set a value for .metadata.namespace
+		namespace?: string
+
+		// Server specifies the URL of the target cluster's Kubernetes
+		// control plane API. This must be set if Name is not set.
+		server?: string
+	}]
+
+	// NamespaceResourceBlacklist contains list of blacklisted
+	// namespace level resources
+	namespaceResourceBlacklist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// NamespaceResourceWhitelist contains list of whitelisted
+	// namespace level resources
+	namespaceResourceWhitelist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// OrphanedResources specifies if controller should monitor
+	// orphaned resources of apps in this project
+	orphanedResources?: {
+		// Ignore contains a list of resources that are to be excluded
+		// from orphaned resources monitoring
+		ignore?: [...{
+			group?: string
+			kind?:  string
+			name?:  string
+		}]
+
+		// Warn indicates if warning condition should be created for apps
+		// which have orphaned resources
+		warn?: bool
+	}
+
+	// PermitOnlyProjectScopedClusters determines whether destinations
+	// can only reference clusters which are project-scoped
+	permitOnlyProjectScopedClusters?: bool
+
+	// Roles are user defined RBAC roles associated with this project
+	roles?: [...{
+		// Description is a description of the role
+		description?: string
+
+		// Groups are a list of OIDC group claims bound to this role
+		groups?: [...string]
+
+		// JWTTokens are a list of generated JWT tokens bound to this role
+		jwtTokens?: [...{
+			exp?: int
+			iat:  int
+			id?:  string
+		}]
+
+		// Name is a name for this role
+		name: string
+
+		// Policies Stores a list of casbin formatted strings that define
+		// access policies for the role in the project
+		policies?: [...string]
+	}]
+
+	// SignatureKeys contains a list of PGP key IDs that commits in
+	// Git must be signed with in order to be allowed for sync
+	signatureKeys?: [...{
+		// The ID of the key in hexadecimal notation
+		keyID: string
+	}]
+
+	// SourceNamespaces defines the namespaces application resources
+	// are allowed to be created in
+	sourceNamespaces?: [...string]
+
+	// SourceRepos contains list of repository URLs which can be used
+	// for deployment
+	sourceRepos?: [...string]
+
+	// SyncWindows controls when syncs can be run for apps in this
+	// project
+	syncWindows?: [...{
+		// Applications contains a list of applications that the window
+		// will apply to
+		applications?: [...string]
+
+		// Clusters contains a list of clusters that the window will apply
+		// to
+		clusters?: [...string]
+
+		// Duration is the amount of time the sync window will be open
+		duration?: string
+
+		// Kind defines if the window allows or blocks syncs
+		kind?: string
+
+		// ManualSync enables manual syncs when they would otherwise be
+		// blocked
+		manualSync?: bool
+
+		// Namespaces contains a list of namespaces that the window will
+		// apply to
+		namespaces?: [...string]
+
+		// Schedule is the time the window will begin, specified in cron
+		// format
+		schedule?: string
+
+		// TimeZone of the sync that will be applied to the schedule
+		timeZone?: string
+	}]
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/podmonitor/v1/types_gen.cue

@@ -0,0 +1,546 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// PodMonitor defines monitoring for a set of pods.
+#PodMonitor: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "PodMonitor"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Pod selection for target discovery by
+	// Prometheus.
+	spec!: #PodMonitorSpec
+}
+
+// Specification of desired Pod selection for target discovery by
+// Prometheus.
+#PodMonitorSpec: {
+	attachMetadata?: {
+		// When set to true, Prometheus must have the `get` permission on
+		// the `Nodes` objects.
+		node?: bool
+	}
+
+	// The label to use to retrieve the job name from. `jobLabel`
+	// selects the label from the associated Kubernetes `Pod` object
+	// which will be used as the `job` label for all metrics.
+	// For example if `jobLabel` is set to `foo` and the Kubernetes
+	// `Pod` object is labeled with `foo: bar`, then Prometheus adds
+	// the `job="bar"` label to all ingested metrics.
+	// If the value of this field is empty, the `job` label of the
+	// metrics defaults to the namespace and name of the PodMonitor
+	// object (e.g. `<namespace>/<name>`).
+	jobLabel?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelValueLengthLimit?: int
+
+	// Selector to select which namespaces the Kubernetes `Pods`
+	// objects are discovered from.
+	namespaceSelector?: {
+		// Boolean describing whether all namespaces are selected in
+		// contrast to a list restricting them.
+		any?: bool
+
+		// List of namespace names to select from.
+		matchNames?: [...string]
+	}
+
+	// List of endpoints part of this PodMonitor.
+	podMetricsEndpoints?: [...{
+		// `authorization` configures the Authorization header credentials
+		// to use when scraping the target.
+		// Cannot be set at the same time as `basicAuth`, or `oauth2`.
+		authorization?: {
+			// Selects a key of a Secret in the namespace that contains the
+			// credentials for authentication.
+			credentials?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Defines the authentication type. The value is case-insensitive.
+			// "Basic" is not a supported value.
+			// Default: "Bearer"
+			type?: string
+		}
+
+		// `basicAuth` configures the Basic Authentication credentials to
+		// use when scraping the target.
+		// Cannot be set at the same time as `authorization`, or `oauth2`.
+		basicAuth?: {
+			// `password` specifies a key of a Secret containing the password
+			// for authentication.
+			password?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `username` specifies a key of a Secret containing the username
+			// for authentication.
+			username?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// `bearerTokenSecret` specifies a key of a Secret containing the
+		// bearer token for scraping targets. The secret needs to be in
+		// the same namespace as the PodMonitor object and readable by
+		// the Prometheus Operator.
+		// Deprecated: use `authorization` instead.
+		bearerTokenSecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `enableHttp2` can be used to disable HTTP2 when scraping the
+		// target.
+		enableHttp2?: bool
+
+		// When true, the pods which are not running (e.g. either in
+		// Failed or Succeeded state) are dropped during the target
+		// discovery.
+		// If unset, the filtering is enabled.
+		// More info:
+		// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+		filterRunning?: bool
+
+		// `followRedirects` defines whether the scrape requests should
+		// follow HTTP 3xx redirects.
+		followRedirects?: bool
+
+		// When true, `honorLabels` preserves the metric's labels when
+		// they collide with the target's labels.
+		honorLabels?: bool
+
+		// `honorTimestamps` controls whether Prometheus preserves the
+		// timestamps when exposed by the target.
+		honorTimestamps?: bool
+
+		// Interval at which Prometheus scrapes the metrics from the
+		// target.
+		// If empty, Prometheus uses the global scrape interval.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// `metricRelabelings` configures the relabeling rules to apply to
+		// the samples before ingestion.
+		metricRelabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// `oauth2` configures the OAuth2 settings to use when scraping
+		// the target.
+		// It requires Prometheus >= 2.27.0.
+		// Cannot be set at the same time as `authorization`, or
+		// `basicAuth`.
+		oauth2?: {
+			// `clientId` specifies a key of a Secret or ConfigMap containing
+			// the OAuth2 client's ID.
+			clientId: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// `clientSecret` specifies a key of a Secret containing the
+			// OAuth2 client's secret.
+			clientSecret: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `endpointParams` configures the HTTP parameters to append to
+			// the token URL.
+			endpointParams?: {
+				[string]: string
+			}
+
+			// `scopes` defines the OAuth2 scopes used for the token request.
+			scopes?: [...string]
+
+			// `tokenURL` configures the URL to fetch the token from.
+			tokenUrl: strings.MinRunes(1)
+		}
+
+		// `params` define optional HTTP URL parameters.
+		params?: {
+			[string]: [...string]
+		}
+
+		// HTTP path from which to scrape for metrics.
+		// If empty, Prometheus uses the default value (e.g. `/metrics`).
+		path?: string
+
+		// Name of the Pod port which this endpoint refers to.
+		// It takes precedence over `targetPort`.
+		port?: string
+
+		// `proxyURL` configures the HTTP Proxy URL (e.g.
+		// "http://proxyserver:2195") to go through when scraping the
+		// target.
+		proxyUrl?: string
+
+		// `relabelings` configures the relabeling rules to apply the
+		// target's metadata labels.
+		// The Operator automatically adds relabelings for a few standard
+		// Kubernetes fields.
+		// The original scrape job's name is available via the
+		// `__tmp_prometheus_job_name` label.
+		// More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+		relabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// HTTP scheme to use for scraping.
+		// `http` and `https` are the expected values unless you rewrite
+		// the `__scheme__` label via relabeling.
+		// If empty, Prometheus uses the default value `http`.
+		scheme?: "http" | "https"
+
+		// Timeout after which Prometheus considers the scrape to be
+		// failed.
+		// If empty, Prometheus uses the global scrape timeout unless it
+		// is less than the target's scrape interval value in which the
+		// latter is used.
+		scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Name or number of the target port of the `Pod` object behind
+		// the Service, the port must be specified with container port
+		// property.
+		// Deprecated: use 'port' instead.
+		targetPort?: (int | string) & {
+			string
+		}
+
+		// TLS configuration to use when scraping the target.
+		tlsConfig?: {
+			// Certificate authority used when verifying server certificates.
+			ca?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Client certificate to present when doing client-authentication.
+			cert?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Disable target certificate validation.
+			insecureSkipVerify?: bool
+
+			// Secret containing the client key file for the targets.
+			keySecret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Used to verify the hostname for the targets.
+			serverName?: string
+		}
+
+		// `trackTimestampsStaleness` defines whether Prometheus tracks
+		// staleness of the metrics that have an explicit timestamp
+		// present in scraped data. Has no effect if `honorTimestamps` is
+		// false.
+		// It requires Prometheus >= v2.48.0.
+		trackTimestampsStaleness?: bool
+	}]
+
+	// `podTargetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Pod` object onto the ingested
+	// metrics.
+	podTargetLabels?: [...string]
+
+	// `sampleLimit` defines a per-scrape limit on the number of
+	// scraped samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Label selector to select the Kubernetes `Pod` objects.
+	selector: {
+		// matchExpressions is a list of label selector requirements. The
+		// requirements are ANDed.
+		matchExpressions?: [...{
+			// key is the label key that the selector applies to.
+			key: string
+
+			// operator represents a key's relationship to a set of values.
+			// Valid operators are In, NotIn, Exists and DoesNotExist.
+			operator: string
+
+			// values is an array of string values. If the operator is In or
+			// NotIn, the values array must be non-empty. If the operator is
+			// Exists or DoesNotExist, the values array must be empty. This
+			// array is replaced during a strategic merge patch.
+			values?: [...string]
+		}]
+
+		// matchLabels is a map of {key,value} pairs. A single {key,value}
+		// in the matchLabels map is equivalent to an element of
+		// matchExpressions, whose key field is "key", the operator is
+		// "In", and the values array contains only "value". The
+		// requirements are ANDed.
+		matchLabels?: {
+			[string]: string
+		}
+	}
+
+	// `targetLimit` defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/probe/v1/types_gen.cue

@@ -0,0 +1,536 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// Probe defines monitoring for a set of static targets or
+// ingresses.
+#Probe: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "Probe"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Ingress selection for target discovery
+	// by Prometheus.
+	spec!: #ProbeSpec
+}
+
+// Specification of desired Ingress selection for target discovery
+// by Prometheus.
+#ProbeSpec: {
+	// Authorization section for this endpoint
+	authorization?: {
+		// Selects a key of a Secret in the namespace that contains the
+		// credentials for authentication.
+		credentials?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// Defines the authentication type. The value is case-insensitive.
+		// "Basic" is not a supported value.
+		// Default: "Bearer"
+		type?: string
+	}
+
+	// BasicAuth allow an endpoint to authenticate over basic
+	// authentication. More info:
+	// https://prometheus.io/docs/operating/configuration/#endpoint
+	basicAuth?: {
+		// `password` specifies a key of a Secret containing the password
+		// for authentication.
+		password?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `username` specifies a key of a Secret containing the username
+		// for authentication.
+		username?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+	}
+
+	// Secret to mount to read bearer token for scraping targets. The
+	// secret needs to be in the same namespace as the probe and
+	// accessible by the Prometheus Operator.
+	bearerTokenSecret?: {
+		// The key of the secret to select from. Must be a valid secret
+		// key.
+		key: string
+
+		// Name of the referent. More info:
+		// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+		// TODO: Add other useful fields. apiVersion, kind, uid?
+		name?: string
+
+		// Specify whether the Secret or its key must be defined
+		optional?: bool
+	}
+
+	// Interval at which targets are probed using the configured
+	// prober. If not specified Prometheus' global scrape interval is
+	// used.
+	interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+	// The job name assigned to scraped metrics by default.
+	jobName?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample. Only valid in Prometheus versions 2.27.0 and newer.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample. Only valid in Prometheus versions 2.27.0 and
+	// newer.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample. Only valid in Prometheus versions
+	// 2.27.0 and newer.
+	labelValueLengthLimit?: int
+
+	// MetricRelabelConfigs to apply to samples before ingestion.
+	metricRelabelings?: [...{
+		// Action to perform based on the regex matching.
+		// `Uppercase` and `Lowercase` actions require Prometheus >=
+		// v2.36.0. `DropEqual` and `KeepEqual` actions require
+		// Prometheus >= v2.41.0.
+		// Default: "Replace"
+		action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+		// Modulus to take of the hash of the source label values.
+		// Only applicable when the action is `HashMod`.
+		modulus?: int
+
+		// Regular expression against which the extracted value is
+		// matched.
+		regex?: string
+
+		// Replacement value against which a Replace action is performed
+		// if the regular expression matches.
+		// Regex capture groups are available.
+		replacement?: string
+
+		// Separator is the string between concatenated SourceLabels.
+		separator?: string
+
+		// The source labels select values from existing labels. Their
+		// content is concatenated using the configured Separator and
+		// matched against the configured regular expression.
+		sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+		// Label to which the resulting string is written in a
+		// replacement.
+		// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+		// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+		// Regex capture groups are available.
+		targetLabel?: string
+	}]
+
+	// The module to use for probing specifying how to probe the
+	// target. Example module configuring in the blackbox exporter:
+	// https://github.com/prometheus/blackbox_exporter/blob/master/example.yml
+	module?: string
+
+	// OAuth2 for the URL. Only valid in Prometheus versions 2.27.0
+	// and newer.
+	oauth2?: {
+		// `clientId` specifies a key of a Secret or ConfigMap containing
+		// the OAuth2 client's ID.
+		clientId: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// `clientSecret` specifies a key of a Secret containing the
+		// OAuth2 client's secret.
+		clientSecret: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `endpointParams` configures the HTTP parameters to append to
+		// the token URL.
+		endpointParams?: {
+			[string]: string
+		}
+
+		// `scopes` defines the OAuth2 scopes used for the token request.
+		scopes?: [...string]
+
+		// `tokenURL` configures the URL to fetch the token from.
+		tokenUrl: strings.MinRunes(1)
+	}
+
+	// Specification for the prober to use for probing targets. The
+	// prober.URL parameter is required. Targets cannot be probed if
+	// left empty.
+	prober?: {
+		// Path to collect metrics from. Defaults to `/probe`.
+		path?: string | *"/probe"
+
+		// Optional ProxyURL.
+		proxyUrl?: string
+
+		// HTTP scheme to use for scraping. `http` and `https` are the
+		// expected values unless you rewrite the `__scheme__` label via
+		// relabeling. If empty, Prometheus uses the default value
+		// `http`.
+		scheme?: "http" | "https"
+
+		// Mandatory URL of the prober.
+		url: string
+	}
+
+	// SampleLimit defines per-scrape limit on number of scraped
+	// samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Timeout for scraping metrics from the Prometheus exporter. If
+	// not specified, the Prometheus global scrape timeout is used.
+	scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+	// TargetLimit defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+
+	// Targets defines a set of static or dynamically discovered
+	// targets to probe.
+	targets?: {
+		// ingress defines the Ingress objects to probe and the relabeling
+		// configuration. If `staticConfig` is also defined,
+		// `staticConfig` takes precedence.
+		ingress?: {
+			// From which namespaces to select Ingress objects.
+			namespaceSelector?: {
+				// Boolean describing whether all namespaces are selected in
+				// contrast to a list restricting them.
+				any?: bool
+
+				// List of namespace names to select from.
+				matchNames?: [...string]
+			}
+
+			// RelabelConfigs to apply to the label set of the target before
+			// it gets scraped. The original ingress address is available via
+			// the `__tmp_prometheus_ingress_address` label. It can be used
+			// to customize the probed URL. The original scrape job's name is
+			// available via the `__tmp_prometheus_job_name` label. More
+			// info:
+			// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+			relabelingConfigs?: [...{
+				// Action to perform based on the regex matching.
+				// `Uppercase` and `Lowercase` actions require Prometheus >=
+				// v2.36.0. `DropEqual` and `KeepEqual` actions require
+				// Prometheus >= v2.41.0.
+				// Default: "Replace"
+				action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+				// Modulus to take of the hash of the source label values.
+				// Only applicable when the action is `HashMod`.
+				modulus?: int
+
+				// Regular expression against which the extracted value is
+				// matched.
+				regex?: string
+
+				// Replacement value against which a Replace action is performed
+				// if the regular expression matches.
+				// Regex capture groups are available.
+				replacement?: string
+
+				// Separator is the string between concatenated SourceLabels.
+				separator?: string
+
+				// The source labels select values from existing labels. Their
+				// content is concatenated using the configured Separator and
+				// matched against the configured regular expression.
+				sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+				// Label to which the resulting string is written in a
+				// replacement.
+				// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+				// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+				// Regex capture groups are available.
+				targetLabel?: string
+			}]
+
+			// Selector to select the Ingress objects.
+			selector?: {
+				// matchExpressions is a list of label selector requirements. The
+				// requirements are ANDed.
+				matchExpressions?: [...{
+					// key is the label key that the selector applies to.
+					key: string
+
+					// operator represents a key's relationship to a set of values.
+					// Valid operators are In, NotIn, Exists and DoesNotExist.
+					operator: string
+
+					// values is an array of string values. If the operator is In or
+					// NotIn, the values array must be non-empty. If the operator is
+					// Exists or DoesNotExist, the values array must be empty. This
+					// array is replaced during a strategic merge patch.
+					values?: [...string]
+				}]
+
+				// matchLabels is a map of {key,value} pairs. A single {key,value}
+				// in the matchLabels map is equivalent to an element of
+				// matchExpressions, whose key field is "key", the operator is
+				// "In", and the values array contains only "value". The
+				// requirements are ANDed.
+				matchLabels?: {
+					[string]: string
+				}
+			}
+		}
+
+		// staticConfig defines the static list of targets to probe and
+		// the relabeling configuration. If `ingress` is also defined,
+		// `staticConfig` takes precedence. More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config.
+		staticConfig?: {
+			// Labels assigned to all metrics scraped from the targets.
+			labels?: {
+				[string]: string
+			}
+
+			// RelabelConfigs to apply to the label set of the targets before
+			// it gets scraped. More info:
+			// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+			relabelingConfigs?: [...{
+				// Action to perform based on the regex matching.
+				// `Uppercase` and `Lowercase` actions require Prometheus >=
+				// v2.36.0. `DropEqual` and `KeepEqual` actions require
+				// Prometheus >= v2.41.0.
+				// Default: "Replace"
+				action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+				// Modulus to take of the hash of the source label values.
+				// Only applicable when the action is `HashMod`.
+				modulus?: int
+
+				// Regular expression against which the extracted value is
+				// matched.
+				regex?: string
+
+				// Replacement value against which a Replace action is performed
+				// if the regular expression matches.
+				// Regex capture groups are available.
+				replacement?: string
+
+				// Separator is the string between concatenated SourceLabels.
+				separator?: string
+
+				// The source labels select values from existing labels. Their
+				// content is concatenated using the configured Separator and
+				// matched against the configured regular expression.
+				sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+				// Label to which the resulting string is written in a
+				// replacement.
+				// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+				// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+				// Regex capture groups are available.
+				targetLabel?: string
+			}]
+
+			// The list of hosts to probe.
+			static?: [...string]
+		}
+	}
+
+	// TLS configuration to use when scraping the endpoint.
+	tlsConfig?: {
+		// Certificate authority used when verifying server certificates.
+		ca?: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// Client certificate to present when doing client-authentication.
+		cert?: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// Disable target certificate validation.
+		insecureSkipVerify?: bool
+
+		// Secret containing the client key file for the targets.
+		keySecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// Used to verify the hostname for the targets.
+		serverName?: string
+	}
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/prometheusrule/v1/types_gen.cue

@@ -0,0 +1,100 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// PrometheusRule defines recording and alerting rules for a
+// Prometheus instance
+#PrometheusRule: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "PrometheusRule"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired alerting rule definitions for
+	// Prometheus.
+	spec!: #PrometheusRuleSpec
+}
+#PrometheusRuleSpec: {
+	// Content of Prometheus rule file
+	groups?: [...{
+		// Interval determines how often rules in the group are evaluated.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Limit the number of alerts an alerting rule and series a
+		// recording rule can produce. Limit is supported starting with
+		// Prometheus >= 2.31 and Thanos Ruler >= 0.24.
+		limit?: int
+
+		// Name of the rule group.
+		name: strings.MinRunes(1)
+
+		// PartialResponseStrategy is only used by ThanosRuler and will be
+		// ignored by Prometheus instances. More info:
+		// https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response
+		partial_response_strategy?: =~"^(?i)(abort|warn)?$"
+
+		// List of alerting and recording rules.
+		rules?: [...{
+			// Name of the alert. Must be a valid label value. Only one of
+			// `record` and `alert` must be set.
+			alert?: string
+
+			// Annotations to add to each alert. Only valid for alerting
+			// rules.
+			annotations?: {
+				[string]: string
+			}
+
+			// PromQL expression to evaluate.
+			expr: (int | string) & {
+				string
+			}
+
+			// Alerts are considered firing once they have been returned for
+			// this long.
+			for?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+			// KeepFiringFor defines how long an alert will continue firing
+			// after the condition that triggered it has cleared.
+			keep_firing_for?: strings.MinRunes(1) & {
+				=~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+			}
+
+			// Labels to add or overwrite.
+			labels?: {
+				[string]: string
+			}
+
+			// Name of the time series to output to. Must be a valid metric
+			// name. Only one of `record` and `alert` must be set.
+			record?: string
+		}]
+	}]
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/servicemonitor/v1/types_gen.cue

@@ -0,0 +1,566 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// ServiceMonitor defines monitoring for a set of services.
+#ServiceMonitor: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "ServiceMonitor"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Service selection for target discovery
+	// by Prometheus.
+	spec!: #ServiceMonitorSpec
+}
+
+// Specification of desired Service selection for target discovery
+// by Prometheus.
+#ServiceMonitorSpec: {
+	attachMetadata?: {
+		// When set to true, Prometheus must have the `get` permission on
+		// the `Nodes` objects.
+		node?: bool
+	}
+
+	// List of endpoints part of this ServiceMonitor.
+	endpoints?: [...{
+		// `authorization` configures the Authorization header credentials
+		// to use when scraping the target.
+		// Cannot be set at the same time as `basicAuth`, or `oauth2`.
+		authorization?: {
+			// Selects a key of a Secret in the namespace that contains the
+			// credentials for authentication.
+			credentials?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Defines the authentication type. The value is case-insensitive.
+			// "Basic" is not a supported value.
+			// Default: "Bearer"
+			type?: string
+		}
+
+		// `basicAuth` configures the Basic Authentication credentials to
+		// use when scraping the target.
+		// Cannot be set at the same time as `authorization`, or `oauth2`.
+		basicAuth?: {
+			// `password` specifies a key of a Secret containing the password
+			// for authentication.
+			password?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `username` specifies a key of a Secret containing the username
+			// for authentication.
+			username?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// File to read bearer token for scraping the target.
+		// Deprecated: use `authorization` instead.
+		bearerTokenFile?: string
+
+		// `bearerTokenSecret` specifies a key of a Secret containing the
+		// bearer token for scraping targets. The secret needs to be in
+		// the same namespace as the ServiceMonitor object and readable
+		// by the Prometheus Operator.
+		// Deprecated: use `authorization` instead.
+		bearerTokenSecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `enableHttp2` can be used to disable HTTP2 when scraping the
+		// target.
+		enableHttp2?: bool
+
+		// When true, the pods which are not running (e.g. either in
+		// Failed or Succeeded state) are dropped during the target
+		// discovery.
+		// If unset, the filtering is enabled.
+		// More info:
+		// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+		filterRunning?: bool
+
+		// `followRedirects` defines whether the scrape requests should
+		// follow HTTP 3xx redirects.
+		followRedirects?: bool
+
+		// When true, `honorLabels` preserves the metric's labels when
+		// they collide with the target's labels.
+		honorLabels?: bool
+
+		// `honorTimestamps` controls whether Prometheus preserves the
+		// timestamps when exposed by the target.
+		honorTimestamps?: bool
+
+		// Interval at which Prometheus scrapes the metrics from the
+		// target.
+		// If empty, Prometheus uses the global scrape interval.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// `metricRelabelings` configures the relabeling rules to apply to
+		// the samples before ingestion.
+		metricRelabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// `oauth2` configures the OAuth2 settings to use when scraping
+		// the target.
+		// It requires Prometheus >= 2.27.0.
+		// Cannot be set at the same time as `authorization`, or
+		// `basicAuth`.
+		oauth2?: {
+			// `clientId` specifies a key of a Secret or ConfigMap containing
+			// the OAuth2 client's ID.
+			clientId: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// `clientSecret` specifies a key of a Secret containing the
+			// OAuth2 client's secret.
+			clientSecret: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `endpointParams` configures the HTTP parameters to append to
+			// the token URL.
+			endpointParams?: {
+				[string]: string
+			}
+
+			// `scopes` defines the OAuth2 scopes used for the token request.
+			scopes?: [...string]
+
+			// `tokenURL` configures the URL to fetch the token from.
+			tokenUrl: strings.MinRunes(1)
+		}
+
+		// params define optional HTTP URL parameters.
+		params?: {
+			[string]: [...string]
+		}
+
+		// HTTP path from which to scrape for metrics.
+		// If empty, Prometheus uses the default value (e.g. `/metrics`).
+		path?: string
+
+		// Name of the Service port which this endpoint refers to.
+		// It takes precedence over `targetPort`.
+		port?: string
+
+		// `proxyURL` configures the HTTP Proxy URL (e.g.
+		// "http://proxyserver:2195") to go through when scraping the
+		// target.
+		proxyUrl?: string
+
+		// `relabelings` configures the relabeling rules to apply the
+		// target's metadata labels.
+		// The Operator automatically adds relabelings for a few standard
+		// Kubernetes fields.
+		// The original scrape job's name is available via the
+		// `__tmp_prometheus_job_name` label.
+		// More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+		relabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// HTTP scheme to use for scraping.
+		// `http` and `https` are the expected values unless you rewrite
+		// the `__scheme__` label via relabeling.
+		// If empty, Prometheus uses the default value `http`.
+		scheme?: "http" | "https"
+
+		// Timeout after which Prometheus considers the scrape to be
+		// failed.
+		// If empty, Prometheus uses the global scrape timeout unless it
+		// is less than the target's scrape interval value in which the
+		// latter is used.
+		scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Name or number of the target port of the `Pod` object behind
+		// the Service. The port must be specified with the container's
+		// port property.
+		targetPort?: (int | string) & {
+			string
+		}
+
+		// TLS configuration to use when scraping the target.
+		tlsConfig?: {
+			// Certificate authority used when verifying server certificates.
+			ca?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Path to the CA cert in the Prometheus container to use for the
+			// targets.
+			caFile?: string
+
+			// Client certificate to present when doing client-authentication.
+			cert?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Path to the client cert file in the Prometheus container for
+			// the targets.
+			certFile?: string
+
+			// Disable target certificate validation.
+			insecureSkipVerify?: bool
+
+			// Path to the client key file in the Prometheus container for the
+			// targets.
+			keyFile?: string
+
+			// Secret containing the client key file for the targets.
+			keySecret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Used to verify the hostname for the targets.
+			serverName?: string
+		}
+
+		// `trackTimestampsStaleness` defines whether Prometheus tracks
+		// staleness of the metrics that have an explicit timestamp
+		// present in scraped data. Has no effect if `honorTimestamps` is
+		// false.
+		// It requires Prometheus >= v2.48.0.
+		trackTimestampsStaleness?: bool
+	}]
+
+	// `jobLabel` selects the label from the associated Kubernetes
+	// `Service` object which will be used as the `job` label for all
+	// metrics.
+	// For example if `jobLabel` is set to `foo` and the Kubernetes
+	// `Service` object is labeled with `foo: bar`, then Prometheus
+	// adds the `job="bar"` label to all ingested metrics.
+	// If the value of this field is empty or if the label doesn't
+	// exist for the given Service, the `job` label of the metrics
+	// defaults to the name of the associated Kubernetes `Service`.
+	jobLabel?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelValueLengthLimit?: int
+
+	// Selector to select which namespaces the Kubernetes `Endpoints`
+	// objects are discovered from.
+	namespaceSelector?: {
+		// Boolean describing whether all namespaces are selected in
+		// contrast to a list restricting them.
+		any?: bool
+
+		// List of namespace names to select from.
+		matchNames?: [...string]
+	}
+
+	// `podTargetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Pod` object onto the ingested
+	// metrics.
+	podTargetLabels?: [...string]
+
+	// `sampleLimit` defines a per-scrape limit on the number of
+	// scraped samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Label selector to select the Kubernetes `Endpoints` objects.
+	selector: {
+		// matchExpressions is a list of label selector requirements. The
+		// requirements are ANDed.
+		matchExpressions?: [...{
+			// key is the label key that the selector applies to.
+			key: string
+
+			// operator represents a key's relationship to a set of values.
+			// Valid operators are In, NotIn, Exists and DoesNotExist.
+			operator: string
+
+			// values is an array of string values. If the operator is In or
+			// NotIn, the values array must be non-empty. If the operator is
+			// Exists or DoesNotExist, the values array must be empty. This
+			// array is replaced during a strategic merge patch.
+			values?: [...string]
+		}]
+
+		// matchLabels is a map of {key,value} pairs. A single {key,value}
+		// in the matchLabels map is equivalent to an element of
+		// matchExpressions, whose key field is "key", the operator is
+		// "In", and the values array contains only "value". The
+		// requirements are ANDed.
+		matchLabels?: {
+			[string]: string
+		}
+	}
+
+	// `targetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Service` object onto the ingested
+	// metrics.
+	targetLabels?: [...string]
+
+	// `targetLimit` defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+}

docs/examples/platforms/reference/certificates.cue

@@ -1,5 +1,21 @@
 package holos
 
+#PlatformServers: {
+	for cluster in #Platform.clusters {
+		(cluster.name): {
+			"https-istio-ingress-httpbin": {
+				let cert = #PlatformCerts[cluster.name+"-httpbin"]
+				hosts: [for host in cert.spec.dnsNames {"istio-ingress/\(host)"}]
+				port: name:          "https-istio-ingress-httpbin"
+				port: number:        443
+				port: protocol:      "HTTPS"
+				tls: credentialName: cert.spec.secretName
+				tls: mode:           "SIMPLE"
+			}
+		}
+	}
+}
+
 #PlatformCerts: {
 	// Globally scoped platform services are defined here.
 	login: #PlatformCert & {

docs/examples/platforms/reference/clusters/accounts/iam/iam.cue

@@ -4,4 +4,4 @@ package holos
 #InputKeys: project: "iam"
 
 // Shared dependencies for all components in this collection.
-#DependsOn: namespaces: name: "\(#StageName)-secrets-namespaces"
+#DependsOn: namespaces: name: "\(#StageName)-secrets-stores"

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/README.md

@@ -0,0 +1,10 @@
+To deploy monitoring:
+
+> **_NOTE:_** For more detailed instructions on deploying, see the [documentation on installing Monitoring](https://access.crunchydata.com/documentation/postgres-operator/latest/installation/monitoring/kustomize).
+
+1. verify the namespace is correct in kustomization.yaml 
+2. If you are deploying in openshift, comment out the fsGroup line under securityContext in the following files:
+  - `alertmanager/deployment.yaml`
+  - `grafana/deployment.yaml`
+  - `prometheus/deployment.yaml`
+3. kubectl apply -k .

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/config/alertmanager.yml

@@ -0,0 +1,78 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+# Based on upstream example file found here: https://github.com/prometheus/alertmanager/blob/master/doc/examples/simple.yml
+global:
+    smtp_smarthost: 'localhost: 25'
+    smtp_require_tls: false
+    smtp_from: 'Alertmanager <abc@yahoo.com>'
+#    smtp_smarthost: 'smtp.example.com:587'
+#    smtp_from: 'Alertmanager <abc@yahoo.com>'
+#    smtp_auth_username: '<username>'
+#    smtp_auth_password: '<password>'
+
+# templates:
+# - '/etc/alertmanager/template/*.tmpl'
+
+inhibit_rules:
+# Apply inhibition of warning if the alertname for the same system and service is already critical
+- source_match:
+    severity: 'critical'
+  target_match:
+    severity: 'warning'
+  equal: ['alertname', 'job', 'service']
+
+receivers:
+- name: 'default-receiver'
+  email_configs:
+  - to: 'example@crunchydata.com'
+    send_resolved: true
+
+## Examples of alternative alert receivers. See documentation for more info on how to configure these fully
+#- name: 'pagerduty-dba'
+#  pagerduty_configs:
+#      - service_key: <RANDOMKEYSTUFF>
+
+#- name: 'pagerduty-sre'
+#  pagerduty_configs:
+#      - service_key: <RANDOMKEYSTUFF>
+
+#- name: 'dba-team'
+#  email_configs:
+#  - to: 'example-dba-team@crunchydata.com'
+#    send_resolved: true
+
+#- name: 'sre-team'
+#  email_configs:
+#  - to: 'example-sre-team@crunchydata.com'
+#    send_resolved: true
+
+route:
+    receiver: default-receiver
+    group_by: [severity, service, job, alertname]
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 24h
+
+## Example routes to show how to route outgoing alerts based on the content of that alert
+#    routes:
+#    - match_re:
+#        service: ^(postgresql|mysql|oracle)$
+#      receiver: dba-team
+#      # sub route to send critical dba alerts to pagerduty
+#      routes:
+#      - match:
+#          severity: critical
+#        receiver: pagerduty-dba
+#
+#    - match:
+#        service: system
+#      receiver: sre-team
+#      # sub route to send critical sre alerts to pagerduty
+#      routes:
+#      - match:
+#          severity: critical
+#        receiver: pagerduty-sre

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/deployment.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crunchy-alertmanager
+spec:
+  selector: {}
+  template:
+    spec:
+      containers:
+      - name: alertmanager
+        image: prom/alertmanager:v0.24.0
+        args:
+        - --config.file=/etc/alertmanager/alertmanager.yml
+        - --storage.path=/alertmanager
+        - --log.level=info
+        - --cluster.advertise-address=0.0.0.0:9093
+        livenessProbe:
+          httpGet:
+            path: /-/healthy
+            port: 9093
+          initialDelaySeconds: 25
+          periodSeconds: 20
+        ports:
+        - containerPort: 9093
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 9093
+        volumeMounts:
+        - mountPath: /etc/alertmanager
+          name: alertmanagerconf
+        - mountPath: /alertmanager
+          name: alertmanagerdata
+      securityContext:
+        fsGroup: 26
+        # supplementalGroups:
+        # - 65534
+      serviceAccountName: alertmanager
+      volumes:
+      - name: alertmanagerdata
+        persistentVolumeClaim:
+          claimName: alertmanagerdata
+      - name: alertmanagerconf
+        configMap:
+          defaultMode: 420
+          name: alertmanager-config

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/kustomization.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: crunchy-alertmanager
+
+resources:
+- deployment.yaml
+- pvc.yaml
+- service.yaml
+- serviceaccount.yaml
+
+configMapGenerator:
+- name: alertmanager-config
+  files:
+  - config/alertmanager.yml
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: alertmanagerdata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crunchy-alertmanager
+spec:
+  type: ClusterIP
+  ports:
+  - name: alertmanager
+    port: 9093

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: alertmanager

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/config/crunchy_grafana_dashboards.yml

@@ -0,0 +1,16 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+apiVersion: 1
+
+providers:
+- name: 'crunchy_dashboards'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards
+  options:
+    path: /etc/grafana/provisioning/dashboards

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/config/crunchy_grafana_datasource.yml

@@ -0,0 +1,18 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+# config file version
+apiVersion: 1
+
+datasources:
+  - name: PROMETHEUS
+    type: prometheus
+    access: proxy
+    url: http://$PROM_HOST:$PROM_PORT
+    isDefault: True
+    editable: False
+    orgId: 1
+    version: 1

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+configMapGenerator:
+- name: grafana-dashboards
+  files:
+  - pgbackrest.json
+  - pod_details.json
+  - postgresql_overview.json
+  - postgresql_details.json
+  - postgresql_service_health.json
+  - prometheus_alerts.json
+  - query_statistics.json
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/pgbackrest.json

@@ -0,0 +1,687 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069660860,
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "title": "",
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "dtdhms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "/^Value$/",
+          "values": false
+        },
+        "text": {
+          "valueSize": 45
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "time()-ccp_backrest_oldest_full_backup_time_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Recovery window",
+          "refId": "A"
+        }
+      ],
+      "title": "Recovery Window",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Differential Backup": "dark-blue",
+        "Full": "dark-green",
+        "Full Backup": "dark-green",
+        "Incremental": "light-blue",
+        "Incremental Backup": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 3
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": false
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_incr_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment,instance,ip,pod)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental Backup",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_diff_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential Backup",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_full_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full Backup",
+          "refId": "C"
+        },
+        {
+          "expr": "min(ccp_archive_command_status_seconds_since_last_archive{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "WAL Archive",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Time Since",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 3
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"incr\"}) without (deployment,instance,pod,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"diff\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"full\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Backup Runtimes",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 5,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"incr\"}) without (deployment, instance,pod,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"diff\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"full\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Backup Size",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Archive age": "blue",
+        "Archive count": "green",
+        "Differential": "dark-blue",
+        "Failed count": "red",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 3,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg(idelta(ccp_archive_command_status_failed_count{pg_cluster=\"[[cluster]]\", role=\"master\"}[1m])) without (instance,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Failed count",
+          "refId": "A"
+        },
+        {
+          "expr": "avg(idelta(ccp_archive_command_status_archived_count{pg_cluster=\"[[cluster]]\", role=\"master\"}[1m])) without (instance,pod, ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Archive count",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "WAL Stats",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "cluster",
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-2w",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "pgBackRest",
+  "uid": "2fcFZ6PGk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/postgresql_overview.json

@@ -0,0 +1,237 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069480601,
+  "links": [],
+  "panels": [
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "links": [
+            {
+              "targetBlank": true,
+              "title": "Cluster Details",
+              "url": "d/fMip0cuMk/postgresqldetails?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Backup Details",
+              "url": "d/2fcFZ6PGk/pgbackrest?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Pod Details",
+              "url": "d/4auP6Mk7k/pod-details?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Query Statistics",
+              "url": "d/ZKoTOHDGk/query-statistics?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Service Health",
+              "url": "d/dhG1wgsMz/postgresql-service-health?$__all_variables"
+            }
+          ],
+          "mappings": [
+            {
+              "from": "0",
+              "id": 0,
+              "text": "DOWN",
+              "to": "99",
+              "type": 2
+            },
+            {
+              "from": "100",
+              "id": 1,
+              "text": "Standalone Cluster",
+              "to": "199",
+              "type": 2
+            },
+            {
+              "from": "200",
+              "id": 2,
+              "text": "HA CLUSTER",
+              "to": "1000",
+              "type": 2
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "#bf1b00",
+                "value": null
+              },
+              {
+                "color": "#eab839",
+                "value": 10
+              },
+              {
+                "color": "#56A64B",
+                "value": 100
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {
+          "valueSize": 30
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "repeat": "cluster",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "$hashKey": "object:243",
+          "expr": "sum(pg_up{pg_cluster=~\"$cluster\"})*100+sum(ccp_is_in_recovery_status{pg_cluster=~\"$cluster\"})",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{cluster}}",
+          "metric": "up",
+          "refId": "A",
+          "step": 2
+        }
+      ],
+      "title": "$cluster - Overview",
+      "type": "stat"
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allFormat": "glob",
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 1,
+        "includeAll": true,
+        "label": "cluster",
+        "multi": true,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "PostgreSQL Overview",
+  "uid": "D2X39SlGk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/postgresql_service_health.json

@@ -0,0 +1,649 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069909806,
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "title": "",
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(ccp_connection_stats_total{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip) / sum(ccp_connection_stats_max_connections{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Connections",
+          "refId": "C"
+        },
+        {
+          "expr": "100 - 100 * avg(ccp_nodemx_data_disk_available_bytes{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip) / avg(ccp_nodemx_data_disk_total_bytes{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"})  without (pod,instance,ip)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Mount:{{mount_point}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Saturation (pct used)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "percent",
+          "label": null,
+          "logBase": 1,
+          "max": "100",
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "cacheTimeout": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 18,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "  sum(irate(ccp_stat_database_xact_commit{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m])) \n+ sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Transactions",
+          "refId": "A"
+        },
+        {
+          "expr": "max(ccp_connection_stats_active{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip,dbname)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Active connections",
+          "refId": "C"
+        },
+        {
+          "expr": "sum(irate(ccp_pg_stat_statements_total_calls_count{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Queries",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Traffic",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0.001",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "Errors",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]) without(pod,instance,ip))",
+          "format": "time_series",
+          "hide": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Rollbacks",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(irate(ccp_stat_database_deadlocks{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))  without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Deadlock ",
+          "refId": "D"
+        },
+        {
+          "expr": "sum(irate(ccp_stat_database_conflicts{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m])) without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Conflicts",
+          "refId": "B"
+        },
+        {
+          "expr": "max(pg_exporter_last_scrape_error{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "scrape error",
+          "refId": "C"
+        },
+        {
+          "expr": "max(clamp_max(ccp_archive_command_status_seconds_since_last_fail{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"},1)) without (instance,pod,ip)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "archive error",
+          "refId": "E"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Errors",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": "",
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [
+        {
+          "alias": "/Max:/",
+          "color": "#E02F44",
+          "nullPointMode": "null as zero"
+        },
+        {
+          "alias": "/Avg:/",
+          "color": "#8AB8FF"
+        }
+      ],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(ccp_pg_stat_statements_total_mean_exec_time_ms{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Avg: {{exported_role}}({{dbname}})",
+          "refId": "A"
+        },
+        {
+          "expr": "max(ccp_pg_stat_statements_top_max_exec_time_ms{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip,query,queryid)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Max: {{exported_role}}({{dbname}})",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Query Duration",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "ms",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values({pg_cluster=\"[[cluster]]\"},role)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "role",
+        "options": [],
+        "query": {
+          "query": "label_values({pg_cluster=\"[[cluster]]\"},role)",
+          "refId": "PROMETHEUS-role-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "PostgreSQL Service Health",
+  "uid": "dhG1wgsMz",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/prometheus_alerts.json

@@ -0,0 +1,961 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Show current firing and pending alerts, and  severity alert counts.",
+  "editable": false,
+  "gnetId": 4181,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [
+    {
+      "icon": "external link",
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 10,
+      "panels": [],
+      "repeat": null,
+      "title": "Environment Summary",
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 0,
+        "y": 1
+      },
+      "id": 6,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(count by (kubernetes_namespace) (pg_up))",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "Namespaces",
+          "refId": "A"
+        }
+      ],
+      "title": "Namespaces",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 4,
+        "y": 1
+      },
+      "id": 13,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(count by (pg_cluster) (pg_up))",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "PostgreSQL Clusters",
+          "refId": "A"
+        }
+      ],
+      "title": "PG Clusters",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 8,
+        "y": 1
+      },
+      "id": 14,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(pg_up)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "PostgreSQL Clusters",
+          "refId": "A"
+        }
+      ],
+      "title": "PG Instances",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 3
+      },
+      "id": 11,
+      "panels": [],
+      "repeat": null,
+      "title": "Alert Summary",
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-red",
+                "value": null
+              },
+              {
+                "color": "#F2495C",
+                "value": 1
+              },
+              {
+                "color": "#F2495C"
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 0,
+        "y": 4
+      },
+      "id": 2,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "bucketAggs": [
+            {
+              "id": "2",
+              "settings": {
+                "interval": "auto",
+                "min_doc_count": 0,
+                "trimEdges": 0
+              },
+              "type": "date_histogram"
+            }
+          ],
+          "dsType": "elasticsearch",
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"critical\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Critical",
+          "metrics": [
+            {
+              "field": "select field",
+              "id": "1",
+              "type": "count"
+            }
+          ],
+          "refId": "A"
+        }
+      ],
+      "title": "Critical",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-orange",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 4,
+        "y": 4
+      },
+      "id": 5,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"warning\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Warning",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "#299c46",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 8,
+        "y": 4
+      },
+      "id": 9,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"info\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Info",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 6
+      },
+      "id": 12,
+      "panels": [],
+      "repeat": null,
+      "title": "Alerts",
+      "type": "row"
+    },
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": null,
+            "displayMode": "auto",
+            "filterable": true
+          },
+          "decimals": 2,
+          "displayName": "",
+          "mappings": [
+            {
+              "from": "",
+              "id": 1,
+              "text": "",
+              "to": "",
+              "type": 1,
+              "value": ""
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "blue",
+                "value": 100
+              },
+              {
+                "color": "#EAB839",
+                "value": 200
+              },
+              {
+                "color": "red",
+                "value": 300
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity_num"
+            },
+            "properties": [
+              {
+                "id": "custom.displayMode",
+                "value": "color-background"
+              },
+              {
+                "id": "custom.width",
+                "value": 124
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Time"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 170
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 119
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertname"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 206
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertstate"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 128
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 24,
+        "x": 0,
+        "y": 7
+      },
+      "id": 1,
+      "links": [],
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "ALERTS{alertstate='firing'} > 0",
+          "format": "table",
+          "instant": true,
+          "interval": "2s",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Firing",
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {
+            "reducers": []
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Value": true,
+              "__name__": true,
+              "alertstate": false,
+              "deployment": false,
+              "exp_type": true,
+              "fs_type": true,
+              "instance": true,
+              "job": true,
+              "kubernetes_namespace": true,
+              "mount_point": true,
+              "server": true,
+              "service": true,
+              "severity_num": false
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 16,
+              "__name__": 3,
+              "alertname": 4,
+              "alertstate": 5,
+              "deployment": 7,
+              "exp_type": 9,
+              "instance": 10,
+              "ip": 11,
+              "job": 12,
+              "kubernetes_namespace": 13,
+              "pg_cluster": 6,
+              "pod": 8,
+              "role": 14,
+              "service": 15,
+              "severity": 2,
+              "severity_num": 1
+            },
+            "renameByName": {
+              "Time": "",
+              "__name__": "",
+              "severity": "",
+              "severity_num": ""
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": null,
+            "filterable": true
+          },
+          "decimals": 2,
+          "displayName": "",
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/(instance|__name__|Time|alertstate|job|type|Value)/"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              },
+              {
+                "id": "decimals",
+                "value": 2
+              },
+              {
+                "id": "custom.align",
+                "value": null
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Time"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": null
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity_num"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 126
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 115
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertname"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 207
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertstate"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 131
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "id": 3,
+      "links": [],
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "ALERTS{alertstate=\"pending\"}",
+          "format": "table",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Alerts (1 week)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Value": true,
+              "__name__": true,
+              "exp_type": true,
+              "instance": true,
+              "job": true,
+              "kubernetes_namespace": true,
+              "service": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 16,
+              "__name__": 3,
+              "alertname": 4,
+              "alertstate": 5,
+              "deployment": 7,
+              "exp_type": 8,
+              "instance": 9,
+              "ip": 11,
+              "job": 12,
+              "kubernetes_namespace": 13,
+              "pg_cluster": 6,
+              "pod": 10,
+              "role": 14,
+              "service": 15,
+              "severity": 2,
+              "severity_num": 1
+            },
+            "renameByName": {}
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "15m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Prometheus Alerts",
+  "uid": "lwxXsZsMk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/deployment.yaml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crunchy-grafana
+spec:
+  selector: {}
+  template:
+    spec:
+      containers:
+      - name: grafana
+        image: grafana/grafana:9.2.20
+        ports:
+        - containerPort: 3000
+        env:
+        - name: GF_PATHS_DATA
+          value: /data/grafana/data
+        - name: GF_SECURITY_ADMIN_USER__FILE
+          value: /conf/admin/username
+        - name: GF_SECURITY_ADMIN_PASSWORD__FILE
+          value: /conf/admin/password
+        - name: PROM_HOST
+          value: crunchy-prometheus
+        - name: PROM_PORT
+          value: "9090"
+        livenessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
+          initialDelaySeconds: 25
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
+        volumeMounts:
+        - mountPath: /data
+          name: grafanadata
+        - mountPath: /conf/admin
+          name: grafana-admin
+        - mountPath: /etc/grafana/provisioning/datasources
+          name: grafana-datasources
+        - mountPath: /etc/grafana/provisioning/dashboards
+          name: grafana-dashboards
+      securityContext:
+        fsGroup: 26
+        # supplementalGroups:
+        # - 65534
+      serviceAccountName: grafana
+      volumes:
+      - name: grafanadata
+        persistentVolumeClaim:
+          claimName: grafanadata
+      - name: grafana-admin
+        secret:
+          defaultMode: 420
+          secretName: grafana-admin
+      - name: grafana-datasources
+        configMap:
+          defaultMode: 420
+          name: grafana-datasources
+      - name: grafana-dashboards
+        configMap:
+          defaultMode: 420
+          name: grafana-dashboards

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/kustomization.yaml

@@ -0,0 +1,33 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: crunchy-grafana
+
+resources:
+- deployment.yaml
+- pvc.yaml
+- service.yaml
+- serviceaccount.yaml
+- dashboards
+
+configMapGenerator:
+- name: grafana-datasources
+  files:
+  - config/crunchy_grafana_datasource.yml
+- name: grafana-dashboards
+  behavior: merge
+  files:
+  - config/crunchy_grafana_dashboards.yml
+
+secretGenerator:
+- name: grafana-admin
+  literals:
+  - password=admin
+  - username=admin
+  type: Opaque
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafanadata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crunchy-grafana
+spec:
+  type: ClusterIP
+  ports:
+  - name: grafana
+    port: 3000

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grafana

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: prod-iam
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/name: prod-iam-obs
+    vendor: holos
+
+resources:
+- grafana
+- prometheus
+- alertmanager

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/monitoring.cue

@@ -0,0 +1,9 @@
+package holos
+
+spec: components: KustomizeBuildList: [
+	#KustomizeBuild & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "prod-iam-obs"
+	},
+]

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/clusterrole.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- resources:
+  - pods
+  apiGroups:
+  - ""
+  verbs:
+  - get
+  - list
+  - watch

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/clusterrolebinding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/config/crunchy-alert-rules-pg.yml

@@ -0,0 +1,418 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+groups:
+- name: alert-rules
+  rules:
+
+########## EXPORTER RULES ##########
+  - alert: PGExporterScrapeError
+    expr: pg_exporter_last_scrape_error > 0
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: 'Postgres Exporter running on {{ $labels.job }} (instance: {{ $labels.instance }}) is encountering scrape errors processing queries. Error count: ( {{ $value }} )'
+
+
+########## SYSTEM RULES ##########
+  - alert: ExporterDown
+    expr: avg_over_time(up[5m]) < 0.5
+    for: 10s
+    labels:
+      service: system
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'Metrics exporter service for {{ $labels.job }} running on {{ $labels.instance }} has been down at least 50% of the time for the last 5 minutes. Service may be flapping or down.'
+      summary: 'Prometheus Exporter Service Down'
+
+
+########## POSTGRESQL RULES ##########
+  - alert: PGIsUp
+    expr: pg_up < 1
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: 'postgres_exporter running on {{ $labels.job }} is unable to communicate with the configured database'
+
+
+# Example to check for current version of PostgreSQL. Metric returns the version that the exporter is running on, so you can set a rule to check for the minimum version you'd like all systems to be on. Number returned is the 6 digit integer representation contained in the setting "server_version_num".
+#
+#  - alert: PGMinimumVersion
+#    expr:  ccp_postgresql_version_current < 110005
+#    for: 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      summary: '{{ $labels.job }} is not running at least version 11.5 of PostgreSQL'
+
+
+# Whether a system switches from primary to replica or vice versa must be configured per named job.
+# No way to tell what value a system is supposed to be without a rule expression for that specific system
+# 2 to 1 means it changed from primary to replica. 1 to 2 means it changed from replica to primary
+# Set this alert for each system that you want to monitor a recovery status change
+# Below is an example for a target job called "Replica" and watches for the value to change above 1 which means it's no longer a replica
+#
+#  - alert: PGRecoveryStatusSwitch_Replica
+#    expr: ccp_is_in_recovery_status{job="Replica"} > 1
+#    for: 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      summary: '{{ $labels.job }} has changed from replica to primary'
+
+
+# Absence alerts must be configured per named job, otherwise there's no way to know which job is down
+# Below is an example for a target job called "Prod"
+#  - alert: PGConnectionAbsent_Prod
+#    expr: absent(ccp_connection_stats_max_connections{job="Prod"})
+#    for: 10s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Connection metric is absent from target (Prod). Check that postgres_exporter can connect to PostgreSQL.'
+
+
+# Optional monitor for changes to pg_settings (postgresql.conf) system catalog.
+# A similar metric is available for monitoring pg_hba.conf. See ccp_hba_settings_checksum().
+# If metric returns 0, then NO settings have changed for either pg_settings since last known valid state
+# If metric returns 1, then pg_settings have changed since last known valid state
+# To see what may have changed, check the monitor.pg_settings_checksum table for a history of config state.
+#  - alert: PGSettingsChecksum
+#    expr: ccp_pg_settings_checksum > 0
+#    for 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Configuration settings on {{ $labels.job }} have changed from previously known valid state. To reset current config to a valid state after alert fires, run monitor.pg_settings_checksum_set_valid().'
+#      summary: 'PGSQL Instance settings checksum'
+
+
+# Monitor for data block checksum failures. Only works in PG12+
+#  - alert: PGDataChecksum
+#    expr: ccp_data_checksum_failure > 0
+#    for 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: '{{ $labels.job }} has at least one data checksum failure in database {{ $labels.dbname }}. See pg_stat_database system catalog for more information.'
+#      summary: 'PGSQL Data Checksum failure'
+
+  - alert: PGIdleTxn
+    expr: ccp_connection_stats_max_idle_in_txn_time > 300
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} has at least one session idle in transaction for over 5 minutes.'
+      summary: 'PGSQL Instance idle transactions'
+
+  - alert: PGIdleTxn
+    expr: ccp_connection_stats_max_idle_in_txn_time > 900
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} has at least one session idle in transaction for over 15 minutes.'
+      summary: 'PGSQL Instance idle transactions'
+
+  - alert: PGQueryTime
+    expr: ccp_connection_stats_max_query_time > 43200
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} has at least one query running for over 12 hours.'
+      summary: 'PGSQL Max Query Runtime'
+
+  - alert: PGQueryTime
+    expr: ccp_connection_stats_max_query_time > 86400
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} has at least one query running for over 1 day.'
+      summary: 'PGSQL Max Query Runtime'
+
+  - alert: PGConnPerc
+    expr: 100 * (ccp_connection_stats_total / ccp_connection_stats_max_connections) > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} is using 75% or more of available connections ({{ $value }}%)'
+      summary: 'PGSQL Instance connections'
+
+  - alert: PGConnPerc
+    expr: 100 * (ccp_connection_stats_total / ccp_connection_stats_max_connections) > 90
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} is using 90% or more of available connections ({{ $value }}%)'
+      summary: 'PGSQL Instance connections'
+
+  - alert: DiskFillPredict
+    expr: predict_linear(ccp_nodemx_data_disk_available_bytes{mount_point!~"tmpfs"}[1h], 24 * 3600) < 0 and 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 70
+    for: 5m
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      summary: 'Disk predicted to be full in 24 hours'
+      description: 'Disk on {{ $labels.pg_cluster }}:{{ $labels.kubernetes_pod_name }} is predicted to fill in 24 hrs based on current usage'
+
+  - alert: PGClusterRoleChange
+    expr: count by (pg_cluster) (ccp_is_in_recovery_status != ignoring(instance,ip,pod,role) (ccp_is_in_recovery_status offset 5m)) >= 1
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: '{{ $labels.pg_cluster }} has had a switchover/failover event. Please check this cluster for more details'
+
+  - alert: PGDiskSize
+    expr: 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.deployment }} over 75% disk usage at mount point "{{ $labels.mount_point }}": {{ $value }}%'
+      summary: PGSQL Instance usage warning
+
+  - alert: PGDiskSize
+    expr: 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 90
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.deployment }} over 90% disk usage at mount point "{{ $labels.mount_point }}": {{ $value }}%'
+      summary: 'PGSQL Instance size critical'
+
+  - alert: PGReplicationByteLag
+    expr: ccp_replication_lag_size_bytes > 5.24288e+07
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has at least one replica lagging over 50MB behind.'
+      summary: 'PGSQL Instance replica lag warning'
+
+  - alert: PGReplicationByteLag
+    expr: ccp_replication_lag_size_bytes > 1.048576e+08
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has at least one replica lagging over 100MB behind.'
+      summary: 'PGSQL Instance replica lag warning'
+
+  - alert: PGReplicationSlotsInactive
+    expr: ccp_replication_slots_active == 0
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has one or more inactive replication slots'
+      summary: 'PGSQL Instance inactive replication slot'
+
+  - alert: PGXIDWraparound
+    expr: ccp_transaction_wraparound_percent_towards_wraparound > 50
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 50% towards transaction id wraparound.'
+      summary: 'PGSQL Instance {{ $labels.job }} transaction id wraparound imminent'
+
+  - alert: PGXIDWraparound
+    expr: ccp_transaction_wraparound_percent_towards_wraparound > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 75% towards transaction id wraparound.'
+      summary: 'PGSQL Instance transaction id wraparound imminent'
+
+  - alert: PGEmergencyVacuum
+    expr: ccp_transaction_wraparound_percent_towards_emergency_autovac > 110
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 110% beyond autovacuum_freeze_max_age value. Autovacuum may need tuning to better keep up.'
+      summary: 'PGSQL Instance emergency vacuum imminent'
+
+  - alert: PGEmergencyVacuum
+    expr: ccp_transaction_wraparound_percent_towards_emergency_autovac > 125
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 125% beyond autovacuum_freeze_max_age value. Autovacuum needs tuning to better keep up.'
+      summary: 'PGSQL Instance emergency vacuum imminent'
+
+  - alert: PGArchiveCommandStatus
+    expr: ccp_archive_command_status_seconds_since_last_fail > 300
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'PGSQL Instance {{ $labels.job }} has a recent failing archive command'
+        summary: 'Seconds since the last recorded failure of the archive_command'
+
+  - alert: PGSequenceExhaustion
+    expr: ccp_sequence_exhaustion_count > 0
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'Count of sequences on instance {{ $labels.job }} at over 75% usage: {{ $value }}. Run following query to see full sequence status: SELECT * FROM monitor.sequence_status() WHERE percent >= 75'
+
+  - alert: PGSettingsPendingRestart
+    expr: ccp_settings_pending_restart_count > 0
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'One or more settings in the pg_settings system catalog on system {{ $labels.job }} are in a pending_restart state. Check the system catalog for which settings are pending and review postgresql.conf for changes.'
+
+########## PGBACKREST RULES ##########
+#
+# Uncomment and customize one or more of these rules to monitor your pgbackrest backups.
+# Full backups are considered the equivalent of both differentials and incrementals since both are based on the last full
+#   And differentials are considered incrementals since incrementals will be based off the last diff if one exists
+#   This avoid false alerts, for example when you don't run diff/incr backups on the days that you run a full
+# Stanza should also be set if different intervals are expected for each stanza.
+#   Otherwise rule will be applied to all stanzas returned on target system if not set.
+#
+# Relevant metric names are:
+#   ccp_backrest_last_full_backup_time_since_completion_seconds
+#   ccp_backrest_last_incr_backup_time_since_completion_seconds
+#   ccp_backrest_last_diff_backup_time_since_completion_seconds
+#
+# To avoid false positives on backup time alerts, 12 hours are added onto each threshold to allow a buffer if the backup runtime varies from day to day.
+#    Further adjustment may be needed depending on your backup runtimes/schedule.
+#
+#  - alert: PGBackRestLastCompletedFull_main
+#    expr: ccp_backrest_last_full_backup_time_since_completion_seconds{stanza="main"} > 648000
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Full backup for stanza [main] on system {{ $labels.job }} has not completed in the last week.'
+#
+#  - alert: PGBackRestLastCompletedIncr_main
+#    expr: ccp_backrest_last_incr_backup_time_since_completion_seconds{stanza="main"} > 129600
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Incremental backup for stanza [main] on system {{ $labels.job }} has not completed in the last 24 hours.'
+#
+#
+# Runtime monitoring is handled with a single metric:
+#
+#   ccp_backrest_last_info_backup_runtime_seconds
+#
+# Runtime monitoring should have the "backup_type" label set.
+#   Otherwise the rule will apply to the last run of all backup types returned (full, diff, incr)
+# Stanza should also be set if runtimes per stanza have different expected times
+#
+#  - alert: PGBackRestLastRuntimeFull_main
+#    expr: ccp_backrest_last_info_backup_runtime_seconds{backup_type="full", stanza="main"} > 14400
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Expected runtime of full backup for stanza [main] has exceeded 4 hours'
+#
+#  - alert: PGBackRestLastRuntimeDiff_main
+#    expr: ccp_backrest_last_info_backup_runtime_seconds{backup_type="diff", stanza="main"} > 3600
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Expected runtime of diff backup for stanza [main] has exceeded 1 hour'
+##
+#
+## If the pgbackrest command fails to run, the metric disappears from the exporter output and the alert never fires.
+## An absence alert must be configured explicitly for each target (job) that backups are being monitored.
+## Checking for absence of just the full backup type should be sufficient (no need for diff/incr).
+## Note that while the backrest check command failing will likely also cause a scrape error alert, the addition of this
+## check gives a clearer answer as to what is causing it and that something is wrong with the backups.
+#
+#  - alert: PGBackrestAbsentFull_Prod
+#    expr: absent(ccp_backrest_last_full_backup_time_since_completion_seconds{job="Prod"})
+#    for: 10s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Backup Full status missing for Prod. Check that pgbackrest info command is working on target system.'

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/config/prometheus.yml

@@ -0,0 +1,85 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+---
+global:
+  scrape_interval: 15s
+  scrape_timeout: 15s
+  evaluation_interval: 5s
+
+scrape_configs:
+- job_name: 'crunchy-postgres-exporter'
+  kubernetes_sd_configs:
+  - role: pod
+    selectors:
+    - role: pod
+      label: postgres-operator.crunchydata.com/crunchy-postgres-exporter=true
+  relabel_configs:
+  # Keep exporter port and drop all others
+  - source_labels: [__meta_kubernetes_pod_container_port_number]
+    action: keep
+    regex: 9187
+  # Set label for namespace
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: kubernetes_namespace
+  # Set label for pod name
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  # Convert namespace and cluster name to pg_cluster=namespace:cluster
+  - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_cluster]
+    target_label: pg_cluster
+    separator: ":"
+    replacement: '$1$2'
+  # Convert kubernetes pod ip to ip
+  - source_labels: [__meta_kubernetes_pod_ip]
+    target_label: ip
+  # Convert postgres-operator.crunchydata.com/instance to deployment
+  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_instance]
+    target_label: deployment
+  # Convert postgres-operator.crunchydata.com/role to role
+  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_role]
+    target_label: role
+
+- job_name: 'crunchy-postgres-exporter-v4'
+  kubernetes_sd_configs:
+  - role: pod
+    selectors:
+    - role: pod
+      label: crunchy-postgres-exporter=true
+
+  relabel_configs:
+  # Keep exporter port and drop all others
+  - source_labels: [__meta_kubernetes_pod_container_port_number]
+    action: keep
+    regex: 9187
+  # Set label for namespace
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: kubernetes_namespace
+  # Set label for pod name
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  # Convert namespace and cluster name to pg_cluster=namespace:cluster
+  - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_pg_cluster]
+    target_label: pg_cluster
+    separator: ":"
+    replacement: '$1$2'
+  # Convert kubernetes pod ip to ip
+  - source_labels: [__meta_kubernetes_pod_ip]
+    target_label: ip
+  # Set deployment_name as deployment label
+  - source_labels: [__meta_kubernetes_pod_label_deployment_name]
+    target_label: deployment
+  # Set label for role
+  - source_labels: [__meta_kubernetes_pod_label_role]
+    target_label: role
+rule_files:
+  - /etc/prometheus/alert-rules.d/*.yml
+alerting:
+  alertmanagers:
+  - kubernetes_sd_configs:
+    - role: pod
+      selectors:
+      - role: pod
+        label: app.kubernetes.io/component=crunchy-alertmanager

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/deployment.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crunchy-prometheus
+spec:
+  selector: {}
+  template:
+    spec:
+      containers:
+      - name: prometheus
+        image: prom/prometheus:v2.39.2
+        ports:
+        - containerPort: 9090
+        livenessProbe:
+          httpGet:
+            path: /-/healthy
+            port: 9090
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 9090
+        volumeMounts:
+        - mountPath: /etc/prometheus
+          name: prometheusconf
+        - mountPath: /prometheus
+          name: prometheusdata
+        - mountPath: /etc/prometheus/alert-rules.d
+          name: alertmanagerrules
+      securityContext:
+        fsGroup: 26
+        # supplementalGroups:
+        # - 65534
+      serviceAccountName: prometheus
+      volumes:
+      - name: prometheusconf
+        configMap:
+          defaultMode: 420
+          name: crunchy-prometheus
+      - name: prometheusdata
+        persistentVolumeClaim:
+          claimName: prometheusdata
+      - name: alertmanagerrules
+        configMap:
+          defaultMode: 420
+          name: alert-rules-config

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: crunchy-prometheus
+
+resources:
+- deployment.yaml
+- pvc.yaml
+- service.yaml
+- serviceaccount.yaml
+- clusterrole.yaml
+- clusterrolebinding.yaml
+
+configMapGenerator:
+- name: crunchy-prometheus
+  files:
+  - config/prometheus.yml
+- name: alert-rules-config
+  files:
+  - config/crunchy-alert-rules-pg.yml
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: prometheusdata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crunchy-prometheus
+spec:
+  type: ClusterIP
+  ports:
+  - name: prometheus
+    port: 9090

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/postgres/postgres.cue

@@ -37,7 +37,7 @@ spec: components: KubernetesObjectsList: [
 	#KubernetesObjects & {
 		metadata: name: "prod-iam-postgres"
 
-		_dependsOn: "prod-secrets-namespaces": _
+		_dependsOn: "prod-secrets-stores":     _
 		_dependsOn: "prod-iam-postgres-certs": _
 		apiObjectMap: OBJECTS.apiObjectMap
 	},
@@ -68,7 +68,7 @@ let OBJECTS = #APIObjects & {
 					replicas: 2
 					dataVolumeClaimSpec: {
 						accessModes: ["ReadWriteOnce"]
-						resources: requests: storage: "10Gi"
+						resources: requests: storage: "20Gi"
 					}
 				}]
 				standby: {
@@ -80,6 +80,8 @@ let OBJECTS = #APIObjects & {
 						enabled: true
 					}
 				}
+				// Monitoring configuration
+				monitoring: pgmonitor: exporter: image: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-5.5.1-0"
 				// Restore from backup if and only if the cluster is primary
 				if Cluster.primary {
 					dataSource: pgbackrest: {
@@ -124,7 +126,7 @@ let OBJECTS = #APIObjects & {
 						"\(BucketRepoName)-cipher-type": "aes-256-cbc"
 						// "The convention we recommend for setting this variable is /pgbackrest/$NAMESPACE/$CLUSTER_NAME/repoN"
 						// Ref: https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/backups#understanding-backup-configuration-and-basic-operations
-						"\(BucketRepoName)-path": "/pgbackrest/\(#TargetNamespace)/\(metadata.name)/\(manual.repoName)"
+						"\(BucketRepoName)-path": "/pgbackrest/\(metadata.namespace)/\(metadata.name)/\(manual.repoName)"
 					}
 					repos: [
 						{
@@ -165,7 +167,7 @@ let HighlyAvailable = {
 			replicas: 2
 			dataVolumeClaimSpec: {
 				accessModes: ["ReadWriteOnce"]
-				resources: requests: storage: string | *"10Gi"
+				resources: requests: storage: string | *"20Gi"
 			}
 			affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: [{
 				weight: 1

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel.cue

@@ -1,7 +1,9 @@
 package holos
 
-#InstancePrefix:  "prod-iam"
-#TargetNamespace: #InstancePrefix + "-zitadel"
+#InstancePrefix: "prod-iam"
+
+// The namespace is managed by a project.
+#TargetNamespace: _Projects.iam.environments.prod.namespace
 
 // _DBName is the database name used across multiple holos components in this project
 _DBName: "zitadel"

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/values.cue

@@ -74,7 +74,7 @@ package holos
 		repository: "ghcr.io/zitadel/zitadel"
 		pullPolicy: "IfNotPresent"
 		// Overrides the image tag whose default is the chart appVersion.
-		tag: ""
+		tag: string | *""
 	}
 
 	chownImage: {

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/values.holos.cue

@@ -1,6 +1,10 @@
 package holos
 
 #Values: {
+	// https://github.com/zitadel/zitadel/releases
+	// Overrides the image tag whose default is the chart appVersion.
+	image: tag: "v2.49.1"
+
 	// Database credentials
 	// Refer to https://access.crunchydata.com/documentation/postgres-operator/5.2.0/architecture/user-management/
 	// Refer to https://zitadel.com/docs/self-hosting/manage/database#postgres
@@ -74,6 +78,12 @@ package holos
 			ExternalPort:   443
 			TLS: Enabled: false
 
+			// Fix AuthProxy JWKS Error - Jwks doesn't have key to match kid or alg from Jwt
+			// Refer to: https://github.com/holos-run/holos/issues/96
+			// Refer to: https://github.com/zitadel/zitadel/discussions/7464
+			SystemDefaults: KeyConfig: PrivateKeyLifetime: "999999h"
+			SystemDefaults: KeyConfig: PublicKeyLifetime:  "999999h"
+
 			// Database connection credentials are injected via environment variables from the db-pguser-db secret.
 			Database: postgres: {
 				MaxOpenConns:    25

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/zitadel.cue

@@ -16,7 +16,7 @@ spec: components: HelmChartList: [
 		enableHooks: true
 		chart: {
 			name:    Name
-			version: "7.9.0"
+			version: "7.11.0"
 			repository: {
 				name: Name
 				url:  "https://charts.zitadel.com"

docs/examples/platforms/reference/clusters/foundation/cloud/argocd/argocd.cue

@@ -0,0 +1,84 @@
+package holos
+
+import "encoding/yaml"
+
+let ArgoCD = "argocd"
+let Namespace = "prod-platform"
+
+spec: components: HelmChartList: [
+	#HelmChart & {
+		_dependsOn: "prod-secrets-stores": _
+		namespace: Namespace
+
+		metadata: name: "\(namespace)-\(ArgoCD)"
+
+		chart: {
+			name:    "argo-cd"
+			release: "argocd"
+			version: "6.7.8"
+			repository: {
+				name: "argocd"
+				url:  "https://argoproj.github.io/argo-helm"
+			}
+		}
+
+		_values: #ArgoCDValues & {
+			kubeVersionOverride: "1.29.0"
+			global: domain: "argocd.\(#ClusterName).\(#Platform.org.domain)"
+			dex: enabled:   false
+			// for integration with istio
+			configs: params: "server.insecure": true
+			configs: cm: {
+				"admin.enabled": false
+				"oidc.config":   yaml.Marshal(OIDCConfig)
+			}
+		}
+
+		// Holos overlay objects
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		// ExternalSecret: "deploy-key": _
+		VirtualService: (ArgoCD): {
+			metadata: name:      ArgoCD
+			metadata: namespace: Namespace
+			spec: hosts: [
+				ArgoCD + ".\(#Platform.org.domain)",
+				ArgoCD + ".\(#ClusterName).\(#Platform.org.domain)",
+			]
+			spec: gateways: ["istio-ingress/default"]
+			spec: http: [{route: [{destination: {
+				host: "argocd-server.\(Namespace).svc.cluster.local"
+				port: number: 80
+			}}]}]
+		}
+	}
+}
+let IstioInject = [{op: "add", path: "/spec/template/metadata/labels/sidecar.istio.io~1inject", value: "true"}]
+
+#Kustomize: _patches: {
+	mesh: {
+		target: {
+			group:   "apps"
+			version: "v1"
+			kind:    "Deployment"
+			name:    "argocd-server"
+		}
+		patch: yaml.Marshal(IstioInject)
+	}
+}
+
+// Probably shouldn't use the authproxy struct and should instead define an identity provider struct.
+let AuthProxySpec = #AuthProxySpec & #Platform.authproxy
+
+let OIDCConfig = {
+	name:     "Holos Platform"
+	issuer:   AuthProxySpec.issuer
+	clientID: #Platform.argocd.clientID
+	requestedIDTokenClaims: groups: essential: true
+	requestedScopes: ["openid", "profile", "email", "groups", "urn:zitadel:iam:org:domain:primary:\(AuthProxySpec.orgDomain)"]
+	enablePKCEAuthentication: true
+}

docs/examples/platforms/reference/clusters/foundation/cloud/certmanager/certmanager.cue

@@ -0,0 +1,43 @@
+package holos
+
+// https://cert-manager.io/docs/
+
+#TargetNamespace: "cert-manager"
+
+spec: components: HelmChartList: [
+	#HelmChart & {
+		metadata: name: "prod-mesh-certmanager"
+
+		_dependsOn: "prod-secrets-namespaces": _
+		namespace: #TargetNamespace
+		_values: #Values & {
+			installCRDs: true
+			startupapicheck: enabled: false
+			// Must not use kube-system on gke autopilot.  GKE Warden authz blocks access.
+			global: leaderElection: namespace: #TargetNamespace
+		}
+		chart: {
+			name:    "cert-manager"
+			version: "1.14.3"
+			repository: {
+				name: "jetstack"
+				url:  "https://charts.jetstack.io"
+			}
+		}
+	},
+]
+
+// https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#min-max-requests
+#PodResources: {
+	requests: {
+		cpu:                 string | *"250m"
+		memory:              string | *"512Mi"
+		"ephemeral-storage": string | *"100Mi"
+	}
+}
+
+// https://cloud.google.com/kubernetes-engine/docs/how-to/autopilot-spot-pods
+#NodeSelector: {
+	// "kubernetes.io/os":          "linux"
+	// "cloud.google.com/gke-spot": "true"
+}

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/gateway/gateway.cue

@@ -7,48 +7,53 @@ let Name = "gateway"
 #InputKeys: component: Name
 #TargetNamespace: "istio-ingress"
 
-let LoginCert = #PlatformCerts.login
-
 spec: components: KubernetesObjectsList: [
 	#KubernetesObjects & {
-		_dependsOn: "prod-secrets-namespaces": _
-		_dependsOn: "prod-mesh-istio-base":    _
-		_dependsOn: "prod-mesh-ingress":       _
+		_dependsOn: "prod-secrets-stores":  _
+		_dependsOn: "prod-mesh-istio-base": _
+		_dependsOn: "prod-mesh-ingress":    _
 
 		metadata: name: "\(#InstancePrefix)-\(Name)"
 		apiObjectMap: OBJECTS.apiObjectMap
 	},
 ]
 
+// GatewayServers represents all hosts for all VirtualServices in the cluster attached to Gateway/default
+// NOTE: This is a critical structure because the default Gateway should be used in most cases.
+let GatewayServers = {
+	for Project in _Projects {
+		for server in (#ProjectTemplate & {project: Project}).ClusterGatewayServers {
+			(server.port.name): server
+		}
+	}
+
+	for k, svc in #OptionalServices {
+		if svc.enabled && list.Contains(svc.clusterNames, #ClusterName) {
+			for server in svc.servers {
+				(server.port.name): server
+			}
+		}
+	}
+
+	if #PlatformServers[#ClusterName] != _|_ {
+		for server in #PlatformServers[#ClusterName] {
+			(server.port.name): server
+		}
+	}
+}
+
 let OBJECTS = #APIObjects & {
 	apiObjects: {
-		ExternalSecret: login: #ExternalSecret & {
-			_name: "login"
-		}
 		Gateway: default: #Gateway & {
 			metadata: name:      "default"
 			metadata: namespace: #TargetNamespace
+
 			spec: selector: istio: "ingressgateway"
-			spec: servers: [
-				{
-					hosts: [for dnsName in LoginCert.spec.dnsNames {"prod-iam-zitadel/\(dnsName)"}]
-					port: name:          "https-prod-iam-login"
-					port: number:        443
-					port: protocol:      "HTTPS"
-					tls: credentialName: LoginCert.spec.secretName
-					tls: mode:           "SIMPLE"
-				},
-			]
+			spec: servers: [for x in GatewayServers {x}]
 		}
 
 		for k, svc in #OptionalServices {
 			if svc.enabled && list.Contains(svc.clusterNames, #ClusterName) {
-				Gateway: "\(svc.name)": #Gateway & {
-					metadata: name:      svc.name
-					metadata: namespace: #TargetNamespace
-					spec: selector: istio: "ingressgateway"
-					spec: servers: [for s in svc.servers {s}]
-				}
 				for k, s in svc.servers {
 					ExternalSecret: "\(s.tls.credentialName)": _
 				}

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/httpbin/httpbin.cue

@@ -3,7 +3,6 @@ package holos
 let Name = "httpbin"
 let ComponentName = "\(#InstancePrefix)-\(Name)"
 
-let SecretName = #InputKeys.cluster + "-" + Name
 let MatchLabels = {
 	app:                          Name
 	"app.kubernetes.io/instance": ComponentName
@@ -18,7 +17,7 @@ let Metadata = {
 
 #TargetNamespace: "istio-ingress"
 
-let Cert = #PlatformCerts[SecretName]
+let Cert = #PlatformCerts["\(#ClusterName)-httpbin"]
 
 spec: components: KubernetesObjectsList: [
 	#KubernetesObjects & {
@@ -63,24 +62,10 @@ let OBJECTS = #APIObjects & {
 				{port: 80, targetPort: 8080, protocol: "TCP", name: "http"},
 			]
 		}
-		Gateway: httpbin: #Gateway & {
-			metadata: Metadata
-			spec: selector: istio: "ingressgateway"
-			spec: servers: [
-				{
-					hosts: [for host in Cert.spec.dnsNames {"\(#TargetNamespace)/\(host)"}]
-					port: name:          "https-\(ComponentName)"
-					port: number:        443
-					port: protocol:      "HTTPS"
-					tls: credentialName: Cert.spec.secretName
-					tls: mode:           "SIMPLE"
-				},
-			]
-		}
 		VirtualService: httpbin: #VirtualService & {
 			metadata: Metadata
 			spec: hosts: [for host in Cert.spec.dnsNames {host}]
-			spec: gateways: ["\(#TargetNamespace)/\(Name)"]
+			spec: gateways: ["istio-ingress/default"]
 			spec: http: [{route: [{destination: host: Name}]}]
 		}
 	}

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/ingress/ingress.cue

@@ -56,6 +56,8 @@ spec: components: HelmChartList: [
 		apiObjectMap: _IngressAuthProxy.Deployment.apiObjectMap
 		// Auth Policy
 		apiObjectMap: _IngressAuthProxy.Policy.apiObjectMap
+		// Auth Policy Exclusions
+		apiObjectMap: _AuthPolicyRules.objects.apiObjectMap
 	},
 ]
 

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/istiod/meshconfig.cue

@@ -1,7 +1,7 @@
 package holos
 
 // Ingress Gateway default auth proxy
-let Provider = _IngressAuthProxy.authproxy.provider
+let Provider = _IngressAuthProxy.AuthProxySpec.provider
 let Service = _IngressAuthProxy.service
 #MeshConfig: extensionProviderMap: (Provider): envoyExtAuthzHttp: service: Service
 

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/mesh.cue

@@ -9,10 +9,10 @@ import "encoding/yaml"
 // The ingress gateway auth proxy is used by multiple cue instances.
 // AUTHPROXY configures one oauth2-proxy deployment for each host in each stage of a project.  Multiple deployments per stage are used to narrow down the cookie domain.
 _IngressAuthProxy: {
-	Name:      "authproxy"
-	Namespace: "istio-ingress"
-	service:   "\(Name).\(Namespace).svc.cluster.local"
-	authproxy: #IngressAuthProxySpec | *#Platform.authproxy
+	Name:          "authproxy"
+	Namespace:     "istio-ingress"
+	service:       "\(Name).\(Namespace).svc.cluster.local"
+	AuthProxySpec: #AuthProxySpec & #Platform.authproxy
 
 	Domains: [DOMAIN=string]: {name: DOMAIN}
 	Domains: (#Platform.org.domain):                    _
@@ -48,8 +48,8 @@ _IngressAuthProxy: {
 						id:                    "Holos Platform"
 						name:                  "Holos Platform"
 						provider:              "oidc"
-						scope:                 "openid profile email groups offline_access urn:zitadel:iam:org:domain:primary:\(authproxy.orgDomain)"
-						clientID:              authproxy.clientID
+						scope:                 "openid profile email groups offline_access urn:zitadel:iam:org:domain:primary:\(AuthProxySpec.orgDomain)"
+						clientID:              AuthProxySpec.clientID
 						clientSecretFile:      "/dev/null"
 						code_challenge_method: "S256"
 						loginURLParameters: [{
@@ -57,7 +57,7 @@ _IngressAuthProxy: {
 							name: "approval_prompt"
 						}]
 						oidcConfig: {
-							issuerURL: authproxy.issuer
+							issuerURL: AuthProxySpec.issuer
 							audienceClaims: ["aud"]
 							emailClaim:  "email"
 							groupsClaim: "groups"
@@ -95,7 +95,7 @@ _IngressAuthProxy: {
 								}]
 								args: [
 									// callback url is proxy prefix + /callback
-									"--proxy-prefix=" + authproxy.proxyPrefix,
+									"--proxy-prefix=" + AuthProxySpec.proxyPrefix,
 									"--email-domain=*",
 									"--session-store-type=redis",
 									"--redis-connection-url=redis://\(RedisMetadata.name):6379",
@@ -155,7 +155,7 @@ _IngressAuthProxy: {
 				spec: hosts: ["*"]
 				spec: gateways: ["istio-ingress/default"]
 				spec: http: [{
-					match: [{uri: prefix: authproxy.proxyPrefix}]
+					match: [{uri: prefix: AuthProxySpec.proxyPrefix}]
 					route: [{
 						destination: host: Name
 						destination: port: number: 4180
@@ -254,24 +254,70 @@ _IngressAuthProxy: {
 			RequestAuthentication: (Name): #RequestAuthentication & {
 				metadata: Metadata & {name: Name}
 				spec: jwtRules: [{
-					audiences: ["\(authproxy.projectID)"]
+					audiences: ["\(AuthProxySpec.projectID)"]
 					forwardOriginalToken: true
-					fromHeaders: [{name: authproxy.idTokenHeader}]
-					issuer: authproxy.issuer
+					fromHeaders: [{name: AuthProxySpec.idTokenHeader}]
+					issuer: AuthProxySpec.issuer
 				}]
 				spec: selector: matchLabels: istio: "ingressgateway"
 			}
 			AuthorizationPolicy: "\(Name)-custom": {
+				_description: "Route all requests through the auth proxy by default"
+
 				metadata: Metadata & {name: "\(Name)-custom"}
 				spec: {
 					action: "CUSTOM"
-					provider: name: authproxy.provider
-					// bypass the external authorizer when the id token is already in the request.
-					// the RequestAuthentication rule will verify the token.
-					rules: [{when: [{key: "request.headers[\(authproxy.idTokenHeader)]", notValues: ["*"]}]}]
+					provider: name: AuthProxySpec.provider
+					rules: [
+						{
+							to: [{
+								operation: notHosts: [
+									// Never send requests for the login service through the authorizer, would block login.
+									AuthProxySpec.issuerHost,
+									// Exclude hosts with specialized rules from the catch-all.
+									for x in _AuthPolicyRules.hosts {x.name},
+								]
+							}]
+							when: [
+								{
+									// bypass the external authorizer when the id token is already in the request.
+									// the RequestAuthentication rule will verify the token.
+									key: "request.headers[\(AuthProxySpec.idTokenHeader)]"
+									notValues: ["*"]
+								},
+							]
+						},
+					]
 					selector: matchLabels: istio: "ingressgateway"
 				}
 			}
 		}
 	}
 }
+
+_AuthPolicyRules: #AuthPolicyRules & {
+	hosts: {
+		let Vault = "vault.core.ois.run"
+		(Vault): {
+			slug: "vault"
+			// Rules for when to route requests through the auth proxy
+			spec: rules: [
+				{
+					to: [{
+						operation: hosts: [Vault]
+						operation: paths: ["/ui", "/ui/*"]
+					}]
+				},
+				{
+					to: [{
+						operation: hosts: [Vault]
+					}]
+					when: [{
+						key: "request.headers[x-vault-request]"
+						notValues: ["true"]
+					}]
+				},
+			]
+		}
+	}
+}

docs/examples/platforms/reference/clusters/foundation/cloud/obs/obs.cue

@@ -0,0 +1,132 @@
+package holos
+
+// import "encoding/yaml"
+
+let Namespace = "prod-platform"
+
+// FYI: kube-prometheus-stack is a large umbrella chart what brings in other large charts like
+// [grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana).
+// This may make affect maintainability.  Consider breaking the integration down into
+// constituent charts represented as holos component instances.
+spec: components: HelmChartList: [
+	#HelmChart & {
+		_dependsOn: "prod-secrets-stores": _
+		namespace: Namespace
+
+		metadata: name: "\(namespace)-obs"
+
+		chart: {
+			name:    "kube-prometheus-stack"
+			release: "obs"
+			version: "57.2.0"
+			repository: {
+				name: "prometheus-community"
+				url:  "https://prometheus-community.github.io/helm-charts"
+			}
+		}
+
+		_values: #KubePromStackValues & {
+			fullnameOverride: "obs"
+
+			// https://github.com/prometheus-community/helm-charts/tree/kube-prometheus-stack-57.2.0/charts/kube-prometheus-stack#alternatives
+			prometheusOperator: admissionWebhooks: certManager: enabled: true
+
+			prometheus: prometheusSpec: {
+				// Pick up all ServiceMonitors in the monitoring namespace.  Note this means
+				// only one Prometheus should be installed in the monitoring namespace.
+				// See https://github.com/prometheus-community/helm-charts/issues/1911
+				serviceMonitorSelectorNilUsesHelmValues: false
+
+				// Unnecessary?
+				// externalUrl: "https://\(Hosts.prometheus.name)"
+
+				storageSpec: volumeClaimTemplate: spec: {
+					accessModes: ["ReadWriteOnce"]
+					resources: requests: storage: "10Gi"
+				}
+			}
+
+			grafana: {
+				enabled: true
+				persistence: {
+					enabled: true
+					type:    "sts"
+					accessModes: ["ReadWriteOnce"]
+					size: "4Gi"
+					finalizers: ["kubernetes.io/pvc-protection"]
+				}
+
+				"grafana.ini": {
+					analytics: check_for_updates: false
+					grafana_net: url:             "https://\(#Platform.org.emailDomain)"
+					server: domain:               "\(Hosts.grafana.name)"
+					// necessary to get the correct oidc redirect uri
+					server: root_url:           "https://\(server.domain)/"
+					auth: oauth_auto_login:     true
+					auth: disable_login_form:   true
+					auth: disable_signout_menu: true
+					"auth.generic_oauth": {
+						let OIDC = #Platform.oauthClients.grafana.spec
+						enabled:             true
+						name:                "Holos Platform"
+						enable_login_token:  true
+						auto_sign_up:        true
+						client_id:           OIDC.clientID
+						scopes:              OIDC.scopes
+						auth_url:            OIDC.authorization_endpoint
+						token_url:           OIDC.token_endpoint
+						api_url:             OIDC.userinfo_endpoint
+						use_pkce:            true
+						name_attribute_path: name
+						// TODO: Lift the admin, editor, and viewer group names up to the plaform config struct.
+						role_attribute_path: "contains(groups[*], 'prod-cluster-admin') && 'Admin' || contains(groups[*], 'prod-cluster-editor') && 'Editor' || 'Viewer'"
+					}
+				}
+			}
+		}
+
+		// Holos overlay objects
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		// ExternalSecret: "deploy-key": _
+		VirtualService: (Hosts.prometheus.service): {
+			metadata: name:      Hosts.prometheus.service
+			metadata: namespace: Namespace
+			spec: hosts: [for host in Hosts.prometheus.hosts {host.name}]
+			spec: gateways: ["istio-ingress/default"]
+			spec: http: [{route: [{destination: {
+				host: "obs-prometheus.\(Namespace).svc.cluster.local"
+				port: number: 9090
+			}}]}]
+		}
+		VirtualService: (Hosts.grafana.service): {
+			metadata: name:      Hosts.grafana.service
+			metadata: namespace: Namespace
+			spec: hosts: [for host in Hosts.grafana.hosts {host.name}]
+			spec: gateways: ["istio-ingress/default"]
+			spec: http: [{route: [{destination: {
+				host: "obs-grafana.\(Namespace).svc.cluster.local"
+				port: number: 80
+			}}]}]
+		}
+	}
+}
+
+let Hosts = {
+	[Service=_]: {
+		name: string | *"\(service).\(#ClusterName).\(#Platform.org.domain)"
+		let Name = name
+		hosts: {
+			cluster: name:  Name
+			platform: name: "\(service).\(#Platform.org.domain)"
+		}
+		service: Service
+	}
+
+	prometheus: _
+	grafana:    _
+}

docs/examples/platforms/reference/clusters/foundation/metal/ceph/ceph.cue

@@ -10,7 +10,7 @@ package holos
 
 spec: components: HelmChartList: [
 	#HelmChart & {
-		_dependsOn: "prod-secrets-namespaces": _
+		_dependsOn: "prod-secrets-stores": _
 
 		metadata: name: "prod-metal-ceph"
 

docs/examples/platforms/reference/clusters/optional/vault/vault.cue

@@ -67,7 +67,7 @@ let OBJECTS = #APIObjects & {
 			metadata: name:      Name
 			metadata: namespace: #TargetNamespace
 			spec: hosts: [for cert in Vault.certs {cert.spec.commonName}]
-			spec: gateways: ["istio-ingress/\(Name)"]
+			spec: gateways: ["istio-ingress/default"]
 			spec: http: [
 				{
 					route: [

docs/examples/platforms/reference/clusters/projects.cue

@@ -2,14 +2,17 @@ package holos
 
 #Project: authProxyOrgDomain: "openinfrastructure.co"
 
+let ZitadelProjectID = 257713952794870157
+
 _Projects: #Projects & {
 	// The platform project is required and where platform services reside.  ArgoCD, Grafana, Prometheus, etc...
 	platform: {
-		resourceId: 257713952794870157
-		clusters: k1: _
+		resourceId: ZitadelProjectID
+		// platform level services typically run in the core cluster pair.
+		clusters: core1: _
+		clusters: core2: _
+		// for development, probably wouldn't run these services in the workload clusters.
 		clusters: k2: _
-		stages: dev: authProxyClientID:  "260887327029658738@holos_platform"
-		stages: prod: authProxyClientID: "260887404288738416@holos_platform"
 		// Services hosted in the platform project
 		hosts: argocd:     _
 		hosts: grafana:    _
@@ -17,11 +20,9 @@ _Projects: #Projects & {
 	}
 
 	holos: {
-		resourceId: 260446255245690199
+		resourceId: ZitadelProjectID
 		clusters: k1: _
 		clusters: k2: _
-		stages: dev: authProxyClientID:  "260505543108527218@holos"
-		stages: prod: authProxyClientID: "260506079325128023@holos"
 		environments: {
 			prod: stage: "prod"
 			dev: stage:  "dev"
@@ -32,13 +33,12 @@ _Projects: #Projects & {
 	}
 
 	iam: {
-		resourceId: 260582480954787159
+		resourceId: ZitadelProjectID
+		hosts: login: _
 		clusters: {
 			core1: _
 			core2: _
 		}
-		stages: dev: authProxyClientID:  "260582521186616432@iam"
-		stages: prod: authProxyClientID: "260582633862399090@iam"
 	}
 }
 

docs/examples/platforms/reference/clusters/provisioner/iam/zitadel/postgres-certs/postgres-certs.cue

@@ -8,7 +8,7 @@ package holos
 
 // Refer to [Using Cert Manager to Deploy TLS for Postgres on Kubernetes](https://www.crunchydata.com/blog/using-cert-manager-to-deploy-tls-for-postgres-on-kubernetes)
 
-#TargetNamespace: "prod-iam-zitadel"
+#TargetNamespace: _Projects.iam.environments.prod.namespace
 #InputKeys: component: "postgres-certs"
 
 let DBName = "zitadel"

docs/examples/platforms/reference/namespaces.cue

@@ -22,7 +22,6 @@ let Privileged = {
 	{name: "istio-ingress"} & Restricted,
 	{name: "cert-manager"},
 	{name: "argocd"},
-	{name: "prod-iam-zitadel"},
 	{name: "arc-system"},
 	{name: "arc-runner"},
 	// https://github.com/CrunchyData/postgres-operator-examples/blob/main/kustomize/install/namespace/namespace.yaml

docs/examples/platforms/reference/platform_projects.cue

@@ -55,6 +55,18 @@ import "encoding/yaml"
 		}
 	}
 
+	// ClusterGatewayServers provides a struct of Gateway servers for the current cluster.
+	// This is intended for Gateway/default to add all servers to the default gateway.
+	ClusterGatewayServers: {
+		if project.clusters[#ClusterName] != _|_ {
+			for Stage in project.stages {
+				for server in GatewayServers[Stage.name] {
+					(server.port.name): server
+				}
+			}
+		}
+	}
+
 	workload: resources: {
 		// Provide resources only if the project is managed on --cluster-name
 		if project.clusters[#ClusterName] != _|_ {
@@ -64,10 +76,6 @@ import "encoding/yaml"
 				// Istio Gateway
 				"\(stage.slug)-gateway": #KubernetesObjects & {
 					apiObjectMap: (#APIObjects & {
-						apiObjects: Gateway: (stage.slug): #Gateway & {
-							spec: servers: [for server in GatewayServers[stage.name] {server}]
-						}
-
 						for host in GatewayServers[stage.name] {
 							apiObjects: ExternalSecret: (host.tls.credentialName): metadata: namespace: "istio-ingress"
 						}
@@ -75,24 +83,32 @@ import "encoding/yaml"
 				}
 
 				// Manage auth-proxy in each stage
-				"\(stage.slug)-authproxy": #KubernetesObjects & {
-					apiObjectMap: (#APIObjects & {
-						apiObjects: (AUTHPROXY & {stage: Stage, project: Project, servers: GatewayServers[stage.name]}).apiObjects
-					}).apiObjectMap
+				if project.features.authproxy.enabled {
+					"\(stage.slug)-authproxy": #KubernetesObjects & {
+						apiObjectMap: (#APIObjects & {
+							apiObjects: (AUTHPROXY & {stage: Stage, project: Project, servers: GatewayServers[stage.name]}).apiObjects
+						}).apiObjectMap
+					}
+
+					for Env in project.environments if Env.stage == stage.name {
+						"\(Env.slug)-authpolicy": #KubernetesObjects & {
+							// Manage auth policy in each env
+							apiObjectMap: (#APIObjects & {
+								apiObjects: (AUTHPOLICY & {env: Env, project: Project, servers: GatewayServers[stage.name]}).apiObjects
+							}).apiObjectMap
+						}
+					}
 				}
 
 				// Manage httpbin in each environment
-				for Env in project.environments if Env.stage == stage.name {
-					"\(Env.slug)-httpbin": #KubernetesObjects & {
-						let Project = project
-						apiObjectMap: (#APIObjects & {
-							apiObjects: (HTTPBIN & {env: Env, project: Project}).apiObjects
-						}).apiObjectMap
-
-						// Manage auth policy in each env
-						apiObjectMap: (#APIObjects & {
-							apiObjects: (AUTHPOLICY & {env: Env, project: Project, servers: GatewayServers[stage.name]}).apiObjects
-						}).apiObjectMap
+				if project.features.httpbin.enabled {
+					for Env in project.environments if Env.stage == stage.name {
+						"\(Env.slug)-httpbin": #KubernetesObjects & {
+							let Project = project
+							apiObjectMap: (#APIObjects & {
+								apiObjects: (HTTPBIN & {env: Env, project: Project}).apiObjects
+							}).apiObjectMap
+						}
 					}
 				}
 			}
@@ -178,7 +194,7 @@ let HTTPBIN = {
 			let Project = project
 			let Env = env
 			spec: hosts: [for host in (#EnvHosts & {project: Project, env: Env}).hosts {host.name}]
-			spec: gateways: ["istio-ingress/\(env.stageSlug)"]
+			spec: gateways: ["istio-ingress/default"]
 			spec: http: [{route: [{destination: host: Name}]}]
 		}
 	}
@@ -194,6 +210,14 @@ let AUTHPROXY = {
 	let Project = project
 	let Stage = stage
 
+	let AuthProxySpec = #AuthProxySpec & {
+		namespace: stage.namespace
+		projectID: project.resourceId
+		clientID:  stage.authProxyClientID
+		orgDomain: project.authProxyOrgDomain
+		provider:  stage.extAuthzProviderName
+	}
+
 	let Metadata = {
 		name:      Name
 		namespace: stage.namespace
@@ -224,15 +248,15 @@ let AUTHPROXY = {
 			data: "config.yaml": yaml.Marshal(AuthProxyConfig)
 			let AuthProxyConfig = {
 				injectResponseHeaders: [{
-					name: "x-oidc-id-token"
+					name: AuthProxySpec.idTokenHeader
 					values: [{claim: "id_token"}]
 				}]
 				providers: [{
 					id:                    "Holos Platform"
 					name:                  "Holos Platform"
 					provider:              "oidc"
-					scope:                 "openid profile email groups offline_access urn:zitadel:iam:org:domain:primary:\(project.authProxyOrgDomain)"
-					clientID:              stage.authProxyClientID
+					scope:                 "openid profile email groups offline_access urn:zitadel:iam:org:domain:primary:\(AuthProxySpec.orgDomain)"
+					clientID:              AuthProxySpec.clientID
 					clientSecretFile:      "/dev/null"
 					code_challenge_method: "S256"
 					loginURLParameters: [{
@@ -240,7 +264,7 @@ let AUTHPROXY = {
 						name: "approval_prompt"
 					}]
 					oidcConfig: {
-						issuerURL: project.authProxyIssuer
+						issuerURL: AuthProxySpec.issuer
 						audienceClaims: ["aud"]
 						emailClaim:  "email"
 						groupsClaim: "groups"
@@ -285,7 +309,7 @@ let AUTHPROXY = {
 							}]
 							args: [
 								// callback url is proxy prefix + /callback
-								"--proxy-prefix=" + project.authProxyPrefix,
+								"--proxy-prefix=" + AuthProxySpec.proxyPrefix,
 								"--email-domain=*",
 								"--session-store-type=redis",
 								"--redis-connection-url=redis://\(RedisMetadata.name):6379",
@@ -343,9 +367,9 @@ let AUTHPROXY = {
 		VirtualService: (Name): #VirtualService & {
 			metadata: Metadata
 			spec: hosts: ["*"]
-			spec: gateways: ["istio-ingress/\(stage.slug)"]
+			spec: gateways: ["istio-ingress/default"]
 			spec: http: [{
-				match: [{uri: prefix: project.authProxyPrefix}]
+				match: [{uri: prefix: AuthProxySpec.proxyPrefix}]
 				route: [{
 					destination: host: Name
 					destination: port: number: 4180
@@ -447,6 +471,14 @@ let AUTHPOLICY = {
 	let stage = project.stages[env.stage]
 	let Env = env
 
+	let AuthProxySpec = #AuthProxySpec & {
+		namespace: stage.namespace
+		projectID: project.resourceId
+		clientID:  stage.authProxyClientID
+		orgDomain: project.authProxyOrgDomain
+		provider:  stage.extAuthzProviderName
+	}
+
 	let Metadata = {
 		name:      string
 		namespace: env.namespace
@@ -469,16 +501,16 @@ let AUTHPOLICY = {
 		for host in Hosts {host.name},
 		for host in Hosts {host.name + ":*"},
 	]
-	let MatchLabels = {"security.holos.run/authproxy": stage.extAuthzProviderName}
+	let MatchLabels = {"security.holos.run/authproxy": AuthProxySpec.provider}
 
 	apiObjects: {
 		RequestAuthentication: (Name): #RequestAuthentication & {
 			metadata: Metadata & {name: Name}
 			spec: jwtRules: [{
-				audiences: [stage.authProxyClientID]
+				audiences: [AuthProxySpec.clientID]
 				forwardOriginalToken: true
-				fromHeaders: [{name: "x-oidc-id-token"}]
-				issuer: project.authProxyIssuer
+				fromHeaders: [{name: AuthProxySpec.idTokenHeader}]
+				issuer: AuthProxySpec.issuer
 			}]
 			spec: selector: matchLabels: MatchLabels
 		}
@@ -487,8 +519,19 @@ let AUTHPOLICY = {
 			spec: {
 				action: "CUSTOM"
 				// send the request to the auth proxy
-				provider: name: stage.extAuthzProviderName
-				rules: [{to: [{operation: hosts: HostList}]}]
+				provider: name: AuthProxySpec.provider
+				rules: [{
+					to: [{operation: hosts: HostList}]
+					when: [
+						{
+							key: "request.headers[\(AuthProxySpec.idTokenHeader)]"
+							notValues: ["*"]
+						},
+						{
+							key: "request.headers[host]"
+							notValues: [AuthProxySpec.issuerHost]
+						},
+					]}]
 				selector: matchLabels: MatchLabels
 			}
 		}

docs/examples/project_types.cue

@@ -26,8 +26,6 @@ import "strings"
 	}
 	domain: string | *#Platform.org.domain
 
-	// authProxyPrefix is the path routed to the ext auth proxy.
-	authProxyPrefix: string | *"/holos/oidc"
 	// authProxyOrgDomain is the primary org domain for zitadel.
 	authProxyOrgDomain: string | *#Platform.org.domain
 	// authProxyIssuer is the issuer url
@@ -63,6 +61,8 @@ import "strings"
 
 	// features is YAGNI maybe? 
 	features: [Name=string]: #Feature & {name: Name}
+	features: authproxy: _
+	features: httpbin:   _
 }
 
 // #Cluster defines a cluster
@@ -127,7 +127,7 @@ import "strings"
 #Feature: {
 	name:        string
 	description: string
-	enabled:     *true | false
+	enabled:     true | *false
 }
 
 #ProjectTemplate: {

docs/examples/schema.cue

@@ -1,5 +1,7 @@
 package holos
 
+import "strings"
+
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	appsv1 "k8s.io/api/apps/v1"
@@ -192,6 +194,8 @@ _apiVersion: "holos.run/v1alpha1"
 		name: string
 		// e.g. "example.com"
 		domain: string
+		// e.g. "example.com"
+		emailDomain: string | *domain
 		// e.g. "Example"
 		displayName: string
 		// e.g. "platform@example.com"
@@ -226,24 +230,51 @@ _apiVersion: "holos.run/v1alpha1"
 		name: string & ID
 	}
 	// authproxy configures the auth proxy attached to the default ingress gateway in the istio-ingress namespace.
-	authproxy: #IngressAuthProxySpec
+	authproxy: #AuthProxySpec & {
+		namespace: "istio-ingress"
+		provider:  "ingressauth"
+	}
+
+	oauthClients: [Name=_]: #OAuthClientSpec & {name: Name}
 }
 
-#IngressAuthProxySpec: {
+#OAuthClientSpec: {
+	name:      string
+	orgDomain: string | *#Platform.org.emailDomain
+	spec: {
+		issuer:   string | *"https://login.\(#Platform.org.domain)"
+		clientID: string | *name
+		scopes:   string | *strings.Join(scopesList, " ")
+		scopesList: ["openid", "profile", "email", "groups", "urn:zitadel:iam:org:domain:primary:\(orgDomain)"]
+		jwks_uri:               string | *"\(issuer)/oauth/v2/keys"
+		authorization_endpoint: string | *"\(issuer)/oauth/v2/authorize"
+		token_endpoint:         string | *"\(issuer)/oauth/v2/token"
+		introspection_endpoint: string | *"\(issuer)/oauth/v2/introspect"
+		userinfo_endpoint:      string | *"\(issuer)/oauth/v1/userinfo"
+		revocation_endpoint:    string | *"\(issuer)/oauth/v2/revoke"
+		end_session_endpoint:   string | *"\(issuer)/oauth/v1/end_session"
+	}
+}
+
+#AuthProxySpec: {
 	// projectID is the zitadel project resource id.
 	projectID: number
 	// clientID is the zitadel application client id.
 	clientID: string
+	// namespace is the namespace
+	namespace: string
+	// provider is the istio extension provider name in the mesh config.
+	provider: string
 	// orgDomain is the zitadel organization domain for logins.
 	orgDomain: string | *#Platform.org.domain
+	// issuerHost is the Host: header value of the oidc issuer
+	issuerHost: string | *"login.\(#Platform.org.domain)"
 	// issuer is the oidc identity provider issuer url
-	issuer: string | *"https://login.\(#Platform.org.domain)"
+	issuer: string | *"https://\(issuerHost)"
 	// path is the oauth2-proxy --proxy-prefix value.  The default callback url is the Host: value with a path of /holos/oidc/callback
-	proxyPrefix: string | *"/holos/oidc"
-	// provider is the istio extension provider name in the mesh config.
-	provider: "ingressauth"
+	proxyPrefix: string | *"/holos/authproxy/\(namespace)"
 	// idTokenHeader represents the header where the id token is placed
-	idTokenHeader: "x-oidc-id-token"
+	idTokenHeader: string | *"x-oidc-id-token"
 }
 
 // ManagedNamespace is a namespace to manage across all clusters in the holos platform.

docs/runbooks/postgres-backup.md

@@ -0,0 +1,150 @@
+# Postgres Full Backup
+
+Suppose you delete all objects in the S3 bucket hosting all postgres backups.  You want to take a full backup ASAP of an existing PostgreSQL database.
+
+The normal method of annotating the `postgrescluster` resource will not work because the job will error:
+
+```
+❯ kubectl annotate postgrescluster zitadel postgres-operator.crunchydata.com/pgbackrest-backup="$(date)" --overwrite
+postgrescluster.postgres-operator.crunchydata.com/zitadel annotated
+```
+
+Backup fails:
+
+```
+❯ k get pods
+NAME                                    READY   STATUS      RESTARTS      AGE
+zitadel-backup-hk7w-76bfk               0/1     Error       0             65s
+zitadel-backup-hk7w-d55v6               0/1     Error       0             44s
+zitadel-backup-hk7w-l9dwm               0/1     Error       0             76s
+zitadel-backup-hk7w-zcg69               0/1     Error       0             3s
+zitadel-pgbouncer-d9f8cffc-nx8lq        2/2     Running     0             49m
+zitadel-pgbouncer-d9f8cffc-s7g7x        2/2     Running     0             49m
+zitadel-pgha1-2xv2-0                    5/5     Running     0             48m
+zitadel-pgha1-78f4-0                    5/5     Running     0             49m
+zitadel-repo-host-0                     2/2     Running     0             49m
+```
+
+Error is: `FileMissingError: unable to open missing file '/pgbackrest/prod-iam/zitadel/repo2/backup/db/backup.info.copy' for read`
+
+```
+time="2024-04-08T00:02:11Z" level=info msg="crunchy-pgbackrest starts"
+time="2024-04-08T00:02:11Z" level=info msg="debug flag set to false"
+time="2024-04-08T00:02:12Z" level=info msg="backrest backup command requested"
+time="2024-04-08T00:02:12Z" level=info msg="command to execute is [pgbackrest backup --stanza=db --repo=2 --type=full]"
+time="2024-04-08T00:02:12Z" level=info msg="output=[]"
+time="2024-04-08T00:02:12Z" level=info msg="stderr=[ERROR: [055]: unable to load info file '/pgbackrest/prod-iam/zitadel/repo2/backup/db/backup.info' or '/pgbackrest/prod-iam/zitadel/repo2/backup/db/backup.info.copy':\n       FileMissingError: unable to open missing file '/pgbackrest/prod-iam/zitadel/repo2/backup/db/backup.info' for read\n       FileMissingError: unable to open missing file '/pgbackrest/prod-iam/zitadel/repo2/backup/db/backup.info.copy' for read\n       HINT: backup.info cannot be opened and is required to perform a backup.\n       HINT: has a stanza-create been performed?\n]"
+time="2024-04-08T00:02:12Z" level=fatal msg="command terminated with exit code 55"
+```
+
+## Fix Process
+
+We need to edit the postgrescluster.  We're going to have the controller re-initialize the backup repository from scratch by removing it and re-adding it.
+
+First, suspend flux:
+
+```
+flux suspend ks prod-iam-zitadel prod-iam-postgres
+```
+
+Save the config to two files:
+
+```
+kubectl get postgresclusters.postgres-operator.crunchydata.com zitadel -o yaml > orig.yaml
+cp orig.yaml new.yaml
+```
+
+Remove the follow fields and re-apply the cluster.  This will leave the cluster running and available while the controller reconciles the repo configuration:
+
+```diff
+--- orig.yaml	2024-04-07 17:08:26.834715820 -0700
++++ new.yaml	2024-04-07 17:08:57.418546067 -0700
+@@ -4,6 +4,4 @@
+   annotations:
+     holos.run/description: ""
+-    postgres-operator.crunchydata.com/pgbackrest-backup: Sun 07 Apr 2024 05:01:35
+-      PM PDT
+   creationTimestamp: "2024-04-07T23:10:44Z"
+   finalizers:
+@@ -26,12 +24,5 @@
+         repo1-retention-full: "1"
+         repo2-cipher-type: aes-256-cbc
+-        repo2-path: /pgbackrest/prod-iam/zitadel/repo2
+-        repo2-retention-full: "14"
+-        repo2-retention-full-type: time
+       image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.49-0
+-      manual:
+-        options:
+-        - --type=full
+-        repoName: repo2
+       repos:
+       - name: repo1
+@@ -43,12 +34,4 @@
+               requests:
+                 storage: 4Gi
+-      - name: repo2
+-        s3:
+-          bucket: ois-zitadel-backups
+-          endpoint: s3.dualstack.us-east-2.amazonaws.com
+-          region: us-east-2
+-        schedules:
+-          differential: 0 1 * * 1-6
+-          full: 0 1 * * 0
+       restore:
+         enabled: true
+```
+
+Apply the config and wait for the controller to reconcile:
+
+```
+k apply --server-side=true -f new.yaml --force-conflicts
+```
+
+Check for reconciliation:
+
+```
+kubectl -n postgres-operator logs -l app.kubernetes.io/name=pgo | tail -1
+```
+
+```
+time="2024-04-08T00:10:03Z" level=debug msg="reconciled cluster" controller=postgrescluster controllerGroup=postgres-operator.crunchydata.com controllerKind=PostgresCluster name=zitadel namespace=prod-iam postgresCluster=prod-iam/zitadel reconcileID=cc8c8eb7-9787-4504-8ecd-a04ec84fbc0b version=5.5.1-0-amd64
+```
+
+Re-add the repo host configuration
+
+```
+grep -v 'resourceVersion:' orig.yaml | k apply --server-side=true --force-conflicts -f-
+```
+
+```
+postgrescluster.postgres-operator.crunchydata.com/zitadel serverside-applied
+```
+
+The full backup should be running and writing to S3 now:
+
+```
+kubectl logs -l postgres-operator.crunchydata.com/pgbackrest-backup=manual
+```
+
+```
+time="2024-04-08T00:12:54Z" level=info msg="crunchy-pgbackrest starts"
+time="2024-04-08T00:12:54Z" level=info msg="debug flag set to false"
+time="2024-04-08T00:12:54Z" level=info msg="backrest backup command requested"
+time="2024-04-08T00:12:54Z" level=info msg="command to execute is [pgbackrest backup --stanza=db --repo=2 --type=full]"
+time="2024-04-08T00:16:02Z" level=info msg="output=[]"
+time="2024-04-08T00:16:02Z" level=info msg="stderr=[]"
+time="2024-04-08T00:16:02Z" level=info msg="crunchy-pgbackrest ends"
+```
+
+Finally, resume flux:
+
+```
+flux resume ks prod-iam-postgres prod-iam-zitadel
+```
+
+```
+► resuming kustomization prod-iam-postgres in flux-system namespace
+✔ kustomization resumed
+► resuming kustomization prod-iam-zitadel in flux-system namespace
+✔ kustomization resumed
+```

go.mod

@@ -3,37 +3,70 @@ module github.com/holos-run/holos
 go 1.21.5
 
 require (
+	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1
+	connectrpc.com/connect v1.16.0
+	connectrpc.com/validate v0.1.0
 	cuelang.org/go v0.8.0
+	entgo.io/ent v0.13.1
+	github.com/coreos/go-oidc/v3 v3.10.0
+	github.com/go-jose/go-jose/v3 v3.0.3
+	github.com/gofrs/uuid v4.4.0+incompatible
+	github.com/google/uuid v1.5.0
+	github.com/jackc/pgx/v5 v5.5.5
+	github.com/lmittmann/tint v1.0.4
 	github.com/mattn/go-isatty v0.0.20
+	github.com/prometheus/client_golang v1.19.0
 	github.com/rogpeppe/go-internal v1.12.0
+	github.com/sethvargo/go-retry v0.2.4
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/net v0.22.0
 	golang.org/x/tools v0.19.0
+	google.golang.org/protobuf v1.33.0
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2
 	k8s.io/client-go v0.29.2
 	k8s.io/kubectl v0.29.2
+	modernc.org/sqlite v1.29.6
 	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
+	ariga.io/atlas v0.19.1-0.20240203083654-5948b60a8e43 // indirect
 	cuelabs.dev/go/oci/ociregistry v0.0.0-20240314152124-224736b49f2e // indirect
+	github.com/agext/levenshtein v1.2.1 // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect
+	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bufbuild/protovalidate-go v0.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/apd/v3 v3.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/emicklei/proto v1.10.0 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
+	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-openapi/inflect v0.19.0 // indirect
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/cel-go v0.17.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.5.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/hashicorp/hcl/v2 v2.13.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/lib/pq v1.10.9 // indirect
@@ -42,26 +75,43 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/onsi/ginkgo/v2 v2.15.0 // indirect
 	github.com/onsi/gomega v1.31.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 // indirect
-	golang.org/x/net v0.22.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/stoewer/go-strcase v1.3.0 // indirect
+	github.com/zclconf/go-cty v1.8.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect
+	golang.org/x/mod v0.16.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231206194836-bf4651e18aa8 // indirect
 	k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	modernc.org/libc v1.41.0 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/token v1.1.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

go.sum

@@ -1,22 +1,58 @@
+ariga.io/atlas v0.19.1-0.20240203083654-5948b60a8e43 h1:GwdJbXydHCYPedeeLt4x/lrlIISQ4JTH1mRWuE5ZZ14=
+ariga.io/atlas v0.19.1-0.20240203083654-5948b60a8e43/go.mod h1:uj3pm+hUTVN/X5yfdBexHlZv+1Xu5u5ZbZx7+CDavNU=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1 h1:2IGhRovxlsOIQgx2ekZWo4wTPAYpck41+18ICxs37is=
+buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1/go.mod h1:Tgn5bgL220vkFOI0KPStlcClPeOJzAv4uT+V8JXGUnw=
+connectrpc.com/connect v1.16.0 h1:rdtfQjZ0OyFkWPTegBNcH7cwquGAN1WzyJy80oFNibg=
+connectrpc.com/connect v1.16.0/go.mod h1:XpZAduBQUySsb4/KO5JffORVkDI4B6/EYPi7N8xpNZw=
+connectrpc.com/validate v0.1.0 h1:r55jirxMK7HO/xZwVHj3w2XkVFarsUM77ZDy367NtH4=
+connectrpc.com/validate v0.1.0/go.mod h1:GU47c9/x/gd+u9wRSPkrQOP46gx2rMN+Wo37EHgI3Ow=
 cuelabs.dev/go/oci/ociregistry v0.0.0-20240314152124-224736b49f2e h1:GwCVItFUPxwdsEYnlUcJ6PJxOjTeFFCKOh6QWg4oAzQ=
 cuelabs.dev/go/oci/ociregistry v0.0.0-20240314152124-224736b49f2e/go.mod h1:ApHceQLLwcOkCEXM1+DyCXTHEJhNGDpJ2kmV6axsx24=
 cuelang.org/go v0.8.0 h1:fO1XPe/SUGtc7dhnGnTPbpIDoQm/XxhDtoSF7jzO01c=
 cuelang.org/go v0.8.0/go.mod h1:CoDbYolfMms4BhWUlhD+t5ORnihR7wvjcfgyO9lL5FI=
+entgo.io/ent v0.13.1 h1:uD8QwN1h6SNphdCCzmkMN3feSUzNnVvV/WIkHKMbzOE=
+entgo.io/ent v0.13.1/go.mod h1:qCEmo+biw3ccBn9OyL4ZK5dfpwg++l1Gxwac5B1206A=
+github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
+github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
+github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
+github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bufbuild/protovalidate-go v0.3.0 h1:t9zKgM//9VtPnP0TvyFqWubLQtSbwLwEUVOxgtX9/os=
+github.com/bufbuild/protovalidate-go v0.3.0/go.mod h1:4mZkDYMGJlnHHQ9rPOhVEZ4bA13iOJBRLzywxy8f/lo=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
 github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
+github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU=
+github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw=
 github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
+github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
+github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
 github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
 github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
+github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
+github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U=
+github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
 github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4=
+github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4=
 github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
 github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ=
 github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA=
@@ -29,14 +65,23 @@ github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7
 github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
+github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
+github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/cel-go v0.17.4 h1:9556LOjSyIZlgnT0oaCYGq2uk9BM6fzuTXhzYHskonk=
+github.com/google/cel-go v0.17.4/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -44,20 +89,33 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
 github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hashicorp/hcl/v2 v2.13.0 h1:0Apadu1w6M11dyGFxWnmhhcMjkbAiKCv7G1r/2QgCNc=
+github.com/hashicorp/hcl/v2 v2.13.0/go.mod h1:e4z5nxYlWNPdDSNYX+ph14EvWYMFm3eP0zIUqPc2jr0=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
+github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -69,10 +127,16 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
+github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
 github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -82,6 +146,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
 github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
 github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo=
@@ -94,42 +162,73 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 h1:sadMIsgmHpEOGbUs6VtHBXRR1OHevnj7hLx9ZcdNGW4=
 github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
+github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
+github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
+github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA=
+github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
 golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
@@ -138,6 +237,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -145,17 +247,27 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
 golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
@@ -165,19 +277,26 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
 golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
+google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
+google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
@@ -202,6 +321,22 @@ k8s.io/kubectl v0.29.2 h1:uaDYaBhumvkwz0S2XHt36fK0v5IdNgL7HyUniwb2IUo=
 k8s.io/kubectl v0.29.2/go.mod h1:BhizuYBGcKaHWyq+G7txGw2fXg576QbPrrnQdQDZgqI=
 k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI=
 k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
+modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk=
+modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
+modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4=
+modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=

internal/server/cli.go

@@ -0,0 +1,110 @@
+package server
+
+import (
+	"context"
+	_ "embed"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/sethvargo/go-retry"
+	"github.com/spf13/cobra"
+
+	"github.com/holos-run/holos/internal/server/db"
+	"github.com/holos-run/holos/internal/server/middleware/authn"
+	"github.com/holos-run/holos/internal/server/server"
+	"github.com/holos-run/holos/internal/server/signals"
+	"github.com/holos-run/holos/pkg/errors"
+	"github.com/holos-run/holos/pkg/holos"
+)
+
+//go:embed help/root.txt
+var helpLong string
+
+// New builds a root cobra command with flags linked to the Config field.
+func New(cfg *holos.Config) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "server",
+		Short: "server",
+		Long:  helpLong,
+		// We handle our own errors.
+		SilenceUsage:  true,
+		SilenceErrors: true,
+		// Hidden because it annoys users trying to complete component
+		CompletionOptions: cobra.CompletionOptions{
+			HiddenDefaultCmd: true,
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			if ctx == nil {
+				ctx = context.Background()
+			}
+
+			log := cfg.Logger()
+			log.DebugContext(ctx, "hello", "lifecycle", "start")
+
+			// Connect to the database
+			conn, err := db.Client(cfg)
+			if err != nil {
+				return errors.Wrap(fmt.Errorf("could not create db client: %w", err))
+			}
+			defer func() {
+				if closeError := conn.Client.Close(); closeError != nil {
+					log.ErrorContext(ctx, "could not close database", "err", closeError)
+				}
+			}()
+
+			// Retry until network is online or limit reached
+			backoff := retry.NewFibonacci(1 * time.Second)
+			backoff = retry.WithCappedDuration(5*time.Second, backoff)
+			backoff = retry.WithMaxDuration(30*time.Second, backoff)
+			// Ping the database
+			ping := func(ctx context.Context) error {
+				plog := slog.With("database", conn.Driver.Dialect(), "check", "network")
+				plog.DebugContext(ctx, "ping")
+				if pingErr := conn.DB.PingContext(ctx); pingErr != nil {
+					plog.DebugContext(ctx, "retryable: could not ping", "ok", false, "err", pingErr)
+					return retry.RetryableError(errors.Wrap(pingErr))
+				}
+				plog.DebugContext(ctx, "pong", "ok", true)
+				return nil
+			}
+			if err = retry.Do(ctx, backoff, ping); err != nil {
+				return errors.Wrap(err)
+			}
+
+			// Automatic migration
+			if err = conn.Client.Schema.Create(ctx); err != nil {
+				return errors.Wrap(err)
+			}
+			log.InfoContext(ctx, "schema created", "database", conn.Driver.Dialect())
+
+			// Authentication (Identity Verifier)
+			// We may pass an instrumented *http.Client via ctx in the future.
+			verifier, err := authn.NewVerifier(ctx, log, cfg.ServerConfig.OIDCIssuer())
+			if err != nil {
+				return errors.Wrap(fmt.Errorf("could not create identity verifier: %w", err))
+			}
+
+			// Start the server
+			srv, err := server.NewServer(cfg, conn.Client, verifier)
+			if err != nil {
+				return errors.Wrap(fmt.Errorf("could not start server: %w", err))
+			}
+
+			if cfg.ServerConfig.ListenAndServe() {
+				httpServer, healthy, ready := srv.ListenAndServe()
+				stopCh := signals.SetupSignalHandler()
+				sd := signals.NewShutdown(15*time.Second, log)
+				sd.Graceful(stopCh, httpServer, healthy, ready)
+			}
+
+			return nil
+		},
+	}
+
+	// Add flags valid for all subcommands
+	cmd.Flags().SortFlags = false
+	cmd.PersistentFlags().AddGoFlagSet(cfg.ServerFlagSet())
+	return cmd
+}

internal/server/db/ent.go

@@ -0,0 +1,55 @@
+// Package db manages database client and schema migrations to interact with entities.
+package db
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"entgo.io/ent/dialect"
+	"github.com/holos-run/holos/internal/server/ent"
+	"github.com/holos-run/holos/pkg/errors"
+	"github.com/holos-run/holos/pkg/holos"
+)
+
+// Conn holds database connection info
+type Conn struct {
+	Client *ent.Client
+	DB     *sql.DB
+	Driver dialect.Driver
+}
+
+func Client(cfg *holos.Config) (Conn, error) {
+	var clientFactory ClientFactory
+	if cfg.ServerConfig == nil || cfg.ServerConfig.DatabaseURI() == "" {
+		clientFactory = NewMemoryClientFactory(cfg)
+	} else {
+		clientFactory = NewPGXClientFactory(cfg)
+	}
+	return clientFactory.New()
+}
+
+type ClientFactory interface {
+	New() (Conn, error)
+}
+
+// withHooks adds our custom hooks to the database client.
+func withHooks(client *ent.Client) *ent.Client {
+	client.Use(setUpdatedToCreatedOnCreate)
+	return client
+}
+
+// setUpdatedToCreatedOnCreate ensures the updatedAt field matches the createdAt field on creation.
+func setUpdatedToCreatedOnCreate(next ent.Mutator) ent.Mutator {
+	return ent.MutateFunc(func(ctx context.Context, m ent.Mutation) (ent.Value, error) {
+		if createdAt, ok := m.Field("created_at"); ok {
+			if _, ok := m.Field("updated_at"); ok && m.Op().Is(ent.OpCreate) {
+				err := m.SetField("updated_at", createdAt.(time.Time))
+				if err != nil {
+					return nil, errors.Wrap(err)
+				}
+			}
+		}
+		return next.Mutate(ctx, m)
+	})
+}

internal/server/db/pg.go

@@ -0,0 +1,35 @@
+package db
+
+import (
+	"database/sql"
+	"fmt"
+
+	"entgo.io/ent/dialect"
+	entsql "entgo.io/ent/dialect/sql"
+	"github.com/holos-run/holos/internal/server/ent"
+	"github.com/holos-run/holos/pkg/errors"
+	"github.com/holos-run/holos/pkg/holos"
+	_ "github.com/jackc/pgx/v5/stdlib"
+)
+
+// NewPGXClientFactory returns a PGXClientFactory implementation of ClientFactory
+func NewPGXClientFactory(cfg *holos.Config) *PGXClientFactory {
+	return &PGXClientFactory{cfg: cfg}
+}
+
+// PGXClientFactory produces pgx clients suitable for live workloads
+type PGXClientFactory struct {
+	cfg *holos.Config
+}
+
+// New returns a new ent.Client using pgx with PostgreSQL
+func (mc *PGXClientFactory) New() (Conn, error) {
+	uri := mc.cfg.ServerConfig.DatabaseURI()
+	db, err := sql.Open("pgx", uri)
+	if err != nil {
+		return Conn{}, errors.Wrap(fmt.Errorf("could not open pgx: %w", err))
+	}
+	drv := entsql.OpenDB(dialect.Postgres, db)
+	client := withHooks(ent.NewClient(ent.Driver(drv)))
+	return Conn{client, db, drv}, nil
+}

internal/server/db/sqlite.go

@@ -0,0 +1,65 @@
+package db
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+	"log/slog"
+
+	esql "entgo.io/ent/dialect/sql"
+	"github.com/holos-run/holos/internal/server/ent"
+	"github.com/holos-run/holos/pkg/errors"
+	"github.com/holos-run/holos/pkg/holos"
+	"modernc.org/sqlite"
+)
+
+// NewMemoryClientFactory returns a MemoryClientFactory implementation of ClientFactory
+func NewMemoryClientFactory(cfg *holos.Config) *MemoryClientFactory {
+	return &MemoryClientFactory{cfg: cfg}
+}
+
+// MemoryClientFactory produces simple in-memory sqlite database clients for development and testing.
+type MemoryClientFactory struct {
+	cfg *holos.Config
+}
+
+func (mc *MemoryClientFactory) New() (Conn, error) {
+	log := mc.cfg.Logger()
+	db, err := sql.Open("sqlite3", "file:db.sqlite3?mode=memory&cache=shared")
+	if err != nil {
+		log.Debug("could not open sql connection", "err", err)
+		return Conn{}, errors.Wrap(err)
+	}
+	// Fix database is locked errors when testing with sqlite3 in-memory and parallel test cases.
+	db.SetMaxOpenConns(1)
+	drv := esql.OpenDB("sqlite3", db)
+	client := withHooks(ent.NewClient(ent.Driver(drv)))
+	return Conn{client, db, drv}, nil
+}
+
+// sqliteDriver sets PRAGMA foreign_keys = on for each new connection with modernc.org/sqlite
+// See: https://github.com/ent/ent/discussions/1667#discussioncomment-1132296
+type sqliteDriver struct {
+	*sqlite.Driver
+}
+
+func (d sqliteDriver) Open(name string) (driver.Conn, error) {
+	conn, err := d.Driver.Open(name)
+	if err != nil {
+		return conn, err
+	}
+	c := conn.(interface {
+		Exec(stmt string, args []driver.Value) (driver.Result, error)
+	})
+	if _, err := c.Exec("PRAGMA foreign_keys = on;", nil); err != nil {
+		if errClose := conn.Close(); errClose != nil {
+			slog.Error("could not close", "err", errClose)
+		}
+		return nil, fmt.Errorf("could not enable foreign keys: %w", err)
+	}
+	return conn, nil
+}
+
+func init() {
+	sql.Register("sqlite3", sqliteDriver{Driver: &sqlite.Driver{}})
+}

internal/server/db/sqlite_test.go

@@ -0,0 +1,30 @@
+package db
+
+import (
+	"context"
+	"testing"
+
+	"github.com/holos-run/holos/internal/server/testutils"
+	"github.com/holos-run/holos/pkg/holos"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMemoryClientFactory(t *testing.T) {
+	t.Run("MemoryClientFactory", func(t *testing.T) {
+		cfg := holos.New(holos.Logger(testutils.TestLogger(t)))
+		mcf := MemoryClientFactory{cfg: cfg}
+		conn, err := mcf.New()
+		assert.NoError(t, err)
+		client := conn.Client
+		assert.NoError(t, client.Schema.Create(context.Background()))
+
+		// Create something
+		t.Run("CreateUser", func(t *testing.T) {
+			uc := client.User.Create().
+				SetName("Foo").
+				SetEmail("foo@example.com")
+			_, err := uc.Save(context.Background())
+			assert.NoError(t, err)
+		})
+	})
+}

internal/server/ent/client.go

@@ -0,0 +1,517 @@
+// Code generated by ent, DO NOT EDIT.
+
+package ent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"reflect"
+
+	"github.com/gofrs/uuid"
+	"github.com/holos-run/holos/internal/server/ent/migrate"
+
+	"entgo.io/ent"
+	"entgo.io/ent/dialect"
+	"entgo.io/ent/dialect/sql"
+	"entgo.io/ent/dialect/sql/sqlgraph"
+	"github.com/holos-run/holos/internal/server/ent/user"
+	"github.com/holos-run/holos/internal/server/ent/useridentity"
+)
+
+// Client is the client that holds all ent builders.
+type Client struct {
+	config
+	// Schema is the client for creating, migrating and dropping schema.
+	Schema *migrate.Schema
+	// User is the client for interacting with the User builders.
+	User *UserClient
+	// UserIdentity is the client for interacting with the UserIdentity builders.
+	UserIdentity *UserIdentityClient
+}
+
+// NewClient creates a new client configured with the given options.
+func NewClient(opts ...Option) *Client {
+	client := &Client{config: newConfig(opts...)}
+	client.init()
+	return client
+}
+
+func (c *Client) init() {
+	c.Schema = migrate.NewSchema(c.driver)
+	c.User = NewUserClient(c.config)
+	c.UserIdentity = NewUserIdentityClient(c.config)
+}
+
+type (
+	// config is the configuration for the client and its builder.
+	config struct {
+		// driver used for executing database requests.
+		driver dialect.Driver
+		// debug enable a debug logging.
+		debug bool
+		// log used for logging on debug mode.
+		log func(...any)
+		// hooks to execute on mutations.
+		hooks *hooks
+		// interceptors to execute on queries.
+		inters *inters
+	}
+	// Option function to configure the client.
+	Option func(*config)
+)
+
+// newConfig creates a new config for the client.
+func newConfig(opts ...Option) config {
+	cfg := config{log: log.Println, hooks: &hooks{}, inters: &inters{}}
+	cfg.options(opts...)
+	return cfg
+}
+
+// options applies the options on the config object.
+func (c *config) options(opts ...Option) {
+	for _, opt := range opts {
+		opt(c)
+	}
+	if c.debug {
+		c.driver = dialect.Debug(c.driver, c.log)
+	}
+}
+
+// Debug enables debug logging on the ent.Driver.
+func Debug() Option {
+	return func(c *config) {
+		c.debug = true
+	}
+}
+
+// Log sets the logging function for debug mode.
+func Log(fn func(...any)) Option {
+	return func(c *config) {
+		c.log = fn
+	}
+}
+
+// Driver configures the client driver.
+func Driver(driver dialect.Driver) Option {
+	return func(c *config) {
+		c.driver = driver
+	}
+}
+
+// Open opens a database/sql.DB specified by the driver name and
+// the data source name, and returns a new client attached to it.
+// Optional parameters can be added for configuring the client.
+func Open(driverName, dataSourceName string, options ...Option) (*Client, error) {
+	switch driverName {
+	case dialect.MySQL, dialect.Postgres, dialect.SQLite:
+		drv, err := sql.Open(driverName, dataSourceName)
+		if err != nil {
+			return nil, err
+		}
+		return NewClient(append(options, Driver(drv))...), nil
+	default:
+		return nil, fmt.Errorf("unsupported driver: %q", driverName)
+	}
+}
+
+// ErrTxStarted is returned when trying to start a new transaction from a transactional client.
+var ErrTxStarted = errors.New("ent: cannot start a transaction within a transaction")
+
+// Tx returns a new transactional client. The provided context
+// is used until the transaction is committed or rolled back.
+func (c *Client) Tx(ctx context.Context) (*Tx, error) {
+	if _, ok := c.driver.(*txDriver); ok {
+		return nil, ErrTxStarted
+	}
+	tx, err := newTx(ctx, c.driver)
+	if err != nil {
+		return nil, fmt.Errorf("ent: starting a transaction: %w", err)
+	}
+	cfg := c.config
+	cfg.driver = tx
+	return &Tx{
+		ctx:          ctx,
+		config:       cfg,
+		User:         NewUserClient(cfg),
+		UserIdentity: NewUserIdentityClient(cfg),
+	}, nil
+}
+
+// BeginTx returns a transactional client with specified options.
+func (c *Client) BeginTx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) {
+	if _, ok := c.driver.(*txDriver); ok {
+		return nil, errors.New("ent: cannot start a transaction within a transaction")
+	}
+	tx, err := c.driver.(interface {
+		BeginTx(context.Context, *sql.TxOptions) (dialect.Tx, error)
+	}).BeginTx(ctx, opts)
+	if err != nil {
+		return nil, fmt.Errorf("ent: starting a transaction: %w", err)
+	}
+	cfg := c.config
+	cfg.driver = &txDriver{tx: tx, drv: c.driver}
+	return &Tx{
+		ctx:          ctx,
+		config:       cfg,
+		User:         NewUserClient(cfg),
+		UserIdentity: NewUserIdentityClient(cfg),
+	}, nil
+}
+
+// Debug returns a new debug-client. It's used to get verbose logging on specific operations.
+//
+//	client.Debug().
+//		User.
+//		Query().
+//		Count(ctx)
+func (c *Client) Debug() *Client {
+	if c.debug {
+		return c
+	}
+	cfg := c.config
+	cfg.driver = dialect.Debug(c.driver, c.log)
+	client := &Client{config: cfg}
+	client.init()
+	return client
+}
+
+// Close closes the database connection and prevents new queries from starting.
+func (c *Client) Close() error {
+	return c.driver.Close()
+}
+
+// Use adds the mutation hooks to all the entity clients.
+// In order to add hooks to a specific client, call: `client.Node.Use(...)`.
+func (c *Client) Use(hooks ...Hook) {
+	c.User.Use(hooks...)
+	c.UserIdentity.Use(hooks...)
+}
+
+// Intercept adds the query interceptors to all the entity clients.
+// In order to add interceptors to a specific client, call: `client.Node.Intercept(...)`.
+func (c *Client) Intercept(interceptors ...Interceptor) {
+	c.User.Intercept(interceptors...)
+	c.UserIdentity.Intercept(interceptors...)
+}
+
+// Mutate implements the ent.Mutator interface.
+func (c *Client) Mutate(ctx context.Context, m Mutation) (Value, error) {
+	switch m := m.(type) {
+	case *UserMutation:
+		return c.User.mutate(ctx, m)
+	case *UserIdentityMutation:
+		return c.UserIdentity.mutate(ctx, m)
+	default:
+		return nil, fmt.Errorf("ent: unknown mutation type %T", m)
+	}
+}
+
+// UserClient is a client for the User schema.
+type UserClient struct {
+	config
+}
+
+// NewUserClient returns a client for the User from the given config.
+func NewUserClient(c config) *UserClient {
+	return &UserClient{config: c}
+}
+
+// Use adds a list of mutation hooks to the hooks stack.
+// A call to `Use(f, g, h)` equals to `user.Hooks(f(g(h())))`.
+func (c *UserClient) Use(hooks ...Hook) {
+	c.hooks.User = append(c.hooks.User, hooks...)
+}
+
+// Intercept adds a list of query interceptors to the interceptors stack.
+// A call to `Intercept(f, g, h)` equals to `user.Intercept(f(g(h())))`.
+func (c *UserClient) Intercept(interceptors ...Interceptor) {
+	c.inters.User = append(c.inters.User, interceptors...)
+}
+
+// Create returns a builder for creating a User entity.
+func (c *UserClient) Create() *UserCreate {
+	mutation := newUserMutation(c.config, OpCreate)
+	return &UserCreate{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// CreateBulk returns a builder for creating a bulk of User entities.
+func (c *UserClient) CreateBulk(builders ...*UserCreate) *UserCreateBulk {
+	return &UserCreateBulk{config: c.config, builders: builders}
+}
+
+// MapCreateBulk creates a bulk creation builder from the given slice. For each item in the slice, the function creates
+// a builder and applies setFunc on it.
+func (c *UserClient) MapCreateBulk(slice any, setFunc func(*UserCreate, int)) *UserCreateBulk {
+	rv := reflect.ValueOf(slice)
+	if rv.Kind() != reflect.Slice {
+		return &UserCreateBulk{err: fmt.Errorf("calling to UserClient.MapCreateBulk with wrong type %T, need slice", slice)}
+	}
+	builders := make([]*UserCreate, rv.Len())
+	for i := 0; i < rv.Len(); i++ {
+		builders[i] = c.Create()
+		setFunc(builders[i], i)
+	}
+	return &UserCreateBulk{config: c.config, builders: builders}
+}
+
+// Update returns an update builder for User.
+func (c *UserClient) Update() *UserUpdate {
+	mutation := newUserMutation(c.config, OpUpdate)
+	return &UserUpdate{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// UpdateOne returns an update builder for the given entity.
+func (c *UserClient) UpdateOne(u *User) *UserUpdateOne {
+	mutation := newUserMutation(c.config, OpUpdateOne, withUser(u))
+	return &UserUpdateOne{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// UpdateOneID returns an update builder for the given id.
+func (c *UserClient) UpdateOneID(id uuid.UUID) *UserUpdateOne {
+	mutation := newUserMutation(c.config, OpUpdateOne, withUserID(id))
+	return &UserUpdateOne{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// Delete returns a delete builder for User.
+func (c *UserClient) Delete() *UserDelete {
+	mutation := newUserMutation(c.config, OpDelete)
+	return &UserDelete{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// DeleteOne returns a builder for deleting the given entity.
+func (c *UserClient) DeleteOne(u *User) *UserDeleteOne {
+	return c.DeleteOneID(u.ID)
+}
+
+// DeleteOneID returns a builder for deleting the given entity by its id.
+func (c *UserClient) DeleteOneID(id uuid.UUID) *UserDeleteOne {
+	builder := c.Delete().Where(user.ID(id))
+	builder.mutation.id = &id
+	builder.mutation.op = OpDeleteOne
+	return &UserDeleteOne{builder}
+}
+
+// Query returns a query builder for User.
+func (c *UserClient) Query() *UserQuery {
+	return &UserQuery{
+		config: c.config,
+		ctx:    &QueryContext{Type: TypeUser},
+		inters: c.Interceptors(),
+	}
+}
+
+// Get returns a User entity by its id.
+func (c *UserClient) Get(ctx context.Context, id uuid.UUID) (*User, error) {
+	return c.Query().Where(user.ID(id)).Only(ctx)
+}
+
+// GetX is like Get, but panics if an error occurs.
+func (c *UserClient) GetX(ctx context.Context, id uuid.UUID) *User {
+	obj, err := c.Get(ctx, id)
+	if err != nil {
+		panic(err)
+	}
+	return obj
+}
+
+// QueryIdentities queries the identities edge of a User.
+func (c *UserClient) QueryIdentities(u *User) *UserIdentityQuery {
+	query := (&UserIdentityClient{config: c.config}).Query()
+	query.path = func(context.Context) (fromV *sql.Selector, _ error) {
+		id := u.ID
+		step := sqlgraph.NewStep(
+			sqlgraph.From(user.Table, user.FieldID, id),
+			sqlgraph.To(useridentity.Table, useridentity.FieldID),
+			sqlgraph.Edge(sqlgraph.O2M, false, user.IdentitiesTable, user.IdentitiesColumn),
+		)
+		fromV = sqlgraph.Neighbors(u.driver.Dialect(), step)
+		return fromV, nil
+	}
+	return query
+}
+
+// Hooks returns the client hooks.
+func (c *UserClient) Hooks() []Hook {
+	return c.hooks.User
+}
+
+// Interceptors returns the client interceptors.
+func (c *UserClient) Interceptors() []Interceptor {
+	return c.inters.User
+}
+
+func (c *UserClient) mutate(ctx context.Context, m *UserMutation) (Value, error) {
+	switch m.Op() {
+	case OpCreate:
+		return (&UserCreate{config: c.config, hooks: c.Hooks(), mutation: m}).Save(ctx)
+	case OpUpdate:
+		return (&UserUpdate{config: c.config, hooks: c.Hooks(), mutation: m}).Save(ctx)
+	case OpUpdateOne:
+		return (&UserUpdateOne{config: c.config, hooks: c.Hooks(), mutation: m}).Save(ctx)
+	case OpDelete, OpDeleteOne:
+		return (&UserDelete{config: c.config, hooks: c.Hooks(), mutation: m}).Exec(ctx)
+	default:
+		return nil, fmt.Errorf("ent: unknown User mutation op: %q", m.Op())
+	}
+}
+
+// UserIdentityClient is a client for the UserIdentity schema.
+type UserIdentityClient struct {
+	config
+}
+
+// NewUserIdentityClient returns a client for the UserIdentity from the given config.
+func NewUserIdentityClient(c config) *UserIdentityClient {
+	return &UserIdentityClient{config: c}
+}
+
+// Use adds a list of mutation hooks to the hooks stack.
+// A call to `Use(f, g, h)` equals to `useridentity.Hooks(f(g(h())))`.
+func (c *UserIdentityClient) Use(hooks ...Hook) {
+	c.hooks.UserIdentity = append(c.hooks.UserIdentity, hooks...)
+}
+
+// Intercept adds a list of query interceptors to the interceptors stack.
+// A call to `Intercept(f, g, h)` equals to `useridentity.Intercept(f(g(h())))`.
+func (c *UserIdentityClient) Intercept(interceptors ...Interceptor) {
+	c.inters.UserIdentity = append(c.inters.UserIdentity, interceptors...)
+}
+
+// Create returns a builder for creating a UserIdentity entity.
+func (c *UserIdentityClient) Create() *UserIdentityCreate {
+	mutation := newUserIdentityMutation(c.config, OpCreate)
+	return &UserIdentityCreate{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// CreateBulk returns a builder for creating a bulk of UserIdentity entities.
+func (c *UserIdentityClient) CreateBulk(builders ...*UserIdentityCreate) *UserIdentityCreateBulk {
+	return &UserIdentityCreateBulk{config: c.config, builders: builders}
+}
+
+// MapCreateBulk creates a bulk creation builder from the given slice. For each item in the slice, the function creates
+// a builder and applies setFunc on it.
+func (c *UserIdentityClient) MapCreateBulk(slice any, setFunc func(*UserIdentityCreate, int)) *UserIdentityCreateBulk {
+	rv := reflect.ValueOf(slice)
+	if rv.Kind() != reflect.Slice {
+		return &UserIdentityCreateBulk{err: fmt.Errorf("calling to UserIdentityClient.MapCreateBulk with wrong type %T, need slice", slice)}
+	}
+	builders := make([]*UserIdentityCreate, rv.Len())
+	for i := 0; i < rv.Len(); i++ {
+		builders[i] = c.Create()
+		setFunc(builders[i], i)
+	}
+	return &UserIdentityCreateBulk{config: c.config, builders: builders}
+}
+
+// Update returns an update builder for UserIdentity.
+func (c *UserIdentityClient) Update() *UserIdentityUpdate {
+	mutation := newUserIdentityMutation(c.config, OpUpdate)
+	return &UserIdentityUpdate{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// UpdateOne returns an update builder for the given entity.
+func (c *UserIdentityClient) UpdateOne(ui *UserIdentity) *UserIdentityUpdateOne {
+	mutation := newUserIdentityMutation(c.config, OpUpdateOne, withUserIdentity(ui))
+	return &UserIdentityUpdateOne{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// UpdateOneID returns an update builder for the given id.
+func (c *UserIdentityClient) UpdateOneID(id uuid.UUID) *UserIdentityUpdateOne {
+	mutation := newUserIdentityMutation(c.config, OpUpdateOne, withUserIdentityID(id))
+	return &UserIdentityUpdateOne{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// Delete returns a delete builder for UserIdentity.
+func (c *UserIdentityClient) Delete() *UserIdentityDelete {
+	mutation := newUserIdentityMutation(c.config, OpDelete)
+	return &UserIdentityDelete{config: c.config, hooks: c.Hooks(), mutation: mutation}
+}
+
+// DeleteOne returns a builder for deleting the given entity.
+func (c *UserIdentityClient) DeleteOne(ui *UserIdentity) *UserIdentityDeleteOne {
+	return c.DeleteOneID(ui.ID)
+}
+
+// DeleteOneID returns a builder for deleting the given entity by its id.
+func (c *UserIdentityClient) DeleteOneID(id uuid.UUID) *UserIdentityDeleteOne {
+	builder := c.Delete().Where(useridentity.ID(id))
+	builder.mutation.id = &id
+	builder.mutation.op = OpDeleteOne
+	return &UserIdentityDeleteOne{builder}
+}
+
+// Query returns a query builder for UserIdentity.
+func (c *UserIdentityClient) Query() *UserIdentityQuery {
+	return &UserIdentityQuery{
+		config: c.config,
+		ctx:    &QueryContext{Type: TypeUserIdentity},
+		inters: c.Interceptors(),
+	}
+}
+
+// Get returns a UserIdentity entity by its id.
+func (c *UserIdentityClient) Get(ctx context.Context, id uuid.UUID) (*UserIdentity, error) {
+	return c.Query().Where(useridentity.ID(id)).Only(ctx)
+}
+
+// GetX is like Get, but panics if an error occurs.
+func (c *UserIdentityClient) GetX(ctx context.Context, id uuid.UUID) *UserIdentity {
+	obj, err := c.Get(ctx, id)
+	if err != nil {
+		panic(err)
+	}
+	return obj
+}
+
+// QueryUser queries the user edge of a UserIdentity.
+func (c *UserIdentityClient) QueryUser(ui *UserIdentity) *UserQuery {
+	query := (&UserClient{config: c.config}).Query()
+	query.path = func(context.Context) (fromV *sql.Selector, _ error) {
+		id := ui.ID
+		step := sqlgraph.NewStep(
+			sqlgraph.From(useridentity.Table, useridentity.FieldID, id),
+			sqlgraph.To(user.Table, user.FieldID),
+			sqlgraph.Edge(sqlgraph.M2O, true, useridentity.UserTable, useridentity.UserColumn),
+		)
+		fromV = sqlgraph.Neighbors(ui.driver.Dialect(), step)
+		return fromV, nil
+	}
+	return query
+}
+
+// Hooks returns the client hooks.
+func (c *UserIdentityClient) Hooks() []Hook {
+	return c.hooks.UserIdentity
+}
+
+// Interceptors returns the client interceptors.
+func (c *UserIdentityClient) Interceptors() []Interceptor {
+	return c.inters.UserIdentity
+}
+
+func (c *UserIdentityClient) mutate(ctx context.Context, m *UserIdentityMutation) (Value, error) {
+	switch m.Op() {
+	case OpCreate:
+		return (&UserIdentityCreate{config: c.config, hooks: c.Hooks(), mutation: m}).Save(ctx)
+	case OpUpdate:
+		return (&UserIdentityUpdate{config: c.config, hooks: c.Hooks(), mutation: m}).Save(ctx)
+	case OpUpdateOne:
+		return (&UserIdentityUpdateOne{config: c.config, hooks: c.Hooks(), mutation: m}).Save(ctx)
+	case OpDelete, OpDeleteOne:
+		return (&UserIdentityDelete{config: c.config, hooks: c.Hooks(), mutation: m}).Exec(ctx)
+	default:
+		return nil, fmt.Errorf("ent: unknown UserIdentity mutation op: %q", m.Op())
+	}
+}
+
+// hooks and interceptors per client, for fast access.
+type (
+	hooks struct {
+		User, UserIdentity []ent.Hook
+	}
+	inters struct {
+		User, UserIdentity []ent.Interceptor
+	}
+)

internal/server/ent/ent.go

@@ -0,0 +1,610 @@
+// Code generated by ent, DO NOT EDIT.
+
+package ent
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+
+	"entgo.io/ent"
+	"entgo.io/ent/dialect/sql"
+	"entgo.io/ent/dialect/sql/sqlgraph"
+	"github.com/holos-run/holos/internal/server/ent/user"
+	"github.com/holos-run/holos/internal/server/ent/useridentity"
+)
+
+// ent aliases to avoid import conflicts in user's code.
+type (
+	Op            = ent.Op
+	Hook          = ent.Hook
+	Value         = ent.Value
+	Query         = ent.Query
+	QueryContext  = ent.QueryContext
+	Querier       = ent.Querier
+	QuerierFunc   = ent.QuerierFunc
+	Interceptor   = ent.Interceptor
+	InterceptFunc = ent.InterceptFunc
+	Traverser     = ent.Traverser
+	TraverseFunc  = ent.TraverseFunc
+	Policy        = ent.Policy
+	Mutator       = ent.Mutator
+	Mutation      = ent.Mutation
+	MutateFunc    = ent.MutateFunc
+)
+
+type clientCtxKey struct{}
+
+// FromContext returns a Client stored inside a context, or nil if there isn't one.
+func FromContext(ctx context.Context) *Client {
+	c, _ := ctx.Value(clientCtxKey{}).(*Client)
+	return c
+}
+
+// NewContext returns a new context with the given Client attached.
+func NewContext(parent context.Context, c *Client) context.Context {
+	return context.WithValue(parent, clientCtxKey{}, c)
+}
+
+type txCtxKey struct{}
+
+// TxFromContext returns a Tx stored inside a context, or nil if there isn't one.
+func TxFromContext(ctx context.Context) *Tx {
+	tx, _ := ctx.Value(txCtxKey{}).(*Tx)
+	return tx
+}
+
+// NewTxContext returns a new context with the given Tx attached.
+func NewTxContext(parent context.Context, tx *Tx) context.Context {
+	return context.WithValue(parent, txCtxKey{}, tx)
+}
+
+// OrderFunc applies an ordering on the sql selector.
+// Deprecated: Use Asc/Desc functions or the package builders instead.
+type OrderFunc func(*sql.Selector)
+
+var (
+	initCheck   sync.Once
+	columnCheck sql.ColumnCheck
+)
+
+// columnChecker checks if the column exists in the given table.
+func checkColumn(table, column string) error {
+	initCheck.Do(func() {
+		columnCheck = sql.NewColumnCheck(map[string]func(string) bool{
+			user.Table:         user.ValidColumn,
+			useridentity.Table: useridentity.ValidColumn,
+		})
+	})
+	return columnCheck(table, column)
+}
+
+// Asc applies the given fields in ASC order.
+func Asc(fields ...string) func(*sql.Selector) {
+	return func(s *sql.Selector) {
+		for _, f := range fields {
+			if err := checkColumn(s.TableName(), f); err != nil {
+				s.AddError(&ValidationError{Name: f, err: fmt.Errorf("ent: %w", err)})
+			}
+			s.OrderBy(sql.Asc(s.C(f)))
+		}
+	}
+}
+
+// Desc applies the given fields in DESC order.
+func Desc(fields ...string) func(*sql.Selector) {
+	return func(s *sql.Selector) {
+		for _, f := range fields {
+			if err := checkColumn(s.TableName(), f); err != nil {
+				s.AddError(&ValidationError{Name: f, err: fmt.Errorf("ent: %w", err)})
+			}
+			s.OrderBy(sql.Desc(s.C(f)))
+		}
+	}
+}
+
+// AggregateFunc applies an aggregation step on the group-by traversal/selector.
+type AggregateFunc func(*sql.Selector) string
+
+// As is a pseudo aggregation function for renaming another other functions with custom names. For example:
+//
+//	GroupBy(field1, field2).
+//	Aggregate(ent.As(ent.Sum(field1), "sum_field1"), (ent.As(ent.Sum(field2), "sum_field2")).
+//	Scan(ctx, &v)
+func As(fn AggregateFunc, end string) AggregateFunc {
+	return func(s *sql.Selector) string {
+		return sql.As(fn(s), end)
+	}
+}
+
+// Count applies the "count" aggregation function on each group.
+func Count() AggregateFunc {
+	return func(s *sql.Selector) string {
+		return sql.Count("*")
+	}
+}
+
+// Max applies the "max" aggregation function on the given field of each group.
+func Max(field string) AggregateFunc {
+	return func(s *sql.Selector) string {
+		if err := checkColumn(s.TableName(), field); err != nil {
+			s.AddError(&ValidationError{Name: field, err: fmt.Errorf("ent: %w", err)})
+			return ""
+		}
+		return sql.Max(s.C(field))
+	}
+}
+
+// Mean applies the "mean" aggregation function on the given field of each group.
+func Mean(field string) AggregateFunc {
+	return func(s *sql.Selector) string {
+		if err := checkColumn(s.TableName(), field); err != nil {
+			s.AddError(&ValidationError{Name: field, err: fmt.Errorf("ent: %w", err)})
+			return ""
+		}
+		return sql.Avg(s.C(field))
+	}
+}
+
+// Min applies the "min" aggregation function on the given field of each group.
+func Min(field string) AggregateFunc {
+	return func(s *sql.Selector) string {
+		if err := checkColumn(s.TableName(), field); err != nil {
+			s.AddError(&ValidationError{Name: field, err: fmt.Errorf("ent: %w", err)})
+			return ""
+		}
+		return sql.Min(s.C(field))
+	}
+}
+
+// Sum applies the "sum" aggregation function on the given field of each group.
+func Sum(field string) AggregateFunc {
+	return func(s *sql.Selector) string {
+		if err := checkColumn(s.TableName(), field); err != nil {
+			s.AddError(&ValidationError{Name: field, err: fmt.Errorf("ent: %w", err)})
+			return ""
+		}
+		return sql.Sum(s.C(field))
+	}
+}
+
+// ValidationError returns when validating a field or edge fails.
+type ValidationError struct {
+	Name string // Field or edge name.
+	err  error
+}
+
+// Error implements the error interface.
+func (e *ValidationError) Error() string {
+	return e.err.Error()
+}
+
+// Unwrap implements the errors.Wrapper interface.
+func (e *ValidationError) Unwrap() error {
+	return e.err
+}
+
+// IsValidationError returns a boolean indicating whether the error is a validation error.
+func IsValidationError(err error) bool {
+	if err == nil {
+		return false
+	}
+	var e *ValidationError
+	return errors.As(err, &e)
+}
+
+// NotFoundError returns when trying to fetch a specific entity and it was not found in the database.
+type NotFoundError struct {
+	label string
+}
+
+// Error implements the error interface.
+func (e *NotFoundError) Error() string {
+	return "ent: " + e.label + " not found"
+}
+
+// IsNotFound returns a boolean indicating whether the error is a not found error.
+func IsNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	var e *NotFoundError
+	return errors.As(err, &e)
+}
+
+// MaskNotFound masks not found error.
+func MaskNotFound(err error) error {
+	if IsNotFound(err) {
+		return nil
+	}
+	return err
+}
+
+// NotSingularError returns when trying to fetch a singular entity and more then one was found in the database.
+type NotSingularError struct {
+	label string
+}
+
+// Error implements the error interface.
+func (e *NotSingularError) Error() string {
+	return "ent: " + e.label + " not singular"
+}
+
+// IsNotSingular returns a boolean indicating whether the error is a not singular error.
+func IsNotSingular(err error) bool {
+	if err == nil {
+		return false
+	}
+	var e *NotSingularError
+	return errors.As(err, &e)
+}
+
+// NotLoadedError returns when trying to get a node that was not loaded by the query.
+type NotLoadedError struct {
+	edge string
+}
+
+// Error implements the error interface.
+func (e *NotLoadedError) Error() string {
+	return "ent: " + e.edge + " edge was not loaded"
+}
+
+// IsNotLoaded returns a boolean indicating whether the error is a not loaded error.
+func IsNotLoaded(err error) bool {
+	if err == nil {
+		return false
+	}
+	var e *NotLoadedError
+	return errors.As(err, &e)
+}
+
+// ConstraintError returns when trying to create/update one or more entities and
+// one or more of their constraints failed. For example, violation of edge or
+// field uniqueness.
+type ConstraintError struct {
+	msg  string
+	wrap error
+}
+
+// Error implements the error interface.
+func (e ConstraintError) Error() string {
+	return "ent: constraint failed: " + e.msg
+}
+
+// Unwrap implements the errors.Wrapper interface.
+func (e *ConstraintError) Unwrap() error {
+	return e.wrap
+}
+
+// IsConstraintError returns a boolean indicating whether the error is a constraint failure.
+func IsConstraintError(err error) bool {
+	if err == nil {
+		return false
+	}
+	var e *ConstraintError
+	return errors.As(err, &e)
+}
+
+// selector embedded by the different Select/GroupBy builders.
+type selector struct {
+	label string
+	flds  *[]string
+	fns   []AggregateFunc
+	scan  func(context.Context, any) error
+}
+
+// ScanX is like Scan, but panics if an error occurs.
+func (s *selector) ScanX(ctx context.Context, v any) {
+	if err := s.scan(ctx, v); err != nil {
+		panic(err)
+	}
+}
+
+// Strings returns list of strings from a selector. It is only allowed when selecting one field.
+func (s *selector) Strings(ctx context.Context) ([]string, error) {
+	if len(*s.flds) > 1 {
+		return nil, errors.New("ent: Strings is not achievable when selecting more than 1 field")
+	}
+	var v []string
+	if err := s.scan(ctx, &v); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+// StringsX is like Strings, but panics if an error occurs.
+func (s *selector) StringsX(ctx context.Context) []string {
+	v, err := s.Strings(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// String returns a single string from a selector. It is only allowed when selecting one field.
+func (s *selector) String(ctx context.Context) (_ string, err error) {
+	var v []string
+	if v, err = s.Strings(ctx); err != nil {
+		return
+	}
+	switch len(v) {
+	case 1:
+		return v[0], nil
+	case 0:
+		err = &NotFoundError{s.label}
+	default:
+		err = fmt.Errorf("ent: Strings returned %d results when one was expected", len(v))
+	}
+	return
+}
+
+// StringX is like String, but panics if an error occurs.
+func (s *selector) StringX(ctx context.Context) string {
+	v, err := s.String(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// Ints returns list of ints from a selector. It is only allowed when selecting one field.
+func (s *selector) Ints(ctx context.Context) ([]int, error) {
+	if len(*s.flds) > 1 {
+		return nil, errors.New("ent: Ints is not achievable when selecting more than 1 field")
+	}
+	var v []int
+	if err := s.scan(ctx, &v); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+// IntsX is like Ints, but panics if an error occurs.
+func (s *selector) IntsX(ctx context.Context) []int {
+	v, err := s.Ints(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// Int returns a single int from a selector. It is only allowed when selecting one field.
+func (s *selector) Int(ctx context.Context) (_ int, err error) {
+	var v []int
+	if v, err = s.Ints(ctx); err != nil {
+		return
+	}
+	switch len(v) {
+	case 1:
+		return v[0], nil
+	case 0:
+		err = &NotFoundError{s.label}
+	default:
+		err = fmt.Errorf("ent: Ints returned %d results when one was expected", len(v))
+	}
+	return
+}
+
+// IntX is like Int, but panics if an error occurs.
+func (s *selector) IntX(ctx context.Context) int {
+	v, err := s.Int(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// Float64s returns list of float64s from a selector. It is only allowed when selecting one field.
+func (s *selector) Float64s(ctx context.Context) ([]float64, error) {
+	if len(*s.flds) > 1 {
+		return nil, errors.New("ent: Float64s is not achievable when selecting more than 1 field")
+	}
+	var v []float64
+	if err := s.scan(ctx, &v); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+// Float64sX is like Float64s, but panics if an error occurs.
+func (s *selector) Float64sX(ctx context.Context) []float64 {
+	v, err := s.Float64s(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// Float64 returns a single float64 from a selector. It is only allowed when selecting one field.
+func (s *selector) Float64(ctx context.Context) (_ float64, err error) {
+	var v []float64
+	if v, err = s.Float64s(ctx); err != nil {
+		return
+	}
+	switch len(v) {
+	case 1:
+		return v[0], nil
+	case 0:
+		err = &NotFoundError{s.label}
+	default:
+		err = fmt.Errorf("ent: Float64s returned %d results when one was expected", len(v))
+	}
+	return
+}
+
+// Float64X is like Float64, but panics if an error occurs.
+func (s *selector) Float64X(ctx context.Context) float64 {
+	v, err := s.Float64(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// Bools returns list of bools from a selector. It is only allowed when selecting one field.
+func (s *selector) Bools(ctx context.Context) ([]bool, error) {
+	if len(*s.flds) > 1 {
+		return nil, errors.New("ent: Bools is not achievable when selecting more than 1 field")
+	}
+	var v []bool
+	if err := s.scan(ctx, &v); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+// BoolsX is like Bools, but panics if an error occurs.
+func (s *selector) BoolsX(ctx context.Context) []bool {
+	v, err := s.Bools(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// Bool returns a single bool from a selector. It is only allowed when selecting one field.
+func (s *selector) Bool(ctx context.Context) (_ bool, err error) {
+	var v []bool
+	if v, err = s.Bools(ctx); err != nil {
+		return
+	}
+	switch len(v) {
+	case 1:
+		return v[0], nil
+	case 0:
+		err = &NotFoundError{s.label}
+	default:
+		err = fmt.Errorf("ent: Bools returned %d results when one was expected", len(v))
+	}
+	return
+}
+
+// BoolX is like Bool, but panics if an error occurs.
+func (s *selector) BoolX(ctx context.Context) bool {
+	v, err := s.Bool(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// withHooks invokes the builder operation with the given hooks, if any.
+func withHooks[V Value, M any, PM interface {
+	*M
+	Mutation
+}](ctx context.Context, exec func(context.Context) (V, error), mutation PM, hooks []Hook) (value V, err error) {
+	if len(hooks) == 0 {
+		return exec(ctx)
+	}
+	var mut Mutator = MutateFunc(func(ctx context.Context, m Mutation) (Value, error) {
+		mutationT, ok := any(m).(PM)
+		if !ok {
+			return nil, fmt.Errorf("unexpected mutation type %T", m)
+		}
+		// Set the mutation to the builder.
+		*mutation = *mutationT
+		return exec(ctx)
+	})
+	for i := len(hooks) - 1; i >= 0; i-- {
+		if hooks[i] == nil {
+			return value, fmt.Errorf("ent: uninitialized hook (forgotten import ent/runtime?)")
+		}
+		mut = hooks[i](mut)
+	}
+	v, err := mut.Mutate(ctx, mutation)
+	if err != nil {
+		return value, err
+	}
+	nv, ok := v.(V)
+	if !ok {
+		return value, fmt.Errorf("unexpected node type %T returned from %T", v, mutation)
+	}
+	return nv, nil
+}
+
+// setContextOp returns a new context with the given QueryContext attached (including its op) in case it does not exist.
+func setContextOp(ctx context.Context, qc *QueryContext, op string) context.Context {
+	if ent.QueryFromContext(ctx) == nil {
+		qc.Op = op
+		ctx = ent.NewQueryContext(ctx, qc)
+	}
+	return ctx
+}
+
+func querierAll[V Value, Q interface {
+	sqlAll(context.Context, ...queryHook) (V, error)
+}]() Querier {
+	return QuerierFunc(func(ctx context.Context, q Query) (Value, error) {
+		query, ok := q.(Q)
+		if !ok {
+			return nil, fmt.Errorf("unexpected query type %T", q)
+		}
+		return query.sqlAll(ctx)
+	})
+}
+
+func querierCount[Q interface {
+	sqlCount(context.Context) (int, error)
+}]() Querier {
+	return QuerierFunc(func(ctx context.Context, q Query) (Value, error) {
+		query, ok := q.(Q)
+		if !ok {
+			return nil, fmt.Errorf("unexpected query type %T", q)
+		}
+		return query.sqlCount(ctx)
+	})
+}
+
+func withInterceptors[V Value](ctx context.Context, q Query, qr Querier, inters []Interceptor) (v V, err error) {
+	for i := len(inters) - 1; i >= 0; i-- {
+		qr = inters[i].Intercept(qr)
+	}
+	rv, err := qr.Query(ctx, q)
+	if err != nil {
+		return v, err
+	}
+	vt, ok := rv.(V)
+	if !ok {
+		return v, fmt.Errorf("unexpected type %T returned from %T. expected type: %T", vt, q, v)
+	}
+	return vt, nil
+}
+
+func scanWithInterceptors[Q1 ent.Query, Q2 interface {
+	sqlScan(context.Context, Q1, any) error
+}](ctx context.Context, rootQuery Q1, selectOrGroup Q2, inters []Interceptor, v any) error {
+	rv := reflect.ValueOf(v)
+	var qr Querier = QuerierFunc(func(ctx context.Context, q Query) (Value, error) {
+		query, ok := q.(Q1)
+		if !ok {
+			return nil, fmt.Errorf("unexpected query type %T", q)
+		}
+		if err := selectOrGroup.sqlScan(ctx, query, v); err != nil {
+			return nil, err
+		}
+		if k := rv.Kind(); k == reflect.Pointer && rv.Elem().CanInterface() {
+			return rv.Elem().Interface(), nil
+		}
+		return v, nil
+	})
+	for i := len(inters) - 1; i >= 0; i-- {
+		qr = inters[i].Intercept(qr)
+	}
+	vv, err := qr.Query(ctx, rootQuery)
+	if err != nil {
+		return err
+	}
+	switch rv2 := reflect.ValueOf(vv); {
+	case rv.IsNil(), rv2.IsNil(), rv.Kind() != reflect.Pointer:
+	case rv.Type() == rv2.Type():
+		rv.Elem().Set(rv2.Elem())
+	case rv.Elem().Type() == rv2.Type():
+		rv.Elem().Set(rv2)
+	}
+	return nil
+}
+
+// queryHook describes an internal hook for the different sqlAll methods.
+type queryHook func(context.Context, *sqlgraph.QuerySpec)

internal/server/ent/enttest/enttest.go

@@ -0,0 +1,84 @@
+// Code generated by ent, DO NOT EDIT.
+
+package enttest
+
+import (
+	"context"
+
+	"github.com/holos-run/holos/internal/server/ent"
+	// required by schema hooks.
+	_ "github.com/holos-run/holos/internal/server/ent/runtime"
+
+	"entgo.io/ent/dialect/sql/schema"
+	"github.com/holos-run/holos/internal/server/ent/migrate"
+)
+
+type (
+	// TestingT is the interface that is shared between
+	// testing.T and testing.B and used by enttest.
+	TestingT interface {
+		FailNow()
+		Error(...any)
+	}
+
+	// Option configures client creation.
+	Option func(*options)
+
+	options struct {
+		opts        []ent.Option
+		migrateOpts []schema.MigrateOption
+	}
+)
+
+// WithOptions forwards options to client creation.
+func WithOptions(opts ...ent.Option) Option {
+	return func(o *options) {
+		o.opts = append(o.opts, opts...)
+	}
+}
+
+// WithMigrateOptions forwards options to auto migration.
+func WithMigrateOptions(opts ...schema.MigrateOption) Option {
+	return func(o *options) {
+		o.migrateOpts = append(o.migrateOpts, opts...)
+	}
+}
+
+func newOptions(opts []Option) *options {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+	return o
+}
+
+// Open calls ent.Open and auto-run migration.
+func Open(t TestingT, driverName, dataSourceName string, opts ...Option) *ent.Client {
+	o := newOptions(opts)
+	c, err := ent.Open(driverName, dataSourceName, o.opts...)
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+	migrateSchema(t, c, o)
+	return c
+}
+
+// NewClient calls ent.NewClient and auto-run migration.
+func NewClient(t TestingT, opts ...Option) *ent.Client {
+	o := newOptions(opts)
+	c := ent.NewClient(o.opts...)
+	migrateSchema(t, c, o)
+	return c
+}
+func migrateSchema(t TestingT, c *ent.Client, o *options) {
+	tables, err := schema.CopyTables(migrate.Tables)
+	if err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+	if err := migrate.Create(context.Background(), c.Schema, tables, o.migrateOpts...); err != nil {
+		t.Error(err)
+		t.FailNow()
+	}
+}

internal/server/ent/generate.go

@@ -0,0 +1,3 @@
+package ent
+
+//go:generate go run -mod=mod entgo.io/ent/cmd/ent generate --feature sql/upsert ./schema

internal/server/ent/hook/hook.go

@@ -0,0 +1,211 @@
+// Code generated by ent, DO NOT EDIT.
+
+package hook
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/holos-run/holos/internal/server/ent"
+)
+
+// The UserFunc type is an adapter to allow the use of ordinary
+// function as User mutator.
+type UserFunc func(context.Context, *ent.UserMutation) (ent.Value, error)
+
+// Mutate calls f(ctx, m).
+func (f UserFunc) Mutate(ctx context.Context, m ent.Mutation) (ent.Value, error) {
+	if mv, ok := m.(*ent.UserMutation); ok {
+		return f(ctx, mv)
+	}
+	return nil, fmt.Errorf("unexpected mutation type %T. expect *ent.UserMutation", m)
+}
+
+// The UserIdentityFunc type is an adapter to allow the use of ordinary
+// function as UserIdentity mutator.
+type UserIdentityFunc func(context.Context, *ent.UserIdentityMutation) (ent.Value, error)
+
+// Mutate calls f(ctx, m).
+func (f UserIdentityFunc) Mutate(ctx context.Context, m ent.Mutation) (ent.Value, error) {
+	if mv, ok := m.(*ent.UserIdentityMutation); ok {
+		return f(ctx, mv)
+	}
+	return nil, fmt.Errorf("unexpected mutation type %T. expect *ent.UserIdentityMutation", m)
+}
+
+// Condition is a hook condition function.
+type Condition func(context.Context, ent.Mutation) bool
+
+// And groups conditions with the AND operator.
+func And(first, second Condition, rest ...Condition) Condition {
+	return func(ctx context.Context, m ent.Mutation) bool {
+		if !first(ctx, m) || !second(ctx, m) {
+			return false
+		}
+		for _, cond := range rest {
+			if !cond(ctx, m) {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// Or groups conditions with the OR operator.
+func Or(first, second Condition, rest ...Condition) Condition {
+	return func(ctx context.Context, m ent.Mutation) bool {
+		if first(ctx, m) || second(ctx, m) {
+			return true
+		}
+		for _, cond := range rest {
+			if cond(ctx, m) {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+// Not negates a given condition.
+func Not(cond Condition) Condition {
+	return func(ctx context.Context, m ent.Mutation) bool {
+		return !cond(ctx, m)
+	}
+}
+
+// HasOp is a condition testing mutation operation.
+func HasOp(op ent.Op) Condition {
+	return func(_ context.Context, m ent.Mutation) bool {
+		return m.Op().Is(op)
+	}
+}
+
+// HasAddedFields is a condition validating `.AddedField` on fields.
+func HasAddedFields(field string, fields ...string) Condition {
+	return func(_ context.Context, m ent.Mutation) bool {
+		if _, exists := m.AddedField(field); !exists {
+			return false
+		}
+		for _, field := range fields {
+			if _, exists := m.AddedField(field); !exists {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// HasClearedFields is a condition validating `.FieldCleared` on fields.
+func HasClearedFields(field string, fields ...string) Condition {
+	return func(_ context.Context, m ent.Mutation) bool {
+		if exists := m.FieldCleared(field); !exists {
+			return false
+		}
+		for _, field := range fields {
+			if exists := m.FieldCleared(field); !exists {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// HasFields is a condition validating `.Field` on fields.
+func HasFields(field string, fields ...string) Condition {
+	return func(_ context.Context, m ent.Mutation) bool {
+		if _, exists := m.Field(field); !exists {
+			return false
+		}
+		for _, field := range fields {
+			if _, exists := m.Field(field); !exists {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// If executes the given hook under condition.
+//
+//	hook.If(ComputeAverage, And(HasFields(...), HasAddedFields(...)))
+func If(hk ent.Hook, cond Condition) ent.Hook {
+	return func(next ent.Mutator) ent.Mutator {
+		return ent.MutateFunc(func(ctx context.Context, m ent.Mutation) (ent.Value, error) {
+			if cond(ctx, m) {
+				return hk(next).Mutate(ctx, m)
+			}
+			return next.Mutate(ctx, m)
+		})
+	}
+}
+
+// On executes the given hook only for the given operation.
+//
+//	hook.On(Log, ent.Delete|ent.Create)
+func On(hk ent.Hook, op ent.Op) ent.Hook {
+	return If(hk, HasOp(op))
+}
+
+// Unless skips the given hook only for the given operation.
+//
+//	hook.Unless(Log, ent.Update|ent.UpdateOne)
+func Unless(hk ent.Hook, op ent.Op) ent.Hook {
+	return If(hk, Not(HasOp(op)))
+}
+
+// FixedError is a hook returning a fixed error.
+func FixedError(err error) ent.Hook {
+	return func(ent.Mutator) ent.Mutator {
+		return ent.MutateFunc(func(context.Context, ent.Mutation) (ent.Value, error) {
+			return nil, err
+		})
+	}
+}
+
+// Reject returns a hook that rejects all operations that match op.
+//
+//	func (T) Hooks() []ent.Hook {
+//		return []ent.Hook{
+//			Reject(ent.Delete|ent.Update),
+//		}
+//	}
+func Reject(op ent.Op) ent.Hook {
+	hk := FixedError(fmt.Errorf("%s operation is not allowed", op))
+	return On(hk, op)
+}
+
+// Chain acts as a list of hooks and is effectively immutable.
+// Once created, it will always hold the same set of hooks in the same order.
+type Chain struct {
+	hooks []ent.Hook
+}
+
+// NewChain creates a new chain of hooks.
+func NewChain(hooks ...ent.Hook) Chain {
+	return Chain{append([]ent.Hook(nil), hooks...)}
+}
+
+// Hook chains the list of hooks and returns the final hook.
+func (c Chain) Hook() ent.Hook {
+	return func(mutator ent.Mutator) ent.Mutator {
+		for i := len(c.hooks) - 1; i >= 0; i-- {
+			mutator = c.hooks[i](mutator)
+		}
+		return mutator
+	}
+}
+
+// Append extends a chain, adding the specified hook
+// as the last ones in the mutation flow.
+func (c Chain) Append(hooks ...ent.Hook) Chain {
+	newHooks := make([]ent.Hook, 0, len(c.hooks)+len(hooks))
+	newHooks = append(newHooks, c.hooks...)
+	newHooks = append(newHooks, hooks...)
+	return Chain{newHooks}
+}
+
+// Extend extends a chain, adding the specified chain
+// as the last ones in the mutation flow.
+func (c Chain) Extend(chain Chain) Chain {
+	return c.Append(chain.hooks...)
+}

internal/server/ent/migrate/migrate.go

@@ -0,0 +1,64 @@
+// Code generated by ent, DO NOT EDIT.
+
+package migrate
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"entgo.io/ent/dialect"
+	"entgo.io/ent/dialect/sql/schema"
+)
+
+var (
+	// WithGlobalUniqueID sets the universal ids options to the migration.
+	// If this option is enabled, ent migration will allocate a 1<<32 range
+	// for the ids of each entity (table).
+	// Note that this option cannot be applied on tables that already exist.
+	WithGlobalUniqueID = schema.WithGlobalUniqueID
+	// WithDropColumn sets the drop column option to the migration.
+	// If this option is enabled, ent migration will drop old columns
+	// that were used for both fields and edges. This defaults to false.
+	WithDropColumn = schema.WithDropColumn
+	// WithDropIndex sets the drop index option to the migration.
+	// If this option is enabled, ent migration will drop old indexes
+	// that were defined in the schema. This defaults to false.
+	// Note that unique constraints are defined using `UNIQUE INDEX`,
+	// and therefore, it's recommended to enable this option to get more
+	// flexibility in the schema changes.
+	WithDropIndex = schema.WithDropIndex
+	// WithForeignKeys enables creating foreign-key in schema DDL. This defaults to true.
+	WithForeignKeys = schema.WithForeignKeys
+)
+
+// Schema is the API for creating, migrating and dropping a schema.
+type Schema struct {
+	drv dialect.Driver
+}
+
+// NewSchema creates a new schema client.
+func NewSchema(drv dialect.Driver) *Schema { return &Schema{drv: drv} }
+
+// Create creates all schema resources.
+func (s *Schema) Create(ctx context.Context, opts ...schema.MigrateOption) error {
+	return Create(ctx, s, Tables, opts...)
+}
+
+// Create creates all table resources using the given schema driver.
+func Create(ctx context.Context, s *Schema, tables []*schema.Table, opts ...schema.MigrateOption) error {
+	migrate, err := schema.NewMigrate(s.drv, opts...)
+	if err != nil {
+		return fmt.Errorf("ent/migrate: %w", err)
+	}
+	return migrate.Create(ctx, tables...)
+}
+
+// WriteTo writes the schema changes to w instead of running them against the database.
+//
+//	if err := client.Schema.WriteTo(context.Background(), os.Stdout); err != nil {
+//		log.Fatal(err)
+//	}
+func (s *Schema) WriteTo(ctx context.Context, w io.Writer, opts ...schema.MigrateOption) error {
+	return Create(ctx, &Schema{drv: &schema.WriteDriver{Writer: w, Driver: s.drv}}, Tables, opts...)
+}

internal/server/ent/migrate/schema.go

@@ -0,0 +1,68 @@
+// Code generated by ent, DO NOT EDIT.
+
+package migrate
+
+import (
+	"entgo.io/ent/dialect/sql/schema"
+	"entgo.io/ent/schema/field"
+)
+
+var (
+	// UsersColumns holds the columns for the "users" table.
+	UsersColumns = []*schema.Column{
+		{Name: "id", Type: field.TypeUUID},
+		{Name: "created_at", Type: field.TypeTime},
+		{Name: "updated_at", Type: field.TypeTime},
+		{Name: "email", Type: field.TypeString, Unique: true},
+		{Name: "email_verified", Type: field.TypeBool, Default: false},
+		{Name: "name", Type: field.TypeString},
+	}
+	// UsersTable holds the schema information for the "users" table.
+	UsersTable = &schema.Table{
+		Name:       "users",
+		Columns:    UsersColumns,
+		PrimaryKey: []*schema.Column{UsersColumns[0]},
+	}
+	// UserIdentitiesColumns holds the columns for the "user_identities" table.
+	UserIdentitiesColumns = []*schema.Column{
+		{Name: "id", Type: field.TypeUUID},
+		{Name: "created_at", Type: field.TypeTime},
+		{Name: "updated_at", Type: field.TypeTime},
+		{Name: "iss", Type: field.TypeString},
+		{Name: "sub", Type: field.TypeString},
+		{Name: "email", Type: field.TypeString},
+		{Name: "email_verified", Type: field.TypeBool, Default: false},
+		{Name: "name", Type: field.TypeString, Nullable: true},
+		{Name: "user_id", Type: field.TypeUUID},
+	}
+	// UserIdentitiesTable holds the schema information for the "user_identities" table.
+	UserIdentitiesTable = &schema.Table{
+		Name:       "user_identities",
+		Columns:    UserIdentitiesColumns,
+		PrimaryKey: []*schema.Column{UserIdentitiesColumns[0]},
+		ForeignKeys: []*schema.ForeignKey{
+			{
+				Symbol:     "user_identities_users_identities",
+				Columns:    []*schema.Column{UserIdentitiesColumns[8]},
+				RefColumns: []*schema.Column{UsersColumns[0]},
+				OnDelete:   schema.Cascade,
+			},
+		},
+		Indexes: []*schema.Index{
+			{
+				Name:    "useridentity_iss_sub",
+				Unique:  true,
+				Columns: []*schema.Column{UserIdentitiesColumns[3], UserIdentitiesColumns[4]},
+			},
+		},
+	}
+	// Tables holds all the tables in the schema.
+	Tables = []*schema.Table{
+		UsersTable,
+		UserIdentitiesTable,
+	}
+)
+
+func init() {
+	UserIdentitiesTable.ForeignKeys[0].RefTable = UsersTable
+}

internal/server/ent/predicate/predicate.go

@@ -0,0 +1,13 @@
+// Code generated by ent, DO NOT EDIT.
+
+package predicate
+
+import (
+	"entgo.io/ent/dialect/sql"
+)
+
+// User is the predicate function for user builders.
+type User func(*sql.Selector)
+
+// UserIdentity is the predicate function for useridentity builders.
+type UserIdentity func(*sql.Selector)

internal/server/ent/runtime.go

@@ -0,0 +1,84 @@
+// Code generated by ent, DO NOT EDIT.
+
+package ent
+
+import (
+	"time"
+
+	"github.com/gofrs/uuid"
+	"github.com/holos-run/holos/internal/server/ent/schema"
+	"github.com/holos-run/holos/internal/server/ent/user"
+	"github.com/holos-run/holos/internal/server/ent/useridentity"
+)
+
+// The init function reads all schema descriptors with runtime code
+// (default values, validators, hooks and policies) and stitches it
+// to their package variables.
+func init() {
+	userMixin := schema.User{}.Mixin()
+	userMixinFields0 := userMixin[0].Fields()
+	_ = userMixinFields0
+	userMixinFields1 := userMixin[1].Fields()
+	_ = userMixinFields1
+	userFields := schema.User{}.Fields()
+	_ = userFields
+	// userDescCreatedAt is the schema descriptor for created_at field.
+	userDescCreatedAt := userMixinFields1[0].Descriptor()
+	// user.DefaultCreatedAt holds the default value on creation for the created_at field.
+	user.DefaultCreatedAt = userDescCreatedAt.Default.(func() time.Time)
+	// userDescUpdatedAt is the schema descriptor for updated_at field.
+	userDescUpdatedAt := userMixinFields1[1].Descriptor()
+	// user.DefaultUpdatedAt holds the default value on creation for the updated_at field.
+	user.DefaultUpdatedAt = userDescUpdatedAt.Default.(func() time.Time)
+	// user.UpdateDefaultUpdatedAt holds the default value on update for the updated_at field.
+	user.UpdateDefaultUpdatedAt = userDescUpdatedAt.UpdateDefault.(func() time.Time)
+	// userDescEmail is the schema descriptor for email field.
+	userDescEmail := userFields[0].Descriptor()
+	// user.EmailValidator is a validator for the "email" field. It is called by the builders before save.
+	user.EmailValidator = userDescEmail.Validators[0].(func(string) error)
+	// userDescEmailVerified is the schema descriptor for email_verified field.
+	userDescEmailVerified := userFields[1].Descriptor()
+	// user.DefaultEmailVerified holds the default value on creation for the email_verified field.
+	user.DefaultEmailVerified = userDescEmailVerified.Default.(bool)
+	// userDescID is the schema descriptor for id field.
+	userDescID := userMixinFields0[0].Descriptor()
+	// user.DefaultID holds the default value on creation for the id field.
+	user.DefaultID = userDescID.Default.(func() uuid.UUID)
+	useridentityMixin := schema.UserIdentity{}.Mixin()
+	useridentityMixinFields0 := useridentityMixin[0].Fields()
+	_ = useridentityMixinFields0
+	useridentityMixinFields1 := useridentityMixin[1].Fields()
+	_ = useridentityMixinFields1
+	useridentityFields := schema.UserIdentity{}.Fields()
+	_ = useridentityFields
+	// useridentityDescCreatedAt is the schema descriptor for created_at field.
+	useridentityDescCreatedAt := useridentityMixinFields1[0].Descriptor()
+	// useridentity.DefaultCreatedAt holds the default value on creation for the created_at field.
+	useridentity.DefaultCreatedAt = useridentityDescCreatedAt.Default.(func() time.Time)
+	// useridentityDescUpdatedAt is the schema descriptor for updated_at field.
+	useridentityDescUpdatedAt := useridentityMixinFields1[1].Descriptor()
+	// useridentity.DefaultUpdatedAt holds the default value on creation for the updated_at field.
+	useridentity.DefaultUpdatedAt = useridentityDescUpdatedAt.Default.(func() time.Time)
+	// useridentity.UpdateDefaultUpdatedAt holds the default value on update for the updated_at field.
+	useridentity.UpdateDefaultUpdatedAt = useridentityDescUpdatedAt.UpdateDefault.(func() time.Time)
+	// useridentityDescIss is the schema descriptor for iss field.
+	useridentityDescIss := useridentityFields[0].Descriptor()
+	// useridentity.IssValidator is a validator for the "iss" field. It is called by the builders before save.
+	useridentity.IssValidator = useridentityDescIss.Validators[0].(func(string) error)
+	// useridentityDescSub is the schema descriptor for sub field.
+	useridentityDescSub := useridentityFields[1].Descriptor()
+	// useridentity.SubValidator is a validator for the "sub" field. It is called by the builders before save.
+	useridentity.SubValidator = useridentityDescSub.Validators[0].(func(string) error)
+	// useridentityDescEmail is the schema descriptor for email field.
+	useridentityDescEmail := useridentityFields[2].Descriptor()
+	// useridentity.EmailValidator is a validator for the "email" field. It is called by the builders before save.
+	useridentity.EmailValidator = useridentityDescEmail.Validators[0].(func(string) error)
+	// useridentityDescEmailVerified is the schema descriptor for email_verified field.
+	useridentityDescEmailVerified := useridentityFields[3].Descriptor()
+	// useridentity.DefaultEmailVerified holds the default value on creation for the email_verified field.
+	useridentity.DefaultEmailVerified = useridentityDescEmailVerified.Default.(bool)
+	// useridentityDescID is the schema descriptor for id field.
+	useridentityDescID := useridentityMixinFields0[0].Descriptor()
+	// useridentity.DefaultID holds the default value on creation for the id field.
+	useridentity.DefaultID = useridentityDescID.Default.(func() uuid.UUID)
+}

internal/server/ent/runtime/runtime.go

@@ -0,0 +1,10 @@
+// Code generated by ent, DO NOT EDIT.
+
+package runtime
+
+// The schema-stitching logic is generated in github.com/holos-run/holos/internal/server/ent/runtime.go
+
+const (
+	Version = "v0.13.1"                                         // Version of ent codegen.
+	Sum     = "h1:uD8QwN1h6SNphdCCzmkMN3feSUzNnVvV/WIkHKMbzOE=" // Sum of ent codegen.
+)

internal/server/ent/schema/application.go

@@ -0,0 +1 @@
+package schema

internal/server/ent/schema/mixin.go

@@ -0,0 +1,39 @@
+package schema
+
+import (
+	"entgo.io/ent"
+	"entgo.io/ent/schema/field"
+	"entgo.io/ent/schema/mixin"
+	"github.com/gofrs/uuid"
+	"time"
+)
+
+func newUUID() uuid.UUID {
+	return uuid.Must(uuid.NewV7())
+}
+
+type BaseMixin struct {
+	mixin.Schema
+}
+
+func (BaseMixin) Fields() []ent.Field {
+	return []ent.Field{
+		field.UUID("id", uuid.UUID{}).Default(newUUID),
+	}
+}
+
+// TimeMixin adds created_at and updated_at fields.
+type TimeMixin struct {
+	mixin.Schema
+}
+
+func (TimeMixin) Fields() []ent.Field {
+	return []ent.Field{
+		field.Time("created_at").
+			Immutable().
+			Default(time.Now),
+		field.Time("updated_at").
+			Default(time.Now).
+			UpdateDefault(time.Now),
+	}
+}

internal/server/ent/schema/user.go

@@ -0,0 +1,89 @@
+package schema
+
+import (
+	"entgo.io/ent"
+	"entgo.io/ent/dialect/entsql"
+	"entgo.io/ent/schema/edge"
+	"entgo.io/ent/schema/field"
+	"entgo.io/ent/schema/index"
+)
+
+// User holds the schema definition for the User entity, the internal
+// representation and identity of a single human user. Users are scoped
+// globally.
+type User struct {
+	ent.Schema
+}
+
+func (User) Mixin() []ent.Mixin {
+	return []ent.Mixin{
+		BaseMixin{},
+		TimeMixin{},
+	}
+}
+
+// Fields of the User.
+func (User) Fields() []ent.Field {
+	return []ent.Field{
+		field.String("email").NotEmpty().Unique(),
+		field.Bool("email_verified").Default(false),
+		field.String("name"),
+	}
+}
+
+// Edges of the User.
+func (User) Edges() []ent.Edge {
+	return []ent.Edge{
+		edge.To("identities", UserIdentity.Type).
+			StorageKey(edge.Column("user_id")).
+			Annotations(entsql.OnDelete(entsql.Cascade)),
+	}
+}
+
+// UserIdentity holds the schema definition for the UserIdentity entity, a
+// representation of an identity from an oidc provider typically implemented as
+// an oidc id token.
+type UserIdentity struct {
+	ent.Schema
+}
+
+func (UserIdentity) Mixin() []ent.Mixin {
+	return []ent.Mixin{
+		BaseMixin{},
+		TimeMixin{},
+	}
+}
+
+// Fields of the UserIdentity. Assumes an oidc id token is available for the user with
+// the openid scope `iss`, `aud`, `exp`, `iat`, and `at_hash` claims. Assumes
+// the profile scope as well, with claims family_name, given_name, middle_name,
+// nickname, picture, and updated_at. Claims are defined by the [standard
+// claims][1].
+//
+// [1]: https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims
+func (UserIdentity) Fields() []ent.Field {
+	return []ent.Field{
+		field.String("iss").NotEmpty(),
+		field.String("sub").NotEmpty(),
+		field.String("email").NotEmpty(),
+		field.Bool("email_verified").Default(false),
+		field.String("name").Optional().Nillable(),
+	}
+}
+
+func (UserIdentity) Indexes() []ent.Index {
+	return []ent.Index{
+		index.Fields("iss", "sub").Unique(),
+	}
+}
+
+// Edges of the UserIdentity.
+func (UserIdentity) Edges() []ent.Edge {
+	return []ent.Edge{
+		edge.From("user", User.Type).
+			Ref("identities").
+			Required().
+			Immutable().
+			Unique(),
+	}
+}

internal/server/ent/tx.go

@@ -0,0 +1,213 @@
+// Code generated by ent, DO NOT EDIT.
+
+package ent
+
+import (
+	"context"
+	"sync"
+
+	"entgo.io/ent/dialect"
+)
+
+// Tx is a transactional client that is created by calling Client.Tx().
+type Tx struct {
+	config
+	// User is the client for interacting with the User builders.
+	User *UserClient
+	// UserIdentity is the client for interacting with the UserIdentity builders.
+	UserIdentity *UserIdentityClient
+
+	// lazily loaded.
+	client     *Client
+	clientOnce sync.Once
+	// ctx lives for the life of the transaction. It is
+	// the same context used by the underlying connection.
+	ctx context.Context
+}
+
+type (
+	// Committer is the interface that wraps the Commit method.
+	Committer interface {
+		Commit(context.Context, *Tx) error
+	}
+
+	// The CommitFunc type is an adapter to allow the use of ordinary
+	// function as a Committer. If f is a function with the appropriate
+	// signature, CommitFunc(f) is a Committer that calls f.
+	CommitFunc func(context.Context, *Tx) error
+
+	// CommitHook defines the "commit middleware". A function that gets a Committer
+	// and returns a Committer. For example:
+	//
+	//	hook := func(next ent.Committer) ent.Committer {
+	//		return ent.CommitFunc(func(ctx context.Context, tx *ent.Tx) error {
+	//			// Do some stuff before.
+	//			if err := next.Commit(ctx, tx); err != nil {
+	//				return err
+	//			}
+	//			// Do some stuff after.
+	//			return nil
+	//		})
+	//	}
+	//
+	CommitHook func(Committer) Committer
+)
+
+// Commit calls f(ctx, m).
+func (f CommitFunc) Commit(ctx context.Context, tx *Tx) error {
+	return f(ctx, tx)
+}
+
+// Commit commits the transaction.
+func (tx *Tx) Commit() error {
+	txDriver := tx.config.driver.(*txDriver)
+	var fn Committer = CommitFunc(func(context.Context, *Tx) error {
+		return txDriver.tx.Commit()
+	})
+	txDriver.mu.Lock()
+	hooks := append([]CommitHook(nil), txDriver.onCommit...)
+	txDriver.mu.Unlock()
+	for i := len(hooks) - 1; i >= 0; i-- {
+		fn = hooks[i](fn)
+	}
+	return fn.Commit(tx.ctx, tx)
+}
+
+// OnCommit adds a hook to call on commit.
+func (tx *Tx) OnCommit(f CommitHook) {
+	txDriver := tx.config.driver.(*txDriver)
+	txDriver.mu.Lock()
+	txDriver.onCommit = append(txDriver.onCommit, f)
+	txDriver.mu.Unlock()
+}
+
+type (
+	// Rollbacker is the interface that wraps the Rollback method.
+	Rollbacker interface {
+		Rollback(context.Context, *Tx) error
+	}
+
+	// The RollbackFunc type is an adapter to allow the use of ordinary
+	// function as a Rollbacker. If f is a function with the appropriate
+	// signature, RollbackFunc(f) is a Rollbacker that calls f.
+	RollbackFunc func(context.Context, *Tx) error
+
+	// RollbackHook defines the "rollback middleware". A function that gets a Rollbacker
+	// and returns a Rollbacker. For example:
+	//
+	//	hook := func(next ent.Rollbacker) ent.Rollbacker {
+	//		return ent.RollbackFunc(func(ctx context.Context, tx *ent.Tx) error {
+	//			// Do some stuff before.
+	//			if err := next.Rollback(ctx, tx); err != nil {
+	//				return err
+	//			}
+	//			// Do some stuff after.
+	//			return nil
+	//		})
+	//	}
+	//
+	RollbackHook func(Rollbacker) Rollbacker
+)
+
+// Rollback calls f(ctx, m).
+func (f RollbackFunc) Rollback(ctx context.Context, tx *Tx) error {
+	return f(ctx, tx)
+}
+
+// Rollback rollbacks the transaction.
+func (tx *Tx) Rollback() error {
+	txDriver := tx.config.driver.(*txDriver)
+	var fn Rollbacker = RollbackFunc(func(context.Context, *Tx) error {
+		return txDriver.tx.Rollback()
+	})
+	txDriver.mu.Lock()
+	hooks := append([]RollbackHook(nil), txDriver.onRollback...)
+	txDriver.mu.Unlock()
+	for i := len(hooks) - 1; i >= 0; i-- {
+		fn = hooks[i](fn)
+	}
+	return fn.Rollback(tx.ctx, tx)
+}
+
+// OnRollback adds a hook to call on rollback.
+func (tx *Tx) OnRollback(f RollbackHook) {
+	txDriver := tx.config.driver.(*txDriver)
+	txDriver.mu.Lock()
+	txDriver.onRollback = append(txDriver.onRollback, f)
+	txDriver.mu.Unlock()
+}
+
+// Client returns a Client that binds to current transaction.
+func (tx *Tx) Client() *Client {
+	tx.clientOnce.Do(func() {
+		tx.client = &Client{config: tx.config}
+		tx.client.init()
+	})
+	return tx.client
+}
+
+func (tx *Tx) init() {
+	tx.User = NewUserClient(tx.config)
+	tx.UserIdentity = NewUserIdentityClient(tx.config)
+}
+
+// txDriver wraps the given dialect.Tx with a nop dialect.Driver implementation.
+// The idea is to support transactions without adding any extra code to the builders.
+// When a builder calls to driver.Tx(), it gets the same dialect.Tx instance.
+// Commit and Rollback are nop for the internal builders and the user must call one
+// of them in order to commit or rollback the transaction.
+//
+// If a closed transaction is embedded in one of the generated entities, and the entity
+// applies a query, for example: User.QueryXXX(), the query will be executed
+// through the driver which created this transaction.
+//
+// Note that txDriver is not goroutine safe.
+type txDriver struct {
+	// the driver we started the transaction from.
+	drv dialect.Driver
+	// tx is the underlying transaction.
+	tx dialect.Tx
+	// completion hooks.
+	mu         sync.Mutex
+	onCommit   []CommitHook
+	onRollback []RollbackHook
+}
+
+// newTx creates a new transactional driver.
+func newTx(ctx context.Context, drv dialect.Driver) (*txDriver, error) {
+	tx, err := drv.Tx(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &txDriver{tx: tx, drv: drv}, nil
+}
+
+// Tx returns the transaction wrapper (txDriver) to avoid Commit or Rollback calls
+// from the internal builders. Should be called only by the internal builders.
+func (tx *txDriver) Tx(context.Context) (dialect.Tx, error) { return tx, nil }
+
+// Dialect returns the dialect of the driver we started the transaction from.
+func (tx *txDriver) Dialect() string { return tx.drv.Dialect() }
+
+// Close is a nop close.
+func (*txDriver) Close() error { return nil }
+
+// Commit is a nop commit for the internal builders.
+// User must call `Tx.Commit` in order to commit the transaction.
+func (*txDriver) Commit() error { return nil }
+
+// Rollback is a nop rollback for the internal builders.
+// User must call `Tx.Rollback` in order to rollback the transaction.
+func (*txDriver) Rollback() error { return nil }
+
+// Exec calls tx.Exec.
+func (tx *txDriver) Exec(ctx context.Context, query string, args, v any) error {
+	return tx.tx.Exec(ctx, query, args, v)
+}
+
+// Query calls tx.Query.
+func (tx *txDriver) Query(ctx context.Context, query string, args, v any) error {
+	return tx.tx.Query(ctx, query, args, v)
+}
+
+var _ dialect.Driver = (*txDriver)(nil)