(#175) Make holos render --cluster-name flag optional

Problem:
Rendering the whole platform doesn't need a cluster name.

Solution:
Make the flag optional, do not set the cue tag if it's empty.

Result:
Holos renders the platform resource and proceeds to the point where we
need to implement the iteration over platform components, passing the
platform model to each one and rendering the component.

.github/workflows/lint.yaml

@@ -17,12 +17,33 @@ jobs:
     name: lint
     runs-on: gha-rs
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
           go-version: stable
-          cache: false
+
+      - name: Install Packages
+        run: sudo apt update && sudo apt -qq -y install git curl zip unzip tar bzip2 make
+
+      - name: Install Tools
+        run: |
+          set -x
+          make tools
+          make buf
+          go generate ./...
+          make frontend
+          go mod tidy
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v4
         with:
           version: latest
+          skip-pkg-cache: true

.github/workflows/release.yaml

@@ -14,23 +14,49 @@ jobs:
   goreleaser:
     runs-on: gha-rs
     steps:
+      # Must come before Checkout, otherwise goreleaser fails
       - name: Provide GPG and Git
-        run: sudo apt update && sudo apt -qq -y install gnupg git
+        run: sudo apt update && sudo apt -qq -y install gnupg git curl zip unzip tar bzip2 make
+
+      # Must come after git executable is provided
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
+      # Necessary to run these outside of goreleaser, otherwise
+      # /home/runner/_work/holos/holos/internal/frontend/node_modules/.bin/protoc-gen-connect-query is not in PATH
+      - name: Install Tools
+        run: |
+          set -x
+          make tools
+          make buf
+          go generate ./...
+          make frontend
+          go mod tidy
+
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@v6
         with:
           gpg_private_key: ${{ secrets.GPG_CODE_SIGNING_SECRETKEY }}
           passphrase: ${{ secrets.GPG_CODE_SIGNING_PASSPHRASE }}
+
       - name: List keys
         run: gpg -K
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
+
+      - name: Git diff
+        run: git diff
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v5
         with:

.github/workflows/test.yaml

@@ -18,13 +18,18 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
           go-version: stable
 
-      - name: Provide unzip for Helm
-        run: sudo apt update && sudo apt -qq -y install curl zip unzip tar bzip2
+      - name: Install Packages
+        run: sudo apt update && sudo apt -qq -y install git curl zip unzip tar bzip2 make
 
       - name: Set up Helm
         uses: azure/setup-helm@v4
@@ -32,5 +37,14 @@ jobs:
       - name: Set up Kubectl
         uses: azure/setup-kubectl@v3
 
+      - name: Install Tools
+        run: |
+          set -x
+          make tools
+          make buf
+          go generate ./...
+          make frontend
+          go mod tidy
+
       - name: Test
         run: ./scripts/test

.gitignore

@@ -1,8 +1,9 @@
-bin/
+/bin/
 vendor/
 .idea/
 coverage.out
-dist/
+/dist/
 *.hold/
 /deploy/
 .vscode/
+tmp/

.goreleaser.yaml

@@ -10,10 +10,8 @@ version: 1
 
 before:
   hooks:
-    # You may remove this if you don't use go modules.
-    - go mod tidy
-    # you may remove this if you don't need go generate
     - go generate ./...
+    - go mod tidy
 
 builds:
   - main: ./cmd/holos
@@ -23,6 +21,9 @@ builds:
       - linux
       - windows
       - darwin
+    goarch:
+      - amd64
+      - arm64
 
 signs:
   - artifacts: checksum

Makefile

@@ -4,7 +4,7 @@ PROJ=holos
 ORG_PATH=github.com/holos-run
 REPO_PATH=$(ORG_PATH)/$(PROJ)
 
-VERSION := $(shell cat pkg/version/embedded/major pkg/version/embedded/minor pkg/version/embedded/patch | xargs printf "%s.%s.%s")
+VERSION := $(shell cat version/embedded/major version/embedded/minor version/embedded/patch | xargs printf "%s.%s.%s")
 BIN_NAME := holos
 
 DOCKER_REPO=quay.io/openinfrastructure/holos
@@ -12,11 +12,14 @@ IMAGE_NAME=$(DOCKER_REPO)
 
 $( shell mkdir -p bin)
 
+# For buf plugin protoc-gen-connect-es
+export PATH := $(PWD)/internal/frontend/holos/node_modules/.bin:$(PATH)
+
 GIT_COMMIT=$(shell git rev-parse HEAD)
 GIT_TREE_STATE=$(shell test -n "`git status --porcelain`" && echo "dirty" || echo "clean")
 BUILD_DATE=$(shell date -Iseconds)
 
-LD_FLAGS="-w -X ${ORG_PATH}/${PROJ}/pkg/version.GitCommit=${GIT_COMMIT} -X ${ORG_PATH}/${PROJ}/pkg/version.GitTreeState=${GIT_TREE_STATE} -X ${ORG_PATH}/${PROJ}/pkg/version.BuildDate=${BUILD_DATE}"
+LD_FLAGS="-w -X ${ORG_PATH}/${PROJ}/version.GitCommit=${GIT_COMMIT} -X ${ORG_PATH}/${PROJ}/version.GitTreeState=${GIT_TREE_STATE} -X ${ORG_PATH}/${PROJ}/version.BuildDate=${BUILD_DATE}"
 
 .PHONY: default
 default: test
@@ -58,14 +61,26 @@ vet: ## Vet Go code.
 
 .PHONY: gencue
 gencue: ## Generate CUE definitions
-	cd docs/examples && cue get go github.com/holos-run/holos/api/...
+	cd internal/generate/platforms && cue get go github.com/holos-run/holos/api/v1alpha1/...
+
+.PHONY: rmgen
+rmgen: ## Remove generated code
+	git rm -rf service/gen/ internal/frontend/holos/src/app/gen/ || true
+	rm -rf service/gen/ internal/frontend/holos/src/app/gen/
+	git rm -rf internal/ent/
+	rm -rf internal/ent/
+	git restore --staged internal/ent/generate.go internal/ent/schema/
+	git restore internal/ent/generate.go internal/ent/schema/
+
+.PHONY: regenerate
+regenerate: generate ## Re-generate code (delete and re-create)
 
 .PHONY: generate
-generate: ## Generate code.
+generate: buf gencue ## Generate code.
 	go generate ./...
 
 .PHONY: build
-build: generate ## Build holos executable.
+build: generate frontend ## Build holos executable.
 	@echo "building ${BIN_NAME} ${VERSION}"
 	@echo "GOPATH=${GOPATH}"
 	go build -trimpath -o bin/$(BIN_NAME) -ldflags $(LD_FLAGS) $(REPO_PATH)/cmd/$(BIN_NAME)
@@ -84,6 +99,8 @@ test: ## Run tests.
 
 .PHONY: lint
 lint: ## Run linters.
+	buf lint
+	cd internal/frontend/holos && ng lint
 	golangci-lint run
 
 .PHONY: coverage
@@ -94,6 +111,40 @@ coverage: test  ## Test coverage profile.
 snapshot:  ## Go release snapshot
 	goreleaser release --snapshot --clean
 
+.PHONY: buf
+buf: ## buf generate
+	cd service && buf dep update
+	buf generate
+
+.PHONY: tools
+tools: go-deps frontend-deps  ## install tool dependencies
+
+.PHONY: go-deps
+go-deps: ## tool versions pinned in tools.go
+	go install github.com/bufbuild/buf/cmd/buf
+	go install github.com/fullstorydev/grpcurl/cmd/grpcurl
+	go install google.golang.org/protobuf/cmd/protoc-gen-go
+	go install connectrpc.com/connect/cmd/protoc-gen-connect-go
+	go install honnef.co/go/tools/cmd/staticcheck@latest
+	# curl https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash
+
+.PHONY: frontend-deps
+frontend-deps: ## Setup npm and vite
+	cd internal/frontend/holos && npm install
+	cd internal/frontend/holos && npm install --save-dev @bufbuild/buf @connectrpc/protoc-gen-connect-es
+	cd internal/frontend/holos && npm install @connectrpc/connect @connectrpc/connect-web @bufbuild/protobuf
+	# https://github.com/connectrpc/connect-query-es/blob/1350b6f07b6aead81793917954bdb1cc3ce09df9/packages/protoc-gen-connect-query/README.md?plain=1#L23
+	cd internal/frontend/holos && npm install --save-dev @connectrpc/protoc-gen-connect-query @bufbuild/protoc-gen-es
+	cd internal/frontend/holos && npm install @connectrpc/connect-query @bufbuild/protobuf
+
+
+.PHONY: frontend
+frontend: buf
+	cd internal/frontend/holos && rm -rf dist
+	mkdir -p internal/frontend/holos/dist
+	cd internal/frontend/holos && ng build
+	touch internal/frontend/frontend.go
+
 .PHONY: help
 help:  ## Display this help menu.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

Tiltfile

@@ -0,0 +1,311 @@
+# -*- mode: Python -*-
+# This Tiltfile manages a Go project with live leload in Kubernetes
+
+listen_port = 3000
+metrics_port = 9090
+
+# Use our wrapper to set the kube namespace
+if os.getenv('TILT_WRAPPER') != '1':
+    fail("could not run, ./hack/tilt/bin/tilt was not used to start tilt")
+
+# AWS Account to work in
+aws_account = '271053619184'
+aws_region = 'us-east-2'
+
+# Resource ids
+holos_backend = 'Holos Backend'
+pg_admin = 'pgAdmin'
+pg_cluster = 'PostgresCluster'
+pg_svc = 'Database Pod'
+compile_id = 'Go Build'
+auth_id = 'Auth Policy'
+lint_id = 'Run Linters'
+tests_id = 'Run Tests'
+
+# PostgresCluster resource name in k8s
+pg_cluster_name = 'holos'
+# Database name inside the PostgresCluster
+pg_database_name = 'holos'
+# PGAdmin name
+pg_admin_name = 'pgadmin'
+
+# Default Registry.
+# See: https://github.com/tilt-dev/tilt.build/blob/master/docs/choosing_clusters.md#manual-configuration
+# Note, Tilt will append the image name to the registry uri path
+default_registry('{account}.dkr.ecr.{region}.amazonaws.com/holos-run/holos-server'.format(account=aws_account, region=aws_region))
+
+# Set a name prefix specific to the user.  Multiple developers share the tilt-holos namespace.
+developer = os.getenv('USER')
+holos_server = 'holos'
+# See ./hack/tilt/bin/tilt
+namespace = os.getenv('NAMESPACE')
+# We always develop against the k1 cluster.
+os.putenv('KUBECONFIG', os.path.abspath('./hack/tilt/kubeconfig'))
+# The context defined in ./hack/tilt/kubeconfig
+allow_k8s_contexts('sso@k1')
+allow_k8s_contexts('sso@k2')
+allow_k8s_contexts('sso@k3')
+allow_k8s_contexts('sso@k4')
+allow_k8s_contexts('sso@k5')
+# PG db connection for localhost -> k8s port-forward
+os.putenv('PGHOST', 'localhost')
+os.putenv('PGPORT', '15432')
+# We always develop in the dev aws account.
+os.putenv('AWS_CONFIG_FILE', os.path.abspath('./hack/tilt/aws.config'))
+os.putenv('AWS_ACCOUNT', aws_account)
+os.putenv('AWS_DEFAULT_REGION', aws_region)
+os.putenv('AWS_PROFILE', 'dev-holos')
+os.putenv('AWS_SDK_LOAD_CONFIG', '1')
+# Authenticate to AWS ECR when tilt up is run by the developer
+local_resource('AWS Credentials', './hack/tilt/aws-login.sh', auto_init=True)
+
+# Extensions are open-source, pre-packaged functions that extend Tilt
+#
+#   More info: https://github.com/tilt-dev/tilt-extensions
+#   More info: https://docs.tilt.dev/extensions.html
+load('ext://restart_process', 'docker_build_with_restart')
+load('ext://k8s_attach', 'k8s_attach')
+load('ext://git_resource', 'git_checkout')
+load('ext://uibutton', 'cmd_button')
+
+# Paths edited by the developer Tilt watches to trigger compilation.
+# Generated files should be excluded to avoid an infinite build loop.
+developer_paths = [
+    './cmd',
+    './internal/server',
+    './internal/ent/schema',
+    './frontend/package-lock.json',
+    './frontend/src',
+    './go.mod',
+    './pkg',
+    './service/holos',
+]
+
+# Builds the holos-server executable
+local_resource(compile_id, 'make build', deps=developer_paths)
+
+# Build Docker image
+#   Tilt will automatically associate image builds with the resource(s)
+#   that reference them (e.g. via Kubernetes or Docker Compose YAML).
+#
+#   More info: https://docs.tilt.dev/api.html#api.docker_build
+#
+docker_build_with_restart(
+    'holos',
+    context='.',
+    entrypoint=[
+        '/app/bin/holos',
+        'server',
+        '--listen-port={}'.format(listen_port),
+        '--oidc-issuer=https://login.ois.run',
+        '--oidc-audience=262096764402729854@holos_platform',
+        '--log-level=debug',
+        '--metrics-port={}'.format(metrics_port),
+    ],
+    dockerfile='./hack/tilt/Dockerfile',
+    only=['./bin'],
+    # (Recommended) Updating a running container in-place
+    # https://docs.tilt.dev/live_update_reference.html
+    live_update=[
+        # Sync files from host to container
+        sync('./bin', '/app/bin'),
+        # Wait for aws-login https://github.com/tilt-dev/tilt/issues/3048
+        sync('./tilt/aws-login.last', '/dev/null'),
+        # Execute commands in the container when paths change
+        # run('/app/hack/codegen.sh', trigger=['./app/api'])
+    ],
+)
+
+
+# Run local commands
+#   Local commands can be helpful for one-time tasks like installing
+#   project prerequisites. They can also manage long-lived processes
+#   for non-containerized services or dependencies.
+#
+#   More info: https://docs.tilt.dev/local_resource.html
+#
+# local_resource('install-helm',
+#                cmd='which helm > /dev/null || brew install helm',
+#                # `cmd_bat`, when present, is used instead of `cmd` on Windows.
+#                cmd_bat=[
+#                    'powershell.exe',
+#                    '-Noninteractive',
+#                    '-Command',
+#                    '& {if (!(Get-Command helm -ErrorAction SilentlyContinue)) {scoop install helm}}'
+#                ]
+# )
+
+# Teach tilt about our custom resources (Note, this may be intended for workloads)
+# k8s_kind('authorizationpolicy')
+# k8s_kind('requestauthentication')
+# k8s_kind('virtualservice')
+k8s_kind('pgadmin')
+
+
+# Troubleshooting
+def resource_name(id):
+    print('resource: {}'.format(id))
+    return id.name
+
+
+workload_to_resource_function(resource_name)
+
+# Apply Kubernetes manifests
+#   Tilt will build & push any necessary images, re-deploying your
+#   resources as they change.
+#
+#   More info: https://docs.tilt.dev/api.html#api.k8s_yaml
+#
+
+def holos_yaml():
+    """Return a k8s Deployment personalized for the developer."""
+    k8s_yaml_template = str(read_file('./hack/tilt/k8s.yaml'))
+    return k8s_yaml_template.format(
+        name=holos_server,
+        developer=developer,
+        namespace=namespace,
+        listen_port=listen_port,
+        metrics_port=metrics_port,
+        tz=os.getenv('TZ'),
+    )
+
+# Customize a Kubernetes resource
+#   By default, Kubernetes resource names are automatically assigned
+#   based on objects in the YAML manifests, e.g. Deployment name.
+#
+#   Tilt strives for sane defaults, so calling k8s_resource is
+#   optional, and you only need to pass the arguments you want to
+#   override.
+#
+#   More info: https://docs.tilt.dev/api.html#api.k8s_resource
+#
+k8s_yaml(blob(holos_yaml()))
+
+# Backend server process
+k8s_resource(
+    workload=holos_server,
+    new_name=holos_backend,
+    objects=[
+        '{}:serviceaccount'.format(holos_server),
+        '{}:servicemonitor'.format(holos_server),
+    ],
+    resource_deps=[compile_id],
+    links=[
+        link('https://{}.app.dev.k2.holos.run/ui/'.format(developer), "Holos Web UI")
+    ],
+)
+
+
+# AuthorizationPolicy - Beyond Corp functionality
+k8s_resource(
+    new_name=auth_id,
+    objects=[
+      '{}:virtualservice'.format(holos_server),
+    ],
+)
+
+# Database
+# Note: Tilt confuses the backup pods with the database server pods, so this code is careful to tease the pods
+# apart so logs are streamed correctly.
+# See: https://github.com/tilt-dev/tilt.specs/blob/master/resource_assembly.md
+
+# pgAdmin Web UI
+k8s_resource(
+    workload=pg_admin_name,
+    new_name=pg_admin,
+    port_forwards=[
+        port_forward(15050, 5050, pg_admin),
+    ],
+)
+
+# Disabled because these don't group resources nicely
+# k8s_kind('postgrescluster')
+
+# Postgres database in-cluster
+k8s_resource(
+    new_name=pg_cluster,
+    objects=['holos:postgrescluster'],
+)
+
+# Needed to select the database by label
+# https://docs.tilt.dev/api.html#api.k8s_custom_deploy
+k8s_custom_deploy(
+    pg_svc,
+    apply_cmd=['./hack/tilt/k8s-get-db-sts', pg_cluster_name],
+    delete_cmd=['echo', 'Skipping delete.  Object managed by custom resource.'],
+    deps=[],
+)
+k8s_resource(
+    pg_svc,
+    port_forwards=[
+        port_forward(15432, 5432, 'psql'),
+    ],
+    resource_deps=[pg_cluster]
+)
+
+
+# Run tests
+local_resource(
+    tests_id,
+    'make test',
+    allow_parallel=True,
+    auto_init=False,
+    deps=developer_paths,
+)
+
+# Run linter
+local_resource(
+    lint_id,
+    'make lint',
+    allow_parallel=True,
+    auto_init=False,
+    deps=developer_paths,
+)
+
+# UI Buttons for helpful things.
+# Icons: https://fonts.google.com/icons
+os.putenv("GH_FORCE_TTY", "80%")
+cmd_button(
+    '{}:go-test-failfast'.format(tests_id),
+    argv=['./hack/tilt/go-test-failfast'],
+    resource=tests_id,
+    icon_name='quiz',
+    text='Fail Fast',
+)
+cmd_button(
+    '{}:issues'.format(holos_server),
+    argv=['./hack/tilt/gh-issues'],
+    resource=holos_backend,
+    icon_name='folder_data',
+    text='Issues',
+)
+cmd_button(
+    '{}:gh-issue-view'.format(holos_server),
+    argv=['./hack/tilt/gh-issue-view'],
+    resource=holos_backend,
+    icon_name='task',
+    text='View Issue',
+)
+cmd_button(
+    '{}:get-pgdb-creds'.format(holos_server),
+    argv=['./hack/tilt/get-pgdb-creds', pg_cluster_name, pg_database_name],
+    resource=pg_svc,
+    icon_name='lock_open_right',
+    text='DB Creds',
+)
+cmd_button(
+    '{}:get-pgdb-creds'.format(pg_admin_name),
+    argv=['./hack/tilt/get-pgdb-creds', pg_cluster_name, pg_database_name],
+    resource=pg_admin,
+    icon_name='lock_open_right',
+    text='DB Creds',
+)
+cmd_button(
+    '{}:get-pgadmin-creds'.format(pg_admin_name),
+    argv=['./hack/tilt/get-pgadmin-creds', pg_admin_name],
+    resource=pg_admin,
+    icon_name='lock_open_right',
+    text='pgAdmin Login',
+)
+
+print("✨ Tiltfile evaluated")

api/v1alpha1/buildplan.go

@@ -38,3 +38,14 @@ func (bp *BuildPlan) Validate() error {
 	}
 	return nil
 }
+
+func (bp *BuildPlan) ResultCapacity() (count int) {
+	if bp == nil {
+		return 0
+	}
+	count = len(bp.Spec.Components.HelmChartList) +
+		len(bp.Spec.Components.KubernetesObjectsList) +
+		len(bp.Spec.Components.KustomizeBuildList) +
+		len(bp.Spec.Components.Resources)
+	return count
+}

api/v1alpha1/form.go

@@ -0,0 +1,13 @@
+package v1alpha1
+
+import object "github.com/holos-run/holos/service/gen/holos/object/v1alpha1"
+
+// Form represents a collection of Formly json powered form.
+type Form struct {
+	TypeMeta `json:",inline" yaml:",inline"`
+	Spec     FormSpec `json:"spec" yaml:"spec"`
+}
+
+type FormSpec struct {
+	Form object.Form `json:"form" yaml:"form"`
+}

api/v1alpha1/helm.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 
 	"github.com/holos-run/holos"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 // A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
@@ -42,7 +42,7 @@ func (hc *HelmChart) Render(ctx context.Context, path holos.InstancePath) (*Resu
 	}
 	result.addObjectMap(ctx, hc.APIObjectMap)
 	if err := result.kustomize(ctx); err != nil {
-		return nil, wrapper.Wrap(fmt.Errorf("could not kustomize: %w", err))
+		return nil, errors.Wrap(fmt.Errorf("could not kustomize: %w", err))
 	}
 	return &result, nil
 }
@@ -64,13 +64,13 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.InstancePat
 			out, err := util.RunCmd(ctx, "helm", "repo", "add", repo.Name, repo.URL)
 			if err != nil {
 				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-				return wrapper.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
+				return errors.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
 			}
 			// Update repository
 			out, err = util.RunCmd(ctx, "helm", "repo", "update", repo.Name)
 			if err != nil {
 				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-				return wrapper.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
+				return errors.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
 			}
 		} else {
 			log.DebugContext(ctx, "no chart repository url proceeding assuming oci chart")
@@ -85,13 +85,13 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.InstancePat
 	// Write values file
 	tempDir, err := os.MkdirTemp("", "holos")
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+		return errors.Wrap(fmt.Errorf("could not make temp dir: %w", err))
 	}
 	defer util.Remove(ctx, tempDir)
 
 	valuesPath := filepath.Join(tempDir, "values.yaml")
 	if err := os.WriteFile(valuesPath, []byte(hc.ValuesContent), 0644); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not write values: %w", err))
+		return errors.Wrap(fmt.Errorf("could not write values: %w", err))
 	}
 	log.DebugContext(ctx, "helm: wrote values", "path", valuesPath, "bytes", len(hc.ValuesContent))
 
@@ -112,7 +112,7 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.InstancePat
 				err = fmt.Errorf("%s: %w", line, err)
 			}
 		}
-		return wrapper.Wrap(fmt.Errorf("could not run helm template: %w", err))
+		return errors.Wrap(fmt.Errorf("could not run helm template: %w", err))
 	}
 
 	r.accumulatedOutput = helmOut.Stdout.String()
@@ -126,7 +126,7 @@ func cacheChart(ctx context.Context, path holos.InstancePath, chartDir string, c
 
 	cacheTemp, err := os.MkdirTemp(string(path), chartDir)
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+		return errors.Wrap(fmt.Errorf("could not make temp dir: %w", err))
 	}
 	defer util.Remove(ctx, cacheTemp)
 
@@ -136,14 +136,30 @@ func cacheChart(ctx context.Context, path holos.InstancePath, chartDir string, c
 	}
 	helmOut, err := util.RunCmd(ctx, "helm", "pull", "--destination", cacheTemp, "--untar=true", "--version", chart.Version, chartName)
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not run helm pull: %w", err))
+		return errors.Wrap(fmt.Errorf("could not run helm pull: %w", err))
 	}
 	log.Debug("helm pull", "stdout", helmOut.Stdout, "stderr", helmOut.Stderr)
 
 	cachePath := filepath.Join(string(path), chartDir)
-	if err := os.Rename(cacheTemp, cachePath); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not rename: %w", err))
+
+	if err := os.MkdirAll(cachePath, 0777); err != nil {
+		return errors.Wrap(fmt.Errorf("could not mkdir: %w", err))
 	}
+
+	items, err := os.ReadDir(cacheTemp)
+	if err != nil {
+		return errors.Wrap(fmt.Errorf("could not read directory: %w", err))
+	}
+
+	for _, item := range items {
+		src := filepath.Join(cacheTemp, item.Name())
+		dst := filepath.Join(cachePath, item.Name())
+		log.DebugContext(ctx, "rename", "src", src, "dst", dst)
+		if err := os.Rename(src, dst); err != nil {
+			return errors.Wrap(fmt.Errorf("could not rename: %w", err))
+		}
+	}
+
 	log.InfoContext(ctx, "cached", "chart", chart.Name, "version", chart.Version, "path", cachePath)
 
 	return nil

api/v1alpha1/kustomize.go

@@ -4,9 +4,9 @@ import (
 	"context"
 
 	"github.com/holos-run/holos"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 const KustomizeBuildKind = "KustomizeBuild"
@@ -37,7 +37,7 @@ func (kb *KustomizeBuild) Render(ctx context.Context, path holos.InstancePath) (
 	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", string(path))
 	if err != nil {
 		log.ErrorContext(ctx, kOut.Stderr.String())
-		return nil, wrapper.Wrap(err)
+		return nil, errors.Wrap(err)
 	}
 	// Replace the accumulated output
 	result.accumulatedOutput = kOut.Stdout.String()

api/v1alpha1/platform.go

@@ -0,0 +1,22 @@
+package v1alpha1
+
+import "google.golang.org/protobuf/types/known/structpb"
+
+// Platform represents a platform to manage.  A Platform resource informs holos
+// which components to build.  The platform resource also acts as a container
+// for the platform model form values provided by the PlatformService.  The
+// primary use case is to collect the cluster names, cluster types, platform
+// model, and holos components to build into one resource.
+type Platform struct {
+	TypeMeta `json:",inline" yaml:",inline"`
+	Metadata ObjectMeta   `json:"metadata,omitempty" yaml:"metadata,omitempty"`
+	Spec     PlatformSpec `json:"spec,omitempty" yaml:"spec,omitempty"`
+}
+
+// PlatformSpec represents the platform build plan specification.
+type PlatformSpec struct {
+	// Model represents the platform model holos gets from from the
+	// holos.platform.v1alpha1.PlatformService.GetPlatform method and provides to
+	// CUE using a tag.
+	Model structpb.Struct `json:"model,omitempty" yaml:"model,omitempty"`
+}

api/v1alpha1/result.go

@@ -7,9 +7,9 @@ import (
 	"path/filepath"
 	"slices"
 
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 // Result is the build result for display or writing.  Holos components Render the Result as a data pipeline.
@@ -19,6 +19,14 @@ type Result struct {
 	accumulatedOutput string
 }
 
+// Continue returns true if Skip is true indicating the result is to be skipped over.
+func (r *Result) Continue() bool {
+	if r == nil {
+		return false
+	}
+	return r.Skip
+}
+
 func (r *Result) Name() string {
 	return r.Metadata.Name
 }
@@ -32,6 +40,11 @@ func (r *Result) KustomizationFilename(writeTo string, cluster string) string {
 	return filepath.Join(writeTo, "clusters", cluster, "holos", "components", r.Metadata.Name+"-kustomization.gen.yaml")
 }
 
+// KustomizationContent returns the kustomization file contents to write.
+func (r *Result) KustomizationContent() string {
+	return r.KsContent
+}
+
 // AccumulatedOutput returns the accumulated rendered output.
 func (r *Result) AccumulatedOutput() string {
 	return r.accumulatedOutput
@@ -82,7 +95,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	}
 	tempDir, err := os.MkdirTemp("", "holos.kustomize")
 	if err != nil {
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	defer util.Remove(ctx, tempDir)
 
@@ -91,7 +104,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	b := []byte(r.AccumulatedOutput())
 	b = util.EnsureNewline(b)
 	if err := os.WriteFile(target, b, 0644); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not write resources: %w", err))
+		return errors.Wrap(fmt.Errorf("could not write resources: %w", err))
 	}
 	log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
 
@@ -99,12 +112,12 @@ func (r *Result) kustomize(ctx context.Context) error {
 	for file, content := range r.KustomizeFiles {
 		target := filepath.Join(tempDir, file)
 		if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
-			return wrapper.Wrap(err)
+			return errors.Wrap(err)
 		}
 		b := []byte(content)
 		b = util.EnsureNewline(b)
 		if err := os.WriteFile(target, b, 0644); err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not write: %w", err))
+			return errors.Wrap(fmt.Errorf("could not write: %w", err))
 		}
 		log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
 	}
@@ -113,7 +126,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", tempDir)
 	if err != nil {
 		log.ErrorContext(ctx, kOut.Stderr.String())
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	// Replace the accumulated output
 	r.accumulatedOutput = kOut.Stdout.String()
@@ -126,12 +139,12 @@ func (r *Result) Save(ctx context.Context, path string, content string) error {
 	dir := filepath.Dir(path)
 	if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
 		log.WarnContext(ctx, "could not mkdir", "path", dir, "err", err)
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	// Write the kube api objects
 	if err := os.WriteFile(path, []byte(content), os.FileMode(0644)); err != nil {
 		log.WarnContext(ctx, "could not write", "path", path, "err", err)
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	log.DebugContext(ctx, "out: wrote "+path, "action", "write", "path", path, "status", "ok")
 	return nil

api/v1alpha1/typemeta.go

@@ -8,3 +8,13 @@ type TypeMeta struct {
 func (tm *TypeMeta) GetKind() string {
 	return tm.Kind
 }
+
+func (tm *TypeMeta) GetAPIVersion() string {
+	return tm.Kind
+}
+
+// Discriminator is an interface to discriminate the kind api object.
+type Discriminator interface {
+	GetKind() string
+	GetAPIVersion() string
+}

buf.gen.yaml

@@ -0,0 +1,20 @@
+# Generates gRPC and ConnectRPC bindings for Go and TypeScript
+#
+# Note: protoc-gen-connect-query is the primary method of wiring up the React
+# frontend.
+version: v1
+plugins:
+  - plugin: go
+    out: service/gen
+    opt: paths=source_relative
+  - plugin: connect-go
+    out: service/gen
+    opt: paths=source_relative
+  - plugin: es
+    out: internal/frontend/holos/src/app/gen
+    opt:
+      - target=ts
+  - plugin: connect-es
+    out: internal/frontend/holos/src/app/gen
+    opt:
+      - target=ts

buf.lock

@@ -0,0 +1,8 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: bufbuild
+    repository: protovalidate
+    commit: b983156c5e994cc9892e0ce3e64e17e0
+    digest: shake256:fb47a62989d38c2529bcc5cd86ded43d800eb84cee82b42b9e8a9e815d4ee8134a0fb9d0ce8299b27c2d2bbb7d6ade0c4ad5a8a4d467e1e2c7ca619ae9f634e2

buf.work.yaml

@@ -0,0 +1,3 @@
+version: v1
+directories:
+  - service

cmd/holos/main.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"github.com/holos-run/holos/pkg/cli"
 	"os"
+
+	"github.com/holos-run/holos/internal/cli"
 )
 
 func main() {

cmd/holos/main_test.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"github.com/holos-run/holos/pkg/cli"
-	"github.com/rogpeppe/go-internal/testscript"
 	"os"
 	"testing"
+
+	"github.com/holos-run/holos/internal/cli"
+	"github.com/rogpeppe/go-internal/testscript"
 )
 
 func TestMain(m *testing.M) {

cmd/holos/testdata/version.txt

@@ -1,5 +1,3 @@
 exec holos --version
 # want version with no v on stdout
 stdout -count=1 '^\d+\.\d+\.\d+$'
-# want nothing on stderr
-! stderr .

docs/examples/api/ListPlatform/listplatform.json

@@ -0,0 +1,10 @@
+{
+  "org_id": "018f36fb-e3f7-7f7f-a1c5-c85fb735d215",
+  "field_mask": {
+    "paths": [
+      "id",
+      "name",
+      "displayName"
+    ]
+  }
+}

docs/examples/api/UpdatePlatform/clearform.json

@@ -0,0 +1,8 @@
+{
+  "update_mask": {
+    "paths": ["form"]
+  },
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94"
+  }
+}

docs/examples/api/UpdatePlatform/model.json

@@ -0,0 +1,11 @@
+{
+  "update_mask": {
+    "paths": ["model","name","display_name"]
+  },
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94",
+    "name": "bareplatform",
+    "display_name": "Bare Platform",
+    "model": {}
+  }
+}

docs/examples/api/UpdatePlatform/nomask.json

@@ -0,0 +1,6 @@
+{
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94",
+    "model": {}
+  }
+}

docs/examples/authpolicy.cue

@@ -0,0 +1,45 @@
+package holos
+
+import ap "security.istio.io/authorizationpolicy/v1"
+
+// #AuthPolicyRules represents AuthorizationPolicy rules for hosts that need
+// specialized treatment.  Entries in this struct are excluded from
+// AuthorizationPolicy/authproxy-custom in the istio-ingress namespace.  Entries
+// are added to their own AuthorizationPolicy.
+#AuthPolicyRules: {
+	// AuthProxySpec represents the identity provider configuration
+	AuthProxySpec: #AuthProxySpec & #Platform.authproxy
+
+	// Hosts are hosts that need specialized treatment
+	hosts: {
+		[Name=_]: {
+			// name is the fully qualifed hostname, a Host: header value.
+			name: Name
+			// slug is the resource name prefix
+			slug: string
+			// NoAuthorizationPolicy disables an AuthorizationPolicy for the host
+			NoAuthorizationPolicy: true | *false
+
+			// Refer to https://istio.io/latest/docs/reference/config/security/authorization-policy/#Rule
+			spec: ap.#AuthorizationPolicySpec & {
+				action: "CUSTOM"
+				provider: name: AuthProxySpec.provider
+				selector: matchLabels: istio: "ingressgateway"
+			}
+		}
+	}
+
+	objects: #APIObjects & {
+		for Host in hosts {
+			if Host.NoAuthorizationPolicy == false {
+				apiObjects: {
+					AuthorizationPolicy: "\(Host.slug)-custom": {
+						metadata: namespace: "istio-ingress"
+						metadata: name:      "\(Host.slug)-custom"
+						spec: Host.spec
+					}
+				}
+			}
+		}
+	}
+}

docs/examples/cue.mod/gen/argoproj.io/appproject/v1alpha1/types_gen.cue

@@ -0,0 +1,189 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-argocd/prod-platform-argocd.gen.yaml
+
+package v1alpha1
+
+import "strings"
+
+// AppProject provides a logical grouping of applications,
+// providing controls for: * where the apps may deploy to
+// (cluster whitelist) * what may be deployed (repository
+// whitelist, resource whitelist/blacklist) * who can access
+// these applications (roles, OIDC group claims bindings) * and
+// what they can do (RBAC policies) * automation access to these
+// roles (JWT tokens)
+#AppProject: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "argoproj.io/v1alpha1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "AppProject"
+	metadata: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// AppProjectSpec is the specification of an AppProject
+	spec!: #AppProjectSpec
+}
+
+// AppProjectSpec is the specification of an AppProject
+#AppProjectSpec: {
+	// ClusterResourceBlacklist contains list of blacklisted cluster
+	// level resources
+	clusterResourceBlacklist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// ClusterResourceWhitelist contains list of whitelisted cluster
+	// level resources
+	clusterResourceWhitelist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// Description contains optional project description
+	description?: string
+
+	// Destinations contains list of destinations available for
+	// deployment
+	destinations?: [...{
+		// Name is an alternate way of specifying the target cluster by
+		// its symbolic name. This must be set if Server is not set.
+		name?: string
+
+		// Namespace specifies the target namespace for the application's
+		// resources. The namespace will only be set for namespace-scoped
+		// resources that have not set a value for .metadata.namespace
+		namespace?: string
+
+		// Server specifies the URL of the target cluster's Kubernetes
+		// control plane API. This must be set if Name is not set.
+		server?: string
+	}]
+
+	// NamespaceResourceBlacklist contains list of blacklisted
+	// namespace level resources
+	namespaceResourceBlacklist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// NamespaceResourceWhitelist contains list of whitelisted
+	// namespace level resources
+	namespaceResourceWhitelist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// OrphanedResources specifies if controller should monitor
+	// orphaned resources of apps in this project
+	orphanedResources?: {
+		// Ignore contains a list of resources that are to be excluded
+		// from orphaned resources monitoring
+		ignore?: [...{
+			group?: string
+			kind?:  string
+			name?:  string
+		}]
+
+		// Warn indicates if warning condition should be created for apps
+		// which have orphaned resources
+		warn?: bool
+	}
+
+	// PermitOnlyProjectScopedClusters determines whether destinations
+	// can only reference clusters which are project-scoped
+	permitOnlyProjectScopedClusters?: bool
+
+	// Roles are user defined RBAC roles associated with this project
+	roles?: [...{
+		// Description is a description of the role
+		description?: string
+
+		// Groups are a list of OIDC group claims bound to this role
+		groups?: [...string]
+
+		// JWTTokens are a list of generated JWT tokens bound to this role
+		jwtTokens?: [...{
+			exp?: int
+			iat:  int
+			id?:  string
+		}]
+
+		// Name is a name for this role
+		name: string
+
+		// Policies Stores a list of casbin formatted strings that define
+		// access policies for the role in the project
+		policies?: [...string]
+	}]
+
+	// SignatureKeys contains a list of PGP key IDs that commits in
+	// Git must be signed with in order to be allowed for sync
+	signatureKeys?: [...{
+		// The ID of the key in hexadecimal notation
+		keyID: string
+	}]
+
+	// SourceNamespaces defines the namespaces application resources
+	// are allowed to be created in
+	sourceNamespaces?: [...string]
+
+	// SourceRepos contains list of repository URLs which can be used
+	// for deployment
+	sourceRepos?: [...string]
+
+	// SyncWindows controls when syncs can be run for apps in this
+	// project
+	syncWindows?: [...{
+		// Applications contains a list of applications that the window
+		// will apply to
+		applications?: [...string]
+
+		// Clusters contains a list of clusters that the window will apply
+		// to
+		clusters?: [...string]
+
+		// Duration is the amount of time the sync window will be open
+		duration?: string
+
+		// Kind defines if the window allows or blocks syncs
+		kind?: string
+
+		// ManualSync enables manual syncs when they would otherwise be
+		// blocked
+		manualSync?: bool
+
+		// Namespaces contains a list of namespaces that the window will
+		// apply to
+		namespaces?: [...string]
+
+		// Schedule is the time the window will begin, specified in cron
+		// format
+		schedule?: string
+
+		// TimeZone of the sync that will be applied to the schedule
+		timeZone?: string
+	}]
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/podmonitor/v1/types_gen.cue

@@ -0,0 +1,546 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// PodMonitor defines monitoring for a set of pods.
+#PodMonitor: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "PodMonitor"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Pod selection for target discovery by
+	// Prometheus.
+	spec!: #PodMonitorSpec
+}
+
+// Specification of desired Pod selection for target discovery by
+// Prometheus.
+#PodMonitorSpec: {
+	attachMetadata?: {
+		// When set to true, Prometheus must have the `get` permission on
+		// the `Nodes` objects.
+		node?: bool
+	}
+
+	// The label to use to retrieve the job name from. `jobLabel`
+	// selects the label from the associated Kubernetes `Pod` object
+	// which will be used as the `job` label for all metrics.
+	// For example if `jobLabel` is set to `foo` and the Kubernetes
+	// `Pod` object is labeled with `foo: bar`, then Prometheus adds
+	// the `job="bar"` label to all ingested metrics.
+	// If the value of this field is empty, the `job` label of the
+	// metrics defaults to the namespace and name of the PodMonitor
+	// object (e.g. `<namespace>/<name>`).
+	jobLabel?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelValueLengthLimit?: int
+
+	// Selector to select which namespaces the Kubernetes `Pods`
+	// objects are discovered from.
+	namespaceSelector?: {
+		// Boolean describing whether all namespaces are selected in
+		// contrast to a list restricting them.
+		any?: bool
+
+		// List of namespace names to select from.
+		matchNames?: [...string]
+	}
+
+	// List of endpoints part of this PodMonitor.
+	podMetricsEndpoints?: [...{
+		// `authorization` configures the Authorization header credentials
+		// to use when scraping the target.
+		// Cannot be set at the same time as `basicAuth`, or `oauth2`.
+		authorization?: {
+			// Selects a key of a Secret in the namespace that contains the
+			// credentials for authentication.
+			credentials?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Defines the authentication type. The value is case-insensitive.
+			// "Basic" is not a supported value.
+			// Default: "Bearer"
+			type?: string
+		}
+
+		// `basicAuth` configures the Basic Authentication credentials to
+		// use when scraping the target.
+		// Cannot be set at the same time as `authorization`, or `oauth2`.
+		basicAuth?: {
+			// `password` specifies a key of a Secret containing the password
+			// for authentication.
+			password?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `username` specifies a key of a Secret containing the username
+			// for authentication.
+			username?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// `bearerTokenSecret` specifies a key of a Secret containing the
+		// bearer token for scraping targets. The secret needs to be in
+		// the same namespace as the PodMonitor object and readable by
+		// the Prometheus Operator.
+		// Deprecated: use `authorization` instead.
+		bearerTokenSecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `enableHttp2` can be used to disable HTTP2 when scraping the
+		// target.
+		enableHttp2?: bool
+
+		// When true, the pods which are not running (e.g. either in
+		// Failed or Succeeded state) are dropped during the target
+		// discovery.
+		// If unset, the filtering is enabled.
+		// More info:
+		// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+		filterRunning?: bool
+
+		// `followRedirects` defines whether the scrape requests should
+		// follow HTTP 3xx redirects.
+		followRedirects?: bool
+
+		// When true, `honorLabels` preserves the metric's labels when
+		// they collide with the target's labels.
+		honorLabels?: bool
+
+		// `honorTimestamps` controls whether Prometheus preserves the
+		// timestamps when exposed by the target.
+		honorTimestamps?: bool
+
+		// Interval at which Prometheus scrapes the metrics from the
+		// target.
+		// If empty, Prometheus uses the global scrape interval.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// `metricRelabelings` configures the relabeling rules to apply to
+		// the samples before ingestion.
+		metricRelabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// `oauth2` configures the OAuth2 settings to use when scraping
+		// the target.
+		// It requires Prometheus >= 2.27.0.
+		// Cannot be set at the same time as `authorization`, or
+		// `basicAuth`.
+		oauth2?: {
+			// `clientId` specifies a key of a Secret or ConfigMap containing
+			// the OAuth2 client's ID.
+			clientId: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// `clientSecret` specifies a key of a Secret containing the
+			// OAuth2 client's secret.
+			clientSecret: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `endpointParams` configures the HTTP parameters to append to
+			// the token URL.
+			endpointParams?: {
+				[string]: string
+			}
+
+			// `scopes` defines the OAuth2 scopes used for the token request.
+			scopes?: [...string]
+
+			// `tokenURL` configures the URL to fetch the token from.
+			tokenUrl: strings.MinRunes(1)
+		}
+
+		// `params` define optional HTTP URL parameters.
+		params?: {
+			[string]: [...string]
+		}
+
+		// HTTP path from which to scrape for metrics.
+		// If empty, Prometheus uses the default value (e.g. `/metrics`).
+		path?: string
+
+		// Name of the Pod port which this endpoint refers to.
+		// It takes precedence over `targetPort`.
+		port?: string
+
+		// `proxyURL` configures the HTTP Proxy URL (e.g.
+		// "http://proxyserver:2195") to go through when scraping the
+		// target.
+		proxyUrl?: string
+
+		// `relabelings` configures the relabeling rules to apply the
+		// target's metadata labels.
+		// The Operator automatically adds relabelings for a few standard
+		// Kubernetes fields.
+		// The original scrape job's name is available via the
+		// `__tmp_prometheus_job_name` label.
+		// More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+		relabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// HTTP scheme to use for scraping.
+		// `http` and `https` are the expected values unless you rewrite
+		// the `__scheme__` label via relabeling.
+		// If empty, Prometheus uses the default value `http`.
+		scheme?: "http" | "https"
+
+		// Timeout after which Prometheus considers the scrape to be
+		// failed.
+		// If empty, Prometheus uses the global scrape timeout unless it
+		// is less than the target's scrape interval value in which the
+		// latter is used.
+		scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Name or number of the target port of the `Pod` object behind
+		// the Service, the port must be specified with container port
+		// property.
+		// Deprecated: use 'port' instead.
+		targetPort?: (int | string) & {
+			string
+		}
+
+		// TLS configuration to use when scraping the target.
+		tlsConfig?: {
+			// Certificate authority used when verifying server certificates.
+			ca?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Client certificate to present when doing client-authentication.
+			cert?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Disable target certificate validation.
+			insecureSkipVerify?: bool
+
+			// Secret containing the client key file for the targets.
+			keySecret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Used to verify the hostname for the targets.
+			serverName?: string
+		}
+
+		// `trackTimestampsStaleness` defines whether Prometheus tracks
+		// staleness of the metrics that have an explicit timestamp
+		// present in scraped data. Has no effect if `honorTimestamps` is
+		// false.
+		// It requires Prometheus >= v2.48.0.
+		trackTimestampsStaleness?: bool
+	}]
+
+	// `podTargetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Pod` object onto the ingested
+	// metrics.
+	podTargetLabels?: [...string]
+
+	// `sampleLimit` defines a per-scrape limit on the number of
+	// scraped samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Label selector to select the Kubernetes `Pod` objects.
+	selector: {
+		// matchExpressions is a list of label selector requirements. The
+		// requirements are ANDed.
+		matchExpressions?: [...{
+			// key is the label key that the selector applies to.
+			key: string
+
+			// operator represents a key's relationship to a set of values.
+			// Valid operators are In, NotIn, Exists and DoesNotExist.
+			operator: string
+
+			// values is an array of string values. If the operator is In or
+			// NotIn, the values array must be non-empty. If the operator is
+			// Exists or DoesNotExist, the values array must be empty. This
+			// array is replaced during a strategic merge patch.
+			values?: [...string]
+		}]
+
+		// matchLabels is a map of {key,value} pairs. A single {key,value}
+		// in the matchLabels map is equivalent to an element of
+		// matchExpressions, whose key field is "key", the operator is
+		// "In", and the values array contains only "value". The
+		// requirements are ANDed.
+		matchLabels?: {
+			[string]: string
+		}
+	}
+
+	// `targetLimit` defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/probe/v1/types_gen.cue

@@ -0,0 +1,536 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// Probe defines monitoring for a set of static targets or
+// ingresses.
+#Probe: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "Probe"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Ingress selection for target discovery
+	// by Prometheus.
+	spec!: #ProbeSpec
+}
+
+// Specification of desired Ingress selection for target discovery
+// by Prometheus.
+#ProbeSpec: {
+	// Authorization section for this endpoint
+	authorization?: {
+		// Selects a key of a Secret in the namespace that contains the
+		// credentials for authentication.
+		credentials?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// Defines the authentication type. The value is case-insensitive.
+		// "Basic" is not a supported value.
+		// Default: "Bearer"
+		type?: string
+	}
+
+	// BasicAuth allow an endpoint to authenticate over basic
+	// authentication. More info:
+	// https://prometheus.io/docs/operating/configuration/#endpoint
+	basicAuth?: {
+		// `password` specifies a key of a Secret containing the password
+		// for authentication.
+		password?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `username` specifies a key of a Secret containing the username
+		// for authentication.
+		username?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+	}
+
+	// Secret to mount to read bearer token for scraping targets. The
+	// secret needs to be in the same namespace as the probe and
+	// accessible by the Prometheus Operator.
+	bearerTokenSecret?: {
+		// The key of the secret to select from. Must be a valid secret
+		// key.
+		key: string
+
+		// Name of the referent. More info:
+		// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+		// TODO: Add other useful fields. apiVersion, kind, uid?
+		name?: string
+
+		// Specify whether the Secret or its key must be defined
+		optional?: bool
+	}
+
+	// Interval at which targets are probed using the configured
+	// prober. If not specified Prometheus' global scrape interval is
+	// used.
+	interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+	// The job name assigned to scraped metrics by default.
+	jobName?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample. Only valid in Prometheus versions 2.27.0 and newer.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample. Only valid in Prometheus versions 2.27.0 and
+	// newer.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample. Only valid in Prometheus versions
+	// 2.27.0 and newer.
+	labelValueLengthLimit?: int
+
+	// MetricRelabelConfigs to apply to samples before ingestion.
+	metricRelabelings?: [...{
+		// Action to perform based on the regex matching.
+		// `Uppercase` and `Lowercase` actions require Prometheus >=
+		// v2.36.0. `DropEqual` and `KeepEqual` actions require
+		// Prometheus >= v2.41.0.
+		// Default: "Replace"
+		action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+		// Modulus to take of the hash of the source label values.
+		// Only applicable when the action is `HashMod`.
+		modulus?: int
+
+		// Regular expression against which the extracted value is
+		// matched.
+		regex?: string
+
+		// Replacement value against which a Replace action is performed
+		// if the regular expression matches.
+		// Regex capture groups are available.
+		replacement?: string
+
+		// Separator is the string between concatenated SourceLabels.
+		separator?: string
+
+		// The source labels select values from existing labels. Their
+		// content is concatenated using the configured Separator and
+		// matched against the configured regular expression.
+		sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+		// Label to which the resulting string is written in a
+		// replacement.
+		// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+		// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+		// Regex capture groups are available.
+		targetLabel?: string
+	}]
+
+	// The module to use for probing specifying how to probe the
+	// target. Example module configuring in the blackbox exporter:
+	// https://github.com/prometheus/blackbox_exporter/blob/master/example.yml
+	module?: string
+
+	// OAuth2 for the URL. Only valid in Prometheus versions 2.27.0
+	// and newer.
+	oauth2?: {
+		// `clientId` specifies a key of a Secret or ConfigMap containing
+		// the OAuth2 client's ID.
+		clientId: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// `clientSecret` specifies a key of a Secret containing the
+		// OAuth2 client's secret.
+		clientSecret: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `endpointParams` configures the HTTP parameters to append to
+		// the token URL.
+		endpointParams?: {
+			[string]: string
+		}
+
+		// `scopes` defines the OAuth2 scopes used for the token request.
+		scopes?: [...string]
+
+		// `tokenURL` configures the URL to fetch the token from.
+		tokenUrl: strings.MinRunes(1)
+	}
+
+	// Specification for the prober to use for probing targets. The
+	// prober.URL parameter is required. Targets cannot be probed if
+	// left empty.
+	prober?: {
+		// Path to collect metrics from. Defaults to `/probe`.
+		path?: string | *"/probe"
+
+		// Optional ProxyURL.
+		proxyUrl?: string
+
+		// HTTP scheme to use for scraping. `http` and `https` are the
+		// expected values unless you rewrite the `__scheme__` label via
+		// relabeling. If empty, Prometheus uses the default value
+		// `http`.
+		scheme?: "http" | "https"
+
+		// Mandatory URL of the prober.
+		url: string
+	}
+
+	// SampleLimit defines per-scrape limit on number of scraped
+	// samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Timeout for scraping metrics from the Prometheus exporter. If
+	// not specified, the Prometheus global scrape timeout is used.
+	scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+	// TargetLimit defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+
+	// Targets defines a set of static or dynamically discovered
+	// targets to probe.
+	targets?: {
+		// ingress defines the Ingress objects to probe and the relabeling
+		// configuration. If `staticConfig` is also defined,
+		// `staticConfig` takes precedence.
+		ingress?: {
+			// From which namespaces to select Ingress objects.
+			namespaceSelector?: {
+				// Boolean describing whether all namespaces are selected in
+				// contrast to a list restricting them.
+				any?: bool
+
+				// List of namespace names to select from.
+				matchNames?: [...string]
+			}
+
+			// RelabelConfigs to apply to the label set of the target before
+			// it gets scraped. The original ingress address is available via
+			// the `__tmp_prometheus_ingress_address` label. It can be used
+			// to customize the probed URL. The original scrape job's name is
+			// available via the `__tmp_prometheus_job_name` label. More
+			// info:
+			// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+			relabelingConfigs?: [...{
+				// Action to perform based on the regex matching.
+				// `Uppercase` and `Lowercase` actions require Prometheus >=
+				// v2.36.0. `DropEqual` and `KeepEqual` actions require
+				// Prometheus >= v2.41.0.
+				// Default: "Replace"
+				action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+				// Modulus to take of the hash of the source label values.
+				// Only applicable when the action is `HashMod`.
+				modulus?: int
+
+				// Regular expression against which the extracted value is
+				// matched.
+				regex?: string
+
+				// Replacement value against which a Replace action is performed
+				// if the regular expression matches.
+				// Regex capture groups are available.
+				replacement?: string
+
+				// Separator is the string between concatenated SourceLabels.
+				separator?: string
+
+				// The source labels select values from existing labels. Their
+				// content is concatenated using the configured Separator and
+				// matched against the configured regular expression.
+				sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+				// Label to which the resulting string is written in a
+				// replacement.
+				// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+				// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+				// Regex capture groups are available.
+				targetLabel?: string
+			}]
+
+			// Selector to select the Ingress objects.
+			selector?: {
+				// matchExpressions is a list of label selector requirements. The
+				// requirements are ANDed.
+				matchExpressions?: [...{
+					// key is the label key that the selector applies to.
+					key: string
+
+					// operator represents a key's relationship to a set of values.
+					// Valid operators are In, NotIn, Exists and DoesNotExist.
+					operator: string
+
+					// values is an array of string values. If the operator is In or
+					// NotIn, the values array must be non-empty. If the operator is
+					// Exists or DoesNotExist, the values array must be empty. This
+					// array is replaced during a strategic merge patch.
+					values?: [...string]
+				}]
+
+				// matchLabels is a map of {key,value} pairs. A single {key,value}
+				// in the matchLabels map is equivalent to an element of
+				// matchExpressions, whose key field is "key", the operator is
+				// "In", and the values array contains only "value". The
+				// requirements are ANDed.
+				matchLabels?: {
+					[string]: string
+				}
+			}
+		}
+
+		// staticConfig defines the static list of targets to probe and
+		// the relabeling configuration. If `ingress` is also defined,
+		// `staticConfig` takes precedence. More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config.
+		staticConfig?: {
+			// Labels assigned to all metrics scraped from the targets.
+			labels?: {
+				[string]: string
+			}
+
+			// RelabelConfigs to apply to the label set of the targets before
+			// it gets scraped. More info:
+			// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+			relabelingConfigs?: [...{
+				// Action to perform based on the regex matching.
+				// `Uppercase` and `Lowercase` actions require Prometheus >=
+				// v2.36.0. `DropEqual` and `KeepEqual` actions require
+				// Prometheus >= v2.41.0.
+				// Default: "Replace"
+				action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+				// Modulus to take of the hash of the source label values.
+				// Only applicable when the action is `HashMod`.
+				modulus?: int
+
+				// Regular expression against which the extracted value is
+				// matched.
+				regex?: string
+
+				// Replacement value against which a Replace action is performed
+				// if the regular expression matches.
+				// Regex capture groups are available.
+				replacement?: string
+
+				// Separator is the string between concatenated SourceLabels.
+				separator?: string
+
+				// The source labels select values from existing labels. Their
+				// content is concatenated using the configured Separator and
+				// matched against the configured regular expression.
+				sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+				// Label to which the resulting string is written in a
+				// replacement.
+				// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+				// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+				// Regex capture groups are available.
+				targetLabel?: string
+			}]
+
+			// The list of hosts to probe.
+			static?: [...string]
+		}
+	}
+
+	// TLS configuration to use when scraping the endpoint.
+	tlsConfig?: {
+		// Certificate authority used when verifying server certificates.
+		ca?: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// Client certificate to present when doing client-authentication.
+		cert?: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// Disable target certificate validation.
+		insecureSkipVerify?: bool
+
+		// Secret containing the client key file for the targets.
+		keySecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// Used to verify the hostname for the targets.
+		serverName?: string
+	}
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/prometheusrule/v1/types_gen.cue

@@ -0,0 +1,100 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// PrometheusRule defines recording and alerting rules for a
+// Prometheus instance
+#PrometheusRule: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "PrometheusRule"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired alerting rule definitions for
+	// Prometheus.
+	spec!: #PrometheusRuleSpec
+}
+#PrometheusRuleSpec: {
+	// Content of Prometheus rule file
+	groups?: [...{
+		// Interval determines how often rules in the group are evaluated.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Limit the number of alerts an alerting rule and series a
+		// recording rule can produce. Limit is supported starting with
+		// Prometheus >= 2.31 and Thanos Ruler >= 0.24.
+		limit?: int
+
+		// Name of the rule group.
+		name: strings.MinRunes(1)
+
+		// PartialResponseStrategy is only used by ThanosRuler and will be
+		// ignored by Prometheus instances. More info:
+		// https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response
+		partial_response_strategy?: =~"^(?i)(abort|warn)?$"
+
+		// List of alerting and recording rules.
+		rules?: [...{
+			// Name of the alert. Must be a valid label value. Only one of
+			// `record` and `alert` must be set.
+			alert?: string
+
+			// Annotations to add to each alert. Only valid for alerting
+			// rules.
+			annotations?: {
+				[string]: string
+			}
+
+			// PromQL expression to evaluate.
+			expr: (int | string) & {
+				string
+			}
+
+			// Alerts are considered firing once they have been returned for
+			// this long.
+			for?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+			// KeepFiringFor defines how long an alert will continue firing
+			// after the condition that triggered it has cleared.
+			keep_firing_for?: strings.MinRunes(1) & {
+				=~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+			}
+
+			// Labels to add or overwrite.
+			labels?: {
+				[string]: string
+			}
+
+			// Name of the time series to output to. Must be a valid metric
+			// name. Only one of `record` and `alert` must be set.
+			record?: string
+		}]
+	}]
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/servicemonitor/v1/types_gen.cue

@@ -0,0 +1,566 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// ServiceMonitor defines monitoring for a set of services.
+#ServiceMonitor: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "ServiceMonitor"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Service selection for target discovery
+	// by Prometheus.
+	spec!: #ServiceMonitorSpec
+}
+
+// Specification of desired Service selection for target discovery
+// by Prometheus.
+#ServiceMonitorSpec: {
+	attachMetadata?: {
+		// When set to true, Prometheus must have the `get` permission on
+		// the `Nodes` objects.
+		node?: bool
+	}
+
+	// List of endpoints part of this ServiceMonitor.
+	endpoints?: [...{
+		// `authorization` configures the Authorization header credentials
+		// to use when scraping the target.
+		// Cannot be set at the same time as `basicAuth`, or `oauth2`.
+		authorization?: {
+			// Selects a key of a Secret in the namespace that contains the
+			// credentials for authentication.
+			credentials?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Defines the authentication type. The value is case-insensitive.
+			// "Basic" is not a supported value.
+			// Default: "Bearer"
+			type?: string
+		}
+
+		// `basicAuth` configures the Basic Authentication credentials to
+		// use when scraping the target.
+		// Cannot be set at the same time as `authorization`, or `oauth2`.
+		basicAuth?: {
+			// `password` specifies a key of a Secret containing the password
+			// for authentication.
+			password?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `username` specifies a key of a Secret containing the username
+			// for authentication.
+			username?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// File to read bearer token for scraping the target.
+		// Deprecated: use `authorization` instead.
+		bearerTokenFile?: string
+
+		// `bearerTokenSecret` specifies a key of a Secret containing the
+		// bearer token for scraping targets. The secret needs to be in
+		// the same namespace as the ServiceMonitor object and readable
+		// by the Prometheus Operator.
+		// Deprecated: use `authorization` instead.
+		bearerTokenSecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `enableHttp2` can be used to disable HTTP2 when scraping the
+		// target.
+		enableHttp2?: bool
+
+		// When true, the pods which are not running (e.g. either in
+		// Failed or Succeeded state) are dropped during the target
+		// discovery.
+		// If unset, the filtering is enabled.
+		// More info:
+		// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+		filterRunning?: bool
+
+		// `followRedirects` defines whether the scrape requests should
+		// follow HTTP 3xx redirects.
+		followRedirects?: bool
+
+		// When true, `honorLabels` preserves the metric's labels when
+		// they collide with the target's labels.
+		honorLabels?: bool
+
+		// `honorTimestamps` controls whether Prometheus preserves the
+		// timestamps when exposed by the target.
+		honorTimestamps?: bool
+
+		// Interval at which Prometheus scrapes the metrics from the
+		// target.
+		// If empty, Prometheus uses the global scrape interval.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// `metricRelabelings` configures the relabeling rules to apply to
+		// the samples before ingestion.
+		metricRelabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// `oauth2` configures the OAuth2 settings to use when scraping
+		// the target.
+		// It requires Prometheus >= 2.27.0.
+		// Cannot be set at the same time as `authorization`, or
+		// `basicAuth`.
+		oauth2?: {
+			// `clientId` specifies a key of a Secret or ConfigMap containing
+			// the OAuth2 client's ID.
+			clientId: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// `clientSecret` specifies a key of a Secret containing the
+			// OAuth2 client's secret.
+			clientSecret: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `endpointParams` configures the HTTP parameters to append to
+			// the token URL.
+			endpointParams?: {
+				[string]: string
+			}
+
+			// `scopes` defines the OAuth2 scopes used for the token request.
+			scopes?: [...string]
+
+			// `tokenURL` configures the URL to fetch the token from.
+			tokenUrl: strings.MinRunes(1)
+		}
+
+		// params define optional HTTP URL parameters.
+		params?: {
+			[string]: [...string]
+		}
+
+		// HTTP path from which to scrape for metrics.
+		// If empty, Prometheus uses the default value (e.g. `/metrics`).
+		path?: string
+
+		// Name of the Service port which this endpoint refers to.
+		// It takes precedence over `targetPort`.
+		port?: string
+
+		// `proxyURL` configures the HTTP Proxy URL (e.g.
+		// "http://proxyserver:2195") to go through when scraping the
+		// target.
+		proxyUrl?: string
+
+		// `relabelings` configures the relabeling rules to apply the
+		// target's metadata labels.
+		// The Operator automatically adds relabelings for a few standard
+		// Kubernetes fields.
+		// The original scrape job's name is available via the
+		// `__tmp_prometheus_job_name` label.
+		// More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+		relabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// HTTP scheme to use for scraping.
+		// `http` and `https` are the expected values unless you rewrite
+		// the `__scheme__` label via relabeling.
+		// If empty, Prometheus uses the default value `http`.
+		scheme?: "http" | "https"
+
+		// Timeout after which Prometheus considers the scrape to be
+		// failed.
+		// If empty, Prometheus uses the global scrape timeout unless it
+		// is less than the target's scrape interval value in which the
+		// latter is used.
+		scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Name or number of the target port of the `Pod` object behind
+		// the Service. The port must be specified with the container's
+		// port property.
+		targetPort?: (int | string) & {
+			string
+		}
+
+		// TLS configuration to use when scraping the target.
+		tlsConfig?: {
+			// Certificate authority used when verifying server certificates.
+			ca?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Path to the CA cert in the Prometheus container to use for the
+			// targets.
+			caFile?: string
+
+			// Client certificate to present when doing client-authentication.
+			cert?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Path to the client cert file in the Prometheus container for
+			// the targets.
+			certFile?: string
+
+			// Disable target certificate validation.
+			insecureSkipVerify?: bool
+
+			// Path to the client key file in the Prometheus container for the
+			// targets.
+			keyFile?: string
+
+			// Secret containing the client key file for the targets.
+			keySecret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Used to verify the hostname for the targets.
+			serverName?: string
+		}
+
+		// `trackTimestampsStaleness` defines whether Prometheus tracks
+		// staleness of the metrics that have an explicit timestamp
+		// present in scraped data. Has no effect if `honorTimestamps` is
+		// false.
+		// It requires Prometheus >= v2.48.0.
+		trackTimestampsStaleness?: bool
+	}]
+
+	// `jobLabel` selects the label from the associated Kubernetes
+	// `Service` object which will be used as the `job` label for all
+	// metrics.
+	// For example if `jobLabel` is set to `foo` and the Kubernetes
+	// `Service` object is labeled with `foo: bar`, then Prometheus
+	// adds the `job="bar"` label to all ingested metrics.
+	// If the value of this field is empty or if the label doesn't
+	// exist for the given Service, the `job` label of the metrics
+	// defaults to the name of the associated Kubernetes `Service`.
+	jobLabel?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelValueLengthLimit?: int
+
+	// Selector to select which namespaces the Kubernetes `Endpoints`
+	// objects are discovered from.
+	namespaceSelector?: {
+		// Boolean describing whether all namespaces are selected in
+		// contrast to a list restricting them.
+		any?: bool
+
+		// List of namespace names to select from.
+		matchNames?: [...string]
+	}
+
+	// `podTargetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Pod` object onto the ingested
+	// metrics.
+	podTargetLabels?: [...string]
+
+	// `sampleLimit` defines a per-scrape limit on the number of
+	// scraped samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Label selector to select the Kubernetes `Endpoints` objects.
+	selector: {
+		// matchExpressions is a list of label selector requirements. The
+		// requirements are ANDed.
+		matchExpressions?: [...{
+			// key is the label key that the selector applies to.
+			key: string
+
+			// operator represents a key's relationship to a set of values.
+			// Valid operators are In, NotIn, Exists and DoesNotExist.
+			operator: string
+
+			// values is an array of string values. If the operator is In or
+			// NotIn, the values array must be non-empty. If the operator is
+			// Exists or DoesNotExist, the values array must be empty. This
+			// array is replaced during a strategic merge patch.
+			values?: [...string]
+		}]
+
+		// matchLabels is a map of {key,value} pairs. A single {key,value}
+		// in the matchLabels map is equivalent to an element of
+		// matchExpressions, whose key field is "key", the operator is
+		// "In", and the values array contains only "value". The
+		// requirements are ANDed.
+		matchLabels?: {
+			[string]: string
+		}
+	}
+
+	// `targetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Service` object onto the ingested
+	// metrics.
+	targetLabels?: [...string]
+
+	// `targetLimit` defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+}

docs/examples/cue.mod/usr/k8s.io/api/apps/v1/types.cue

@@ -4,3 +4,8 @@ package v1
 	apiVersion: "apps/v1"
 	kind:       "Deployment"
 }
+
+#StatefulSet: {
+	apiVersion: "apps/v1"
+	kind:       "StatefulSet"
+}

docs/examples/helpers.cue

@@ -18,8 +18,10 @@ import "encoding/yaml"
 		Issuer?: [Name=_]: #Issuer & {metadata: name: Name}
 		Gateway?: [Name=_]: #Gateway & {metadata: name: Name}
 		ConfigMap?: [Name=_]: #ConfigMap & {metadata: name: Name}
+		ServiceAccount?: [Name=_]: #ServiceAccount & {metadata: name: Name}
 
 		Deployment?: [_]:            #Deployment
+		StatefulSet?: [_]:           #StatefulSet
 		RequestAuthentication?: [_]: #RequestAuthentication
 		AuthorizationPolicy?: [_]:   #AuthorizationPolicy
 	}

docs/examples/managednamespaces.cue

@@ -0,0 +1,26 @@
+package holos
+
+// NOTE: Beyond the base reference platform, services should typically be added to #OptionalServices instead of directly to a managed namespace.
+
+// ManagedNamespace is a namespace to manage across all clusters in the holos platform.
+#ManagedNamespace: {
+	namespace: {
+		metadata: {
+			name: string
+			labels: [string]: string
+		}
+	}
+	// clusterNames represents the set of clusters the namespace is managed on.  Usually all clusters.
+	clusterNames: [...string]
+	for cluster in clusterNames {
+		clusters: (cluster): name: cluster
+	}
+}
+
+// #ManagedNamepsaces is the union of all namespaces across all cluster types and optional services.
+// Holos adopts the namespace sameness position of SIG Multicluster, refer to https://github.com/kubernetes/community/blob/dd4c8b704ef1c9c3bfd928c6fa9234276d61ad18/sig-multicluster/namespace-sameness-position-statement.md
+#ManagedNamespaces: {
+	[Name=_]: #ManagedNamespace & {
+		namespace: metadata: name: Name
+	}
+}

docs/examples/meshconfig.cue

@@ -6,9 +6,10 @@ package holos
 	// clusterName is the value of the --cluster-name flag, the cluster currently being manged / rendered.
 	clusterName: string | *#ClusterName
 
-	extensionProviderMap: [Name=_]: {
-		name: Name
-	}
+	// for extAuthzHttp extension providers
+	extensionProviderMap: [Name=_]: #ExtAuthzProxy & {name: Name}
+	// for other extension providers like zipkin
+	extensionProviderExtraMap: [Name=_]: {name: Name}
 
 	config: {
 		accessLogEncoding: string | *"JSON"
@@ -21,7 +22,10 @@ package holos
 		enablePrometheusMerge: false | *true
 		rootNamespace:         string | *"istio-system"
 		trustDomain:           string | *"cluster.local"
-		extensionProviders: [for x in extensionProviderMap {x}]
+		extensionProviders: [
+			for x in extensionProviderMap {x},
+			for y in extensionProviderExtraMap {y},
+		]
 	}
 }
 

docs/examples/optionalservices.cue

@@ -1,6 +1,8 @@
 // Controls optional feature flags for services distributed across multiple holos components.
 // For example, enable issuing certificates in the provisioner cluster when an optional service is
-// enabled for a workload cluster.
+// enabled for a workload cluster.  Another example is NATS, which isn't necessary on all clusters,
+// but is necessary on clusters with a project like holos which depends on NATS.
+
 package holos
 
 import "list"

docs/examples/platforms/holos-saas/clusters/provisioner/choria/broker/broker.cue

@@ -0,0 +1,45 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Broker = "choria-broker"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-platform-issuer": _
+
+		metadata: name: "\(Namespace)-\(Broker)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Broker
+	"app.kubernetes.io/name":     Broker
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		Certificate: "\(Broker)-tls": #Certificate & {
+			metadata: {
+				name:      "\(Broker)-tls"
+				namespace: Namespace
+				labels:    SelectorLabels
+			}
+			spec: {
+				commonName: "\(Broker).\(Namespace).svc.cluster.local"
+				dnsNames: [
+					Broker,
+					"\(Broker).\(Namespace).svc",
+					"\(Broker).\(Namespace).svc.cluster.local",
+					"*.\(Broker)",
+					"*.\(Broker).\(Namespace).svc",
+					"*.\(Broker).\(Namespace).svc.cluster.local",
+				]
+				issuerRef: kind: "ClusterIssuer"
+				issuerRef: name: "platform-issuer"
+				secretName: metadata.name
+				usages: ["signing", "key encipherment", "server auth", "client auth"]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/provisioner/choria/provisioner/provsioner.cue

@@ -0,0 +1,45 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Provisioner = "choria-provisioner"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-platform-issuer": _
+
+		metadata: name: "\(Namespace)-\(Provisioner)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Provisioner
+	"app.kubernetes.io/name":     Provisioner
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		Certificate: "\(Provisioner)-tls": #Certificate & {
+			metadata: {
+				name:      "\(Provisioner)-tls"
+				namespace: Namespace
+				labels:    SelectorLabels
+			}
+			spec: {
+				commonName: "\(Provisioner).\(Namespace).svc.cluster.local"
+				dnsNames: [
+					Provisioner,
+					"\(Provisioner).\(Namespace).svc",
+					"\(Provisioner).\(Namespace).svc.cluster.local",
+					"*.\(Provisioner)",
+					"*.\(Provisioner).\(Namespace).svc",
+					"*.\(Provisioner).\(Namespace).svc.cluster.local",
+				]
+				issuerRef: kind: "ClusterIssuer"
+				issuerRef: name: "platform-issuer"
+				secretName: metadata.name
+				usages: ["signing", "key encipherment", "server auth", "client auth"]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/broker/broker.cue

@@ -0,0 +1,177 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Broker = "choria-broker"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "\(Namespace)-\(Broker)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/part-of": "choria"
+	"app.kubernetes.io/name":    Broker
+}
+
+let Metadata = {
+	name:      Broker
+	namespace: Namespace
+	labels:    SelectorLabels
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		ExternalSecret: "\(Broker)-tls": #ExternalSecret & {
+			metadata: name:      "\(Broker)-tls"
+			metadata: namespace: Namespace
+		}
+		ExternalSecret: "\(Broker)": #ExternalSecret & {
+			metadata: name:      Broker
+			metadata: namespace: Namespace
+		}
+		StatefulSet: "\(Broker)": {
+			metadata: Metadata
+			spec: {
+				selector: matchLabels: SelectorLabels
+				serviceName: Broker
+				template: metadata: labels: SelectorLabels
+				template: spec: {
+					containers: [
+						{
+							name: Broker
+							command: ["choria", "broker", "run", "--config", "/etc/choria/broker.conf"]
+							image:           "registry.choria.io/choria/choria:0.28.0"
+							imagePullPolicy: "IfNotPresent"
+							ports: [
+								{
+									containerPort: 4222
+									name:          "tcp-nats"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 4333
+									name:          "https-wss"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 5222
+									name:          "tcp-cluster"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 8222
+									name:          "http-stats"
+									protocol:      "TCP"
+								},
+							]
+							livenessProbe: httpGet: {
+								path: "/healthz"
+								port: "http-stats"
+							}
+							readinessProbe: livenessProbe
+							resources: {}
+							securityContext: {}
+							volumeMounts: [
+								{
+									mountPath: "/etc/choria"
+									name:      Broker
+								},
+								{
+									mountPath: "/etc/choria-tls"
+									name:      "\(Broker)-tls"
+								},
+							]
+						},
+					]
+					securityContext: {}
+					serviceAccountName: Broker
+					volumes: [
+						{
+							name: Broker
+							secret: secretName: Broker
+						},
+						{
+							name: "\(Broker)-tls"
+							secret: secretName: "\(Broker)-tls"
+						},
+					]
+				}
+			}
+		}
+		ServiceAccount: "\(Broker)": #ServiceAccount & {
+			metadata: Metadata
+		}
+		Service: "\(Broker)": #Service & {
+			metadata: Metadata
+			spec: {
+				type:      "ClusterIP"
+				clusterIP: "None"
+				selector:  SelectorLabels
+				ports: [
+					{
+						name:        "tcp-nats"
+						appProtocol: "tcp"
+						port:        4222
+						protocol:    "TCP"
+						targetPort:  "tcp-nats"
+					},
+					{
+						name:        "tcp-cluster"
+						appProtocol: "tcp"
+						port:        5222
+						protocol:    "TCP"
+						targetPort:  "tcp-cluster"
+					},
+					{
+						name:        "https-wss"
+						appProtocol: "https"
+						port:        443
+						protocol:    "TCP"
+						targetPort:  "https-wss"
+					},
+				]
+			}
+		}
+		DestinationRule: "\(Broker)-wss": #DestinationRule & {
+			_decriptions: "Configures Istio to connect to Choria using a cert issued by the Platform Issuer"
+			metadata:     Metadata
+			spec: host: "\(Broker).\(Namespace).svc.cluster.local"
+			spec: trafficPolicy: tls: {
+				credentialName: "istio-ingress-mtls-cert"
+				mode:           "MUTUAL"
+				// subjectAltNames is important, otherwise istio will fail to verify the
+				// choria broker upstream server.  make sure this matches a value
+				// present in the choria broker's cert.
+				//
+				//  kubectl get secret choria-broker-tls -o json | jq --exit-status
+				//  '.data | map_values(@base64d)' | jq .\"tls.crt\" -r | openssl x509
+				//  -text -noout -in -
+				subjectAltNames: [spec.host]
+			}
+		}
+		VirtualService: "\(Broker)-wss": #VirtualService & {
+			metadata: name:      "\(Broker)-wss"
+			metadata: namespace: Namespace
+			spec: {
+				gateways: ["istio-ingress/default"]
+				hosts: ["jeff.provision.dev.\(#ClusterName).holos.run"]
+				http: [
+					{
+						route: [
+							{
+								destination: {
+									host: "\(Broker).\(Namespace).svc.cluster.local"
+									port: "number": 443
+								}
+							},
+						]
+					},
+				]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/provisioner/Dockerfile

@@ -0,0 +1,18 @@
+FROM registry.choria.io/choria/provisioner:latest
+
+RUN curl -Lo nsc.zip https://github.com/nats-io/nsc/releases/download/v2.8.6/nsc-linux-amd64.zip &&\
+  unzip nsc.zip && \
+  mv nsc /usr/local/bin/nsc && \
+  chmod 755 /usr/local/bin/nsc && \
+  rm -f nsc.zip
+
+# TODO: Add jwt executable
+# TODO: Add helper executable
+
+USER choria
+ENV USER=choria
+
+ENTRYPOINT ["/usr/sbin/choria-provisioner"]
+
+# These two files are expected to be in the provisioner secret.
+CMD ["--config=/etc/provisioner/provisioner.yaml", "--choria-config=/etc/provisioner/choria.cfg"]

docs/examples/platforms/holos-saas/clusters/workload/choria/provisioner/provisioner.cue

@@ -0,0 +1,82 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Provisioner = "choria-provisioner"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "\(Namespace)-\(Provisioner)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Provisioner
+	"app.kubernetes.io/name":     Provisioner
+}
+
+let Metadata = {
+	name:      Provisioner
+	namespace: Namespace
+	labels:    SelectorLabels
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		ExternalSecret: "\(Provisioner)-tls": #ExternalSecret & {
+			metadata: name:      "\(Provisioner)-tls"
+			metadata: namespace: Namespace
+		}
+		ExternalSecret: "\(Provisioner)": #ExternalSecret & {
+			metadata: name:      Provisioner
+			metadata: namespace: Namespace
+		}
+		ServiceAccount: "\(Provisioner)": #ServiceAccount & {
+			metadata: Metadata
+		}
+		Deployment: "\(Provisioner)": {
+			metadata: Metadata
+			spec: {
+				selector: matchLabels: SelectorLabels
+				template: metadata: labels: SelectorLabels
+				template: spec: {
+					containers: [
+						{
+							name: Provisioner
+							command: ["bash", "/etc/provisioner/entrypoint"]
+							// skopeo inspect docker://registry.choria.io/choria/provisioner | jq .RepoTags
+							image:           "registry.choria.io/choria/provisioner:0.15.1"
+							imagePullPolicy: "IfNotPresent"
+							resources: {}
+							securityContext: {}
+							volumeMounts: [
+								{
+									mountPath: "/etc/provisioner"
+									name:      Provisioner
+								},
+								{
+									mountPath: "/etc/provisioner-tls"
+									name:      "\(Provisioner)-tls"
+								},
+							]
+						},
+					]
+					securityContext: {}
+					serviceAccountName: Provisioner
+					volumes: [
+						{
+							name: Provisioner
+							secret: secretName: name
+						},
+						{
+							name: "\(Provisioner)-tls"
+							secret: secretName: name
+						},
+					]
+				}
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/readme.md

@@ -0,0 +1,8 @@
+# Machine Room Provisioner
+
+This sub-tree contains Holos Components to manage a [Choria Provisioner][1]
+system for the use case of provisioning `holos controller` instances.  These
+instances are implementations of Machine Room which are in turn implementations
+of Choria Server, hence why we use Choria Provisioner.
+
+[1]: https://choria-io.github.io/provisioner/

docs/examples/platforms/holos-saas/clusters/workload/nats/crds/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# curl -LO https://github.com/nats-io/nack/releases/latest/download/crds.yml
+resources:
+- crds.yml

docs/examples/platforms/holos-saas/clusters/workload/nats/crds/nats-crds.cue

@@ -0,0 +1,8 @@
+package holos
+
+// NATS NetStream Controller (NACK)
+spec: components: KustomizeBuildList: [
+	#KustomizeBuild & {
+		metadata: name: "prod-nack-crds"
+	},
+]

docs/examples/platforms/holos-saas/clusters/workload/nats/envs/nats.cue

@@ -0,0 +1,62 @@
+package holos
+
+// for Project in _Projects {
+// 	spec: components: resources: (#ProjectTemplate & {project: Project}).workload.resources
+// }
+
+let Namespace = "jeff-holos"
+
+#Kustomization: spec: targetNamespace: Namespace
+
+spec: components: HelmChartList: [
+	#HelmChart & {
+		metadata: name: "jeff-holos-nats"
+		namespace: Namespace
+		_dependsOn: "prod-secrets-stores": _
+		chart: {
+			name:       "nats"
+			version:    "1.1.10"
+			repository: NatsRepository
+		}
+		_values: #NatsValues & {
+			config: {
+				// https://github.com/nats-io/k8s/tree/main/helm/charts/nats#operator-mode-with-nats-resolver
+				resolver: enabled: true
+				resolver: merge: {
+					type:     "full"
+					interval: "2m"
+					timeout:  "1.9s"
+				}
+				merge: {
+					operator:       "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJUSElBTDM2NUtOS0lVVVJDMzNLNFJGQkJVRlFBSTRLS0NQTDJGVDZYVjdNQVhWU1dFNElRIiwiaWF0IjoxNzEzMjIxMzE1LCJpc3MiOiJPREtQM0RZTzc3T1NBRU5IU0FFR0s3WUNFTFBYT1FFWUI3RVFSTVBLWlBNQUxINE5BRUVLSjZDRyIsIm5hbWUiOiJIb2xvcyIsInN1YiI6Ik9ES1AzRFlPNzdPU0FFTkhTQUVHSzdZQ0VMUFhPUUVZQjdFUVJNUEtaUE1BTEg0TkFFRUtKNkNHIiwibmF0cyI6eyJ0eXBlIjoib3BlcmF0b3IiLCJ2ZXJzaW9uIjoyfX0.dQURTb-zIQMc-OYd9328oY887AEnvog6gOXY1-VCsDG3L89nq5x_ks4ME7dJ4Pn-Pvm2eyBi1Jx6ubgkthHgCQ"
+					system_account: "ADIQCYK4K3OKTPODGCLI4PDQ6XBO52MISBPTAIDESEJMLZCMNULDKCCY"
+					resolver_preload: {
+						// NOTEL: Make sure you do not include the trailing , in the SYS_ACCOUNT_JWT
+						"ADIQCYK4K3OKTPODGCLI4PDQ6XBO52MISBPTAIDESEJMLZCMNULDKCCY": "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI2SEVMNlhKSUdWUElMNFBURVI1MkUzTkFITjZLWkVUUUdFTlFVS0JWRzNUWlNLRzVLT09RIiwiaWF0IjoxNzEzMjIxMzE1LCJpc3MiOiJPREtQM0RZTzc3T1NBRU5IU0FFR0s3WUNFTFBYT1FFWUI3RVFSTVBLWlBNQUxINE5BRUVLSjZDRyIsIm5hbWUiOiJTWVMiLCJzdWIiOiJBRElRQ1lLNEszT0tUUE9ER0NMSTRQRFE2WEJPNTJNSVNCUFRBSURFU0VKTUxaQ01OVUxES0NDWSIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTF9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJhdXRob3JpemF0aW9uIjp7fSwidHlwZSI6ImFjY291bnQiLCJ2ZXJzaW9uIjoyfX0.TiGIk8XON394D9SBEowGHY_nTeOyHiM-ihyw6HZs8AngOnYPFXH9OVjsaAf8Poa2k_V84VtH7yVNgNdjBgduDA"
+					}
+				}
+				cluster: enabled:   true
+				jetstream: enabled: true
+				websocket: enabled: true
+				monitor: enabled:   true
+			}
+			promExporter: enabled: true
+			promExporter: podMonitor: enabled: true
+		}
+	},
+	#HelmChart & {
+		metadata: name: "jeff-holos-nack"
+		namespace: Namespace
+		_dependsOn: "jeff-holos-nats": _
+		chart: {
+			name:       "nack"
+			version:    "0.25.2"
+			repository: NatsRepository
+		}
+	},
+]
+
+let NatsRepository = {
+	name: "nats"
+	url:  "https://nats-io.github.io/k8s/helm/charts/"
+}

docs/examples/platforms/holos-saas/clusters/workload/nats/envs/nats.values.cue

@@ -0,0 +1,722 @@
+package holos
+
+#NatsValues: {
+	//###############################################################################
+	// Global options
+	//###############################################################################
+	global: {
+		image: {
+			// global image pull policy to use for all container images in the chart
+			// can be overridden by individual image pullPolicy
+			pullPolicy: null
+			// global list of secret names to use as image pull secrets for all pod specs in the chart
+			// secrets must exist in the same namespace
+			// https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+			pullSecretNames: []
+			// global registry to use for all container images in the chart
+			// can be overridden by individual image registry
+			registry: null
+		}
+
+		// global labels will be applied to all resources deployed by the chart
+		labels: {}
+	}
+
+	//###############################################################################
+	// Common options
+	//###############################################################################
+	// override name of the chart
+	nameOverride: null
+	// override full name of the chart+release
+	fullnameOverride: null
+	// override the namespace that resources are installed into
+	namespaceOverride: null
+
+	// reference a common CA Certificate or Bundle in all nats config `tls` blocks and nats-box contexts
+	// note: `tls.verify` still must be set in the appropriate nats config `tls` blocks to require mTLS
+	tlsCA: {
+		enabled: false
+		// set configMapName in order to mount an existing configMap to dir
+		configMapName: null
+		// set secretName in order to mount an existing secretName to dir
+		secretName: null
+		// directory to mount the configMap or secret to
+		dir: "/etc/nats-ca-cert"
+		// key in the configMap or secret that contains the CA Certificate or Bundle
+		key: "ca.crt"
+	}
+
+	//###############################################################################
+	// NATS Stateful Set and associated resources
+	//###############################################################################
+	//###########################################################
+	// NATS config
+	//###########################################################
+	config: {
+		cluster: {
+			enabled: true | *false
+			port:    6222
+			// must be 2 or higher when jetstream is enabled
+			replicas: 3
+
+			// apply to generated route URLs that connect to other pods in the StatefulSet
+			routeURLs: {
+				// if both user and password are set, they will be added to route URLs
+				// and the cluster authorization block
+				user:     null
+				password: null
+				// set to true to use FQDN in route URLs
+				useFQDN:          false
+				k8sClusterDomain: "cluster.local"
+			}
+
+			tls: {
+				enabled: true | *false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/cluster"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the cluster config
+			// https://docs.nats.io/running-a-nats-service/configuration/clustering/cluster_config
+			merge: {}
+			patch: []
+		}
+
+		jetstream: {
+			enabled: true | *false
+
+			fileStore: {
+				enabled: true
+				dir:     "/data"
+
+				//###########################################################
+				// stateful set -> volume claim templates -> jetstream pvc
+				//###########################################################
+				pvc: {
+					enabled:          true
+					size:             "10Gi"
+					storageClassName: null
+
+					// merge or patch the jetstream pvc
+					// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core
+					merge: {}
+					patch: []
+					// defaults to "{{ include "nats.fullname" $ }}-js"
+					name: null
+				}
+
+				// defaults to the PVC size
+				maxSize: null
+			}
+
+			memoryStore: {
+				enabled: false
+				// ensure that container has a sufficient memory limit greater than maxSize
+				maxSize: "1Gi"
+			}
+
+			// merge or patch the jetstream config
+			// https://docs.nats.io/running-a-nats-service/configuration#jetstream
+			merge: {}
+			patch: []
+		}
+
+		nats: {
+			port: 4222
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/nats"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+		}
+
+		leafnodes: {
+			enabled: false
+			port:    7422
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/leafnodes"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the leafnodes config
+			// https://docs.nats.io/running-a-nats-service/configuration/leafnodes/leafnode_conf
+			merge: {}
+			patch: []
+		}
+
+		websocket: {
+			enabled: true | *false
+			port:    8080
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/websocket"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			//###########################################################
+			// ingress
+			//###########################################################
+			// service must be enabled also
+			ingress: {
+				enabled: false
+				// must contain at least 1 host otherwise ingress will not be created
+				hosts: []
+				path:     "/"
+				pathType: "Exact"
+				// sets to the ingress class name
+				className: null
+				// set to an existing secret name to enable TLS on the ingress; applies to all hosts
+				tlsSecretName: null
+
+				// merge or patch the ingress
+				// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#ingress-v1-networking-k8s-io
+				merge: {}
+				patch: []
+				// defaults to "{{ include "nats.fullname" $ }}-ws"
+				name: null
+			}
+
+			// merge or patch the websocket config
+			// https://docs.nats.io/running-a-nats-service/configuration/websocket/websocket_conf
+			merge: {}
+			patch: []
+		}
+
+		mqtt: {
+			enabled: false
+			port:    1883
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/mqtt"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the mqtt config
+			// https://docs.nats.io/running-a-nats-service/configuration/mqtt/mqtt_config
+			merge: {}
+			patch: []
+		}
+
+		gateway: {
+			enabled: false
+			port:    7222
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/gateway"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the gateway config
+			// https://docs.nats.io/running-a-nats-service/configuration/gateways/gateway#gateway-configuration-block
+			merge: {}
+			patch: []
+		}
+
+		monitor: {
+			enabled: true
+			port:    8222
+			tls: {
+				// config.nats.tls must be enabled also
+				// when enabled, monitoring port will use HTTPS with the options from config.nats.tls
+				enabled: false
+			}
+		}
+
+		profiling: {
+			enabled: false
+			port:    65432
+		}
+
+		resolver: {
+			enabled: true | *false
+			dir:     "/data/resolver"
+
+			//###########################################################
+			// stateful set -> volume claim templates -> resolver pvc
+			//###########################################################
+			pvc: {
+				enabled:          true
+				size:             "1Gi"
+				storageClassName: null
+
+				// merge or patch the pvc
+				// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core
+				merge: {}
+				patch: []
+				// defaults to "{{ include "nats.fullname" $ }}-resolver"
+				name: null
+			}
+
+			// merge or patch the resolver
+			// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/jwt/resolver
+			merge: {
+				type?:     string
+				interval?: string
+				timeout?:  string
+			}
+			patch: []
+		}
+
+		// adds a prefix to the server name, which defaults to the pod name
+		// helpful for ensuring server name is unique in a super cluster
+		serverNamePrefix: ""
+
+		// merge or patch the nats config
+		// https://docs.nats.io/running-a-nats-service/configuration
+		// following special rules apply
+		//  1. strings that start with << and end with >> will be unquoted
+		//     use this for variables and numbers with units
+		//  2. keys ending in $include will be switched to include directives
+		//     keys are sorted alphabetically, use prefix before $includes to control includes ordering
+		//     paths should be relative to /etc/nats-config/nats.conf
+		// example:
+		//
+		//   merge:
+		//     $include: ./my-config.conf
+		//     zzz$include: ./my-config-last.conf
+		//     server_name: nats
+		//     authorization:
+		//       token: << $TOKEN >>
+		//     jetstream:
+		//       max_memory_store: << 1GB >>
+		//
+		// will yield the config:
+		// {
+		//   include ./my-config.conf;
+		//   "authorization": {
+		//     "token": $TOKEN
+		//   },
+		//   "jetstream": {
+		//     "max_memory_store": 1GB
+		//   },
+		//   "server_name": "nats",
+		//   include ./my-config-last.conf;
+		// }
+		merge: {
+			operator?:       string
+			system_account?: string
+			resolver_preload?: [string]: string
+		}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> nats container
+	//###########################################################
+	container: {
+		image: {
+			repository: "nats"
+			tag:        "2.10.12-alpine"
+			pullPolicy: null
+			registry:   null
+		}
+
+		// container port options
+		// must be enabled in the config section also
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#containerport-v1-core
+		ports: {
+			nats: {}
+			leafnodes: {}
+			websocket: {}
+			mqtt: {}
+			cluster: {}
+			gateway: {}
+			monitor: {}
+			profiling: {}
+		}
+
+		// map with key as env var name, value can be string or map
+		// example:
+		//
+		//   env:
+		//     GOMEMLIMIT: 7GiB
+		//     TOKEN:
+		//       valueFrom:
+		//         secretKeyRef:
+		//           name: nats-auth
+		//           key: token
+		env: {}
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> reloader container
+	//###########################################################
+	reloader: {
+		enabled: true
+		image: {
+			repository: "natsio/nats-server-config-reloader"
+			tag:        "0.14.1"
+			pullPolicy: null
+			registry:   null
+		}
+
+		// env var map, see nats.env for an example
+		env: {}
+
+		// all nats container volume mounts with the following prefixes
+		// will be mounted into the reloader container
+		natsVolumeMountPrefixes: ["/etc/"]
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> prom-exporter container
+	//###########################################################
+	// config.monitor must be enabled
+	promExporter: {
+		enabled: true | *false
+		image: {
+			repository: "natsio/prometheus-nats-exporter"
+			tag:        "0.14.0"
+			pullPolicy: null
+			registry:   null
+		}
+
+		port: 7777
+		// env var map, see nats.env for an example
+		env: {}
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+
+		//###########################################################
+		// prometheus pod monitor
+		//###########################################################
+		podMonitor: {
+			enabled: true | *false
+
+			// merge or patch the pod monitor
+			// https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.PodMonitor
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}"
+			name: null
+		}
+	}
+
+	//###########################################################
+	// service
+	//###########################################################
+	service: {
+		enabled: true
+
+		// service port options
+		// additional boolean field enable to control whether port is exposed in the service
+		// must be enabled in the config section also
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceport-v1-core
+		ports: {
+			nats: enabled:      true
+			leafnodes: enabled: true
+			websocket: enabled: true
+			mqtt: enabled:      true
+			cluster: enabled:   false
+			gateway: enabled:   false
+			monitor: enabled:   false
+			profiling: enabled: false
+		}
+
+		// merge or patch the service
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	//###########################################################
+	// other nats extension points
+	//###########################################################
+	// stateful set
+	statefulSet: {
+		// merge or patch the stateful set
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#statefulset-v1-apps
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	// stateful set -> pod template
+	podTemplate: {
+		// adds a hash of the ConfigMap as a pod annotation
+		// this will cause the StatefulSet to roll when the ConfigMap is updated
+		configChecksumAnnotation: true
+
+		// map of topologyKey: topologySpreadConstraint
+		// labelSelector will be added to match StatefulSet pods
+		//
+		// topologySpreadConstraints:
+		//   kubernetes.io/hostname:
+		//     maxSkew: 1
+		//
+		topologySpreadConstraints: {}
+
+		// merge or patch the pod template
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core
+		merge: {}
+		patch: []
+	}
+
+	// headless service
+	headlessService: {
+		// merge or patch the headless service
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}-headless"
+		name: null
+	}
+
+	// config map
+	configMap: {
+		// merge or patch the config map
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#configmap-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}-config"
+		name: null
+	}
+
+	// pod disruption budget
+	podDisruptionBudget: {
+		enabled: true
+		// merge or patch the pod disruption budget
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#poddisruptionbudget-v1-policy
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	// service account
+	serviceAccount: {
+		enabled: false
+		// merge or patch the service account
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceaccount-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	//###########################################################
+	// natsBox
+	//
+	// NATS Box Deployment and associated resources
+	//###########################################################
+	natsBox: {
+		enabled: true
+
+		//###########################################################
+		// NATS contexts
+		//###########################################################
+		contexts: {
+			default: {
+				creds: {
+					// set contents in order to create a secret with the creds file contents
+					contents: null
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-creds/<context-name>
+					dir: null
+					key: "nats.creds"
+				}
+				nkey: {
+					// set contents in order to create a secret with the nkey file contents
+					contents: null
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-nkeys/<context-name>
+					dir: null
+					key: "nats.nk"
+				}
+				// used to connect with client certificates
+				tls: {
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-certs/<context-name>
+					dir:  null
+					cert: "tls.crt"
+					key:  "tls.key"
+				}
+
+				// merge or patch the context
+				// https://docs.nats.io/using-nats/nats-tools/nats_cli#nats-contexts
+				merge: {}
+				patch: []
+			}
+		}
+
+		// name of context to select by default
+		defaultContextName: "default"
+
+		//###########################################################
+		// deployment -> pod template -> nats-box container
+		//###########################################################
+		container: {
+			image: {
+				repository: "natsio/nats-box"
+				tag:        "0.14.2"
+				pullPolicy: null
+				registry:   null
+			}
+
+			// env var map, see nats.env for an example
+			env: {}
+
+			// merge or patch the container
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+			merge: {}
+			patch: []
+		}
+
+		//###########################################################
+		// other nats-box extension points
+		//###########################################################
+		// deployment
+		deployment: {
+			// merge or patch the deployment
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#deployment-v1-apps
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box"
+			name: null
+		}
+
+		// deployment -> pod template
+		podTemplate: {
+			// merge or patch the pod template
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core
+			merge: {}
+			patch: []
+		}
+
+		// contexts secret
+		contextsSecret: {
+			// merge or patch the context secret
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box-contexts"
+			name: null
+		}
+
+		// contents secret
+		contentsSecret: {
+			// merge or patch the contents secret
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box-contents"
+			name: null
+		}
+
+		// service account
+		serviceAccount: {
+			enabled: false
+			// merge or patch the service account
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceaccount-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box"
+			name: null
+		}
+	}
+
+	//###############################################################################
+	// Extra user-defined resources
+	//###############################################################################
+	//
+	// add arbitrary user-generated resources
+	// example:
+	//
+	// config:
+	//   websocket:
+	//     enabled: true
+	// extraResources:
+	// - apiVersion: networking.istio.io/v1beta1
+	//   kind: VirtualService
+	//   metadata:
+	//     name:
+	//       $tplYaml: >
+	//         {{ include "nats.fullname" $ | quote }}
+	//     labels:
+	//       $tplYaml: |
+	//         {{ include "nats.labels" $ }}
+	//   spec:
+	//     hosts:
+	//     - demo.nats.io
+	//     gateways:
+	//     - my-gateway
+	//     http:
+	//     - name: default
+	//       match:
+	//       - name: root
+	//         uri:
+	//           exact: /
+	//       route:
+	//       - destination:
+	//           host:
+	//             $tplYaml: >
+	//               {{ .Values.service.name | quote }}
+	//           port:
+	//             number:
+	//               $tplYaml: >
+	//                 {{ .Values.config.websocket.port }}
+	//
+	extraResources: []
+}

docs/examples/platforms/holos-saas/clusters/workload/nats/scripts/initialize-auth

@@ -0,0 +1,81 @@
+#! /bin/bash
+#
+# This script initializes authorization for a nats cluster.  The process is:
+#
+# Locally:
+# 1. Generate the nats operator jwt.
+# 2. Generate a SYS account jwt issued by the operator.
+# 3. Store both into vault
+#
+# When nats is deployed an ExternalSecret populates auth.conf which is included
+# into nats.conf.  This approach allows helm values to be used for most things
+# except for secrets.
+#
+# Clean up by removing the nsc directory.
+
+set -euo pipefail
+
+tmpdir="$(mktemp -d)"
+finish() {
+  [[ -d "$tmpdir" ]] && rm -rf "$tmpdir"
+}
+trap finish EXIT
+
+PARENT="$(cd "$(dirname $0)" && pwd)"
+: "${OPERATOR_NAME:="Holos"}"
+
+: "${OIX_NAMESPACE:=$(kubectl config view --minify --flatten -ojsonpath='{.contexts[0].context.namespace}')}"
+nsc="${HOME}/.bin/nsc"
+
+ROOT="${PARENT}/${OIX_NAMESPACE}/nsc"
+export NKEYS_PATH="${ROOT}/nkeys"
+export NSC_HOME="${ROOT}/accounts"
+
+mkdir -p "$NKEYS_PATH"
+mkdir -p "$NSC_HOME"
+
+# Install nsc if not already installed
+if ! [[ -x $nsc ]]; then
+  platform="$(kubectl version --output=json | jq .clientVersion.platform -r)"
+  platform="${platform//\//-}"
+  curl -fSLo "${tmpdir}/nsc.zip" "https://github.com/nats-io/nsc/releases/download/v2.8.6/nsc-${platform}.zip"
+  (cd "${tmpdir}" && unzip nsc.zip)
+  sudo install -o 0 -g 0 -m 0755 "${tmpdir}/nsc" $nsc
+fi
+
+echo "export NKEYS_PATH='${NKEYS_PATH}'" > "${ROOT}/nsc.env"
+echo "export NSC_HOME='${NSC_HOME}'" >> "${ROOT}/nsc.env"
+# use kubectl port-forward nats-headless 4222
+echo "export NATS_URL='nats://localhost:4222'" >> "${ROOT}/nsc.env"
+echo "export NATS_CREDS='${ROOT}/nkeys/creds/${OPERATOR_NAME}/SYS/sys.creds'" >> "${ROOT}/nsc.env"
+
+echo "export NATS_CA='${ROOT}/ca.crt'" >> "${ROOT}/nsc.env"
+echo "export NATS_CERT='${ROOT}/tls.crt'" >> "${ROOT}/nsc.env"
+echo "export NATS_KEY='${ROOT}/tls.key'" >> "${ROOT}/nsc.env"
+
+$nsc --data-dir="${ROOT}/stores" list operators
+
+# Create operator
+$nsc add operator --name "${OPERATOR_NAME}"
+# Create system account
+$nsc add account --name SYS
+$nsc add user    --name sys
+# Create account for STAN purposes.
+$nsc add account --name STAN
+$nsc add user    --name stan
+
+# Generate an auth config compatible with the StatefulSet mounting the
+# nats-jwt-pvc PersistentVolumeClaim at path /data/accounts
+$nsc generate config --sys-account SYS --nats-resolver \
+  | sed "s,dir.*jwt',dir: '/data/accounts'" \
+  > "${ROOT}/auth.conf"
+
+# Store the auth config in vault.
+# vault kv put kv/${OIX_CLUSTER_NAME}/kube-namespace/holos-dev/nats-auth-config "auth.conf=@${tmpdir}/auth.conf"
+# Store the SYS creds in vault for use by the nack controller.
+# vault kv put kv/${OIX_CLUSTER_NAME}/kube-namespace/holos-dev/nats-sys-creds "sys.creds=@${OIX_CLUSTER_NAME}/nsc/nkeys/creds/${OPERATOR_NAME}/SYS/sys.creds"
+
+echo "After deploying the nats component, use the get-cert command to fetch the client cert."
+
+echo "Use kubectl port-forward svc/nats-headless 4222" >&2
+echo "source ${ROOT}/nsc.env to make it all work." >&2

docs/examples/platforms/holos-saas/readme.md

@@ -0,0 +1,5 @@
+# Holos
+
+This subtree contains holos components for holos itself.  We strive for minimal dependencies, so this is likely going to contain NATS and/or Postgres resources.
+
+Components depend on the holos project and may iterate over the defined environments in the project stages.

docs/examples/platforms/reference/certificates.cue

@@ -1,5 +1,21 @@
 package holos
 
+#PlatformServers: {
+	for cluster in #Platform.clusters {
+		(cluster.name): {
+			"https-istio-ingress-httpbin": {
+				let cert = #PlatformCerts[cluster.name+"-httpbin"]
+				hosts: [for host in cert.spec.dnsNames {"istio-ingress/\(host)"}]
+				port: name:          "https-istio-ingress-httpbin"
+				port: number:        443
+				port: protocol:      "HTTPS"
+				tls: credentialName: cert.spec.secretName
+				tls: mode:           "SIMPLE"
+			}
+		}
+	}
+}
+
 #PlatformCerts: {
 	// Globally scoped platform services are defined here.
 	login: #PlatformCert & {

docs/examples/platforms/reference/clusters/accounts/iam/iam.cue

@@ -4,4 +4,4 @@ package holos
 #InputKeys: project: "iam"
 
 // Shared dependencies for all components in this collection.
-#DependsOn: namespaces: name: "\(#StageName)-secrets-namespaces"
+#DependsOn: namespaces: name: "\(#StageName)-secrets-stores"

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/README.md

@@ -0,0 +1,10 @@
+To deploy monitoring:
+
+> **_NOTE:_** For more detailed instructions on deploying, see the [documentation on installing Monitoring](https://access.crunchydata.com/documentation/postgres-operator/latest/installation/monitoring/kustomize).
+
+1. verify the namespace is correct in kustomization.yaml 
+2. If you are deploying in openshift, comment out the fsGroup line under securityContext in the following files:
+  - `alertmanager/deployment.yaml`
+  - `grafana/deployment.yaml`
+  - `prometheus/deployment.yaml`
+3. kubectl apply -k .

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/config/alertmanager.yml

@@ -0,0 +1,78 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+# Based on upstream example file found here: https://github.com/prometheus/alertmanager/blob/master/doc/examples/simple.yml
+global:
+    smtp_smarthost: 'localhost: 25'
+    smtp_require_tls: false
+    smtp_from: 'Alertmanager <abc@yahoo.com>'
+#    smtp_smarthost: 'smtp.example.com:587'
+#    smtp_from: 'Alertmanager <abc@yahoo.com>'
+#    smtp_auth_username: '<username>'
+#    smtp_auth_password: '<password>'
+
+# templates:
+# - '/etc/alertmanager/template/*.tmpl'
+
+inhibit_rules:
+# Apply inhibition of warning if the alertname for the same system and service is already critical
+- source_match:
+    severity: 'critical'
+  target_match:
+    severity: 'warning'
+  equal: ['alertname', 'job', 'service']
+
+receivers:
+- name: 'default-receiver'
+  email_configs:
+  - to: 'example@crunchydata.com'
+    send_resolved: true
+
+## Examples of alternative alert receivers. See documentation for more info on how to configure these fully
+#- name: 'pagerduty-dba'
+#  pagerduty_configs:
+#      - service_key: <RANDOMKEYSTUFF>
+
+#- name: 'pagerduty-sre'
+#  pagerduty_configs:
+#      - service_key: <RANDOMKEYSTUFF>
+
+#- name: 'dba-team'
+#  email_configs:
+#  - to: 'example-dba-team@crunchydata.com'
+#    send_resolved: true
+
+#- name: 'sre-team'
+#  email_configs:
+#  - to: 'example-sre-team@crunchydata.com'
+#    send_resolved: true
+
+route:
+    receiver: default-receiver
+    group_by: [severity, service, job, alertname]
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 24h
+
+## Example routes to show how to route outgoing alerts based on the content of that alert
+#    routes:
+#    - match_re:
+#        service: ^(postgresql|mysql|oracle)$
+#      receiver: dba-team
+#      # sub route to send critical dba alerts to pagerduty
+#      routes:
+#      - match:
+#          severity: critical
+#        receiver: pagerduty-dba
+#
+#    - match:
+#        service: system
+#      receiver: sre-team
+#      # sub route to send critical sre alerts to pagerduty
+#      routes:
+#      - match:
+#          severity: critical
+#        receiver: pagerduty-sre

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/deployment.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crunchy-alertmanager
+spec:
+  selector: {}
+  template:
+    spec:
+      containers:
+      - name: alertmanager
+        image: prom/alertmanager:v0.24.0
+        args:
+        - --config.file=/etc/alertmanager/alertmanager.yml
+        - --storage.path=/alertmanager
+        - --log.level=info
+        - --cluster.advertise-address=0.0.0.0:9093
+        livenessProbe:
+          httpGet:
+            path: /-/healthy
+            port: 9093
+          initialDelaySeconds: 25
+          periodSeconds: 20
+        ports:
+        - containerPort: 9093
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 9093
+        volumeMounts:
+        - mountPath: /etc/alertmanager
+          name: alertmanagerconf
+        - mountPath: /alertmanager
+          name: alertmanagerdata
+      securityContext:
+        fsGroup: 26
+        # supplementalGroups:
+        # - 65534
+      serviceAccountName: alertmanager
+      volumes:
+      - name: alertmanagerdata
+        persistentVolumeClaim:
+          claimName: alertmanagerdata
+      - name: alertmanagerconf
+        configMap:
+          defaultMode: 420
+          name: alertmanager-config

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/kustomization.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: crunchy-alertmanager
+
+resources:
+- deployment.yaml
+- pvc.yaml
+- service.yaml
+- serviceaccount.yaml
+
+configMapGenerator:
+- name: alertmanager-config
+  files:
+  - config/alertmanager.yml
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: alertmanagerdata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crunchy-alertmanager
+spec:
+  type: ClusterIP
+  ports:
+  - name: alertmanager
+    port: 9093

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: alertmanager

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/config/crunchy_grafana_dashboards.yml

@@ -0,0 +1,16 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+apiVersion: 1
+
+providers:
+- name: 'crunchy_dashboards'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards
+  options:
+    path: /etc/grafana/provisioning/dashboards

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/config/crunchy_grafana_datasource.yml

@@ -0,0 +1,18 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+# config file version
+apiVersion: 1
+
+datasources:
+  - name: PROMETHEUS
+    type: prometheus
+    access: proxy
+    url: http://$PROM_HOST:$PROM_PORT
+    isDefault: True
+    editable: False
+    orgId: 1
+    version: 1

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+configMapGenerator:
+- name: grafana-dashboards
+  files:
+  - pgbackrest.json
+  - pod_details.json
+  - postgresql_overview.json
+  - postgresql_details.json
+  - postgresql_service_health.json
+  - prometheus_alerts.json
+  - query_statistics.json
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/pgbackrest.json

@@ -0,0 +1,687 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069660860,
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "title": "",
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "dtdhms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "/^Value$/",
+          "values": false
+        },
+        "text": {
+          "valueSize": 45
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "time()-ccp_backrest_oldest_full_backup_time_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Recovery window",
+          "refId": "A"
+        }
+      ],
+      "title": "Recovery Window",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Differential Backup": "dark-blue",
+        "Full": "dark-green",
+        "Full Backup": "dark-green",
+        "Incremental": "light-blue",
+        "Incremental Backup": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 3
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": false
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_incr_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment,instance,ip,pod)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental Backup",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_diff_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential Backup",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_full_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full Backup",
+          "refId": "C"
+        },
+        {
+          "expr": "min(ccp_archive_command_status_seconds_since_last_archive{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "WAL Archive",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Time Since",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 3
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"incr\"}) without (deployment,instance,pod,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"diff\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"full\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Backup Runtimes",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 5,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"incr\"}) without (deployment, instance,pod,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"diff\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"full\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Backup Size",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Archive age": "blue",
+        "Archive count": "green",
+        "Differential": "dark-blue",
+        "Failed count": "red",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 3,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg(idelta(ccp_archive_command_status_failed_count{pg_cluster=\"[[cluster]]\", role=\"master\"}[1m])) without (instance,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Failed count",
+          "refId": "A"
+        },
+        {
+          "expr": "avg(idelta(ccp_archive_command_status_archived_count{pg_cluster=\"[[cluster]]\", role=\"master\"}[1m])) without (instance,pod, ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Archive count",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "WAL Stats",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "cluster",
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-2w",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "pgBackRest",
+  "uid": "2fcFZ6PGk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/postgresql_overview.json

@@ -0,0 +1,237 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069480601,
+  "links": [],
+  "panels": [
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "links": [
+            {
+              "targetBlank": true,
+              "title": "Cluster Details",
+              "url": "d/fMip0cuMk/postgresqldetails?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Backup Details",
+              "url": "d/2fcFZ6PGk/pgbackrest?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Pod Details",
+              "url": "d/4auP6Mk7k/pod-details?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Query Statistics",
+              "url": "d/ZKoTOHDGk/query-statistics?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Service Health",
+              "url": "d/dhG1wgsMz/postgresql-service-health?$__all_variables"
+            }
+          ],
+          "mappings": [
+            {
+              "from": "0",
+              "id": 0,
+              "text": "DOWN",
+              "to": "99",
+              "type": 2
+            },
+            {
+              "from": "100",
+              "id": 1,
+              "text": "Standalone Cluster",
+              "to": "199",
+              "type": 2
+            },
+            {
+              "from": "200",
+              "id": 2,
+              "text": "HA CLUSTER",
+              "to": "1000",
+              "type": 2
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "#bf1b00",
+                "value": null
+              },
+              {
+                "color": "#eab839",
+                "value": 10
+              },
+              {
+                "color": "#56A64B",
+                "value": 100
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {
+          "valueSize": 30
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "repeat": "cluster",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "$hashKey": "object:243",
+          "expr": "sum(pg_up{pg_cluster=~\"$cluster\"})*100+sum(ccp_is_in_recovery_status{pg_cluster=~\"$cluster\"})",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{cluster}}",
+          "metric": "up",
+          "refId": "A",
+          "step": 2
+        }
+      ],
+      "title": "$cluster - Overview",
+      "type": "stat"
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allFormat": "glob",
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 1,
+        "includeAll": true,
+        "label": "cluster",
+        "multi": true,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "PostgreSQL Overview",
+  "uid": "D2X39SlGk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/postgresql_service_health.json

@@ -0,0 +1,649 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069909806,
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "title": "",
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(ccp_connection_stats_total{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip) / sum(ccp_connection_stats_max_connections{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Connections",
+          "refId": "C"
+        },
+        {
+          "expr": "100 - 100 * avg(ccp_nodemx_data_disk_available_bytes{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip) / avg(ccp_nodemx_data_disk_total_bytes{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"})  without (pod,instance,ip)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Mount:{{mount_point}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Saturation (pct used)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "percent",
+          "label": null,
+          "logBase": 1,
+          "max": "100",
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "cacheTimeout": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 18,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "  sum(irate(ccp_stat_database_xact_commit{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m])) \n+ sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Transactions",
+          "refId": "A"
+        },
+        {
+          "expr": "max(ccp_connection_stats_active{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip,dbname)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Active connections",
+          "refId": "C"
+        },
+        {
+          "expr": "sum(irate(ccp_pg_stat_statements_total_calls_count{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Queries",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Traffic",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0.001",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "Errors",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]) without(pod,instance,ip))",
+          "format": "time_series",
+          "hide": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Rollbacks",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(irate(ccp_stat_database_deadlocks{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))  without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Deadlock ",
+          "refId": "D"
+        },
+        {
+          "expr": "sum(irate(ccp_stat_database_conflicts{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m])) without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Conflicts",
+          "refId": "B"
+        },
+        {
+          "expr": "max(pg_exporter_last_scrape_error{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "scrape error",
+          "refId": "C"
+        },
+        {
+          "expr": "max(clamp_max(ccp_archive_command_status_seconds_since_last_fail{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"},1)) without (instance,pod,ip)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "archive error",
+          "refId": "E"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Errors",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": "",
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [
+        {
+          "alias": "/Max:/",
+          "color": "#E02F44",
+          "nullPointMode": "null as zero"
+        },
+        {
+          "alias": "/Avg:/",
+          "color": "#8AB8FF"
+        }
+      ],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(ccp_pg_stat_statements_total_mean_exec_time_ms{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Avg: {{exported_role}}({{dbname}})",
+          "refId": "A"
+        },
+        {
+          "expr": "max(ccp_pg_stat_statements_top_max_exec_time_ms{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip,query,queryid)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Max: {{exported_role}}({{dbname}})",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Query Duration",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "ms",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values({pg_cluster=\"[[cluster]]\"},role)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "role",
+        "options": [],
+        "query": {
+          "query": "label_values({pg_cluster=\"[[cluster]]\"},role)",
+          "refId": "PROMETHEUS-role-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "PostgreSQL Service Health",
+  "uid": "dhG1wgsMz",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/prometheus_alerts.json

@@ -0,0 +1,961 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Show current firing and pending alerts, and  severity alert counts.",
+  "editable": false,
+  "gnetId": 4181,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [
+    {
+      "icon": "external link",
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 10,
+      "panels": [],
+      "repeat": null,
+      "title": "Environment Summary",
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 0,
+        "y": 1
+      },
+      "id": 6,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(count by (kubernetes_namespace) (pg_up))",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "Namespaces",
+          "refId": "A"
+        }
+      ],
+      "title": "Namespaces",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 4,
+        "y": 1
+      },
+      "id": 13,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(count by (pg_cluster) (pg_up))",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "PostgreSQL Clusters",
+          "refId": "A"
+        }
+      ],
+      "title": "PG Clusters",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 8,
+        "y": 1
+      },
+      "id": 14,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(pg_up)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "PostgreSQL Clusters",
+          "refId": "A"
+        }
+      ],
+      "title": "PG Instances",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 3
+      },
+      "id": 11,
+      "panels": [],
+      "repeat": null,
+      "title": "Alert Summary",
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-red",
+                "value": null
+              },
+              {
+                "color": "#F2495C",
+                "value": 1
+              },
+              {
+                "color": "#F2495C"
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 0,
+        "y": 4
+      },
+      "id": 2,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "bucketAggs": [
+            {
+              "id": "2",
+              "settings": {
+                "interval": "auto",
+                "min_doc_count": 0,
+                "trimEdges": 0
+              },
+              "type": "date_histogram"
+            }
+          ],
+          "dsType": "elasticsearch",
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"critical\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Critical",
+          "metrics": [
+            {
+              "field": "select field",
+              "id": "1",
+              "type": "count"
+            }
+          ],
+          "refId": "A"
+        }
+      ],
+      "title": "Critical",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-orange",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 4,
+        "y": 4
+      },
+      "id": 5,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"warning\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Warning",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "#299c46",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 8,
+        "y": 4
+      },
+      "id": 9,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"info\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Info",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 6
+      },
+      "id": 12,
+      "panels": [],
+      "repeat": null,
+      "title": "Alerts",
+      "type": "row"
+    },
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": null,
+            "displayMode": "auto",
+            "filterable": true
+          },
+          "decimals": 2,
+          "displayName": "",
+          "mappings": [
+            {
+              "from": "",
+              "id": 1,
+              "text": "",
+              "to": "",
+              "type": 1,
+              "value": ""
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "blue",
+                "value": 100
+              },
+              {
+                "color": "#EAB839",
+                "value": 200
+              },
+              {
+                "color": "red",
+                "value": 300
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity_num"
+            },
+            "properties": [
+              {
+                "id": "custom.displayMode",
+                "value": "color-background"
+              },
+              {
+                "id": "custom.width",
+                "value": 124
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Time"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 170
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 119
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertname"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 206
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertstate"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 128
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 24,
+        "x": 0,
+        "y": 7
+      },
+      "id": 1,
+      "links": [],
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "ALERTS{alertstate='firing'} > 0",
+          "format": "table",
+          "instant": true,
+          "interval": "2s",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Firing",
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {
+            "reducers": []
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Value": true,
+              "__name__": true,
+              "alertstate": false,
+              "deployment": false,
+              "exp_type": true,
+              "fs_type": true,
+              "instance": true,
+              "job": true,
+              "kubernetes_namespace": true,
+              "mount_point": true,
+              "server": true,
+              "service": true,
+              "severity_num": false
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 16,
+              "__name__": 3,
+              "alertname": 4,
+              "alertstate": 5,
+              "deployment": 7,
+              "exp_type": 9,
+              "instance": 10,
+              "ip": 11,
+              "job": 12,
+              "kubernetes_namespace": 13,
+              "pg_cluster": 6,
+              "pod": 8,
+              "role": 14,
+              "service": 15,
+              "severity": 2,
+              "severity_num": 1
+            },
+            "renameByName": {
+              "Time": "",
+              "__name__": "",
+              "severity": "",
+              "severity_num": ""
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": null,
+            "filterable": true
+          },
+          "decimals": 2,
+          "displayName": "",
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/(instance|__name__|Time|alertstate|job|type|Value)/"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              },
+              {
+                "id": "decimals",
+                "value": 2
+              },
+              {
+                "id": "custom.align",
+                "value": null
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Time"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": null
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity_num"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 126
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 115
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertname"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 207
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertstate"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 131
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "id": 3,
+      "links": [],
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "ALERTS{alertstate=\"pending\"}",
+          "format": "table",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Alerts (1 week)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Value": true,
+              "__name__": true,
+              "exp_type": true,
+              "instance": true,
+              "job": true,
+              "kubernetes_namespace": true,
+              "service": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 16,
+              "__name__": 3,
+              "alertname": 4,
+              "alertstate": 5,
+              "deployment": 7,
+              "exp_type": 8,
+              "instance": 9,
+              "ip": 11,
+              "job": 12,
+              "kubernetes_namespace": 13,
+              "pg_cluster": 6,
+              "pod": 10,
+              "role": 14,
+              "service": 15,
+              "severity": 2,
+              "severity_num": 1
+            },
+            "renameByName": {}
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "15m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Prometheus Alerts",
+  "uid": "lwxXsZsMk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/deployment.yaml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crunchy-grafana
+spec:
+  selector: {}
+  template:
+    spec:
+      containers:
+      - name: grafana
+        image: grafana/grafana:9.2.20
+        ports:
+        - containerPort: 3000
+        env:
+        - name: GF_PATHS_DATA
+          value: /data/grafana/data
+        - name: GF_SECURITY_ADMIN_USER__FILE
+          value: /conf/admin/username
+        - name: GF_SECURITY_ADMIN_PASSWORD__FILE
+          value: /conf/admin/password
+        - name: PROM_HOST
+          value: crunchy-prometheus
+        - name: PROM_PORT
+          value: "9090"
+        livenessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
+          initialDelaySeconds: 25
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
+        volumeMounts:
+        - mountPath: /data
+          name: grafanadata
+        - mountPath: /conf/admin
+          name: grafana-admin
+        - mountPath: /etc/grafana/provisioning/datasources
+          name: grafana-datasources
+        - mountPath: /etc/grafana/provisioning/dashboards
+          name: grafana-dashboards
+      securityContext:
+        fsGroup: 26
+        # supplementalGroups:
+        # - 65534
+      serviceAccountName: grafana
+      volumes:
+      - name: grafanadata
+        persistentVolumeClaim:
+          claimName: grafanadata
+      - name: grafana-admin
+        secret:
+          defaultMode: 420
+          secretName: grafana-admin
+      - name: grafana-datasources
+        configMap:
+          defaultMode: 420
+          name: grafana-datasources
+      - name: grafana-dashboards
+        configMap:
+          defaultMode: 420
+          name: grafana-dashboards

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/kustomization.yaml

@@ -0,0 +1,33 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: crunchy-grafana
+
+resources:
+- deployment.yaml
+- pvc.yaml
+- service.yaml
+- serviceaccount.yaml
+- dashboards
+
+configMapGenerator:
+- name: grafana-datasources
+  files:
+  - config/crunchy_grafana_datasource.yml
+- name: grafana-dashboards
+  behavior: merge
+  files:
+  - config/crunchy_grafana_dashboards.yml
+
+secretGenerator:
+- name: grafana-admin
+  literals:
+  - password=admin
+  - username=admin
+  type: Opaque
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafanadata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crunchy-grafana
+spec:
+  type: ClusterIP
+  ports:
+  - name: grafana
+    port: 3000

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grafana

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: prod-iam
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/name: prod-iam-obs
+    vendor: holos
+
+resources:
+- grafana
+- prometheus
+- alertmanager

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/monitoring.cue

@@ -0,0 +1,9 @@
+package holos
+
+spec: components: KustomizeBuildList: [
+	#KustomizeBuild & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "prod-iam-obs"
+	},
+]

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/clusterrole.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- resources:
+  - pods
+  apiGroups:
+  - ""
+  verbs:
+  - get
+  - list
+  - watch

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/clusterrolebinding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/config/crunchy-alert-rules-pg.yml

@@ -0,0 +1,418 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+groups:
+- name: alert-rules
+  rules:
+
+########## EXPORTER RULES ##########
+  - alert: PGExporterScrapeError
+    expr: pg_exporter_last_scrape_error > 0
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: 'Postgres Exporter running on {{ $labels.job }} (instance: {{ $labels.instance }}) is encountering scrape errors processing queries. Error count: ( {{ $value }} )'
+
+
+########## SYSTEM RULES ##########
+  - alert: ExporterDown
+    expr: avg_over_time(up[5m]) < 0.5
+    for: 10s
+    labels:
+      service: system
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'Metrics exporter service for {{ $labels.job }} running on {{ $labels.instance }} has been down at least 50% of the time for the last 5 minutes. Service may be flapping or down.'
+      summary: 'Prometheus Exporter Service Down'
+
+
+########## POSTGRESQL RULES ##########
+  - alert: PGIsUp
+    expr: pg_up < 1
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: 'postgres_exporter running on {{ $labels.job }} is unable to communicate with the configured database'
+
+
+# Example to check for current version of PostgreSQL. Metric returns the version that the exporter is running on, so you can set a rule to check for the minimum version you'd like all systems to be on. Number returned is the 6 digit integer representation contained in the setting "server_version_num".
+#
+#  - alert: PGMinimumVersion
+#    expr:  ccp_postgresql_version_current < 110005
+#    for: 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      summary: '{{ $labels.job }} is not running at least version 11.5 of PostgreSQL'
+
+
+# Whether a system switches from primary to replica or vice versa must be configured per named job.
+# No way to tell what value a system is supposed to be without a rule expression for that specific system
+# 2 to 1 means it changed from primary to replica. 1 to 2 means it changed from replica to primary
+# Set this alert for each system that you want to monitor a recovery status change
+# Below is an example for a target job called "Replica" and watches for the value to change above 1 which means it's no longer a replica
+#
+#  - alert: PGRecoveryStatusSwitch_Replica
+#    expr: ccp_is_in_recovery_status{job="Replica"} > 1
+#    for: 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      summary: '{{ $labels.job }} has changed from replica to primary'
+
+
+# Absence alerts must be configured per named job, otherwise there's no way to know which job is down
+# Below is an example for a target job called "Prod"
+#  - alert: PGConnectionAbsent_Prod
+#    expr: absent(ccp_connection_stats_max_connections{job="Prod"})
+#    for: 10s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Connection metric is absent from target (Prod). Check that postgres_exporter can connect to PostgreSQL.'
+
+
+# Optional monitor for changes to pg_settings (postgresql.conf) system catalog.
+# A similar metric is available for monitoring pg_hba.conf. See ccp_hba_settings_checksum().
+# If metric returns 0, then NO settings have changed for either pg_settings since last known valid state
+# If metric returns 1, then pg_settings have changed since last known valid state
+# To see what may have changed, check the monitor.pg_settings_checksum table for a history of config state.
+#  - alert: PGSettingsChecksum
+#    expr: ccp_pg_settings_checksum > 0
+#    for 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Configuration settings on {{ $labels.job }} have changed from previously known valid state. To reset current config to a valid state after alert fires, run monitor.pg_settings_checksum_set_valid().'
+#      summary: 'PGSQL Instance settings checksum'
+
+
+# Monitor for data block checksum failures. Only works in PG12+
+#  - alert: PGDataChecksum
+#    expr: ccp_data_checksum_failure > 0
+#    for 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: '{{ $labels.job }} has at least one data checksum failure in database {{ $labels.dbname }}. See pg_stat_database system catalog for more information.'
+#      summary: 'PGSQL Data Checksum failure'
+
+  - alert: PGIdleTxn
+    expr: ccp_connection_stats_max_idle_in_txn_time > 300
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} has at least one session idle in transaction for over 5 minutes.'
+      summary: 'PGSQL Instance idle transactions'
+
+  - alert: PGIdleTxn
+    expr: ccp_connection_stats_max_idle_in_txn_time > 900
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} has at least one session idle in transaction for over 15 minutes.'
+      summary: 'PGSQL Instance idle transactions'
+
+  - alert: PGQueryTime
+    expr: ccp_connection_stats_max_query_time > 43200
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} has at least one query running for over 12 hours.'
+      summary: 'PGSQL Max Query Runtime'
+
+  - alert: PGQueryTime
+    expr: ccp_connection_stats_max_query_time > 86400
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} has at least one query running for over 1 day.'
+      summary: 'PGSQL Max Query Runtime'
+
+  - alert: PGConnPerc
+    expr: 100 * (ccp_connection_stats_total / ccp_connection_stats_max_connections) > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} is using 75% or more of available connections ({{ $value }}%)'
+      summary: 'PGSQL Instance connections'
+
+  - alert: PGConnPerc
+    expr: 100 * (ccp_connection_stats_total / ccp_connection_stats_max_connections) > 90
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} is using 90% or more of available connections ({{ $value }}%)'
+      summary: 'PGSQL Instance connections'
+
+  - alert: DiskFillPredict
+    expr: predict_linear(ccp_nodemx_data_disk_available_bytes{mount_point!~"tmpfs"}[1h], 24 * 3600) < 0 and 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 70
+    for: 5m
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      summary: 'Disk predicted to be full in 24 hours'
+      description: 'Disk on {{ $labels.pg_cluster }}:{{ $labels.kubernetes_pod_name }} is predicted to fill in 24 hrs based on current usage'
+
+  - alert: PGClusterRoleChange
+    expr: count by (pg_cluster) (ccp_is_in_recovery_status != ignoring(instance,ip,pod,role) (ccp_is_in_recovery_status offset 5m)) >= 1
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: '{{ $labels.pg_cluster }} has had a switchover/failover event. Please check this cluster for more details'
+
+  - alert: PGDiskSize
+    expr: 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.deployment }} over 75% disk usage at mount point "{{ $labels.mount_point }}": {{ $value }}%'
+      summary: PGSQL Instance usage warning
+
+  - alert: PGDiskSize
+    expr: 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 90
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.deployment }} over 90% disk usage at mount point "{{ $labels.mount_point }}": {{ $value }}%'
+      summary: 'PGSQL Instance size critical'
+
+  - alert: PGReplicationByteLag
+    expr: ccp_replication_lag_size_bytes > 5.24288e+07
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has at least one replica lagging over 50MB behind.'
+      summary: 'PGSQL Instance replica lag warning'
+
+  - alert: PGReplicationByteLag
+    expr: ccp_replication_lag_size_bytes > 1.048576e+08
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has at least one replica lagging over 100MB behind.'
+      summary: 'PGSQL Instance replica lag warning'
+
+  - alert: PGReplicationSlotsInactive
+    expr: ccp_replication_slots_active == 0
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has one or more inactive replication slots'
+      summary: 'PGSQL Instance inactive replication slot'
+
+  - alert: PGXIDWraparound
+    expr: ccp_transaction_wraparound_percent_towards_wraparound > 50
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 50% towards transaction id wraparound.'
+      summary: 'PGSQL Instance {{ $labels.job }} transaction id wraparound imminent'
+
+  - alert: PGXIDWraparound
+    expr: ccp_transaction_wraparound_percent_towards_wraparound > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 75% towards transaction id wraparound.'
+      summary: 'PGSQL Instance transaction id wraparound imminent'
+
+  - alert: PGEmergencyVacuum
+    expr: ccp_transaction_wraparound_percent_towards_emergency_autovac > 110
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 110% beyond autovacuum_freeze_max_age value. Autovacuum may need tuning to better keep up.'
+      summary: 'PGSQL Instance emergency vacuum imminent'
+
+  - alert: PGEmergencyVacuum
+    expr: ccp_transaction_wraparound_percent_towards_emergency_autovac > 125
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 125% beyond autovacuum_freeze_max_age value. Autovacuum needs tuning to better keep up.'
+      summary: 'PGSQL Instance emergency vacuum imminent'
+
+  - alert: PGArchiveCommandStatus
+    expr: ccp_archive_command_status_seconds_since_last_fail > 300
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'PGSQL Instance {{ $labels.job }} has a recent failing archive command'
+        summary: 'Seconds since the last recorded failure of the archive_command'
+
+  - alert: PGSequenceExhaustion
+    expr: ccp_sequence_exhaustion_count > 0
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'Count of sequences on instance {{ $labels.job }} at over 75% usage: {{ $value }}. Run following query to see full sequence status: SELECT * FROM monitor.sequence_status() WHERE percent >= 75'
+
+  - alert: PGSettingsPendingRestart
+    expr: ccp_settings_pending_restart_count > 0
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'One or more settings in the pg_settings system catalog on system {{ $labels.job }} are in a pending_restart state. Check the system catalog for which settings are pending and review postgresql.conf for changes.'
+
+########## PGBACKREST RULES ##########
+#
+# Uncomment and customize one or more of these rules to monitor your pgbackrest backups.
+# Full backups are considered the equivalent of both differentials and incrementals since both are based on the last full
+#   And differentials are considered incrementals since incrementals will be based off the last diff if one exists
+#   This avoid false alerts, for example when you don't run diff/incr backups on the days that you run a full
+# Stanza should also be set if different intervals are expected for each stanza.
+#   Otherwise rule will be applied to all stanzas returned on target system if not set.
+#
+# Relevant metric names are:
+#   ccp_backrest_last_full_backup_time_since_completion_seconds
+#   ccp_backrest_last_incr_backup_time_since_completion_seconds
+#   ccp_backrest_last_diff_backup_time_since_completion_seconds
+#
+# To avoid false positives on backup time alerts, 12 hours are added onto each threshold to allow a buffer if the backup runtime varies from day to day.
+#    Further adjustment may be needed depending on your backup runtimes/schedule.
+#
+#  - alert: PGBackRestLastCompletedFull_main
+#    expr: ccp_backrest_last_full_backup_time_since_completion_seconds{stanza="main"} > 648000
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Full backup for stanza [main] on system {{ $labels.job }} has not completed in the last week.'
+#
+#  - alert: PGBackRestLastCompletedIncr_main
+#    expr: ccp_backrest_last_incr_backup_time_since_completion_seconds{stanza="main"} > 129600
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Incremental backup for stanza [main] on system {{ $labels.job }} has not completed in the last 24 hours.'
+#
+#
+# Runtime monitoring is handled with a single metric:
+#
+#   ccp_backrest_last_info_backup_runtime_seconds
+#
+# Runtime monitoring should have the "backup_type" label set.
+#   Otherwise the rule will apply to the last run of all backup types returned (full, diff, incr)
+# Stanza should also be set if runtimes per stanza have different expected times
+#
+#  - alert: PGBackRestLastRuntimeFull_main
+#    expr: ccp_backrest_last_info_backup_runtime_seconds{backup_type="full", stanza="main"} > 14400
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Expected runtime of full backup for stanza [main] has exceeded 4 hours'
+#
+#  - alert: PGBackRestLastRuntimeDiff_main
+#    expr: ccp_backrest_last_info_backup_runtime_seconds{backup_type="diff", stanza="main"} > 3600
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Expected runtime of diff backup for stanza [main] has exceeded 1 hour'
+##
+#
+## If the pgbackrest command fails to run, the metric disappears from the exporter output and the alert never fires.
+## An absence alert must be configured explicitly for each target (job) that backups are being monitored.
+## Checking for absence of just the full backup type should be sufficient (no need for diff/incr).
+## Note that while the backrest check command failing will likely also cause a scrape error alert, the addition of this
+## check gives a clearer answer as to what is causing it and that something is wrong with the backups.
+#
+#  - alert: PGBackrestAbsentFull_Prod
+#    expr: absent(ccp_backrest_last_full_backup_time_since_completion_seconds{job="Prod"})
+#    for: 10s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Backup Full status missing for Prod. Check that pgbackrest info command is working on target system.'

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/config/prometheus.yml

@@ -0,0 +1,85 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+---
+global:
+  scrape_interval: 15s
+  scrape_timeout: 15s
+  evaluation_interval: 5s
+
+scrape_configs:
+- job_name: 'crunchy-postgres-exporter'
+  kubernetes_sd_configs:
+  - role: pod
+    selectors:
+    - role: pod
+      label: postgres-operator.crunchydata.com/crunchy-postgres-exporter=true
+  relabel_configs:
+  # Keep exporter port and drop all others
+  - source_labels: [__meta_kubernetes_pod_container_port_number]
+    action: keep
+    regex: 9187
+  # Set label for namespace
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: kubernetes_namespace
+  # Set label for pod name
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  # Convert namespace and cluster name to pg_cluster=namespace:cluster
+  - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_cluster]
+    target_label: pg_cluster
+    separator: ":"
+    replacement: '$1$2'
+  # Convert kubernetes pod ip to ip
+  - source_labels: [__meta_kubernetes_pod_ip]
+    target_label: ip
+  # Convert postgres-operator.crunchydata.com/instance to deployment
+  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_instance]
+    target_label: deployment
+  # Convert postgres-operator.crunchydata.com/role to role
+  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_role]
+    target_label: role
+
+- job_name: 'crunchy-postgres-exporter-v4'
+  kubernetes_sd_configs:
+  - role: pod
+    selectors:
+    - role: pod
+      label: crunchy-postgres-exporter=true
+
+  relabel_configs:
+  # Keep exporter port and drop all others
+  - source_labels: [__meta_kubernetes_pod_container_port_number]
+    action: keep
+    regex: 9187
+  # Set label for namespace
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: kubernetes_namespace
+  # Set label for pod name
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  # Convert namespace and cluster name to pg_cluster=namespace:cluster
+  - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_pg_cluster]
+    target_label: pg_cluster
+    separator: ":"
+    replacement: '$1$2'
+  # Convert kubernetes pod ip to ip
+  - source_labels: [__meta_kubernetes_pod_ip]
+    target_label: ip
+  # Set deployment_name as deployment label
+  - source_labels: [__meta_kubernetes_pod_label_deployment_name]
+    target_label: deployment
+  # Set label for role
+  - source_labels: [__meta_kubernetes_pod_label_role]
+    target_label: role
+rule_files:
+  - /etc/prometheus/alert-rules.d/*.yml
+alerting:
+  alertmanagers:
+  - kubernetes_sd_configs:
+    - role: pod
+      selectors:
+      - role: pod
+        label: app.kubernetes.io/component=crunchy-alertmanager

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/deployment.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crunchy-prometheus
+spec:
+  selector: {}
+  template:
+    spec:
+      containers:
+      - name: prometheus
+        image: prom/prometheus:v2.39.2
+        ports:
+        - containerPort: 9090
+        livenessProbe:
+          httpGet:
+            path: /-/healthy
+            port: 9090
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 9090
+        volumeMounts:
+        - mountPath: /etc/prometheus
+          name: prometheusconf
+        - mountPath: /prometheus
+          name: prometheusdata
+        - mountPath: /etc/prometheus/alert-rules.d
+          name: alertmanagerrules
+      securityContext:
+        fsGroup: 26
+        # supplementalGroups:
+        # - 65534
+      serviceAccountName: prometheus
+      volumes:
+      - name: prometheusconf
+        configMap:
+          defaultMode: 420
+          name: crunchy-prometheus
+      - name: prometheusdata
+        persistentVolumeClaim:
+          claimName: prometheusdata
+      - name: alertmanagerrules
+        configMap:
+          defaultMode: 420
+          name: alert-rules-config

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: crunchy-prometheus
+
+resources:
+- deployment.yaml
+- pvc.yaml
+- service.yaml
+- serviceaccount.yaml
+- clusterrole.yaml
+- clusterrolebinding.yaml
+
+configMapGenerator:
+- name: crunchy-prometheus
+  files:
+  - config/prometheus.yml
+- name: alert-rules-config
+  files:
+  - config/crunchy-alert-rules-pg.yml
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: prometheusdata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crunchy-prometheus
+spec:
+  type: ClusterIP
+  ports:
+  - name: prometheus
+    port: 9090

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/postgres/postgres.cue

@@ -37,7 +37,7 @@ spec: components: KubernetesObjectsList: [
 	#KubernetesObjects & {
 		metadata: name: "prod-iam-postgres"
 
-		_dependsOn: "prod-secrets-namespaces": _
+		_dependsOn: "prod-secrets-stores":     _
 		_dependsOn: "prod-iam-postgres-certs": _
 		apiObjectMap: OBJECTS.apiObjectMap
 	},
@@ -68,7 +68,7 @@ let OBJECTS = #APIObjects & {
 					replicas: 2
 					dataVolumeClaimSpec: {
 						accessModes: ["ReadWriteOnce"]
-						resources: requests: storage: "10Gi"
+						resources: requests: storage: "20Gi"
 					}
 				}]
 				standby: {
@@ -80,6 +80,8 @@ let OBJECTS = #APIObjects & {
 						enabled: true
 					}
 				}
+				// Monitoring configuration
+				monitoring: pgmonitor: exporter: image: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-5.5.1-0"
 				// Restore from backup if and only if the cluster is primary
 				if Cluster.primary {
 					dataSource: pgbackrest: {
@@ -124,7 +126,7 @@ let OBJECTS = #APIObjects & {
 						"\(BucketRepoName)-cipher-type": "aes-256-cbc"
 						// "The convention we recommend for setting this variable is /pgbackrest/$NAMESPACE/$CLUSTER_NAME/repoN"
 						// Ref: https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/backups#understanding-backup-configuration-and-basic-operations
-						"\(BucketRepoName)-path": "/pgbackrest/\(#TargetNamespace)/\(metadata.name)/\(manual.repoName)"
+						"\(BucketRepoName)-path": "/pgbackrest/\(metadata.namespace)/\(metadata.name)/\(manual.repoName)"
 					}
 					repos: [
 						{
@@ -165,7 +167,7 @@ let HighlyAvailable = {
 			replicas: 2
 			dataVolumeClaimSpec: {
 				accessModes: ["ReadWriteOnce"]
-				resources: requests: storage: string | *"10Gi"
+				resources: requests: storage: string | *"20Gi"
 			}
 			affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: [{
 				weight: 1

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel.cue

@@ -1,7 +1,9 @@
 package holos
 
-#InstancePrefix:  "prod-iam"
-#TargetNamespace: #InstancePrefix + "-zitadel"
+#InstancePrefix: "prod-iam"
+
+// The namespace is managed by a project.
+#TargetNamespace: _Projects.iam.environments.prod.namespace
 
 // _DBName is the database name used across multiple holos components in this project
 _DBName: "zitadel"

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/values.cue

@@ -74,7 +74,7 @@ package holos
 		repository: "ghcr.io/zitadel/zitadel"
 		pullPolicy: "IfNotPresent"
 		// Overrides the image tag whose default is the chart appVersion.
-		tag: ""
+		tag: string | *""
 	}
 
 	chownImage: {

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/values.holos.cue

@@ -1,6 +1,10 @@
 package holos
 
 #Values: {
+	// https://github.com/zitadel/zitadel/releases
+	// Overrides the image tag whose default is the chart appVersion.
+	image: tag: "v2.49.1"
+
 	// Database credentials
 	// Refer to https://access.crunchydata.com/documentation/postgres-operator/5.2.0/architecture/user-management/
 	// Refer to https://zitadel.com/docs/self-hosting/manage/database#postgres
@@ -74,6 +78,12 @@ package holos
 			ExternalPort:   443
 			TLS: Enabled: false
 
+			// Fix AuthProxy JWKS Error - Jwks doesn't have key to match kid or alg from Jwt
+			// Refer to: https://github.com/holos-run/holos/issues/96
+			// Refer to: https://github.com/zitadel/zitadel/discussions/7464
+			SystemDefaults: KeyConfig: PrivateKeyLifetime: "999999h"
+			SystemDefaults: KeyConfig: PublicKeyLifetime:  "999999h"
+
 			// Database connection credentials are injected via environment variables from the db-pguser-db secret.
 			Database: postgres: {
 				MaxOpenConns:    25

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/zitadel.cue

@@ -16,7 +16,7 @@ spec: components: HelmChartList: [
 		enableHooks: true
 		chart: {
 			name:    Name
-			version: "7.9.0"
+			version: "7.11.0"
 			repository: {
 				name: Name
 				url:  "https://charts.zitadel.com"

docs/examples/platforms/reference/clusters/foundation/cloud/argocd/argocd.cue

@@ -0,0 +1,85 @@
+package holos
+
+import "encoding/yaml"
+
+let ArgoCD = "argocd"
+let Namespace = "prod-platform"
+
+spec: components: HelmChartList: [
+	#HelmChart & {
+		_dependsOn: "prod-secrets-stores": _
+		namespace: Namespace
+
+		metadata: name: "\(namespace)-\(ArgoCD)"
+
+		chart: {
+			name:    "argo-cd"
+			release: "argocd"
+			version: "6.7.8"
+			repository: {
+				name: "argocd"
+				url:  "https://argoproj.github.io/argo-helm"
+			}
+		}
+
+		_values: #ArgoCDValues & {
+			kubeVersionOverride: "1.29.0"
+			global: domain: "argocd.\(#ClusterName).\(#Platform.org.domain)"
+			dex: enabled:   false
+			// for integration with istio
+			configs: params: "server.insecure": true
+			configs: cm: {
+				"admin.enabled": false
+				"oidc.config":   yaml.Marshal(OIDCConfig)
+			}
+		}
+
+		// Holos overlay objects
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		// ExternalSecret: "deploy-key": _
+		VirtualService: (ArgoCD): {
+			metadata: name:      ArgoCD
+			metadata: namespace: Namespace
+			spec: hosts: [
+				ArgoCD + ".\(#Platform.org.domain)",
+				ArgoCD + ".\(#ClusterName).\(#Platform.org.domain)",
+			]
+			spec: gateways: ["istio-ingress/default"]
+			spec: http: [{route: [{destination: {
+				host: "argocd-server.\(Namespace).svc.cluster.local"
+				port: number: 80
+			}}]}]
+		}
+	}
+}
+let IstioInject = [{op: "add", path: "/spec/template/metadata/labels/sidecar.istio.io~1inject", value: "true"}]
+
+#Kustomize: _patches: {
+	mesh: {
+		target: {
+			group:   "apps"
+			version: "v1"
+			kind:    "Deployment"
+			name:    "argocd-server"
+		}
+		patch: yaml.Marshal(IstioInject)
+	}
+}
+
+let OAuthClient = #Platform.oauthClients.argocd.spec
+
+let OIDCConfig = {
+	name:            "Holos Platform"
+	issuer:          OAuthClient.issuer
+	clientID:        OAuthClient.clientID
+	requestedScopes: OAuthClient.scopesList
+	// Set redirect uri to https://argocd.example.com/pkce/verify
+	enablePKCEAuthentication: true
+
+	requestedIDTokenClaims: groups: essential: true
+}

docs/examples/platforms/reference/clusters/foundation/cloud/certmanager/certmanager.cue

@@ -0,0 +1,43 @@
+package holos
+
+// https://cert-manager.io/docs/
+
+#TargetNamespace: "cert-manager"
+
+spec: components: HelmChartList: [
+	#HelmChart & {
+		metadata: name: "prod-mesh-certmanager"
+
+		_dependsOn: "prod-secrets-namespaces": _
+		namespace: #TargetNamespace
+		_values: #Values & {
+			installCRDs: true
+			startupapicheck: enabled: false
+			// Must not use kube-system on gke autopilot.  GKE Warden authz blocks access.
+			global: leaderElection: namespace: #TargetNamespace
+		}
+		chart: {
+			name:    "cert-manager"
+			version: "1.14.3"
+			repository: {
+				name: "jetstack"
+				url:  "https://charts.jetstack.io"
+			}
+		}
+	},
+]
+
+// https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#min-max-requests
+#PodResources: {
+	requests: {
+		cpu:                 string | *"250m"
+		memory:              string | *"512Mi"
+		"ephemeral-storage": string | *"100Mi"
+	}
+}
+
+// https://cloud.google.com/kubernetes-engine/docs/how-to/autopilot-spot-pods
+#NodeSelector: {
+	// "kubernetes.io/os":          "linux"
+	// "cloud.google.com/gke-spot": "true"
+}