(#175) Fix tests

Split holos render into component and platform.

This patch splits the previous `holos render` command into subcommands.
`holos render component ./path/to/component/` behaves as the previous
`holos render` command and renders an individual component.

The new `holos render platform ./path/to/platform/` subcommand makes
space to render the entire platform using the platform model pulled from
the PlatformService.

Starting with an empty directory:

```sh
holos register user
holos generate platform bare
holos pull platform config .
holos render platform ./platform/
```

```txt
10:01AM INF platform.go:29 ok render component version=0.80.2 path=components/configmap cluster=k1 num=1 total=1 duration=448.133038ms
```

The bare platform has a single component which refers to the platform
model pulled from the PlatformService:

```sh
cat deploy/clusters/mycluster/components/platform-configmap/platform-configmap.gen.yaml
```

```yaml
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: platform
  namespace: default
data:
  platform: |
    spec:
      model:
        cloud:
          providers:
            - cloudflare
        cloudflare:
          email: platform@openinfrastructure.co
        org:
          displayName: Open Infrastructure Services
          name: ois
```

.github/workflows/lint.yaml

@@ -17,12 +17,33 @@ jobs:
     name: lint
     runs-on: gha-rs
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
           go-version: stable
-          cache: false
+
+      - name: Install Packages
+        run: sudo apt update && sudo apt -qq -y install git curl zip unzip tar bzip2 make
+
+      - name: Install Tools
+        run: |
+          set -x
+          make tools
+          make buf
+          go generate ./...
+          make frontend
+          go mod tidy
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v4
         with:
           version: latest
+          skip-pkg-cache: true

.github/workflows/release.yaml

@@ -14,23 +14,49 @@ jobs:
   goreleaser:
     runs-on: gha-rs
     steps:
+      # Must come before Checkout, otherwise goreleaser fails
       - name: Provide GPG and Git
-        run: sudo apt update && sudo apt -qq -y install gnupg git
+        run: sudo apt update && sudo apt -qq -y install gnupg git curl zip unzip tar bzip2 make
+
+      # Must come after git executable is provided
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
+      # Necessary to run these outside of goreleaser, otherwise
+      # /home/runner/_work/holos/holos/internal/frontend/node_modules/.bin/protoc-gen-connect-query is not in PATH
+      - name: Install Tools
+        run: |
+          set -x
+          make tools
+          make buf
+          go generate ./...
+          make frontend
+          go mod tidy
+
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@v6
         with:
           gpg_private_key: ${{ secrets.GPG_CODE_SIGNING_SECRETKEY }}
           passphrase: ${{ secrets.GPG_CODE_SIGNING_PASSPHRASE }}
+
       - name: List keys
         run: gpg -K
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
+
+      - name: Git diff
+        run: git diff
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v5
         with:

.github/workflows/test.yaml

@@ -18,13 +18,18 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
           go-version: stable
 
-      - name: Provide unzip for Helm
-        run: sudo apt update && sudo apt -qq -y install curl zip unzip tar bzip2
+      - name: Install Packages
+        run: sudo apt update && sudo apt -qq -y install git curl zip unzip tar bzip2 make
 
       - name: Set up Helm
         uses: azure/setup-helm@v4
@@ -32,5 +37,14 @@ jobs:
       - name: Set up Kubectl
         uses: azure/setup-kubectl@v3
 
+      - name: Install Tools
+        run: |
+          set -x
+          make tools
+          make buf
+          go generate ./...
+          make frontend
+          go mod tidy
+
       - name: Test
         run: ./scripts/test

.gitignore

@@ -1,7 +1,9 @@
-bin/
+/bin/
 vendor/
 .idea/
 coverage.out
-dist/
+/dist/
 *.hold/
 /deploy/
+.vscode/
+tmp/

.goreleaser.yaml

@@ -10,10 +10,8 @@ version: 1
 
 before:
   hooks:
-    # You may remove this if you don't use go modules.
-    - go mod tidy
-    # you may remove this if you don't need go generate
     - go generate ./...
+    - go mod tidy
 
 builds:
   - main: ./cmd/holos
@@ -23,6 +21,9 @@ builds:
       - linux
       - windows
       - darwin
+    goarch:
+      - amd64
+      - arm64
 
 signs:
   - artifacts: checksum

Makefile

@@ -4,7 +4,7 @@ PROJ=holos
 ORG_PATH=github.com/holos-run
 REPO_PATH=$(ORG_PATH)/$(PROJ)
 
-VERSION := $(shell grep "const Version " pkg/version/version.go | sed -E 's/.*"(.+)"$$/\1/')
+VERSION := $(shell cat version/embedded/major version/embedded/minor version/embedded/patch | xargs printf "%s.%s.%s")
 BIN_NAME := holos
 
 DOCKER_REPO=quay.io/openinfrastructure/holos
@@ -12,11 +12,14 @@ IMAGE_NAME=$(DOCKER_REPO)
 
 $( shell mkdir -p bin)
 
+# For buf plugin protoc-gen-connect-es
+export PATH := $(PWD)/internal/frontend/holos/node_modules/.bin:$(PATH)
+
 GIT_COMMIT=$(shell git rev-parse HEAD)
 GIT_TREE_STATE=$(shell test -n "`git status --porcelain`" && echo "dirty" || echo "clean")
 BUILD_DATE=$(shell date -Iseconds)
 
-LD_FLAGS="-w -X ${ORG_PATH}/${PROJ}/pkg/version.GitCommit=${GIT_COMMIT} -X ${ORG_PATH}/${PROJ}/pkg/version.GitTreeState=${GIT_TREE_STATE} -X ${ORG_PATH}/${PROJ}/pkg/version.BuildDate=${BUILD_DATE}"
+LD_FLAGS="-w -X ${ORG_PATH}/${PROJ}/version.GitCommit=${GIT_COMMIT} -X ${ORG_PATH}/${PROJ}/version.GitTreeState=${GIT_TREE_STATE} -X ${ORG_PATH}/${PROJ}/version.BuildDate=${BUILD_DATE}"
 
 .PHONY: default
 default: test
@@ -39,6 +42,10 @@ bumpmajor: ## Bump the major version.
 	scripts/bump minor 0
 	scripts/bump patch 0
 
+.PHONY: show-version
+show-version: ## Print the full version.
+	@echo $(VERSION)
+
 .PHONY: tidy
 tidy: ## Tidy go module.
 	go mod tidy
@@ -54,14 +61,26 @@ vet: ## Vet Go code.
 
 .PHONY: gencue
 gencue: ## Generate CUE definitions
-	cd docs/examples && cue get go github.com/holos-run/holos/api/...
+	cd internal/generate/platforms && cue get go github.com/holos-run/holos/api/v1alpha1/...
+
+.PHONY: rmgen
+rmgen: ## Remove generated code
+	git rm -rf service/gen/ internal/frontend/holos/src/app/gen/ || true
+	rm -rf service/gen/ internal/frontend/holos/src/app/gen/
+	git rm -rf internal/ent/
+	rm -rf internal/ent/
+	git restore --staged internal/ent/generate.go internal/ent/schema/
+	git restore internal/ent/generate.go internal/ent/schema/
+
+.PHONY: regenerate
+regenerate: generate ## Re-generate code (delete and re-create)
 
 .PHONY: generate
-generate: ## Generate code.
+generate: buf gencue ## Generate code.
 	go generate ./...
 
 .PHONY: build
-build: generate ## Build holos executable.
+build: generate frontend ## Build holos executable.
 	@echo "building ${BIN_NAME} ${VERSION}"
 	@echo "GOPATH=${GOPATH}"
 	go build -trimpath -o bin/$(BIN_NAME) -ldflags $(LD_FLAGS) $(REPO_PATH)/cmd/$(BIN_NAME)
@@ -80,6 +99,8 @@ test: ## Run tests.
 
 .PHONY: lint
 lint: ## Run linters.
+	buf lint
+	cd internal/frontend/holos && ng lint
 	golangci-lint run
 
 .PHONY: coverage
@@ -90,6 +111,40 @@ coverage: test  ## Test coverage profile.
 snapshot:  ## Go release snapshot
 	goreleaser release --snapshot --clean
 
+.PHONY: buf
+buf: ## buf generate
+	cd service && buf dep update
+	buf generate
+
+.PHONY: tools
+tools: go-deps frontend-deps  ## install tool dependencies
+
+.PHONY: go-deps
+go-deps: ## tool versions pinned in tools.go
+	go install github.com/bufbuild/buf/cmd/buf
+	go install github.com/fullstorydev/grpcurl/cmd/grpcurl
+	go install google.golang.org/protobuf/cmd/protoc-gen-go
+	go install connectrpc.com/connect/cmd/protoc-gen-connect-go
+	go install honnef.co/go/tools/cmd/staticcheck@latest
+	# curl https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash
+
+.PHONY: frontend-deps
+frontend-deps: ## Setup npm and vite
+	cd internal/frontend/holos && npm install
+	cd internal/frontend/holos && npm install --save-dev @bufbuild/buf @connectrpc/protoc-gen-connect-es
+	cd internal/frontend/holos && npm install @connectrpc/connect @connectrpc/connect-web @bufbuild/protobuf
+	# https://github.com/connectrpc/connect-query-es/blob/1350b6f07b6aead81793917954bdb1cc3ce09df9/packages/protoc-gen-connect-query/README.md?plain=1#L23
+	cd internal/frontend/holos && npm install --save-dev @connectrpc/protoc-gen-connect-query @bufbuild/protoc-gen-es
+	cd internal/frontend/holos && npm install @connectrpc/connect-query @bufbuild/protobuf
+
+
+.PHONY: frontend
+frontend: buf
+	cd internal/frontend/holos && rm -rf dist
+	mkdir -p internal/frontend/holos/dist
+	cd internal/frontend/holos && ng build
+	touch internal/frontend/frontend.go
+
 .PHONY: help
 help:  ## Display this help menu.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

Tiltfile

@@ -0,0 +1,311 @@
+# -*- mode: Python -*-
+# This Tiltfile manages a Go project with live leload in Kubernetes
+
+listen_port = 3000
+metrics_port = 9090
+
+# Use our wrapper to set the kube namespace
+if os.getenv('TILT_WRAPPER') != '1':
+    fail("could not run, ./hack/tilt/bin/tilt was not used to start tilt")
+
+# AWS Account to work in
+aws_account = '271053619184'
+aws_region = 'us-east-2'
+
+# Resource ids
+holos_backend = 'Holos Backend'
+pg_admin = 'pgAdmin'
+pg_cluster = 'PostgresCluster'
+pg_svc = 'Database Pod'
+compile_id = 'Go Build'
+auth_id = 'Auth Policy'
+lint_id = 'Run Linters'
+tests_id = 'Run Tests'
+
+# PostgresCluster resource name in k8s
+pg_cluster_name = 'holos'
+# Database name inside the PostgresCluster
+pg_database_name = 'holos'
+# PGAdmin name
+pg_admin_name = 'pgadmin'
+
+# Default Registry.
+# See: https://github.com/tilt-dev/tilt.build/blob/master/docs/choosing_clusters.md#manual-configuration
+# Note, Tilt will append the image name to the registry uri path
+default_registry('{account}.dkr.ecr.{region}.amazonaws.com/holos-run/holos-server'.format(account=aws_account, region=aws_region))
+
+# Set a name prefix specific to the user.  Multiple developers share the tilt-holos namespace.
+developer = os.getenv('USER')
+holos_server = 'holos'
+# See ./hack/tilt/bin/tilt
+namespace = os.getenv('NAMESPACE')
+# We always develop against the k1 cluster.
+os.putenv('KUBECONFIG', os.path.abspath('./hack/tilt/kubeconfig'))
+# The context defined in ./hack/tilt/kubeconfig
+allow_k8s_contexts('sso@k1')
+allow_k8s_contexts('sso@k2')
+allow_k8s_contexts('sso@k3')
+allow_k8s_contexts('sso@k4')
+allow_k8s_contexts('sso@k5')
+# PG db connection for localhost -> k8s port-forward
+os.putenv('PGHOST', 'localhost')
+os.putenv('PGPORT', '15432')
+# We always develop in the dev aws account.
+os.putenv('AWS_CONFIG_FILE', os.path.abspath('./hack/tilt/aws.config'))
+os.putenv('AWS_ACCOUNT', aws_account)
+os.putenv('AWS_DEFAULT_REGION', aws_region)
+os.putenv('AWS_PROFILE', 'dev-holos')
+os.putenv('AWS_SDK_LOAD_CONFIG', '1')
+# Authenticate to AWS ECR when tilt up is run by the developer
+local_resource('AWS Credentials', './hack/tilt/aws-login.sh', auto_init=True)
+
+# Extensions are open-source, pre-packaged functions that extend Tilt
+#
+#   More info: https://github.com/tilt-dev/tilt-extensions
+#   More info: https://docs.tilt.dev/extensions.html
+load('ext://restart_process', 'docker_build_with_restart')
+load('ext://k8s_attach', 'k8s_attach')
+load('ext://git_resource', 'git_checkout')
+load('ext://uibutton', 'cmd_button')
+
+# Paths edited by the developer Tilt watches to trigger compilation.
+# Generated files should be excluded to avoid an infinite build loop.
+developer_paths = [
+    './cmd',
+    './internal/server',
+    './internal/ent/schema',
+    './frontend/package-lock.json',
+    './frontend/src',
+    './go.mod',
+    './pkg',
+    './service/holos',
+]
+
+# Builds the holos-server executable
+local_resource(compile_id, 'make build', deps=developer_paths)
+
+# Build Docker image
+#   Tilt will automatically associate image builds with the resource(s)
+#   that reference them (e.g. via Kubernetes or Docker Compose YAML).
+#
+#   More info: https://docs.tilt.dev/api.html#api.docker_build
+#
+docker_build_with_restart(
+    'holos',
+    context='.',
+    entrypoint=[
+        '/app/bin/holos',
+        'server',
+        '--listen-port={}'.format(listen_port),
+        '--oidc-issuer=https://login.ois.run',
+        '--oidc-audience=262096764402729854@holos_platform',
+        '--log-level=debug',
+        '--metrics-port={}'.format(metrics_port),
+    ],
+    dockerfile='./hack/tilt/Dockerfile',
+    only=['./bin'],
+    # (Recommended) Updating a running container in-place
+    # https://docs.tilt.dev/live_update_reference.html
+    live_update=[
+        # Sync files from host to container
+        sync('./bin', '/app/bin'),
+        # Wait for aws-login https://github.com/tilt-dev/tilt/issues/3048
+        sync('./tilt/aws-login.last', '/dev/null'),
+        # Execute commands in the container when paths change
+        # run('/app/hack/codegen.sh', trigger=['./app/api'])
+    ],
+)
+
+
+# Run local commands
+#   Local commands can be helpful for one-time tasks like installing
+#   project prerequisites. They can also manage long-lived processes
+#   for non-containerized services or dependencies.
+#
+#   More info: https://docs.tilt.dev/local_resource.html
+#
+# local_resource('install-helm',
+#                cmd='which helm > /dev/null || brew install helm',
+#                # `cmd_bat`, when present, is used instead of `cmd` on Windows.
+#                cmd_bat=[
+#                    'powershell.exe',
+#                    '-Noninteractive',
+#                    '-Command',
+#                    '& {if (!(Get-Command helm -ErrorAction SilentlyContinue)) {scoop install helm}}'
+#                ]
+# )
+
+# Teach tilt about our custom resources (Note, this may be intended for workloads)
+# k8s_kind('authorizationpolicy')
+# k8s_kind('requestauthentication')
+# k8s_kind('virtualservice')
+k8s_kind('pgadmin')
+
+
+# Troubleshooting
+def resource_name(id):
+    print('resource: {}'.format(id))
+    return id.name
+
+
+workload_to_resource_function(resource_name)
+
+# Apply Kubernetes manifests
+#   Tilt will build & push any necessary images, re-deploying your
+#   resources as they change.
+#
+#   More info: https://docs.tilt.dev/api.html#api.k8s_yaml
+#
+
+def holos_yaml():
+    """Return a k8s Deployment personalized for the developer."""
+    k8s_yaml_template = str(read_file('./hack/tilt/k8s.yaml'))
+    return k8s_yaml_template.format(
+        name=holos_server,
+        developer=developer,
+        namespace=namespace,
+        listen_port=listen_port,
+        metrics_port=metrics_port,
+        tz=os.getenv('TZ'),
+    )
+
+# Customize a Kubernetes resource
+#   By default, Kubernetes resource names are automatically assigned
+#   based on objects in the YAML manifests, e.g. Deployment name.
+#
+#   Tilt strives for sane defaults, so calling k8s_resource is
+#   optional, and you only need to pass the arguments you want to
+#   override.
+#
+#   More info: https://docs.tilt.dev/api.html#api.k8s_resource
+#
+k8s_yaml(blob(holos_yaml()))
+
+# Backend server process
+k8s_resource(
+    workload=holos_server,
+    new_name=holos_backend,
+    objects=[
+        '{}:serviceaccount'.format(holos_server),
+        '{}:servicemonitor'.format(holos_server),
+    ],
+    resource_deps=[compile_id],
+    links=[
+        link('https://{}.app.dev.k2.holos.run/ui/'.format(developer), "Holos Web UI")
+    ],
+)
+
+
+# AuthorizationPolicy - Beyond Corp functionality
+k8s_resource(
+    new_name=auth_id,
+    objects=[
+      '{}:virtualservice'.format(holos_server),
+    ],
+)
+
+# Database
+# Note: Tilt confuses the backup pods with the database server pods, so this code is careful to tease the pods
+# apart so logs are streamed correctly.
+# See: https://github.com/tilt-dev/tilt.specs/blob/master/resource_assembly.md
+
+# pgAdmin Web UI
+k8s_resource(
+    workload=pg_admin_name,
+    new_name=pg_admin,
+    port_forwards=[
+        port_forward(15050, 5050, pg_admin),
+    ],
+)
+
+# Disabled because these don't group resources nicely
+# k8s_kind('postgrescluster')
+
+# Postgres database in-cluster
+k8s_resource(
+    new_name=pg_cluster,
+    objects=['holos:postgrescluster'],
+)
+
+# Needed to select the database by label
+# https://docs.tilt.dev/api.html#api.k8s_custom_deploy
+k8s_custom_deploy(
+    pg_svc,
+    apply_cmd=['./hack/tilt/k8s-get-db-sts', pg_cluster_name],
+    delete_cmd=['echo', 'Skipping delete.  Object managed by custom resource.'],
+    deps=[],
+)
+k8s_resource(
+    pg_svc,
+    port_forwards=[
+        port_forward(15432, 5432, 'psql'),
+    ],
+    resource_deps=[pg_cluster]
+)
+
+
+# Run tests
+local_resource(
+    tests_id,
+    'make test',
+    allow_parallel=True,
+    auto_init=False,
+    deps=developer_paths,
+)
+
+# Run linter
+local_resource(
+    lint_id,
+    'make lint',
+    allow_parallel=True,
+    auto_init=False,
+    deps=developer_paths,
+)
+
+# UI Buttons for helpful things.
+# Icons: https://fonts.google.com/icons
+os.putenv("GH_FORCE_TTY", "80%")
+cmd_button(
+    '{}:go-test-failfast'.format(tests_id),
+    argv=['./hack/tilt/go-test-failfast'],
+    resource=tests_id,
+    icon_name='quiz',
+    text='Fail Fast',
+)
+cmd_button(
+    '{}:issues'.format(holos_server),
+    argv=['./hack/tilt/gh-issues'],
+    resource=holos_backend,
+    icon_name='folder_data',
+    text='Issues',
+)
+cmd_button(
+    '{}:gh-issue-view'.format(holos_server),
+    argv=['./hack/tilt/gh-issue-view'],
+    resource=holos_backend,
+    icon_name='task',
+    text='View Issue',
+)
+cmd_button(
+    '{}:get-pgdb-creds'.format(holos_server),
+    argv=['./hack/tilt/get-pgdb-creds', pg_cluster_name, pg_database_name],
+    resource=pg_svc,
+    icon_name='lock_open_right',
+    text='DB Creds',
+)
+cmd_button(
+    '{}:get-pgdb-creds'.format(pg_admin_name),
+    argv=['./hack/tilt/get-pgdb-creds', pg_cluster_name, pg_database_name],
+    resource=pg_admin,
+    icon_name='lock_open_right',
+    text='DB Creds',
+)
+cmd_button(
+    '{}:get-pgadmin-creds'.format(pg_admin_name),
+    argv=['./hack/tilt/get-pgadmin-creds', pg_admin_name],
+    resource=pg_admin,
+    icon_name='lock_open_right',
+    text='pgAdmin Login',
+)
+
+print("✨ Tiltfile evaluated")

api/v1alpha1/buildplan.go

@@ -19,9 +19,10 @@ type BuildPlanSpec struct {
 }
 
 type BuildPlanComponents struct {
-	HelmChartList         []HelmChart         `json:"helmChartList,omitempty" yaml:"helmChartList,omitempty"`
-	KubernetesObjectsList []KubernetesObjects `json:"kubernetesObjectsList,omitempty" yaml:"kubernetesObjectsList,omitempty"`
-	KustomizeBuildList    []KustomizeBuild    `json:"kustomizeBuildList,omitempty" yaml:"kustomizeBuildList,omitempty"`
+	HelmChartList         []HelmChart                  `json:"helmChartList,omitempty" yaml:"helmChartList,omitempty"`
+	KubernetesObjectsList []KubernetesObjects          `json:"kubernetesObjectsList,omitempty" yaml:"kubernetesObjectsList,omitempty"`
+	KustomizeBuildList    []KustomizeBuild             `json:"kustomizeBuildList,omitempty" yaml:"kustomizeBuildList,omitempty"`
+	Resources             map[string]KubernetesObjects `json:"resources,omitempty" yaml:"resources,omitempty"`
 }
 
 func (bp *BuildPlan) Validate() error {
@@ -37,3 +38,14 @@ func (bp *BuildPlan) Validate() error {
 	}
 	return nil
 }
+
+func (bp *BuildPlan) ResultCapacity() (count int) {
+	if bp == nil {
+		return 0
+	}
+	count = len(bp.Spec.Components.HelmChartList) +
+		len(bp.Spec.Components.KubernetesObjectsList) +
+		len(bp.Spec.Components.KustomizeBuildList) +
+		len(bp.Spec.Components.Resources)
+	return count
+}

api/v1alpha1/form.go

@@ -0,0 +1,13 @@
+package v1alpha1
+
+import object "github.com/holos-run/holos/service/gen/holos/object/v1alpha1"
+
+// Form represents a collection of Formly json powered form.
+type Form struct {
+	TypeMeta `json:",inline" yaml:",inline"`
+	Spec     FormSpec `json:"spec" yaml:"spec"`
+}
+
+type FormSpec struct {
+	Form object.Form `json:"form" yaml:"form"`
+}

api/v1alpha1/helm.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 
 	"github.com/holos-run/holos"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 // A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
@@ -35,21 +35,21 @@ type Repository struct {
 	URL  string `json:"url"`
 }
 
-func (hc *HelmChart) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+func (hc *HelmChart) Render(ctx context.Context, path holos.InstancePath) (*Result, error) {
 	result := Result{HolosComponent: hc.HolosComponent}
 	if err := hc.helm(ctx, &result, path); err != nil {
 		return nil, err
 	}
 	result.addObjectMap(ctx, hc.APIObjectMap)
 	if err := result.kustomize(ctx); err != nil {
-		return nil, wrapper.Wrap(fmt.Errorf("could not kustomize: %w", err))
+		return nil, errors.Wrap(fmt.Errorf("could not kustomize: %w", err))
 	}
 	return &result, nil
 }
 
 // runHelm provides the values produced by CUE to helm template and returns
 // the rendered kubernetes api objects in the result.
-func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.PathComponent) error {
+func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.InstancePath) error {
 	log := logger.FromContext(ctx).With("chart", hc.Chart.Name)
 	if hc.Chart.Name == "" {
 		log.WarnContext(ctx, "skipping helm: no chart name specified, use a different component type")
@@ -64,13 +64,13 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.PathCompone
 			out, err := util.RunCmd(ctx, "helm", "repo", "add", repo.Name, repo.URL)
 			if err != nil {
 				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-				return wrapper.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
+				return errors.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
 			}
 			// Update repository
 			out, err = util.RunCmd(ctx, "helm", "repo", "update", repo.Name)
 			if err != nil {
 				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-				return wrapper.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
+				return errors.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
 			}
 		} else {
 			log.DebugContext(ctx, "no chart repository url proceeding assuming oci chart")
@@ -85,13 +85,13 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.PathCompone
 	// Write values file
 	tempDir, err := os.MkdirTemp("", "holos")
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+		return errors.Wrap(fmt.Errorf("could not make temp dir: %w", err))
 	}
 	defer util.Remove(ctx, tempDir)
 
 	valuesPath := filepath.Join(tempDir, "values.yaml")
 	if err := os.WriteFile(valuesPath, []byte(hc.ValuesContent), 0644); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not write values: %w", err))
+		return errors.Wrap(fmt.Errorf("could not write values: %w", err))
 	}
 	log.DebugContext(ctx, "helm: wrote values", "path", valuesPath, "bytes", len(hc.ValuesContent))
 
@@ -112,7 +112,7 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.PathCompone
 				err = fmt.Errorf("%s: %w", line, err)
 			}
 		}
-		return wrapper.Wrap(fmt.Errorf("could not run helm template: %w", err))
+		return errors.Wrap(fmt.Errorf("could not run helm template: %w", err))
 	}
 
 	r.accumulatedOutput = helmOut.Stdout.String()
@@ -121,12 +121,12 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.PathCompone
 }
 
 // cacheChart stores a cached copy of Chart in the chart subdirectory of path.
-func cacheChart(ctx context.Context, path holos.PathComponent, chartDir string, chart Chart) error {
+func cacheChart(ctx context.Context, path holos.InstancePath, chartDir string, chart Chart) error {
 	log := logger.FromContext(ctx)
 
 	cacheTemp, err := os.MkdirTemp(string(path), chartDir)
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+		return errors.Wrap(fmt.Errorf("could not make temp dir: %w", err))
 	}
 	defer util.Remove(ctx, cacheTemp)
 
@@ -136,14 +136,30 @@ func cacheChart(ctx context.Context, path holos.PathComponent, chartDir string,
 	}
 	helmOut, err := util.RunCmd(ctx, "helm", "pull", "--destination", cacheTemp, "--untar=true", "--version", chart.Version, chartName)
 	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not run helm pull: %w", err))
+		return errors.Wrap(fmt.Errorf("could not run helm pull: %w", err))
 	}
 	log.Debug("helm pull", "stdout", helmOut.Stdout, "stderr", helmOut.Stderr)
 
 	cachePath := filepath.Join(string(path), chartDir)
-	if err := os.Rename(cacheTemp, cachePath); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not rename: %w", err))
+
+	if err := os.MkdirAll(cachePath, 0777); err != nil {
+		return errors.Wrap(fmt.Errorf("could not mkdir: %w", err))
 	}
+
+	items, err := os.ReadDir(cacheTemp)
+	if err != nil {
+		return errors.Wrap(fmt.Errorf("could not read directory: %w", err))
+	}
+
+	for _, item := range items {
+		src := filepath.Join(cacheTemp, item.Name())
+		dst := filepath.Join(cachePath, item.Name())
+		log.DebugContext(ctx, "rename", "src", src, "dst", dst)
+		if err := os.Rename(src, dst); err != nil {
+			return errors.Wrap(fmt.Errorf("could not rename: %w", err))
+		}
+	}
+
 	log.InfoContext(ctx, "cached", "chart", chart.Name, "version", chart.Version, "path", cachePath)
 
 	return nil

api/v1alpha1/kubernetesobjects.go

@@ -2,6 +2,7 @@ package v1alpha1
 
 import (
 	"context"
+
 	"github.com/holos-run/holos"
 )
 
@@ -13,7 +14,7 @@ type KubernetesObjects struct {
 }
 
 // Render produces kubernetes api objects from the APIObjectMap
-func (o *KubernetesObjects) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+func (o *KubernetesObjects) Render(ctx context.Context, path holos.InstancePath) (*Result, error) {
 	result := Result{HolosComponent: o.HolosComponent}
 	result.addObjectMap(ctx, o.APIObjectMap)
 	return &result, nil

api/v1alpha1/kustomize.go

@@ -2,10 +2,11 @@ package v1alpha1
 
 import (
 	"context"
+
 	"github.com/holos-run/holos"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 const KustomizeBuildKind = "KustomizeBuild"
@@ -29,14 +30,14 @@ type KustomizeBuild struct {
 
 // Render produces a Result by executing kubectl kustomize on the holos
 // component path. Useful for processing raw yaml files.
-func (kb *KustomizeBuild) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+func (kb *KustomizeBuild) Render(ctx context.Context, path holos.InstancePath) (*Result, error) {
 	log := logger.FromContext(ctx)
 	result := Result{HolosComponent: kb.HolosComponent}
 	// Run kustomize.
 	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", string(path))
 	if err != nil {
 		log.ErrorContext(ctx, kOut.Stderr.String())
-		return nil, wrapper.Wrap(err)
+		return nil, errors.Wrap(err)
 	}
 	// Replace the accumulated output
 	result.accumulatedOutput = kOut.Stdout.String()

api/v1alpha1/objectmap.go

@@ -1,8 +1,14 @@
 package v1alpha1
 
+// Label is an arbitrary unique identifier.  Defined as a type for clarity and type checking.
+type Label string
+
+// Kind is a kubernetes api object kind. Defined as a type for clarity and type checking.
+type Kind string
+
 // APIObjectMap is the shape of marshalled api objects returned from cue to the
 // holos cli. A map is used to improve the clarity of error messages from cue.
-type APIObjectMap map[string]map[string]string
+type APIObjectMap map[Kind]map[Label]string
 
 // FileContentMap is a map of file names to file contents.
 type FileContentMap map[string]string

api/v1alpha1/platform.go

@@ -0,0 +1,32 @@
+package v1alpha1
+
+import "google.golang.org/protobuf/types/known/structpb"
+
+// Platform represents a platform to manage.  A Platform resource informs holos
+// which components to build.  The platform resource also acts as a container
+// for the platform model form values provided by the PlatformService.  The
+// primary use case is to collect the cluster names, cluster types, platform
+// model, and holos components to build into one resource.
+type Platform struct {
+	TypeMeta `json:",inline" yaml:",inline"`
+	Metadata ObjectMeta   `json:"metadata" yaml:"metadata"`
+	Spec     PlatformSpec `json:"spec" yaml:"spec"`
+}
+
+// PlatformSpec represents the platform build plan specification.
+type PlatformSpec struct {
+	// Model represents the platform model holos gets from from the
+	// holos.platform.v1alpha1.PlatformService.GetPlatform method and provides to
+	// CUE using a tag.
+	Model      structpb.Struct         `json:"model" yaml:"model"`
+	Components []PlatformSpecComponent `json:"components" yaml:"components"`
+}
+
+// PlatformSpecComponent represents a component to build or render with flags to
+// pass, for example the cluster name.
+type PlatformSpecComponent struct {
+	// Path is the path of the component relative to the platform root.
+	Path string `json:"path" yaml:"path"`
+	// Cluster is the cluster name to use when building the component.
+	Cluster string `json:"cluster" yaml:"cluster"`
+}

api/v1alpha1/render.go

@@ -2,12 +2,13 @@ package v1alpha1
 
 import (
 	"context"
+
 	"github.com/holos-run/holos"
 )
 
 type Renderer interface {
 	GetKind() string
-	Render(ctx context.Context, path holos.PathComponent) (*Result, error)
+	Render(ctx context.Context, path holos.InstancePath) (*Result, error)
 }
 
 // Render produces a Result representing the kubernetes api objects to
@@ -16,6 +17,6 @@ type Renderer interface {
 // conceptualized as a data pipeline, for example a component may render a
 // result by first calling helm template, then passing the result through
 // kustomize, then mixing in overlay api objects.
-func Render(ctx context.Context, r Renderer, path holos.PathComponent) (*Result, error) {
+func Render(ctx context.Context, r Renderer, path holos.InstancePath) (*Result, error) {
 	return r.Render(ctx, path)
 }

api/v1alpha1/result.go

@@ -3,12 +3,13 @@ package v1alpha1
 import (
 	"context"
 	"fmt"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
 	"os"
 	"path/filepath"
 	"slices"
+
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 // Result is the build result for display or writing.  Holos components Render the Result as a data pipeline.
@@ -18,6 +19,14 @@ type Result struct {
 	accumulatedOutput string
 }
 
+// Continue returns true if Skip is true indicating the result is to be skipped over.
+func (r *Result) Continue() bool {
+	if r == nil {
+		return false
+	}
+	return r.Skip
+}
+
 func (r *Result) Name() string {
 	return r.Metadata.Name
 }
@@ -31,6 +40,11 @@ func (r *Result) KustomizationFilename(writeTo string, cluster string) string {
 	return filepath.Join(writeTo, "clusters", cluster, "holos", "components", r.Metadata.Name+"-kustomization.gen.yaml")
 }
 
+// KustomizationContent returns the kustomization file contents to write.
+func (r *Result) KustomizationContent() string {
+	return r.KsContent
+}
+
 // AccumulatedOutput returns the accumulated rendered output.
 func (r *Result) AccumulatedOutput() string {
 	return r.accumulatedOutput
@@ -40,7 +54,7 @@ func (r *Result) AccumulatedOutput() string {
 func (r *Result) addObjectMap(ctx context.Context, objectMap APIObjectMap) {
 	log := logger.FromContext(ctx)
 	b := []byte(r.AccumulatedOutput())
-	kinds := make([]string, 0, len(objectMap))
+	kinds := make([]Kind, 0, len(objectMap))
 	// Sort the keys
 	for kind := range objectMap {
 		kinds = append(kinds, kind)
@@ -50,7 +64,7 @@ func (r *Result) addObjectMap(ctx context.Context, objectMap APIObjectMap) {
 	for _, kind := range kinds {
 		v := objectMap[kind]
 		// Sort the keys
-		names := make([]string, 0, len(v))
+		names := make([]Label, 0, len(v))
 		for name := range v {
 			names = append(names, name)
 		}
@@ -81,7 +95,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	}
 	tempDir, err := os.MkdirTemp("", "holos.kustomize")
 	if err != nil {
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	defer util.Remove(ctx, tempDir)
 
@@ -90,7 +104,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	b := []byte(r.AccumulatedOutput())
 	b = util.EnsureNewline(b)
 	if err := os.WriteFile(target, b, 0644); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not write resources: %w", err))
+		return errors.Wrap(fmt.Errorf("could not write resources: %w", err))
 	}
 	log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
 
@@ -98,12 +112,12 @@ func (r *Result) kustomize(ctx context.Context) error {
 	for file, content := range r.KustomizeFiles {
 		target := filepath.Join(tempDir, file)
 		if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
-			return wrapper.Wrap(err)
+			return errors.Wrap(err)
 		}
 		b := []byte(content)
 		b = util.EnsureNewline(b)
 		if err := os.WriteFile(target, b, 0644); err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not write: %w", err))
+			return errors.Wrap(fmt.Errorf("could not write: %w", err))
 		}
 		log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
 	}
@@ -112,7 +126,7 @@ func (r *Result) kustomize(ctx context.Context) error {
 	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", tempDir)
 	if err != nil {
 		log.ErrorContext(ctx, kOut.Stderr.String())
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	// Replace the accumulated output
 	r.accumulatedOutput = kOut.Stdout.String()
@@ -125,12 +139,12 @@ func (r *Result) Save(ctx context.Context, path string, content string) error {
 	dir := filepath.Dir(path)
 	if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
 		log.WarnContext(ctx, "could not mkdir", "path", dir, "err", err)
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	// Write the kube api objects
 	if err := os.WriteFile(path, []byte(content), os.FileMode(0644)); err != nil {
 		log.WarnContext(ctx, "could not write", "path", path, "err", err)
-		return wrapper.Wrap(err)
+		return errors.Wrap(err)
 	}
 	log.DebugContext(ctx, "out: wrote "+path, "action", "write", "path", path, "status", "ok")
 	return nil

api/v1alpha1/typemeta.go

@@ -8,3 +8,13 @@ type TypeMeta struct {
 func (tm *TypeMeta) GetKind() string {
 	return tm.Kind
 }
+
+func (tm *TypeMeta) GetAPIVersion() string {
+	return tm.Kind
+}
+
+// Discriminator is an interface to discriminate the kind api object.
+type Discriminator interface {
+	GetKind() string
+	GetAPIVersion() string
+}

buf.gen.yaml

@@ -0,0 +1,20 @@
+# Generates gRPC and ConnectRPC bindings for Go and TypeScript
+#
+# Note: protoc-gen-connect-query is the primary method of wiring up the React
+# frontend.
+version: v1
+plugins:
+  - plugin: go
+    out: service/gen
+    opt: paths=source_relative
+  - plugin: connect-go
+    out: service/gen
+    opt: paths=source_relative
+  - plugin: es
+    out: internal/frontend/holos/src/app/gen
+    opt:
+      - target=ts
+  - plugin: connect-es
+    out: internal/frontend/holos/src/app/gen
+    opt:
+      - target=ts

buf.lock

@@ -0,0 +1,8 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: bufbuild
+    repository: protovalidate
+    commit: b983156c5e994cc9892e0ce3e64e17e0
+    digest: shake256:fb47a62989d38c2529bcc5cd86ded43d800eb84cee82b42b9e8a9e815d4ee8134a0fb9d0ce8299b27c2d2bbb7d6ade0c4ad5a8a4d467e1e2c7ca619ae9f634e2

buf.work.yaml

@@ -0,0 +1,3 @@
+version: v1
+directories:
+  - service

cmd/holos/main.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"github.com/holos-run/holos/pkg/cli"
 	"os"
+
+	"github.com/holos-run/holos/internal/cli"
 )
 
 func main() {

cmd/holos/main_test.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"github.com/holos-run/holos/pkg/cli"
-	"github.com/rogpeppe/go-internal/testscript"
 	"os"
 	"testing"
+
+	"github.com/holos-run/holos/internal/cli"
+	"github.com/rogpeppe/go-internal/testscript"
 )
 
 func TestMain(m *testing.M) {

cmd/holos/testdata/constraints.txt

@@ -2,6 +2,8 @@
 exec holos build ./foo/... --log-level debug
 stdout '^bf2bc7f9-9ba0-4f9e-9bd2-9a205627eb0b$'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- foo/constraints.cue --
@@ -20,6 +22,7 @@ spec: components: KubernetesObjectsList: [
 package holos
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 #KubernetesObjects: {
 	apiVersion: "holos.run/v1alpha1"

cmd/holos/testdata/issue15_cue_errors.txt

@@ -3,12 +3,15 @@
 stderr 'apiObjectMap.foo.bar: cannot convert incomplete value'
 stderr '/component.cue:\d+:\d+$'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
 package holos
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 apiVersion: "holos.run/v1alpha1"
 kind: "BuildPlan"

cmd/holos/testdata/issue25_apiobjects_cue.txt

@@ -3,6 +3,8 @@ exec holos build .
 stdout '^kind: SecretStore$'
 stdout '# Source: CUE apiObjects.SecretStore.default'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
@@ -13,6 +15,7 @@ kind: "BuildPlan"
 spec: components: KubernetesObjectsList: [{apiObjectMap: #APIObjects.apiObjectMap}]
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 #SecretStore: {
     kind: string

cmd/holos/testdata/issue25_apiobjects_helm.txt

@@ -4,6 +4,8 @@ stdout '^kind: SecretStore$'
 stdout '# Source: CUE apiObjects.SecretStore.default'
 stderr 'skipping helm: no chart name specified'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
@@ -14,6 +16,7 @@ kind: "BuildPlan"
 spec: components: HelmChartList: [{apiObjectMap: #APIObjects.apiObjectMap}]
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 #SecretStore: {
     kind: string

cmd/holos/testdata/issue25_show_object_names.txt

@@ -2,6 +2,8 @@
 ! exec holos build .
 stderr 'apiObjects.secretstore.default.foo: field not allowed'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
@@ -10,6 +12,7 @@ package holos
 apiVersion: "holos.run/v1alpha1"
 kind: "KubernetesObjects"
 cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 #SecretStore: {
     metadata: name: string

cmd/holos/testdata/issue33_helm_stderr.txt

@@ -2,6 +2,8 @@
 ! exec holos build .
 stderr 'Error: execution error at \(zitadel/templates/secret_zitadel-masterkey.yaml:2:4\): Either set .Values.zitadel.masterkey xor .Values.zitadel.masterkeySecretName'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- zitadel.cue --
@@ -12,6 +14,7 @@ kind: "BuildPlan"
 spec: components: HelmChartList: [_HelmChart]
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 _HelmChart: {
   apiVersion: "holos.run/v1alpha1"

cmd/holos/testdata/issue42_kustomize_build_kind.txt

@@ -1,15 +1,18 @@
 # Kustomize is a supported holos component kind
-exec holos render --cluster-name=mycluster . --log-level=debug
+exec holos render component --cluster-name=mycluster . --log-level=debug
 
 # Want generated output
 cmp want.yaml deploy/clusters/mycluster/components/kstest/kstest.gen.yaml
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
 package holos
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 apiVersion: "holos.run/v1alpha1"
 kind: "BuildPlan"

cmd/holos/testdata/issue72_disallow_unknown_fields.txt

@@ -3,11 +3,14 @@
 ! exec holos build .
 stderr 'unknown field \\"TypoKubernetesObjectsList\\"'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
 package holos
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 apiVersion: "holos.run/v1alpha1"
 kind: "BuildPlan"

cmd/holos/testdata/version.txt

@@ -1,5 +1,3 @@
 exec holos --version
 # want version with no v on stdout
 stdout -count=1 '^\d+\.\d+\.\d+$'
-# want nothing on stderr
-! stderr .

docs/examples/api/ListPlatform/listplatform.json

@@ -0,0 +1,10 @@
+{
+  "org_id": "018f36fb-e3f7-7f7f-a1c5-c85fb735d215",
+  "field_mask": {
+    "paths": [
+      "id",
+      "name",
+      "displayName"
+    ]
+  }
+}

docs/examples/api/UpdatePlatform/clearform.json

@@ -0,0 +1,8 @@
+{
+  "update_mask": {
+    "paths": ["form"]
+  },
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94"
+  }
+}

docs/examples/api/UpdatePlatform/model.json

@@ -0,0 +1,11 @@
+{
+  "update_mask": {
+    "paths": ["model","name","display_name"]
+  },
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94",
+    "name": "bareplatform",
+    "display_name": "Bare Platform",
+    "model": {}
+  }
+}

docs/examples/api/UpdatePlatform/nomask.json

@@ -0,0 +1,6 @@
+{
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94",
+    "model": {}
+  }
+}

docs/examples/authpolicy.cue

@@ -0,0 +1,45 @@
+package holos
+
+import ap "security.istio.io/authorizationpolicy/v1"
+
+// #AuthPolicyRules represents AuthorizationPolicy rules for hosts that need
+// specialized treatment.  Entries in this struct are excluded from
+// AuthorizationPolicy/authproxy-custom in the istio-ingress namespace.  Entries
+// are added to their own AuthorizationPolicy.
+#AuthPolicyRules: {
+	// AuthProxySpec represents the identity provider configuration
+	AuthProxySpec: #AuthProxySpec & #Platform.authproxy
+
+	// Hosts are hosts that need specialized treatment
+	hosts: {
+		[Name=_]: {
+			// name is the fully qualifed hostname, a Host: header value.
+			name: Name
+			// slug is the resource name prefix
+			slug: string
+			// NoAuthorizationPolicy disables an AuthorizationPolicy for the host
+			NoAuthorizationPolicy: true | *false
+
+			// Refer to https://istio.io/latest/docs/reference/config/security/authorization-policy/#Rule
+			spec: ap.#AuthorizationPolicySpec & {
+				action: "CUSTOM"
+				provider: name: AuthProxySpec.provider
+				selector: matchLabels: istio: "ingressgateway"
+			}
+		}
+	}
+
+	objects: #APIObjects & {
+		for Host in hosts {
+			if Host.NoAuthorizationPolicy == false {
+				apiObjects: {
+					AuthorizationPolicy: "\(Host.slug)-custom": {
+						metadata: namespace: "istio-ingress"
+						metadata: name:      "\(Host.slug)-custom"
+						spec: Host.spec
+					}
+				}
+			}
+		}
+	}
+}

docs/examples/cue.mod/gen/argoproj.io/appproject/v1alpha1/types_gen.cue

@@ -0,0 +1,189 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-argocd/prod-platform-argocd.gen.yaml
+
+package v1alpha1
+
+import "strings"
+
+// AppProject provides a logical grouping of applications,
+// providing controls for: * where the apps may deploy to
+// (cluster whitelist) * what may be deployed (repository
+// whitelist, resource whitelist/blacklist) * who can access
+// these applications (roles, OIDC group claims bindings) * and
+// what they can do (RBAC policies) * automation access to these
+// roles (JWT tokens)
+#AppProject: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "argoproj.io/v1alpha1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "AppProject"
+	metadata: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// AppProjectSpec is the specification of an AppProject
+	spec!: #AppProjectSpec
+}
+
+// AppProjectSpec is the specification of an AppProject
+#AppProjectSpec: {
+	// ClusterResourceBlacklist contains list of blacklisted cluster
+	// level resources
+	clusterResourceBlacklist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// ClusterResourceWhitelist contains list of whitelisted cluster
+	// level resources
+	clusterResourceWhitelist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// Description contains optional project description
+	description?: string
+
+	// Destinations contains list of destinations available for
+	// deployment
+	destinations?: [...{
+		// Name is an alternate way of specifying the target cluster by
+		// its symbolic name. This must be set if Server is not set.
+		name?: string
+
+		// Namespace specifies the target namespace for the application's
+		// resources. The namespace will only be set for namespace-scoped
+		// resources that have not set a value for .metadata.namespace
+		namespace?: string
+
+		// Server specifies the URL of the target cluster's Kubernetes
+		// control plane API. This must be set if Name is not set.
+		server?: string
+	}]
+
+	// NamespaceResourceBlacklist contains list of blacklisted
+	// namespace level resources
+	namespaceResourceBlacklist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// NamespaceResourceWhitelist contains list of whitelisted
+	// namespace level resources
+	namespaceResourceWhitelist?: [...{
+		group: string
+		kind:  string
+	}]
+
+	// OrphanedResources specifies if controller should monitor
+	// orphaned resources of apps in this project
+	orphanedResources?: {
+		// Ignore contains a list of resources that are to be excluded
+		// from orphaned resources monitoring
+		ignore?: [...{
+			group?: string
+			kind?:  string
+			name?:  string
+		}]
+
+		// Warn indicates if warning condition should be created for apps
+		// which have orphaned resources
+		warn?: bool
+	}
+
+	// PermitOnlyProjectScopedClusters determines whether destinations
+	// can only reference clusters which are project-scoped
+	permitOnlyProjectScopedClusters?: bool
+
+	// Roles are user defined RBAC roles associated with this project
+	roles?: [...{
+		// Description is a description of the role
+		description?: string
+
+		// Groups are a list of OIDC group claims bound to this role
+		groups?: [...string]
+
+		// JWTTokens are a list of generated JWT tokens bound to this role
+		jwtTokens?: [...{
+			exp?: int
+			iat:  int
+			id?:  string
+		}]
+
+		// Name is a name for this role
+		name: string
+
+		// Policies Stores a list of casbin formatted strings that define
+		// access policies for the role in the project
+		policies?: [...string]
+	}]
+
+	// SignatureKeys contains a list of PGP key IDs that commits in
+	// Git must be signed with in order to be allowed for sync
+	signatureKeys?: [...{
+		// The ID of the key in hexadecimal notation
+		keyID: string
+	}]
+
+	// SourceNamespaces defines the namespaces application resources
+	// are allowed to be created in
+	sourceNamespaces?: [...string]
+
+	// SourceRepos contains list of repository URLs which can be used
+	// for deployment
+	sourceRepos?: [...string]
+
+	// SyncWindows controls when syncs can be run for apps in this
+	// project
+	syncWindows?: [...{
+		// Applications contains a list of applications that the window
+		// will apply to
+		applications?: [...string]
+
+		// Clusters contains a list of clusters that the window will apply
+		// to
+		clusters?: [...string]
+
+		// Duration is the amount of time the sync window will be open
+		duration?: string
+
+		// Kind defines if the window allows or blocks syncs
+		kind?: string
+
+		// ManualSync enables manual syncs when they would otherwise be
+		// blocked
+		manualSync?: bool
+
+		// Namespaces contains a list of namespaces that the window will
+		// apply to
+		namespaces?: [...string]
+
+		// Schedule is the time the window will begin, specified in cron
+		// format
+		schedule?: string
+
+		// TimeZone of the sync that will be applied to the schedule
+		timeZone?: string
+	}]
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/buildplan_go_gen.cue

@@ -19,7 +19,8 @@ package v1alpha1
 }
 
 #BuildPlanComponents: {
-	helmCharts?: [...#HelmChart] @go(HelmCharts,[]HelmChart)
-	kubernetesObjects?: [...#KubernetesObjects] @go(KubernetesObjects,[]KubernetesObjects)
-	kustomizeBuilds?: [...#KustomizeBuild] @go(KustomizeBuilds,[]KustomizeBuild)
+	helmChartList?: [...#HelmChart] @go(HelmChartList,[]HelmChart)
+	kubernetesObjectsList?: [...#KubernetesObjects] @go(KubernetesObjectsList,[]KubernetesObjects)
+	kustomizeBuildList?: [...#KustomizeBuild] @go(KustomizeBuildList,[]KustomizeBuild)
+	resources?: {[string]: #KubernetesObjects} @go(Resources,map[string]KubernetesObjects)
 }

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/constants_go_gen.cue

@@ -11,5 +11,5 @@ package v1alpha1
 // ChartDir is the directory name created in the holos component directory to cache a chart.
 #ChartDir: "vendor"
 
-// ResourcesFile is the file name used to store component output when post processing with kustomize.
+// ResourcesFile is the file name used to store component output when post-processing with kustomize.
 #ResourcesFile: "resources.yaml"

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/objectmap_go_gen.cue

@@ -4,6 +4,12 @@
 
 package v1alpha1
 
+// Label is an arbitrary unique identifier.  Defined as a type for clarity and type checking.
+#Label: string
+
+// Kind is a kubernetes api object kind. Defined as a type for clarity and type checking.
+#Kind: string
+
 // APIObjectMap is the shape of marshalled api objects returned from cue to the
 // holos cli. A map is used to improve the clarity of error messages from cue.
 #APIObjectMap: {[string]: [string]: string}

docs/examples/cue.mod/gen/monitoring.coreos.com/podmonitor/v1/types_gen.cue

@@ -0,0 +1,546 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// PodMonitor defines monitoring for a set of pods.
+#PodMonitor: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "PodMonitor"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Pod selection for target discovery by
+	// Prometheus.
+	spec!: #PodMonitorSpec
+}
+
+// Specification of desired Pod selection for target discovery by
+// Prometheus.
+#PodMonitorSpec: {
+	attachMetadata?: {
+		// When set to true, Prometheus must have the `get` permission on
+		// the `Nodes` objects.
+		node?: bool
+	}
+
+	// The label to use to retrieve the job name from. `jobLabel`
+	// selects the label from the associated Kubernetes `Pod` object
+	// which will be used as the `job` label for all metrics.
+	// For example if `jobLabel` is set to `foo` and the Kubernetes
+	// `Pod` object is labeled with `foo: bar`, then Prometheus adds
+	// the `job="bar"` label to all ingested metrics.
+	// If the value of this field is empty, the `job` label of the
+	// metrics defaults to the namespace and name of the PodMonitor
+	// object (e.g. `<namespace>/<name>`).
+	jobLabel?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelValueLengthLimit?: int
+
+	// Selector to select which namespaces the Kubernetes `Pods`
+	// objects are discovered from.
+	namespaceSelector?: {
+		// Boolean describing whether all namespaces are selected in
+		// contrast to a list restricting them.
+		any?: bool
+
+		// List of namespace names to select from.
+		matchNames?: [...string]
+	}
+
+	// List of endpoints part of this PodMonitor.
+	podMetricsEndpoints?: [...{
+		// `authorization` configures the Authorization header credentials
+		// to use when scraping the target.
+		// Cannot be set at the same time as `basicAuth`, or `oauth2`.
+		authorization?: {
+			// Selects a key of a Secret in the namespace that contains the
+			// credentials for authentication.
+			credentials?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Defines the authentication type. The value is case-insensitive.
+			// "Basic" is not a supported value.
+			// Default: "Bearer"
+			type?: string
+		}
+
+		// `basicAuth` configures the Basic Authentication credentials to
+		// use when scraping the target.
+		// Cannot be set at the same time as `authorization`, or `oauth2`.
+		basicAuth?: {
+			// `password` specifies a key of a Secret containing the password
+			// for authentication.
+			password?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `username` specifies a key of a Secret containing the username
+			// for authentication.
+			username?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// `bearerTokenSecret` specifies a key of a Secret containing the
+		// bearer token for scraping targets. The secret needs to be in
+		// the same namespace as the PodMonitor object and readable by
+		// the Prometheus Operator.
+		// Deprecated: use `authorization` instead.
+		bearerTokenSecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `enableHttp2` can be used to disable HTTP2 when scraping the
+		// target.
+		enableHttp2?: bool
+
+		// When true, the pods which are not running (e.g. either in
+		// Failed or Succeeded state) are dropped during the target
+		// discovery.
+		// If unset, the filtering is enabled.
+		// More info:
+		// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+		filterRunning?: bool
+
+		// `followRedirects` defines whether the scrape requests should
+		// follow HTTP 3xx redirects.
+		followRedirects?: bool
+
+		// When true, `honorLabels` preserves the metric's labels when
+		// they collide with the target's labels.
+		honorLabels?: bool
+
+		// `honorTimestamps` controls whether Prometheus preserves the
+		// timestamps when exposed by the target.
+		honorTimestamps?: bool
+
+		// Interval at which Prometheus scrapes the metrics from the
+		// target.
+		// If empty, Prometheus uses the global scrape interval.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// `metricRelabelings` configures the relabeling rules to apply to
+		// the samples before ingestion.
+		metricRelabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// `oauth2` configures the OAuth2 settings to use when scraping
+		// the target.
+		// It requires Prometheus >= 2.27.0.
+		// Cannot be set at the same time as `authorization`, or
+		// `basicAuth`.
+		oauth2?: {
+			// `clientId` specifies a key of a Secret or ConfigMap containing
+			// the OAuth2 client's ID.
+			clientId: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// `clientSecret` specifies a key of a Secret containing the
+			// OAuth2 client's secret.
+			clientSecret: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `endpointParams` configures the HTTP parameters to append to
+			// the token URL.
+			endpointParams?: {
+				[string]: string
+			}
+
+			// `scopes` defines the OAuth2 scopes used for the token request.
+			scopes?: [...string]
+
+			// `tokenURL` configures the URL to fetch the token from.
+			tokenUrl: strings.MinRunes(1)
+		}
+
+		// `params` define optional HTTP URL parameters.
+		params?: {
+			[string]: [...string]
+		}
+
+		// HTTP path from which to scrape for metrics.
+		// If empty, Prometheus uses the default value (e.g. `/metrics`).
+		path?: string
+
+		// Name of the Pod port which this endpoint refers to.
+		// It takes precedence over `targetPort`.
+		port?: string
+
+		// `proxyURL` configures the HTTP Proxy URL (e.g.
+		// "http://proxyserver:2195") to go through when scraping the
+		// target.
+		proxyUrl?: string
+
+		// `relabelings` configures the relabeling rules to apply the
+		// target's metadata labels.
+		// The Operator automatically adds relabelings for a few standard
+		// Kubernetes fields.
+		// The original scrape job's name is available via the
+		// `__tmp_prometheus_job_name` label.
+		// More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+		relabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// HTTP scheme to use for scraping.
+		// `http` and `https` are the expected values unless you rewrite
+		// the `__scheme__` label via relabeling.
+		// If empty, Prometheus uses the default value `http`.
+		scheme?: "http" | "https"
+
+		// Timeout after which Prometheus considers the scrape to be
+		// failed.
+		// If empty, Prometheus uses the global scrape timeout unless it
+		// is less than the target's scrape interval value in which the
+		// latter is used.
+		scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Name or number of the target port of the `Pod` object behind
+		// the Service, the port must be specified with container port
+		// property.
+		// Deprecated: use 'port' instead.
+		targetPort?: (int | string) & {
+			string
+		}
+
+		// TLS configuration to use when scraping the target.
+		tlsConfig?: {
+			// Certificate authority used when verifying server certificates.
+			ca?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Client certificate to present when doing client-authentication.
+			cert?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Disable target certificate validation.
+			insecureSkipVerify?: bool
+
+			// Secret containing the client key file for the targets.
+			keySecret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Used to verify the hostname for the targets.
+			serverName?: string
+		}
+
+		// `trackTimestampsStaleness` defines whether Prometheus tracks
+		// staleness of the metrics that have an explicit timestamp
+		// present in scraped data. Has no effect if `honorTimestamps` is
+		// false.
+		// It requires Prometheus >= v2.48.0.
+		trackTimestampsStaleness?: bool
+	}]
+
+	// `podTargetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Pod` object onto the ingested
+	// metrics.
+	podTargetLabels?: [...string]
+
+	// `sampleLimit` defines a per-scrape limit on the number of
+	// scraped samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Label selector to select the Kubernetes `Pod` objects.
+	selector: {
+		// matchExpressions is a list of label selector requirements. The
+		// requirements are ANDed.
+		matchExpressions?: [...{
+			// key is the label key that the selector applies to.
+			key: string
+
+			// operator represents a key's relationship to a set of values.
+			// Valid operators are In, NotIn, Exists and DoesNotExist.
+			operator: string
+
+			// values is an array of string values. If the operator is In or
+			// NotIn, the values array must be non-empty. If the operator is
+			// Exists or DoesNotExist, the values array must be empty. This
+			// array is replaced during a strategic merge patch.
+			values?: [...string]
+		}]
+
+		// matchLabels is a map of {key,value} pairs. A single {key,value}
+		// in the matchLabels map is equivalent to an element of
+		// matchExpressions, whose key field is "key", the operator is
+		// "In", and the values array contains only "value". The
+		// requirements are ANDed.
+		matchLabels?: {
+			[string]: string
+		}
+	}
+
+	// `targetLimit` defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/probe/v1/types_gen.cue

@@ -0,0 +1,536 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// Probe defines monitoring for a set of static targets or
+// ingresses.
+#Probe: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "Probe"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Ingress selection for target discovery
+	// by Prometheus.
+	spec!: #ProbeSpec
+}
+
+// Specification of desired Ingress selection for target discovery
+// by Prometheus.
+#ProbeSpec: {
+	// Authorization section for this endpoint
+	authorization?: {
+		// Selects a key of a Secret in the namespace that contains the
+		// credentials for authentication.
+		credentials?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// Defines the authentication type. The value is case-insensitive.
+		// "Basic" is not a supported value.
+		// Default: "Bearer"
+		type?: string
+	}
+
+	// BasicAuth allow an endpoint to authenticate over basic
+	// authentication. More info:
+	// https://prometheus.io/docs/operating/configuration/#endpoint
+	basicAuth?: {
+		// `password` specifies a key of a Secret containing the password
+		// for authentication.
+		password?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `username` specifies a key of a Secret containing the username
+		// for authentication.
+		username?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+	}
+
+	// Secret to mount to read bearer token for scraping targets. The
+	// secret needs to be in the same namespace as the probe and
+	// accessible by the Prometheus Operator.
+	bearerTokenSecret?: {
+		// The key of the secret to select from. Must be a valid secret
+		// key.
+		key: string
+
+		// Name of the referent. More info:
+		// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+		// TODO: Add other useful fields. apiVersion, kind, uid?
+		name?: string
+
+		// Specify whether the Secret or its key must be defined
+		optional?: bool
+	}
+
+	// Interval at which targets are probed using the configured
+	// prober. If not specified Prometheus' global scrape interval is
+	// used.
+	interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+	// The job name assigned to scraped metrics by default.
+	jobName?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample. Only valid in Prometheus versions 2.27.0 and newer.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample. Only valid in Prometheus versions 2.27.0 and
+	// newer.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample. Only valid in Prometheus versions
+	// 2.27.0 and newer.
+	labelValueLengthLimit?: int
+
+	// MetricRelabelConfigs to apply to samples before ingestion.
+	metricRelabelings?: [...{
+		// Action to perform based on the regex matching.
+		// `Uppercase` and `Lowercase` actions require Prometheus >=
+		// v2.36.0. `DropEqual` and `KeepEqual` actions require
+		// Prometheus >= v2.41.0.
+		// Default: "Replace"
+		action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+		// Modulus to take of the hash of the source label values.
+		// Only applicable when the action is `HashMod`.
+		modulus?: int
+
+		// Regular expression against which the extracted value is
+		// matched.
+		regex?: string
+
+		// Replacement value against which a Replace action is performed
+		// if the regular expression matches.
+		// Regex capture groups are available.
+		replacement?: string
+
+		// Separator is the string between concatenated SourceLabels.
+		separator?: string
+
+		// The source labels select values from existing labels. Their
+		// content is concatenated using the configured Separator and
+		// matched against the configured regular expression.
+		sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+		// Label to which the resulting string is written in a
+		// replacement.
+		// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+		// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+		// Regex capture groups are available.
+		targetLabel?: string
+	}]
+
+	// The module to use for probing specifying how to probe the
+	// target. Example module configuring in the blackbox exporter:
+	// https://github.com/prometheus/blackbox_exporter/blob/master/example.yml
+	module?: string
+
+	// OAuth2 for the URL. Only valid in Prometheus versions 2.27.0
+	// and newer.
+	oauth2?: {
+		// `clientId` specifies a key of a Secret or ConfigMap containing
+		// the OAuth2 client's ID.
+		clientId: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// `clientSecret` specifies a key of a Secret containing the
+		// OAuth2 client's secret.
+		clientSecret: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `endpointParams` configures the HTTP parameters to append to
+		// the token URL.
+		endpointParams?: {
+			[string]: string
+		}
+
+		// `scopes` defines the OAuth2 scopes used for the token request.
+		scopes?: [...string]
+
+		// `tokenURL` configures the URL to fetch the token from.
+		tokenUrl: strings.MinRunes(1)
+	}
+
+	// Specification for the prober to use for probing targets. The
+	// prober.URL parameter is required. Targets cannot be probed if
+	// left empty.
+	prober?: {
+		// Path to collect metrics from. Defaults to `/probe`.
+		path?: string | *"/probe"
+
+		// Optional ProxyURL.
+		proxyUrl?: string
+
+		// HTTP scheme to use for scraping. `http` and `https` are the
+		// expected values unless you rewrite the `__scheme__` label via
+		// relabeling. If empty, Prometheus uses the default value
+		// `http`.
+		scheme?: "http" | "https"
+
+		// Mandatory URL of the prober.
+		url: string
+	}
+
+	// SampleLimit defines per-scrape limit on number of scraped
+	// samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Timeout for scraping metrics from the Prometheus exporter. If
+	// not specified, the Prometheus global scrape timeout is used.
+	scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+	// TargetLimit defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+
+	// Targets defines a set of static or dynamically discovered
+	// targets to probe.
+	targets?: {
+		// ingress defines the Ingress objects to probe and the relabeling
+		// configuration. If `staticConfig` is also defined,
+		// `staticConfig` takes precedence.
+		ingress?: {
+			// From which namespaces to select Ingress objects.
+			namespaceSelector?: {
+				// Boolean describing whether all namespaces are selected in
+				// contrast to a list restricting them.
+				any?: bool
+
+				// List of namespace names to select from.
+				matchNames?: [...string]
+			}
+
+			// RelabelConfigs to apply to the label set of the target before
+			// it gets scraped. The original ingress address is available via
+			// the `__tmp_prometheus_ingress_address` label. It can be used
+			// to customize the probed URL. The original scrape job's name is
+			// available via the `__tmp_prometheus_job_name` label. More
+			// info:
+			// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+			relabelingConfigs?: [...{
+				// Action to perform based on the regex matching.
+				// `Uppercase` and `Lowercase` actions require Prometheus >=
+				// v2.36.0. `DropEqual` and `KeepEqual` actions require
+				// Prometheus >= v2.41.0.
+				// Default: "Replace"
+				action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+				// Modulus to take of the hash of the source label values.
+				// Only applicable when the action is `HashMod`.
+				modulus?: int
+
+				// Regular expression against which the extracted value is
+				// matched.
+				regex?: string
+
+				// Replacement value against which a Replace action is performed
+				// if the regular expression matches.
+				// Regex capture groups are available.
+				replacement?: string
+
+				// Separator is the string between concatenated SourceLabels.
+				separator?: string
+
+				// The source labels select values from existing labels. Their
+				// content is concatenated using the configured Separator and
+				// matched against the configured regular expression.
+				sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+				// Label to which the resulting string is written in a
+				// replacement.
+				// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+				// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+				// Regex capture groups are available.
+				targetLabel?: string
+			}]
+
+			// Selector to select the Ingress objects.
+			selector?: {
+				// matchExpressions is a list of label selector requirements. The
+				// requirements are ANDed.
+				matchExpressions?: [...{
+					// key is the label key that the selector applies to.
+					key: string
+
+					// operator represents a key's relationship to a set of values.
+					// Valid operators are In, NotIn, Exists and DoesNotExist.
+					operator: string
+
+					// values is an array of string values. If the operator is In or
+					// NotIn, the values array must be non-empty. If the operator is
+					// Exists or DoesNotExist, the values array must be empty. This
+					// array is replaced during a strategic merge patch.
+					values?: [...string]
+				}]
+
+				// matchLabels is a map of {key,value} pairs. A single {key,value}
+				// in the matchLabels map is equivalent to an element of
+				// matchExpressions, whose key field is "key", the operator is
+				// "In", and the values array contains only "value". The
+				// requirements are ANDed.
+				matchLabels?: {
+					[string]: string
+				}
+			}
+		}
+
+		// staticConfig defines the static list of targets to probe and
+		// the relabeling configuration. If `ingress` is also defined,
+		// `staticConfig` takes precedence. More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config.
+		staticConfig?: {
+			// Labels assigned to all metrics scraped from the targets.
+			labels?: {
+				[string]: string
+			}
+
+			// RelabelConfigs to apply to the label set of the targets before
+			// it gets scraped. More info:
+			// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+			relabelingConfigs?: [...{
+				// Action to perform based on the regex matching.
+				// `Uppercase` and `Lowercase` actions require Prometheus >=
+				// v2.36.0. `DropEqual` and `KeepEqual` actions require
+				// Prometheus >= v2.41.0.
+				// Default: "Replace"
+				action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+				// Modulus to take of the hash of the source label values.
+				// Only applicable when the action is `HashMod`.
+				modulus?: int
+
+				// Regular expression against which the extracted value is
+				// matched.
+				regex?: string
+
+				// Replacement value against which a Replace action is performed
+				// if the regular expression matches.
+				// Regex capture groups are available.
+				replacement?: string
+
+				// Separator is the string between concatenated SourceLabels.
+				separator?: string
+
+				// The source labels select values from existing labels. Their
+				// content is concatenated using the configured Separator and
+				// matched against the configured regular expression.
+				sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+				// Label to which the resulting string is written in a
+				// replacement.
+				// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+				// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+				// Regex capture groups are available.
+				targetLabel?: string
+			}]
+
+			// The list of hosts to probe.
+			static?: [...string]
+		}
+	}
+
+	// TLS configuration to use when scraping the endpoint.
+	tlsConfig?: {
+		// Certificate authority used when verifying server certificates.
+		ca?: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// Client certificate to present when doing client-authentication.
+		cert?: {
+			// ConfigMap containing data to use for the targets.
+			configMap?: {
+				// The key to select.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the ConfigMap or its key must be defined
+				optional?: bool
+			}
+
+			// Secret containing data to use for the targets.
+			secret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// Disable target certificate validation.
+		insecureSkipVerify?: bool
+
+		// Secret containing the client key file for the targets.
+		keySecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// Used to verify the hostname for the targets.
+		serverName?: string
+	}
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/prometheusrule/v1/types_gen.cue

@@ -0,0 +1,100 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// PrometheusRule defines recording and alerting rules for a
+// Prometheus instance
+#PrometheusRule: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "PrometheusRule"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired alerting rule definitions for
+	// Prometheus.
+	spec!: #PrometheusRuleSpec
+}
+#PrometheusRuleSpec: {
+	// Content of Prometheus rule file
+	groups?: [...{
+		// Interval determines how often rules in the group are evaluated.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Limit the number of alerts an alerting rule and series a
+		// recording rule can produce. Limit is supported starting with
+		// Prometheus >= 2.31 and Thanos Ruler >= 0.24.
+		limit?: int
+
+		// Name of the rule group.
+		name: strings.MinRunes(1)
+
+		// PartialResponseStrategy is only used by ThanosRuler and will be
+		// ignored by Prometheus instances. More info:
+		// https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response
+		partial_response_strategy?: =~"^(?i)(abort|warn)?$"
+
+		// List of alerting and recording rules.
+		rules?: [...{
+			// Name of the alert. Must be a valid label value. Only one of
+			// `record` and `alert` must be set.
+			alert?: string
+
+			// Annotations to add to each alert. Only valid for alerting
+			// rules.
+			annotations?: {
+				[string]: string
+			}
+
+			// PromQL expression to evaluate.
+			expr: (int | string) & {
+				string
+			}
+
+			// Alerts are considered firing once they have been returned for
+			// this long.
+			for?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+			// KeepFiringFor defines how long an alert will continue firing
+			// after the condition that triggered it has cleared.
+			keep_firing_for?: strings.MinRunes(1) & {
+				=~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+			}
+
+			// Labels to add or overwrite.
+			labels?: {
+				[string]: string
+			}
+
+			// Name of the time series to output to. Must be a valid metric
+			// name. Only one of `record` and `alert` must be set.
+			record?: string
+		}]
+	}]
+}

docs/examples/cue.mod/gen/monitoring.coreos.com/servicemonitor/v1/types_gen.cue

@@ -0,0 +1,566 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/k2/components/prod-platform-monitoring/prod-platform-monitoring.gen.yaml
+
+package v1
+
+import "strings"
+
+// ServiceMonitor defines monitoring for a set of services.
+#ServiceMonitor: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "monitoring.coreos.com/v1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "ServiceMonitor"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// Specification of desired Service selection for target discovery
+	// by Prometheus.
+	spec!: #ServiceMonitorSpec
+}
+
+// Specification of desired Service selection for target discovery
+// by Prometheus.
+#ServiceMonitorSpec: {
+	attachMetadata?: {
+		// When set to true, Prometheus must have the `get` permission on
+		// the `Nodes` objects.
+		node?: bool
+	}
+
+	// List of endpoints part of this ServiceMonitor.
+	endpoints?: [...{
+		// `authorization` configures the Authorization header credentials
+		// to use when scraping the target.
+		// Cannot be set at the same time as `basicAuth`, or `oauth2`.
+		authorization?: {
+			// Selects a key of a Secret in the namespace that contains the
+			// credentials for authentication.
+			credentials?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Defines the authentication type. The value is case-insensitive.
+			// "Basic" is not a supported value.
+			// Default: "Bearer"
+			type?: string
+		}
+
+		// `basicAuth` configures the Basic Authentication credentials to
+		// use when scraping the target.
+		// Cannot be set at the same time as `authorization`, or `oauth2`.
+		basicAuth?: {
+			// `password` specifies a key of a Secret containing the password
+			// for authentication.
+			password?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `username` specifies a key of a Secret containing the username
+			// for authentication.
+			username?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+		}
+
+		// File to read bearer token for scraping the target.
+		// Deprecated: use `authorization` instead.
+		bearerTokenFile?: string
+
+		// `bearerTokenSecret` specifies a key of a Secret containing the
+		// bearer token for scraping targets. The secret needs to be in
+		// the same namespace as the ServiceMonitor object and readable
+		// by the Prometheus Operator.
+		// Deprecated: use `authorization` instead.
+		bearerTokenSecret?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			// TODO: Add other useful fields. apiVersion, kind, uid?
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// `enableHttp2` can be used to disable HTTP2 when scraping the
+		// target.
+		enableHttp2?: bool
+
+		// When true, the pods which are not running (e.g. either in
+		// Failed or Succeeded state) are dropped during the target
+		// discovery.
+		// If unset, the filtering is enabled.
+		// More info:
+		// https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+		filterRunning?: bool
+
+		// `followRedirects` defines whether the scrape requests should
+		// follow HTTP 3xx redirects.
+		followRedirects?: bool
+
+		// When true, `honorLabels` preserves the metric's labels when
+		// they collide with the target's labels.
+		honorLabels?: bool
+
+		// `honorTimestamps` controls whether Prometheus preserves the
+		// timestamps when exposed by the target.
+		honorTimestamps?: bool
+
+		// Interval at which Prometheus scrapes the metrics from the
+		// target.
+		// If empty, Prometheus uses the global scrape interval.
+		interval?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// `metricRelabelings` configures the relabeling rules to apply to
+		// the samples before ingestion.
+		metricRelabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// `oauth2` configures the OAuth2 settings to use when scraping
+		// the target.
+		// It requires Prometheus >= 2.27.0.
+		// Cannot be set at the same time as `authorization`, or
+		// `basicAuth`.
+		oauth2?: {
+			// `clientId` specifies a key of a Secret or ConfigMap containing
+			// the OAuth2 client's ID.
+			clientId: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// `clientSecret` specifies a key of a Secret containing the
+			// OAuth2 client's secret.
+			clientSecret: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// `endpointParams` configures the HTTP parameters to append to
+			// the token URL.
+			endpointParams?: {
+				[string]: string
+			}
+
+			// `scopes` defines the OAuth2 scopes used for the token request.
+			scopes?: [...string]
+
+			// `tokenURL` configures the URL to fetch the token from.
+			tokenUrl: strings.MinRunes(1)
+		}
+
+		// params define optional HTTP URL parameters.
+		params?: {
+			[string]: [...string]
+		}
+
+		// HTTP path from which to scrape for metrics.
+		// If empty, Prometheus uses the default value (e.g. `/metrics`).
+		path?: string
+
+		// Name of the Service port which this endpoint refers to.
+		// It takes precedence over `targetPort`.
+		port?: string
+
+		// `proxyURL` configures the HTTP Proxy URL (e.g.
+		// "http://proxyserver:2195") to go through when scraping the
+		// target.
+		proxyUrl?: string
+
+		// `relabelings` configures the relabeling rules to apply the
+		// target's metadata labels.
+		// The Operator automatically adds relabelings for a few standard
+		// Kubernetes fields.
+		// The original scrape job's name is available via the
+		// `__tmp_prometheus_job_name` label.
+		// More info:
+		// https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+		relabelings?: [...{
+			// Action to perform based on the regex matching.
+			// `Uppercase` and `Lowercase` actions require Prometheus >=
+			// v2.36.0. `DropEqual` and `KeepEqual` actions require
+			// Prometheus >= v2.41.0.
+			// Default: "Replace"
+			action?: "replace" | "Replace" | "keep" | "Keep" | "drop" | "Drop" | "hashmod" | "HashMod" | "labelmap" | "LabelMap" | "labeldrop" | "LabelDrop" | "labelkeep" | "LabelKeep" | "lowercase" | "Lowercase" | "uppercase" | "Uppercase" | "keepequal" | "KeepEqual" | "dropequal" | "DropEqual" | *"replace"
+
+			// Modulus to take of the hash of the source label values.
+			// Only applicable when the action is `HashMod`.
+			modulus?: int
+
+			// Regular expression against which the extracted value is
+			// matched.
+			regex?: string
+
+			// Replacement value against which a Replace action is performed
+			// if the regular expression matches.
+			// Regex capture groups are available.
+			replacement?: string
+
+			// Separator is the string between concatenated SourceLabels.
+			separator?: string
+
+			// The source labels select values from existing labels. Their
+			// content is concatenated using the configured Separator and
+			// matched against the configured regular expression.
+			sourceLabels?: [...=~"^[a-zA-Z_][a-zA-Z0-9_]*$"]
+
+			// Label to which the resulting string is written in a
+			// replacement.
+			// It is mandatory for `Replace`, `HashMod`, `Lowercase`,
+			// `Uppercase`, `KeepEqual` and `DropEqual` actions.
+			// Regex capture groups are available.
+			targetLabel?: string
+		}]
+
+		// HTTP scheme to use for scraping.
+		// `http` and `https` are the expected values unless you rewrite
+		// the `__scheme__` label via relabeling.
+		// If empty, Prometheus uses the default value `http`.
+		scheme?: "http" | "https"
+
+		// Timeout after which Prometheus considers the scrape to be
+		// failed.
+		// If empty, Prometheus uses the global scrape timeout unless it
+		// is less than the target's scrape interval value in which the
+		// latter is used.
+		scrapeTimeout?: =~"^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$"
+
+		// Name or number of the target port of the `Pod` object behind
+		// the Service. The port must be specified with the container's
+		// port property.
+		targetPort?: (int | string) & {
+			string
+		}
+
+		// TLS configuration to use when scraping the target.
+		tlsConfig?: {
+			// Certificate authority used when verifying server certificates.
+			ca?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Path to the CA cert in the Prometheus container to use for the
+			// targets.
+			caFile?: string
+
+			// Client certificate to present when doing client-authentication.
+			cert?: {
+				// ConfigMap containing data to use for the targets.
+				configMap?: {
+					// The key to select.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the ConfigMap or its key must be defined
+					optional?: bool
+				}
+
+				// Secret containing data to use for the targets.
+				secret?: {
+					// The key of the secret to select from. Must be a valid secret
+					// key.
+					key: string
+
+					// Name of the referent. More info:
+					// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+					// TODO: Add other useful fields. apiVersion, kind, uid?
+					name?: string
+
+					// Specify whether the Secret or its key must be defined
+					optional?: bool
+				}
+			}
+
+			// Path to the client cert file in the Prometheus container for
+			// the targets.
+			certFile?: string
+
+			// Disable target certificate validation.
+			insecureSkipVerify?: bool
+
+			// Path to the client key file in the Prometheus container for the
+			// targets.
+			keyFile?: string
+
+			// Secret containing the client key file for the targets.
+			keySecret?: {
+				// The key of the secret to select from. Must be a valid secret
+				// key.
+				key: string
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				// TODO: Add other useful fields. apiVersion, kind, uid?
+				name?: string
+
+				// Specify whether the Secret or its key must be defined
+				optional?: bool
+			}
+
+			// Used to verify the hostname for the targets.
+			serverName?: string
+		}
+
+		// `trackTimestampsStaleness` defines whether Prometheus tracks
+		// staleness of the metrics that have an explicit timestamp
+		// present in scraped data. Has no effect if `honorTimestamps` is
+		// false.
+		// It requires Prometheus >= v2.48.0.
+		trackTimestampsStaleness?: bool
+	}]
+
+	// `jobLabel` selects the label from the associated Kubernetes
+	// `Service` object which will be used as the `job` label for all
+	// metrics.
+	// For example if `jobLabel` is set to `foo` and the Kubernetes
+	// `Service` object is labeled with `foo: bar`, then Prometheus
+	// adds the `job="bar"` label to all ingested metrics.
+	// If the value of this field is empty or if the label doesn't
+	// exist for the given Service, the `job` label of the metrics
+	// defaults to the name of the associated Kubernetes `Service`.
+	jobLabel?: string
+
+	// Per-scrape limit on the number of targets dropped by relabeling
+	// that will be kept in memory. 0 means no limit.
+	// It requires Prometheus >= v2.47.0.
+	keepDroppedTargets?: int
+
+	// Per-scrape limit on number of labels that will be accepted for
+	// a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelLimit?: int
+
+	// Per-scrape limit on length of labels name that will be accepted
+	// for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelNameLengthLimit?: int
+
+	// Per-scrape limit on length of labels value that will be
+	// accepted for a sample.
+	// It requires Prometheus >= v2.27.0.
+	labelValueLengthLimit?: int
+
+	// Selector to select which namespaces the Kubernetes `Endpoints`
+	// objects are discovered from.
+	namespaceSelector?: {
+		// Boolean describing whether all namespaces are selected in
+		// contrast to a list restricting them.
+		any?: bool
+
+		// List of namespace names to select from.
+		matchNames?: [...string]
+	}
+
+	// `podTargetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Pod` object onto the ingested
+	// metrics.
+	podTargetLabels?: [...string]
+
+	// `sampleLimit` defines a per-scrape limit on the number of
+	// scraped samples that will be accepted.
+	sampleLimit?: int
+
+	// The scrape class to apply.
+	scrapeClass?: strings.MinRunes(1)
+
+	// `scrapeProtocols` defines the protocols to negotiate during a
+	// scrape. It tells clients the protocols supported by Prometheus
+	// in order of preference (from most to least preferred).
+	// If unset, Prometheus uses its default value.
+	// It requires Prometheus >= v2.49.0.
+	scrapeProtocols?: [..."PrometheusProto" | "OpenMetricsText0.0.1" | "OpenMetricsText1.0.0" | "PrometheusText0.0.4"]
+
+	// Label selector to select the Kubernetes `Endpoints` objects.
+	selector: {
+		// matchExpressions is a list of label selector requirements. The
+		// requirements are ANDed.
+		matchExpressions?: [...{
+			// key is the label key that the selector applies to.
+			key: string
+
+			// operator represents a key's relationship to a set of values.
+			// Valid operators are In, NotIn, Exists and DoesNotExist.
+			operator: string
+
+			// values is an array of string values. If the operator is In or
+			// NotIn, the values array must be non-empty. If the operator is
+			// Exists or DoesNotExist, the values array must be empty. This
+			// array is replaced during a strategic merge patch.
+			values?: [...string]
+		}]
+
+		// matchLabels is a map of {key,value} pairs. A single {key,value}
+		// in the matchLabels map is equivalent to an element of
+		// matchExpressions, whose key field is "key", the operator is
+		// "In", and the values array contains only "value". The
+		// requirements are ANDed.
+		matchLabels?: {
+			[string]: string
+		}
+	}
+
+	// `targetLabels` defines the labels which are transferred from
+	// the associated Kubernetes `Service` object onto the ingested
+	// metrics.
+	targetLabels?: [...string]
+
+	// `targetLimit` defines a limit on the number of scraped targets
+	// that will be accepted.
+	targetLimit?: int
+}

docs/examples/cue.mod/gen/networking.istio.io/virtualservice/v1beta1/types_gen.cue

@@ -306,19 +306,10 @@ import "strings"
 			// "value"` for prefix-based match - `regex: "value"` for RE2
 			// style regex-based match
 			// (https://github.com/google/re2/wiki/Syntax).
-			uri?: ({} | {
-				exact: _
-			} | {
-				prefix: _
-			} | {
-				regex: _
-			}) & {
+			uri?: {
 				exact?:  string
 				prefix?: string
-
-				// RE2 style regex-based match
-				// (https://github.com/google/re2/wiki/Syntax).
-				regex?: string
+				regex?:  string
 			}
 
 			// withoutHeader has the same syntax with the header, but has

docs/examples/cue.mod/usr/k8s.io/api/apps/v1/types.cue

@@ -4,3 +4,8 @@ package v1
 	apiVersion: "apps/v1"
 	kind:       "Deployment"
 }
+
+#StatefulSet: {
+	apiVersion: "apps/v1"
+	kind:       "StatefulSet"
+}

docs/examples/helpers.cue

@@ -7,14 +7,23 @@ import "encoding/yaml"
 	// apiObjects holds each the api objects produced by cue.
 	apiObjects: {
 		[Kind=_]: {
-			[Name=_]: {
+			[string]: {
 				kind: Kind
 				...
 			}
 		}
+		Namespace?: [Name=_]: #Namespace & {metadata: name: Name}
 		ExternalSecret?: [Name=_]: #ExternalSecret & {_name: Name}
 		VirtualService?: [Name=_]: #VirtualService & {metadata: name: Name}
 		Issuer?: [Name=_]: #Issuer & {metadata: name: Name}
+		Gateway?: [Name=_]: #Gateway & {metadata: name: Name}
+		ConfigMap?: [Name=_]: #ConfigMap & {metadata: name: Name}
+		ServiceAccount?: [Name=_]: #ServiceAccount & {metadata: name: Name}
+
+		Deployment?: [_]:            #Deployment
+		StatefulSet?: [_]:           #StatefulSet
+		RequestAuthentication?: [_]: #RequestAuthentication
+		AuthorizationPolicy?: [_]:   #AuthorizationPolicy
 	}
 
 	// apiObjectMap holds the marshalled representation of apiObjects

docs/examples/managednamespaces.cue

@@ -0,0 +1,26 @@
+package holos
+
+// NOTE: Beyond the base reference platform, services should typically be added to #OptionalServices instead of directly to a managed namespace.
+
+// ManagedNamespace is a namespace to manage across all clusters in the holos platform.
+#ManagedNamespace: {
+	namespace: {
+		metadata: {
+			name: string
+			labels: [string]: string
+		}
+	}
+	// clusterNames represents the set of clusters the namespace is managed on.  Usually all clusters.
+	clusterNames: [...string]
+	for cluster in clusterNames {
+		clusters: (cluster): name: cluster
+	}
+}
+
+// #ManagedNamepsaces is the union of all namespaces across all cluster types and optional services.
+// Holos adopts the namespace sameness position of SIG Multicluster, refer to https://github.com/kubernetes/community/blob/dd4c8b704ef1c9c3bfd928c6fa9234276d61ad18/sig-multicluster/namespace-sameness-position-statement.md
+#ManagedNamespaces: {
+	[Name=_]: #ManagedNamespace & {
+		namespace: metadata: name: Name
+	}
+}

docs/examples/meshconfig.cue

@@ -0,0 +1,54 @@
+package holos
+
+// #MeshConfig provides the istio meshconfig in the config key given projects.
+#MeshConfig: {
+	projects: #Projects
+	// clusterName is the value of the --cluster-name flag, the cluster currently being manged / rendered.
+	clusterName: string | *#ClusterName
+
+	// for extAuthzHttp extension providers
+	extensionProviderMap: [Name=_]: #ExtAuthzProxy & {name: Name}
+	// for other extension providers like zipkin
+	extensionProviderExtraMap: [Name=_]: {name: Name}
+
+	config: {
+		accessLogEncoding: string | *"JSON"
+		accessLogFile:     string | *"/dev/stdout"
+		defaultConfig: {
+			discoveryAddress: string | *"istiod.istio-system.svc:15012"
+			tracing: zipkin: address: string | *"zipkin.istio-system:9411"
+		}
+		defaultProviders: metrics: [...string] | *["prometheus"]
+		enablePrometheusMerge: false | *true
+		rootNamespace:         string | *"istio-system"
+		trustDomain:           string | *"cluster.local"
+		extensionProviders: [
+			for x in extensionProviderMap {x},
+			for y in extensionProviderExtraMap {y},
+		]
+	}
+}
+
+// #ExtAuthzProxy defines the provider configuration for an istio external authorization auth proxy.
+#ExtAuthzProxy: {
+	name: string
+	envoyExtAuthzHttp: {
+		headersToDownstreamOnDeny: [
+			"content-type",
+			"set-cookie",
+		]
+		headersToUpstreamOnAllow: [
+			"authorization",
+			"path",
+			"x-oidc-id-token",
+		]
+		includeAdditionalHeadersInCheck: "X-Auth-Request-Redirect": "%REQ(x-forwarded-proto)%://%REQ(:authority)%%REQ(:path)%%REQ(:query)%"
+		includeRequestHeadersInCheck: [
+			"authorization",
+			"cookie",
+			"x-forwarded-for",
+		]
+		port:    4180
+		service: string
+	}
+}

docs/examples/optionalservices.cue

@@ -1,6 +1,8 @@
 // Controls optional feature flags for services distributed across multiple holos components.
 // For example, enable issuing certificates in the provisioner cluster when an optional service is
-// enabled for a workload cluster.
+// enabled for a workload cluster.  Another example is NATS, which isn't necessary on all clusters,
+// but is necessary on clusters with a project like holos which depends on NATS.
+
 package holos
 
 import "list"

docs/examples/platforms/holos-saas/clusters/provisioner/choria/broker/broker.cue

@@ -0,0 +1,45 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Broker = "choria-broker"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-platform-issuer": _
+
+		metadata: name: "\(Namespace)-\(Broker)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Broker
+	"app.kubernetes.io/name":     Broker
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		Certificate: "\(Broker)-tls": #Certificate & {
+			metadata: {
+				name:      "\(Broker)-tls"
+				namespace: Namespace
+				labels:    SelectorLabels
+			}
+			spec: {
+				commonName: "\(Broker).\(Namespace).svc.cluster.local"
+				dnsNames: [
+					Broker,
+					"\(Broker).\(Namespace).svc",
+					"\(Broker).\(Namespace).svc.cluster.local",
+					"*.\(Broker)",
+					"*.\(Broker).\(Namespace).svc",
+					"*.\(Broker).\(Namespace).svc.cluster.local",
+				]
+				issuerRef: kind: "ClusterIssuer"
+				issuerRef: name: "platform-issuer"
+				secretName: metadata.name
+				usages: ["signing", "key encipherment", "server auth", "client auth"]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/provisioner/choria/provisioner/provsioner.cue

@@ -0,0 +1,45 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Provisioner = "choria-provisioner"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-platform-issuer": _
+
+		metadata: name: "\(Namespace)-\(Provisioner)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Provisioner
+	"app.kubernetes.io/name":     Provisioner
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		Certificate: "\(Provisioner)-tls": #Certificate & {
+			metadata: {
+				name:      "\(Provisioner)-tls"
+				namespace: Namespace
+				labels:    SelectorLabels
+			}
+			spec: {
+				commonName: "\(Provisioner).\(Namespace).svc.cluster.local"
+				dnsNames: [
+					Provisioner,
+					"\(Provisioner).\(Namespace).svc",
+					"\(Provisioner).\(Namespace).svc.cluster.local",
+					"*.\(Provisioner)",
+					"*.\(Provisioner).\(Namespace).svc",
+					"*.\(Provisioner).\(Namespace).svc.cluster.local",
+				]
+				issuerRef: kind: "ClusterIssuer"
+				issuerRef: name: "platform-issuer"
+				secretName: metadata.name
+				usages: ["signing", "key encipherment", "server auth", "client auth"]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/broker/broker.cue

@@ -0,0 +1,177 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Broker = "choria-broker"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "\(Namespace)-\(Broker)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/part-of": "choria"
+	"app.kubernetes.io/name":    Broker
+}
+
+let Metadata = {
+	name:      Broker
+	namespace: Namespace
+	labels:    SelectorLabels
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		ExternalSecret: "\(Broker)-tls": #ExternalSecret & {
+			metadata: name:      "\(Broker)-tls"
+			metadata: namespace: Namespace
+		}
+		ExternalSecret: "\(Broker)": #ExternalSecret & {
+			metadata: name:      Broker
+			metadata: namespace: Namespace
+		}
+		StatefulSet: "\(Broker)": {
+			metadata: Metadata
+			spec: {
+				selector: matchLabels: SelectorLabels
+				serviceName: Broker
+				template: metadata: labels: SelectorLabels
+				template: spec: {
+					containers: [
+						{
+							name: Broker
+							command: ["choria", "broker", "run", "--config", "/etc/choria/broker.conf"]
+							image:           "registry.choria.io/choria/choria:0.28.0"
+							imagePullPolicy: "IfNotPresent"
+							ports: [
+								{
+									containerPort: 4222
+									name:          "tcp-nats"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 4333
+									name:          "https-wss"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 5222
+									name:          "tcp-cluster"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 8222
+									name:          "http-stats"
+									protocol:      "TCP"
+								},
+							]
+							livenessProbe: httpGet: {
+								path: "/healthz"
+								port: "http-stats"
+							}
+							readinessProbe: livenessProbe
+							resources: {}
+							securityContext: {}
+							volumeMounts: [
+								{
+									mountPath: "/etc/choria"
+									name:      Broker
+								},
+								{
+									mountPath: "/etc/choria-tls"
+									name:      "\(Broker)-tls"
+								},
+							]
+						},
+					]
+					securityContext: {}
+					serviceAccountName: Broker
+					volumes: [
+						{
+							name: Broker
+							secret: secretName: Broker
+						},
+						{
+							name: "\(Broker)-tls"
+							secret: secretName: "\(Broker)-tls"
+						},
+					]
+				}
+			}
+		}
+		ServiceAccount: "\(Broker)": #ServiceAccount & {
+			metadata: Metadata
+		}
+		Service: "\(Broker)": #Service & {
+			metadata: Metadata
+			spec: {
+				type:      "ClusterIP"
+				clusterIP: "None"
+				selector:  SelectorLabels
+				ports: [
+					{
+						name:        "tcp-nats"
+						appProtocol: "tcp"
+						port:        4222
+						protocol:    "TCP"
+						targetPort:  "tcp-nats"
+					},
+					{
+						name:        "tcp-cluster"
+						appProtocol: "tcp"
+						port:        5222
+						protocol:    "TCP"
+						targetPort:  "tcp-cluster"
+					},
+					{
+						name:        "https-wss"
+						appProtocol: "https"
+						port:        443
+						protocol:    "TCP"
+						targetPort:  "https-wss"
+					},
+				]
+			}
+		}
+		DestinationRule: "\(Broker)-wss": #DestinationRule & {
+			_decriptions: "Configures Istio to connect to Choria using a cert issued by the Platform Issuer"
+			metadata:     Metadata
+			spec: host: "\(Broker).\(Namespace).svc.cluster.local"
+			spec: trafficPolicy: tls: {
+				credentialName: "istio-ingress-mtls-cert"
+				mode:           "MUTUAL"
+				// subjectAltNames is important, otherwise istio will fail to verify the
+				// choria broker upstream server.  make sure this matches a value
+				// present in the choria broker's cert.
+				//
+				//  kubectl get secret choria-broker-tls -o json | jq --exit-status
+				//  '.data | map_values(@base64d)' | jq .\"tls.crt\" -r | openssl x509
+				//  -text -noout -in -
+				subjectAltNames: [spec.host]
+			}
+		}
+		VirtualService: "\(Broker)-wss": #VirtualService & {
+			metadata: name:      "\(Broker)-wss"
+			metadata: namespace: Namespace
+			spec: {
+				gateways: ["istio-ingress/default"]
+				hosts: ["jeff.provision.dev.\(#ClusterName).holos.run"]
+				http: [
+					{
+						route: [
+							{
+								destination: {
+									host: "\(Broker).\(Namespace).svc.cluster.local"
+									port: "number": 443
+								}
+							},
+						]
+					},
+				]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/provisioner/Dockerfile

@@ -0,0 +1,18 @@
+FROM registry.choria.io/choria/provisioner:latest
+
+RUN curl -Lo nsc.zip https://github.com/nats-io/nsc/releases/download/v2.8.6/nsc-linux-amd64.zip &&\
+  unzip nsc.zip && \
+  mv nsc /usr/local/bin/nsc && \
+  chmod 755 /usr/local/bin/nsc && \
+  rm -f nsc.zip
+
+# TODO: Add jwt executable
+# TODO: Add helper executable
+
+USER choria
+ENV USER=choria
+
+ENTRYPOINT ["/usr/sbin/choria-provisioner"]
+
+# These two files are expected to be in the provisioner secret.
+CMD ["--config=/etc/provisioner/provisioner.yaml", "--choria-config=/etc/provisioner/choria.cfg"]

docs/examples/platforms/holos-saas/clusters/workload/choria/provisioner/provisioner.cue

@@ -0,0 +1,82 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Provisioner = "choria-provisioner"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "\(Namespace)-\(Provisioner)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Provisioner
+	"app.kubernetes.io/name":     Provisioner
+}
+
+let Metadata = {
+	name:      Provisioner
+	namespace: Namespace
+	labels:    SelectorLabels
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		ExternalSecret: "\(Provisioner)-tls": #ExternalSecret & {
+			metadata: name:      "\(Provisioner)-tls"
+			metadata: namespace: Namespace
+		}
+		ExternalSecret: "\(Provisioner)": #ExternalSecret & {
+			metadata: name:      Provisioner
+			metadata: namespace: Namespace
+		}
+		ServiceAccount: "\(Provisioner)": #ServiceAccount & {
+			metadata: Metadata
+		}
+		Deployment: "\(Provisioner)": {
+			metadata: Metadata
+			spec: {
+				selector: matchLabels: SelectorLabels
+				template: metadata: labels: SelectorLabels
+				template: spec: {
+					containers: [
+						{
+							name: Provisioner
+							command: ["bash", "/etc/provisioner/entrypoint"]
+							// skopeo inspect docker://registry.choria.io/choria/provisioner | jq .RepoTags
+							image:           "registry.choria.io/choria/provisioner:0.15.1"
+							imagePullPolicy: "IfNotPresent"
+							resources: {}
+							securityContext: {}
+							volumeMounts: [
+								{
+									mountPath: "/etc/provisioner"
+									name:      Provisioner
+								},
+								{
+									mountPath: "/etc/provisioner-tls"
+									name:      "\(Provisioner)-tls"
+								},
+							]
+						},
+					]
+					securityContext: {}
+					serviceAccountName: Provisioner
+					volumes: [
+						{
+							name: Provisioner
+							secret: secretName: name
+						},
+						{
+							name: "\(Provisioner)-tls"
+							secret: secretName: name
+						},
+					]
+				}
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/readme.md

@@ -0,0 +1,8 @@
+# Machine Room Provisioner
+
+This sub-tree contains Holos Components to manage a [Choria Provisioner][1]
+system for the use case of provisioning `holos controller` instances.  These
+instances are implementations of Machine Room which are in turn implementations
+of Choria Server, hence why we use Choria Provisioner.
+
+[1]: https://choria-io.github.io/provisioner/

docs/examples/platforms/holos-saas/clusters/workload/nats/crds/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# curl -LO https://github.com/nats-io/nack/releases/latest/download/crds.yml
+resources:
+- crds.yml

docs/examples/platforms/holos-saas/clusters/workload/nats/crds/nats-crds.cue

@@ -0,0 +1,8 @@
+package holos
+
+// NATS NetStream Controller (NACK)
+spec: components: KustomizeBuildList: [
+	#KustomizeBuild & {
+		metadata: name: "prod-nack-crds"
+	},
+]

docs/examples/platforms/holos-saas/clusters/workload/nats/envs/nats.cue

@@ -0,0 +1,62 @@
+package holos
+
+// for Project in _Projects {
+// 	spec: components: resources: (#ProjectTemplate & {project: Project}).workload.resources
+// }
+
+let Namespace = "jeff-holos"
+
+#Kustomization: spec: targetNamespace: Namespace
+
+spec: components: HelmChartList: [
+	#HelmChart & {
+		metadata: name: "jeff-holos-nats"
+		namespace: Namespace
+		_dependsOn: "prod-secrets-stores": _
+		chart: {
+			name:       "nats"
+			version:    "1.1.10"
+			repository: NatsRepository
+		}
+		_values: #NatsValues & {
+			config: {
+				// https://github.com/nats-io/k8s/tree/main/helm/charts/nats#operator-mode-with-nats-resolver
+				resolver: enabled: true
+				resolver: merge: {
+					type:     "full"
+					interval: "2m"
+					timeout:  "1.9s"
+				}
+				merge: {
+					operator:       "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJUSElBTDM2NUtOS0lVVVJDMzNLNFJGQkJVRlFBSTRLS0NQTDJGVDZYVjdNQVhWU1dFNElRIiwiaWF0IjoxNzEzMjIxMzE1LCJpc3MiOiJPREtQM0RZTzc3T1NBRU5IU0FFR0s3WUNFTFBYT1FFWUI3RVFSTVBLWlBNQUxINE5BRUVLSjZDRyIsIm5hbWUiOiJIb2xvcyIsInN1YiI6Ik9ES1AzRFlPNzdPU0FFTkhTQUVHSzdZQ0VMUFhPUUVZQjdFUVJNUEtaUE1BTEg0TkFFRUtKNkNHIiwibmF0cyI6eyJ0eXBlIjoib3BlcmF0b3IiLCJ2ZXJzaW9uIjoyfX0.dQURTb-zIQMc-OYd9328oY887AEnvog6gOXY1-VCsDG3L89nq5x_ks4ME7dJ4Pn-Pvm2eyBi1Jx6ubgkthHgCQ"
+					system_account: "ADIQCYK4K3OKTPODGCLI4PDQ6XBO52MISBPTAIDESEJMLZCMNULDKCCY"
+					resolver_preload: {
+						// NOTEL: Make sure you do not include the trailing , in the SYS_ACCOUNT_JWT
+						"ADIQCYK4K3OKTPODGCLI4PDQ6XBO52MISBPTAIDESEJMLZCMNULDKCCY": "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI2SEVMNlhKSUdWUElMNFBURVI1MkUzTkFITjZLWkVUUUdFTlFVS0JWRzNUWlNLRzVLT09RIiwiaWF0IjoxNzEzMjIxMzE1LCJpc3MiOiJPREtQM0RZTzc3T1NBRU5IU0FFR0s3WUNFTFBYT1FFWUI3RVFSTVBLWlBNQUxINE5BRUVLSjZDRyIsIm5hbWUiOiJTWVMiLCJzdWIiOiJBRElRQ1lLNEszT0tUUE9ER0NMSTRQRFE2WEJPNTJNSVNCUFRBSURFU0VKTUxaQ01OVUxES0NDWSIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTF9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJhdXRob3JpemF0aW9uIjp7fSwidHlwZSI6ImFjY291bnQiLCJ2ZXJzaW9uIjoyfX0.TiGIk8XON394D9SBEowGHY_nTeOyHiM-ihyw6HZs8AngOnYPFXH9OVjsaAf8Poa2k_V84VtH7yVNgNdjBgduDA"
+					}
+				}
+				cluster: enabled:   true
+				jetstream: enabled: true
+				websocket: enabled: true
+				monitor: enabled:   true
+			}
+			promExporter: enabled: true
+			promExporter: podMonitor: enabled: true
+		}
+	},
+	#HelmChart & {
+		metadata: name: "jeff-holos-nack"
+		namespace: Namespace
+		_dependsOn: "jeff-holos-nats": _
+		chart: {
+			name:       "nack"
+			version:    "0.25.2"
+			repository: NatsRepository
+		}
+	},
+]
+
+let NatsRepository = {
+	name: "nats"
+	url:  "https://nats-io.github.io/k8s/helm/charts/"
+}

docs/examples/platforms/holos-saas/clusters/workload/nats/envs/nats.values.cue

@@ -0,0 +1,722 @@
+package holos
+
+#NatsValues: {
+	//###############################################################################
+	// Global options
+	//###############################################################################
+	global: {
+		image: {
+			// global image pull policy to use for all container images in the chart
+			// can be overridden by individual image pullPolicy
+			pullPolicy: null
+			// global list of secret names to use as image pull secrets for all pod specs in the chart
+			// secrets must exist in the same namespace
+			// https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+			pullSecretNames: []
+			// global registry to use for all container images in the chart
+			// can be overridden by individual image registry
+			registry: null
+		}
+
+		// global labels will be applied to all resources deployed by the chart
+		labels: {}
+	}
+
+	//###############################################################################
+	// Common options
+	//###############################################################################
+	// override name of the chart
+	nameOverride: null
+	// override full name of the chart+release
+	fullnameOverride: null
+	// override the namespace that resources are installed into
+	namespaceOverride: null
+
+	// reference a common CA Certificate or Bundle in all nats config `tls` blocks and nats-box contexts
+	// note: `tls.verify` still must be set in the appropriate nats config `tls` blocks to require mTLS
+	tlsCA: {
+		enabled: false
+		// set configMapName in order to mount an existing configMap to dir
+		configMapName: null
+		// set secretName in order to mount an existing secretName to dir
+		secretName: null
+		// directory to mount the configMap or secret to
+		dir: "/etc/nats-ca-cert"
+		// key in the configMap or secret that contains the CA Certificate or Bundle
+		key: "ca.crt"
+	}
+
+	//###############################################################################
+	// NATS Stateful Set and associated resources
+	//###############################################################################
+	//###########################################################
+	// NATS config
+	//###########################################################
+	config: {
+		cluster: {
+			enabled: true | *false
+			port:    6222
+			// must be 2 or higher when jetstream is enabled
+			replicas: 3
+
+			// apply to generated route URLs that connect to other pods in the StatefulSet
+			routeURLs: {
+				// if both user and password are set, they will be added to route URLs
+				// and the cluster authorization block
+				user:     null
+				password: null
+				// set to true to use FQDN in route URLs
+				useFQDN:          false
+				k8sClusterDomain: "cluster.local"
+			}
+
+			tls: {
+				enabled: true | *false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/cluster"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the cluster config
+			// https://docs.nats.io/running-a-nats-service/configuration/clustering/cluster_config
+			merge: {}
+			patch: []
+		}
+
+		jetstream: {
+			enabled: true | *false
+
+			fileStore: {
+				enabled: true
+				dir:     "/data"
+
+				//###########################################################
+				// stateful set -> volume claim templates -> jetstream pvc
+				//###########################################################
+				pvc: {
+					enabled:          true
+					size:             "10Gi"
+					storageClassName: null
+
+					// merge or patch the jetstream pvc
+					// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core
+					merge: {}
+					patch: []
+					// defaults to "{{ include "nats.fullname" $ }}-js"
+					name: null
+				}
+
+				// defaults to the PVC size
+				maxSize: null
+			}
+
+			memoryStore: {
+				enabled: false
+				// ensure that container has a sufficient memory limit greater than maxSize
+				maxSize: "1Gi"
+			}
+
+			// merge or patch the jetstream config
+			// https://docs.nats.io/running-a-nats-service/configuration#jetstream
+			merge: {}
+			patch: []
+		}
+
+		nats: {
+			port: 4222
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/nats"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+		}
+
+		leafnodes: {
+			enabled: false
+			port:    7422
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/leafnodes"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the leafnodes config
+			// https://docs.nats.io/running-a-nats-service/configuration/leafnodes/leafnode_conf
+			merge: {}
+			patch: []
+		}
+
+		websocket: {
+			enabled: true | *false
+			port:    8080
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/websocket"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			//###########################################################
+			// ingress
+			//###########################################################
+			// service must be enabled also
+			ingress: {
+				enabled: false
+				// must contain at least 1 host otherwise ingress will not be created
+				hosts: []
+				path:     "/"
+				pathType: "Exact"
+				// sets to the ingress class name
+				className: null
+				// set to an existing secret name to enable TLS on the ingress; applies to all hosts
+				tlsSecretName: null
+
+				// merge or patch the ingress
+				// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#ingress-v1-networking-k8s-io
+				merge: {}
+				patch: []
+				// defaults to "{{ include "nats.fullname" $ }}-ws"
+				name: null
+			}
+
+			// merge or patch the websocket config
+			// https://docs.nats.io/running-a-nats-service/configuration/websocket/websocket_conf
+			merge: {}
+			patch: []
+		}
+
+		mqtt: {
+			enabled: false
+			port:    1883
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/mqtt"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the mqtt config
+			// https://docs.nats.io/running-a-nats-service/configuration/mqtt/mqtt_config
+			merge: {}
+			patch: []
+		}
+
+		gateway: {
+			enabled: false
+			port:    7222
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/gateway"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the gateway config
+			// https://docs.nats.io/running-a-nats-service/configuration/gateways/gateway#gateway-configuration-block
+			merge: {}
+			patch: []
+		}
+
+		monitor: {
+			enabled: true
+			port:    8222
+			tls: {
+				// config.nats.tls must be enabled also
+				// when enabled, monitoring port will use HTTPS with the options from config.nats.tls
+				enabled: false
+			}
+		}
+
+		profiling: {
+			enabled: false
+			port:    65432
+		}
+
+		resolver: {
+			enabled: true | *false
+			dir:     "/data/resolver"
+
+			//###########################################################
+			// stateful set -> volume claim templates -> resolver pvc
+			//###########################################################
+			pvc: {
+				enabled:          true
+				size:             "1Gi"
+				storageClassName: null
+
+				// merge or patch the pvc
+				// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core
+				merge: {}
+				patch: []
+				// defaults to "{{ include "nats.fullname" $ }}-resolver"
+				name: null
+			}
+
+			// merge or patch the resolver
+			// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/jwt/resolver
+			merge: {
+				type?:     string
+				interval?: string
+				timeout?:  string
+			}
+			patch: []
+		}
+
+		// adds a prefix to the server name, which defaults to the pod name
+		// helpful for ensuring server name is unique in a super cluster
+		serverNamePrefix: ""
+
+		// merge or patch the nats config
+		// https://docs.nats.io/running-a-nats-service/configuration
+		// following special rules apply
+		//  1. strings that start with << and end with >> will be unquoted
+		//     use this for variables and numbers with units
+		//  2. keys ending in $include will be switched to include directives
+		//     keys are sorted alphabetically, use prefix before $includes to control includes ordering
+		//     paths should be relative to /etc/nats-config/nats.conf
+		// example:
+		//
+		//   merge:
+		//     $include: ./my-config.conf
+		//     zzz$include: ./my-config-last.conf
+		//     server_name: nats
+		//     authorization:
+		//       token: << $TOKEN >>
+		//     jetstream:
+		//       max_memory_store: << 1GB >>
+		//
+		// will yield the config:
+		// {
+		//   include ./my-config.conf;
+		//   "authorization": {
+		//     "token": $TOKEN
+		//   },
+		//   "jetstream": {
+		//     "max_memory_store": 1GB
+		//   },
+		//   "server_name": "nats",
+		//   include ./my-config-last.conf;
+		// }
+		merge: {
+			operator?:       string
+			system_account?: string
+			resolver_preload?: [string]: string
+		}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> nats container
+	//###########################################################
+	container: {
+		image: {
+			repository: "nats"
+			tag:        "2.10.12-alpine"
+			pullPolicy: null
+			registry:   null
+		}
+
+		// container port options
+		// must be enabled in the config section also
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#containerport-v1-core
+		ports: {
+			nats: {}
+			leafnodes: {}
+			websocket: {}
+			mqtt: {}
+			cluster: {}
+			gateway: {}
+			monitor: {}
+			profiling: {}
+		}
+
+		// map with key as env var name, value can be string or map
+		// example:
+		//
+		//   env:
+		//     GOMEMLIMIT: 7GiB
+		//     TOKEN:
+		//       valueFrom:
+		//         secretKeyRef:
+		//           name: nats-auth
+		//           key: token
+		env: {}
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> reloader container
+	//###########################################################
+	reloader: {
+		enabled: true
+		image: {
+			repository: "natsio/nats-server-config-reloader"
+			tag:        "0.14.1"
+			pullPolicy: null
+			registry:   null
+		}
+
+		// env var map, see nats.env for an example
+		env: {}
+
+		// all nats container volume mounts with the following prefixes
+		// will be mounted into the reloader container
+		natsVolumeMountPrefixes: ["/etc/"]
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> prom-exporter container
+	//###########################################################
+	// config.monitor must be enabled
+	promExporter: {
+		enabled: true | *false
+		image: {
+			repository: "natsio/prometheus-nats-exporter"
+			tag:        "0.14.0"
+			pullPolicy: null
+			registry:   null
+		}
+
+		port: 7777
+		// env var map, see nats.env for an example
+		env: {}
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+
+		//###########################################################
+		// prometheus pod monitor
+		//###########################################################
+		podMonitor: {
+			enabled: true | *false
+
+			// merge or patch the pod monitor
+			// https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.PodMonitor
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}"
+			name: null
+		}
+	}
+
+	//###########################################################
+	// service
+	//###########################################################
+	service: {
+		enabled: true
+
+		// service port options
+		// additional boolean field enable to control whether port is exposed in the service
+		// must be enabled in the config section also
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceport-v1-core
+		ports: {
+			nats: enabled:      true
+			leafnodes: enabled: true
+			websocket: enabled: true
+			mqtt: enabled:      true
+			cluster: enabled:   false
+			gateway: enabled:   false
+			monitor: enabled:   false
+			profiling: enabled: false
+		}
+
+		// merge or patch the service
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	//###########################################################
+	// other nats extension points
+	//###########################################################
+	// stateful set
+	statefulSet: {
+		// merge or patch the stateful set
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#statefulset-v1-apps
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	// stateful set -> pod template
+	podTemplate: {
+		// adds a hash of the ConfigMap as a pod annotation
+		// this will cause the StatefulSet to roll when the ConfigMap is updated
+		configChecksumAnnotation: true
+
+		// map of topologyKey: topologySpreadConstraint
+		// labelSelector will be added to match StatefulSet pods
+		//
+		// topologySpreadConstraints:
+		//   kubernetes.io/hostname:
+		//     maxSkew: 1
+		//
+		topologySpreadConstraints: {}
+
+		// merge or patch the pod template
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core
+		merge: {}
+		patch: []
+	}
+
+	// headless service
+	headlessService: {
+		// merge or patch the headless service
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}-headless"
+		name: null
+	}
+
+	// config map
+	configMap: {
+		// merge or patch the config map
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#configmap-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}-config"
+		name: null
+	}
+
+	// pod disruption budget
+	podDisruptionBudget: {
+		enabled: true
+		// merge or patch the pod disruption budget
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#poddisruptionbudget-v1-policy
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	// service account
+	serviceAccount: {
+		enabled: false
+		// merge or patch the service account
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceaccount-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	//###########################################################
+	// natsBox
+	//
+	// NATS Box Deployment and associated resources
+	//###########################################################
+	natsBox: {
+		enabled: true
+
+		//###########################################################
+		// NATS contexts
+		//###########################################################
+		contexts: {
+			default: {
+				creds: {
+					// set contents in order to create a secret with the creds file contents
+					contents: null
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-creds/<context-name>
+					dir: null
+					key: "nats.creds"
+				}
+				nkey: {
+					// set contents in order to create a secret with the nkey file contents
+					contents: null
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-nkeys/<context-name>
+					dir: null
+					key: "nats.nk"
+				}
+				// used to connect with client certificates
+				tls: {
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-certs/<context-name>
+					dir:  null
+					cert: "tls.crt"
+					key:  "tls.key"
+				}
+
+				// merge or patch the context
+				// https://docs.nats.io/using-nats/nats-tools/nats_cli#nats-contexts
+				merge: {}
+				patch: []
+			}
+		}
+
+		// name of context to select by default
+		defaultContextName: "default"
+
+		//###########################################################
+		// deployment -> pod template -> nats-box container
+		//###########################################################
+		container: {
+			image: {
+				repository: "natsio/nats-box"
+				tag:        "0.14.2"
+				pullPolicy: null
+				registry:   null
+			}
+
+			// env var map, see nats.env for an example
+			env: {}
+
+			// merge or patch the container
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+			merge: {}
+			patch: []
+		}
+
+		//###########################################################
+		// other nats-box extension points
+		//###########################################################
+		// deployment
+		deployment: {
+			// merge or patch the deployment
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#deployment-v1-apps
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box"
+			name: null
+		}
+
+		// deployment -> pod template
+		podTemplate: {
+			// merge or patch the pod template
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core
+			merge: {}
+			patch: []
+		}
+
+		// contexts secret
+		contextsSecret: {
+			// merge or patch the context secret
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box-contexts"
+			name: null
+		}
+
+		// contents secret
+		contentsSecret: {
+			// merge or patch the contents secret
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box-contents"
+			name: null
+		}
+
+		// service account
+		serviceAccount: {
+			enabled: false
+			// merge or patch the service account
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceaccount-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box"
+			name: null
+		}
+	}
+
+	//###############################################################################
+	// Extra user-defined resources
+	//###############################################################################
+	//
+	// add arbitrary user-generated resources
+	// example:
+	//
+	// config:
+	//   websocket:
+	//     enabled: true
+	// extraResources:
+	// - apiVersion: networking.istio.io/v1beta1
+	//   kind: VirtualService
+	//   metadata:
+	//     name:
+	//       $tplYaml: >
+	//         {{ include "nats.fullname" $ | quote }}
+	//     labels:
+	//       $tplYaml: |
+	//         {{ include "nats.labels" $ }}
+	//   spec:
+	//     hosts:
+	//     - demo.nats.io
+	//     gateways:
+	//     - my-gateway
+	//     http:
+	//     - name: default
+	//       match:
+	//       - name: root
+	//         uri:
+	//           exact: /
+	//       route:
+	//       - destination:
+	//           host:
+	//             $tplYaml: >
+	//               {{ .Values.service.name | quote }}
+	//           port:
+	//             number:
+	//               $tplYaml: >
+	//                 {{ .Values.config.websocket.port }}
+	//
+	extraResources: []
+}

docs/examples/platforms/holos-saas/clusters/workload/nats/scripts/initialize-auth

@@ -0,0 +1,81 @@
+#! /bin/bash
+#
+# This script initializes authorization for a nats cluster.  The process is:
+#
+# Locally:
+# 1. Generate the nats operator jwt.
+# 2. Generate a SYS account jwt issued by the operator.
+# 3. Store both into vault
+#
+# When nats is deployed an ExternalSecret populates auth.conf which is included
+# into nats.conf.  This approach allows helm values to be used for most things
+# except for secrets.
+#
+# Clean up by removing the nsc directory.
+
+set -euo pipefail
+
+tmpdir="$(mktemp -d)"
+finish() {
+  [[ -d "$tmpdir" ]] && rm -rf "$tmpdir"
+}
+trap finish EXIT
+
+PARENT="$(cd "$(dirname $0)" && pwd)"
+: "${OPERATOR_NAME:="Holos"}"
+
+: "${OIX_NAMESPACE:=$(kubectl config view --minify --flatten -ojsonpath='{.contexts[0].context.namespace}')}"
+nsc="${HOME}/.bin/nsc"
+
+ROOT="${PARENT}/${OIX_NAMESPACE}/nsc"
+export NKEYS_PATH="${ROOT}/nkeys"
+export NSC_HOME="${ROOT}/accounts"
+
+mkdir -p "$NKEYS_PATH"
+mkdir -p "$NSC_HOME"
+
+# Install nsc if not already installed
+if ! [[ -x $nsc ]]; then
+  platform="$(kubectl version --output=json | jq .clientVersion.platform -r)"
+  platform="${platform//\//-}"
+  curl -fSLo "${tmpdir}/nsc.zip" "https://github.com/nats-io/nsc/releases/download/v2.8.6/nsc-${platform}.zip"
+  (cd "${tmpdir}" && unzip nsc.zip)
+  sudo install -o 0 -g 0 -m 0755 "${tmpdir}/nsc" $nsc
+fi
+
+echo "export NKEYS_PATH='${NKEYS_PATH}'" > "${ROOT}/nsc.env"
+echo "export NSC_HOME='${NSC_HOME}'" >> "${ROOT}/nsc.env"
+# use kubectl port-forward nats-headless 4222
+echo "export NATS_URL='nats://localhost:4222'" >> "${ROOT}/nsc.env"
+echo "export NATS_CREDS='${ROOT}/nkeys/creds/${OPERATOR_NAME}/SYS/sys.creds'" >> "${ROOT}/nsc.env"
+
+echo "export NATS_CA='${ROOT}/ca.crt'" >> "${ROOT}/nsc.env"
+echo "export NATS_CERT='${ROOT}/tls.crt'" >> "${ROOT}/nsc.env"
+echo "export NATS_KEY='${ROOT}/tls.key'" >> "${ROOT}/nsc.env"
+
+$nsc --data-dir="${ROOT}/stores" list operators
+
+# Create operator
+$nsc add operator --name "${OPERATOR_NAME}"
+# Create system account
+$nsc add account --name SYS
+$nsc add user    --name sys
+# Create account for STAN purposes.
+$nsc add account --name STAN
+$nsc add user    --name stan
+
+# Generate an auth config compatible with the StatefulSet mounting the
+# nats-jwt-pvc PersistentVolumeClaim at path /data/accounts
+$nsc generate config --sys-account SYS --nats-resolver \
+  | sed "s,dir.*jwt',dir: '/data/accounts'" \
+  > "${ROOT}/auth.conf"
+
+# Store the auth config in vault.
+# vault kv put kv/${OIX_CLUSTER_NAME}/kube-namespace/holos-dev/nats-auth-config "auth.conf=@${tmpdir}/auth.conf"
+# Store the SYS creds in vault for use by the nack controller.
+# vault kv put kv/${OIX_CLUSTER_NAME}/kube-namespace/holos-dev/nats-sys-creds "sys.creds=@${OIX_CLUSTER_NAME}/nsc/nkeys/creds/${OPERATOR_NAME}/SYS/sys.creds"
+
+echo "After deploying the nats component, use the get-cert command to fetch the client cert."
+
+echo "Use kubectl port-forward svc/nats-headless 4222" >&2
+echo "source ${ROOT}/nsc.env to make it all work." >&2

docs/examples/platforms/holos-saas/readme.md

@@ -0,0 +1,5 @@
+# Holos
+
+This subtree contains holos components for holos itself.  We strive for minimal dependencies, so this is likely going to contain NATS and/or Postgres resources.
+
+Components depend on the holos project and may iterate over the defined environments in the project stages.

docs/examples/platforms/reference/certificates.cue

@@ -1,5 +1,21 @@
 package holos
 
+#PlatformServers: {
+	for cluster in #Platform.clusters {
+		(cluster.name): {
+			"https-istio-ingress-httpbin": {
+				let cert = #PlatformCerts[cluster.name+"-httpbin"]
+				hosts: [for host in cert.spec.dnsNames {"istio-ingress/\(host)"}]
+				port: name:          "https-istio-ingress-httpbin"
+				port: number:        443
+				port: protocol:      "HTTPS"
+				tls: credentialName: cert.spec.secretName
+				tls: mode:           "SIMPLE"
+			}
+		}
+	}
+}
+
 #PlatformCerts: {
 	// Globally scoped platform services are defined here.
 	login: #PlatformCert & {

docs/examples/platforms/reference/clusters/accounts/iam/iam.cue

@@ -4,4 +4,4 @@ package holos
 #InputKeys: project: "iam"
 
 // Shared dependencies for all components in this collection.
-#DependsOn: namespaces: name: "\(#StageName)-secrets-namespaces"
+#DependsOn: namespaces: name: "\(#StageName)-secrets-stores"

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/README.md

@@ -0,0 +1,10 @@
+To deploy monitoring:
+
+> **_NOTE:_** For more detailed instructions on deploying, see the [documentation on installing Monitoring](https://access.crunchydata.com/documentation/postgres-operator/latest/installation/monitoring/kustomize).
+
+1. verify the namespace is correct in kustomization.yaml 
+2. If you are deploying in openshift, comment out the fsGroup line under securityContext in the following files:
+  - `alertmanager/deployment.yaml`
+  - `grafana/deployment.yaml`
+  - `prometheus/deployment.yaml`
+3. kubectl apply -k .

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/config/alertmanager.yml

@@ -0,0 +1,78 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+# Based on upstream example file found here: https://github.com/prometheus/alertmanager/blob/master/doc/examples/simple.yml
+global:
+    smtp_smarthost: 'localhost: 25'
+    smtp_require_tls: false
+    smtp_from: 'Alertmanager <abc@yahoo.com>'
+#    smtp_smarthost: 'smtp.example.com:587'
+#    smtp_from: 'Alertmanager <abc@yahoo.com>'
+#    smtp_auth_username: '<username>'
+#    smtp_auth_password: '<password>'
+
+# templates:
+# - '/etc/alertmanager/template/*.tmpl'
+
+inhibit_rules:
+# Apply inhibition of warning if the alertname for the same system and service is already critical
+- source_match:
+    severity: 'critical'
+  target_match:
+    severity: 'warning'
+  equal: ['alertname', 'job', 'service']
+
+receivers:
+- name: 'default-receiver'
+  email_configs:
+  - to: 'example@crunchydata.com'
+    send_resolved: true
+
+## Examples of alternative alert receivers. See documentation for more info on how to configure these fully
+#- name: 'pagerduty-dba'
+#  pagerduty_configs:
+#      - service_key: <RANDOMKEYSTUFF>
+
+#- name: 'pagerduty-sre'
+#  pagerduty_configs:
+#      - service_key: <RANDOMKEYSTUFF>
+
+#- name: 'dba-team'
+#  email_configs:
+#  - to: 'example-dba-team@crunchydata.com'
+#    send_resolved: true
+
+#- name: 'sre-team'
+#  email_configs:
+#  - to: 'example-sre-team@crunchydata.com'
+#    send_resolved: true
+
+route:
+    receiver: default-receiver
+    group_by: [severity, service, job, alertname]
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 24h
+
+## Example routes to show how to route outgoing alerts based on the content of that alert
+#    routes:
+#    - match_re:
+#        service: ^(postgresql|mysql|oracle)$
+#      receiver: dba-team
+#      # sub route to send critical dba alerts to pagerduty
+#      routes:
+#      - match:
+#          severity: critical
+#        receiver: pagerduty-dba
+#
+#    - match:
+#        service: system
+#      receiver: sre-team
+#      # sub route to send critical sre alerts to pagerduty
+#      routes:
+#      - match:
+#          severity: critical
+#        receiver: pagerduty-sre

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/deployment.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crunchy-alertmanager
+spec:
+  selector: {}
+  template:
+    spec:
+      containers:
+      - name: alertmanager
+        image: prom/alertmanager:v0.24.0
+        args:
+        - --config.file=/etc/alertmanager/alertmanager.yml
+        - --storage.path=/alertmanager
+        - --log.level=info
+        - --cluster.advertise-address=0.0.0.0:9093
+        livenessProbe:
+          httpGet:
+            path: /-/healthy
+            port: 9093
+          initialDelaySeconds: 25
+          periodSeconds: 20
+        ports:
+        - containerPort: 9093
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 9093
+        volumeMounts:
+        - mountPath: /etc/alertmanager
+          name: alertmanagerconf
+        - mountPath: /alertmanager
+          name: alertmanagerdata
+      securityContext:
+        fsGroup: 26
+        # supplementalGroups:
+        # - 65534
+      serviceAccountName: alertmanager
+      volumes:
+      - name: alertmanagerdata
+        persistentVolumeClaim:
+          claimName: alertmanagerdata
+      - name: alertmanagerconf
+        configMap:
+          defaultMode: 420
+          name: alertmanager-config

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/kustomization.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: crunchy-alertmanager
+
+resources:
+- deployment.yaml
+- pvc.yaml
+- service.yaml
+- serviceaccount.yaml
+
+configMapGenerator:
+- name: alertmanager-config
+  files:
+  - config/alertmanager.yml
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: alertmanagerdata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crunchy-alertmanager
+spec:
+  type: ClusterIP
+  ports:
+  - name: alertmanager
+    port: 9093

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/alertmanager/serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: alertmanager

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/config/crunchy_grafana_dashboards.yml

@@ -0,0 +1,16 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+apiVersion: 1
+
+providers:
+- name: 'crunchy_dashboards'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards
+  options:
+    path: /etc/grafana/provisioning/dashboards

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/config/crunchy_grafana_datasource.yml

@@ -0,0 +1,18 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+# config file version
+apiVersion: 1
+
+datasources:
+  - name: PROMETHEUS
+    type: prometheus
+    access: proxy
+    url: http://$PROM_HOST:$PROM_PORT
+    isDefault: True
+    editable: False
+    orgId: 1
+    version: 1

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+configMapGenerator:
+- name: grafana-dashboards
+  files:
+  - pgbackrest.json
+  - pod_details.json
+  - postgresql_overview.json
+  - postgresql_details.json
+  - postgresql_service_health.json
+  - prometheus_alerts.json
+  - query_statistics.json
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/pgbackrest.json

@@ -0,0 +1,687 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069660860,
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "title": "",
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "dtdhms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "/^Value$/",
+          "values": false
+        },
+        "text": {
+          "valueSize": 45
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "time()-ccp_backrest_oldest_full_backup_time_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "Recovery window",
+          "refId": "A"
+        }
+      ],
+      "title": "Recovery Window",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Differential Backup": "dark-blue",
+        "Full": "dark-green",
+        "Full Backup": "dark-green",
+        "Incremental": "light-blue",
+        "Incremental Backup": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 3
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": false
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_incr_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment,instance,ip,pod)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental Backup",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_diff_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential Backup",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_full_backup_time_since_completion_seconds{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full Backup",
+          "refId": "C"
+        },
+        {
+          "expr": "min(ccp_archive_command_status_seconds_since_last_archive{pg_cluster=\"[[cluster]]\", role=\"master\"}) without(deployment, instance,ip,pod)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "WAL Archive",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Time Since",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 3
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"incr\"}) without (deployment,instance,pod,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"diff\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_backup_runtime_seconds{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"full\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Backup Runtimes",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Differential": "dark-blue",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 5,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"incr\"}) without (deployment, instance,pod,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Incremental",
+          "refId": "A"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"diff\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Differential",
+          "refId": "B"
+        },
+        {
+          "expr": "min(ccp_backrest_last_info_repo_backup_size_bytes{pg_cluster=\"[[cluster]]\", role=\"master\", backup_type=\"full\"}) without (deployment,instance,pod,ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Full",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Backup Size",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "Archive age": "blue",
+        "Archive count": "green",
+        "Differential": "dark-blue",
+        "Failed count": "red",
+        "Full": "dark-green",
+        "Incremental": "light-blue"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 3,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "avg(idelta(ccp_archive_command_status_failed_count{pg_cluster=\"[[cluster]]\", role=\"master\"}[1m])) without (instance,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Failed count",
+          "refId": "A"
+        },
+        {
+          "expr": "avg(idelta(ccp_archive_command_status_archived_count{pg_cluster=\"[[cluster]]\", role=\"master\"}[1m])) without (instance,pod, ip)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "Archive count",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "WAL Stats",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": "cluster",
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-2w",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "pgBackRest",
+  "uid": "2fcFZ6PGk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/postgresql_overview.json

@@ -0,0 +1,237 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069480601,
+  "links": [],
+  "panels": [
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "links": [
+            {
+              "targetBlank": true,
+              "title": "Cluster Details",
+              "url": "d/fMip0cuMk/postgresqldetails?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Backup Details",
+              "url": "d/2fcFZ6PGk/pgbackrest?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Pod Details",
+              "url": "d/4auP6Mk7k/pod-details?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Query Statistics",
+              "url": "d/ZKoTOHDGk/query-statistics?$__all_variables"
+            },
+            {
+              "targetBlank": true,
+              "title": "Service Health",
+              "url": "d/dhG1wgsMz/postgresql-service-health?$__all_variables"
+            }
+          ],
+          "mappings": [
+            {
+              "from": "0",
+              "id": 0,
+              "text": "DOWN",
+              "to": "99",
+              "type": 2
+            },
+            {
+              "from": "100",
+              "id": 1,
+              "text": "Standalone Cluster",
+              "to": "199",
+              "type": 2
+            },
+            {
+              "from": "200",
+              "id": 2,
+              "text": "HA CLUSTER",
+              "to": "1000",
+              "type": 2
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "#bf1b00",
+                "value": null
+              },
+              {
+                "color": "#eab839",
+                "value": 10
+              },
+              {
+                "color": "#56A64B",
+                "value": 100
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "maxPerRow": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {
+          "valueSize": 30
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "repeat": "cluster",
+      "repeatDirection": "h",
+      "targets": [
+        {
+          "$hashKey": "object:243",
+          "expr": "sum(pg_up{pg_cluster=~\"$cluster\"})*100+sum(ccp_is_in_recovery_status{pg_cluster=~\"$cluster\"})",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{cluster}}",
+          "metric": "up",
+          "refId": "A",
+          "step": 2
+        }
+      ],
+      "title": "$cluster - Overview",
+      "type": "stat"
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allFormat": "glob",
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 1,
+        "includeAll": true,
+        "label": "cluster",
+        "multi": true,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "PostgreSQL Overview",
+  "uid": "D2X39SlGk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/postgresql_service_health.json

@@ -0,0 +1,649 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1625069909806,
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "external link",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "title": "",
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 6,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(ccp_connection_stats_total{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip) / sum(ccp_connection_stats_max_connections{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Connections",
+          "refId": "C"
+        },
+        {
+          "expr": "100 - 100 * avg(ccp_nodemx_data_disk_available_bytes{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip) / avg(ccp_nodemx_data_disk_total_bytes{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"})  without (pod,instance,ip)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Mount:{{mount_point}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Saturation (pct used)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "percent",
+          "label": null,
+          "logBase": 1,
+          "max": "100",
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "cacheTimeout": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 18,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": false,
+          "expr": "  sum(irate(ccp_stat_database_xact_commit{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m])) \n+ sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Transactions",
+          "refId": "A"
+        },
+        {
+          "expr": "max(ccp_connection_stats_active{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip,dbname)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Active connections",
+          "refId": "C"
+        },
+        {
+          "expr": "sum(irate(ccp_pg_stat_statements_total_calls_count{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Queries",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Traffic",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0.001",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "description": "Errors",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 5,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]) without(pod,instance,ip))",
+          "format": "time_series",
+          "hide": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Rollbacks",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(irate(ccp_stat_database_deadlocks{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m]))  without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Deadlock ",
+          "refId": "D"
+        },
+        {
+          "expr": "sum(irate(ccp_stat_database_conflicts{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}[1m])) without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Conflicts",
+          "refId": "B"
+        },
+        {
+          "expr": "max(pg_exporter_last_scrape_error{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without(pod,instance,ip,dbname)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "scrape error",
+          "refId": "C"
+        },
+        {
+          "expr": "max(clamp_max(ccp_archive_command_status_seconds_since_last_fail{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"},1)) without (instance,pod,ip)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "archive error",
+          "refId": "E"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Errors",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": "",
+          "logBase": 2,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 1,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "sideWidth": 150,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.5",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [
+        {
+          "alias": "/Max:/",
+          "color": "#E02F44",
+          "nullPointMode": "null as zero"
+        },
+        {
+          "alias": "/Avg:/",
+          "color": "#8AB8FF"
+        }
+      ],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "max(ccp_pg_stat_statements_total_mean_exec_time_ms{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Avg: {{exported_role}}({{dbname}})",
+          "refId": "A"
+        },
+        {
+          "expr": "max(ccp_pg_stat_statements_top_max_exec_time_ms{pg_cluster=\"[[cluster]]\",role=\"[[role]]\"}) without (pod,instance,ip,query,queryid)",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Max: {{exported_role}}({{dbname}})",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Query Duration",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "ms",
+          "label": null,
+          "logBase": 2,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values(pg_cluster)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(pg_cluster)",
+          "refId": "PROMETHEUS-cluster-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "PROMETHEUS",
+        "definition": "label_values({pg_cluster=\"[[cluster]]\"},role)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": false,
+        "name": "role",
+        "options": [],
+        "query": {
+          "query": "label_values({pg_cluster=\"[[cluster]]\"},role)",
+          "refId": "PROMETHEUS-role-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "PostgreSQL Service Health",
+  "uid": "dhG1wgsMz",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/dashboards/prometheus_alerts.json

@@ -0,0 +1,961 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "PROMETHEUS",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.4.5"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Show current firing and pending alerts, and  severity alert counts.",
+  "editable": false,
+  "gnetId": 4181,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [
+    {
+      "icon": "external link",
+      "tags": [
+        "vendor=crunchydata"
+      ],
+      "type": "dashboards"
+    }
+  ],
+  "panels": [
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 10,
+      "panels": [],
+      "repeat": null,
+      "title": "Environment Summary",
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 0,
+        "y": 1
+      },
+      "id": 6,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(count by (kubernetes_namespace) (pg_up))",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "Namespaces",
+          "refId": "A"
+        }
+      ],
+      "title": "Namespaces",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 4,
+        "y": 1
+      },
+      "id": 13,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(count by (pg_cluster) (pg_up))",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "PostgreSQL Clusters",
+          "refId": "A"
+        }
+      ],
+      "title": "PG Clusters",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-blue",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 8,
+        "y": 1
+      },
+      "id": 14,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "count(pg_up)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "PostgreSQL Clusters",
+          "refId": "A"
+        }
+      ],
+      "title": "PG Instances",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 3
+      },
+      "id": 11,
+      "panels": [],
+      "repeat": null,
+      "title": "Alert Summary",
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-red",
+                "value": null
+              },
+              {
+                "color": "#F2495C",
+                "value": 1
+              },
+              {
+                "color": "#F2495C"
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 0,
+        "y": 4
+      },
+      "id": 2,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "bucketAggs": [
+            {
+              "id": "2",
+              "settings": {
+                "interval": "auto",
+                "min_doc_count": 0,
+                "trimEdges": 0
+              },
+              "type": "date_histogram"
+            }
+          ],
+          "dsType": "elasticsearch",
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"critical\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Critical",
+          "metrics": [
+            {
+              "field": "select field",
+              "id": "1",
+              "type": "count"
+            }
+          ],
+          "refId": "A"
+        }
+      ],
+      "title": "Critical",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "semi-dark-orange",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 4,
+        "y": 4
+      },
+      "id": 5,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"warning\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "instant": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Warning",
+      "type": "stat"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {},
+          "mappings": [
+            {
+              "id": 0,
+              "op": "=",
+              "text": "N/A",
+              "type": 1,
+              "value": "null"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "#299c46",
+                "value": null
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 4,
+        "x": 8,
+        "y": 4
+      },
+      "id": 9,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "text": {},
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "sum(ALERTS{alertstate=\"firing\",severity=\"info\"} > 0) OR on() vector(0)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Info",
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "datasource": "PROMETHEUS",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 6
+      },
+      "id": 12,
+      "panels": [],
+      "repeat": null,
+      "title": "Alerts",
+      "type": "row"
+    },
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": null,
+            "displayMode": "auto",
+            "filterable": true
+          },
+          "decimals": 2,
+          "displayName": "",
+          "mappings": [
+            {
+              "from": "",
+              "id": 1,
+              "text": "",
+              "to": "",
+              "type": 1,
+              "value": ""
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "blue",
+                "value": 100
+              },
+              {
+                "color": "#EAB839",
+                "value": 200
+              },
+              {
+                "color": "red",
+                "value": 300
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity_num"
+            },
+            "properties": [
+              {
+                "id": "custom.displayMode",
+                "value": "color-background"
+              },
+              {
+                "id": "custom.width",
+                "value": 124
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Time"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 170
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 119
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertname"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 206
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertstate"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 128
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 24,
+        "x": 0,
+        "y": 7
+      },
+      "id": 1,
+      "links": [],
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "ALERTS{alertstate='firing'} > 0",
+          "format": "table",
+          "instant": true,
+          "interval": "2s",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Firing",
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {
+            "reducers": []
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Value": true,
+              "__name__": true,
+              "alertstate": false,
+              "deployment": false,
+              "exp_type": true,
+              "fs_type": true,
+              "instance": true,
+              "job": true,
+              "kubernetes_namespace": true,
+              "mount_point": true,
+              "server": true,
+              "service": true,
+              "severity_num": false
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 16,
+              "__name__": 3,
+              "alertname": 4,
+              "alertstate": 5,
+              "deployment": 7,
+              "exp_type": 9,
+              "instance": 10,
+              "ip": 11,
+              "job": 12,
+              "kubernetes_namespace": 13,
+              "pg_cluster": 6,
+              "pod": 8,
+              "role": 14,
+              "service": 15,
+              "severity": 2,
+              "severity_num": 1
+            },
+            "renameByName": {
+              "Time": "",
+              "__name__": "",
+              "severity": "",
+              "severity_num": ""
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": null,
+            "filterable": true
+          },
+          "decimals": 2,
+          "displayName": "",
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/(instance|__name__|Time|alertstate|job|type|Value)/"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              },
+              {
+                "id": "decimals",
+                "value": 2
+              },
+              {
+                "id": "custom.align",
+                "value": null
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Time"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": null
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity_num"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 126
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "severity"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 115
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertname"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 207
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "alertstate"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 131
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "id": 3,
+      "links": [],
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "expr": "ALERTS{alertstate=\"pending\"}",
+          "format": "table",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Alerts (1 week)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Value": true,
+              "__name__": true,
+              "exp_type": true,
+              "instance": true,
+              "job": true,
+              "kubernetes_namespace": true,
+              "service": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "Value": 16,
+              "__name__": 3,
+              "alertname": 4,
+              "alertstate": 5,
+              "deployment": 7,
+              "exp_type": 8,
+              "instance": 9,
+              "ip": 11,
+              "job": 12,
+              "kubernetes_namespace": 13,
+              "pg_cluster": 6,
+              "pod": 10,
+              "role": 14,
+              "service": 15,
+              "severity": 2,
+              "severity_num": 1
+            },
+            "renameByName": {}
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "15m",
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [
+    "vendor=crunchydata"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Prometheus Alerts",
+  "uid": "lwxXsZsMk",
+  "version": 1
+}

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/deployment.yaml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crunchy-grafana
+spec:
+  selector: {}
+  template:
+    spec:
+      containers:
+      - name: grafana
+        image: grafana/grafana:9.2.20
+        ports:
+        - containerPort: 3000
+        env:
+        - name: GF_PATHS_DATA
+          value: /data/grafana/data
+        - name: GF_SECURITY_ADMIN_USER__FILE
+          value: /conf/admin/username
+        - name: GF_SECURITY_ADMIN_PASSWORD__FILE
+          value: /conf/admin/password
+        - name: PROM_HOST
+          value: crunchy-prometheus
+        - name: PROM_PORT
+          value: "9090"
+        livenessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
+          initialDelaySeconds: 25
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /api/health
+            port: 3000
+        volumeMounts:
+        - mountPath: /data
+          name: grafanadata
+        - mountPath: /conf/admin
+          name: grafana-admin
+        - mountPath: /etc/grafana/provisioning/datasources
+          name: grafana-datasources
+        - mountPath: /etc/grafana/provisioning/dashboards
+          name: grafana-dashboards
+      securityContext:
+        fsGroup: 26
+        # supplementalGroups:
+        # - 65534
+      serviceAccountName: grafana
+      volumes:
+      - name: grafanadata
+        persistentVolumeClaim:
+          claimName: grafanadata
+      - name: grafana-admin
+        secret:
+          defaultMode: 420
+          secretName: grafana-admin
+      - name: grafana-datasources
+        configMap:
+          defaultMode: 420
+          name: grafana-datasources
+      - name: grafana-dashboards
+        configMap:
+          defaultMode: 420
+          name: grafana-dashboards

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/kustomization.yaml

@@ -0,0 +1,33 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/component: crunchy-grafana
+
+resources:
+- deployment.yaml
+- pvc.yaml
+- service.yaml
+- serviceaccount.yaml
+- dashboards
+
+configMapGenerator:
+- name: grafana-datasources
+  files:
+  - config/crunchy_grafana_datasource.yml
+- name: grafana-dashboards
+  behavior: merge
+  files:
+  - config/crunchy_grafana_dashboards.yml
+
+secretGenerator:
+- name: grafana-admin
+  literals:
+  - password=admin
+  - username=admin
+  type: Opaque
+
+generatorOptions:
+  disableNameSuffixHash: true

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafanadata
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: crunchy-grafana
+spec:
+  type: ClusterIP
+  ports:
+  - name: grafana
+    port: 3000

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/grafana/serviceaccount.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grafana

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: prod-iam
+
+labels:
+- includeSelectors: true
+  pairs:
+    app.kubernetes.io/name: prod-iam-obs
+    vendor: holos
+
+resources:
+- grafana
+- prometheus
+- alertmanager

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/monitoring.cue

@@ -0,0 +1,9 @@
+package holos
+
+spec: components: KustomizeBuildList: [
+	#KustomizeBuild & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "prod-iam-obs"
+	},
+]

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/clusterrole.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- resources:
+  - pods
+  apiGroups:
+  - ""
+  verbs:
+  - get
+  - list
+  - watch

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/clusterrolebinding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus

docs/examples/platforms/reference/clusters/accounts/iam/monitoring/prometheus/config/crunchy-alert-rules-pg.yml

@@ -0,0 +1,418 @@
+###
+#
+# Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+groups:
+- name: alert-rules
+  rules:
+
+########## EXPORTER RULES ##########
+  - alert: PGExporterScrapeError
+    expr: pg_exporter_last_scrape_error > 0
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: 'Postgres Exporter running on {{ $labels.job }} (instance: {{ $labels.instance }}) is encountering scrape errors processing queries. Error count: ( {{ $value }} )'
+
+
+########## SYSTEM RULES ##########
+  - alert: ExporterDown
+    expr: avg_over_time(up[5m]) < 0.5
+    for: 10s
+    labels:
+      service: system
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'Metrics exporter service for {{ $labels.job }} running on {{ $labels.instance }} has been down at least 50% of the time for the last 5 minutes. Service may be flapping or down.'
+      summary: 'Prometheus Exporter Service Down'
+
+
+########## POSTGRESQL RULES ##########
+  - alert: PGIsUp
+    expr: pg_up < 1
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: 'postgres_exporter running on {{ $labels.job }} is unable to communicate with the configured database'
+
+
+# Example to check for current version of PostgreSQL. Metric returns the version that the exporter is running on, so you can set a rule to check for the minimum version you'd like all systems to be on. Number returned is the 6 digit integer representation contained in the setting "server_version_num".
+#
+#  - alert: PGMinimumVersion
+#    expr:  ccp_postgresql_version_current < 110005
+#    for: 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      summary: '{{ $labels.job }} is not running at least version 11.5 of PostgreSQL'
+
+
+# Whether a system switches from primary to replica or vice versa must be configured per named job.
+# No way to tell what value a system is supposed to be without a rule expression for that specific system
+# 2 to 1 means it changed from primary to replica. 1 to 2 means it changed from replica to primary
+# Set this alert for each system that you want to monitor a recovery status change
+# Below is an example for a target job called "Replica" and watches for the value to change above 1 which means it's no longer a replica
+#
+#  - alert: PGRecoveryStatusSwitch_Replica
+#    expr: ccp_is_in_recovery_status{job="Replica"} > 1
+#    for: 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      summary: '{{ $labels.job }} has changed from replica to primary'
+
+
+# Absence alerts must be configured per named job, otherwise there's no way to know which job is down
+# Below is an example for a target job called "Prod"
+#  - alert: PGConnectionAbsent_Prod
+#    expr: absent(ccp_connection_stats_max_connections{job="Prod"})
+#    for: 10s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Connection metric is absent from target (Prod). Check that postgres_exporter can connect to PostgreSQL.'
+
+
+# Optional monitor for changes to pg_settings (postgresql.conf) system catalog.
+# A similar metric is available for monitoring pg_hba.conf. See ccp_hba_settings_checksum().
+# If metric returns 0, then NO settings have changed for either pg_settings since last known valid state
+# If metric returns 1, then pg_settings have changed since last known valid state
+# To see what may have changed, check the monitor.pg_settings_checksum table for a history of config state.
+#  - alert: PGSettingsChecksum
+#    expr: ccp_pg_settings_checksum > 0
+#    for 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Configuration settings on {{ $labels.job }} have changed from previously known valid state. To reset current config to a valid state after alert fires, run monitor.pg_settings_checksum_set_valid().'
+#      summary: 'PGSQL Instance settings checksum'
+
+
+# Monitor for data block checksum failures. Only works in PG12+
+#  - alert: PGDataChecksum
+#    expr: ccp_data_checksum_failure > 0
+#    for 60s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: '{{ $labels.job }} has at least one data checksum failure in database {{ $labels.dbname }}. See pg_stat_database system catalog for more information.'
+#      summary: 'PGSQL Data Checksum failure'
+
+  - alert: PGIdleTxn
+    expr: ccp_connection_stats_max_idle_in_txn_time > 300
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} has at least one session idle in transaction for over 5 minutes.'
+      summary: 'PGSQL Instance idle transactions'
+
+  - alert: PGIdleTxn
+    expr: ccp_connection_stats_max_idle_in_txn_time > 900
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} has at least one session idle in transaction for over 15 minutes.'
+      summary: 'PGSQL Instance idle transactions'
+
+  - alert: PGQueryTime
+    expr: ccp_connection_stats_max_query_time > 43200
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} has at least one query running for over 12 hours.'
+      summary: 'PGSQL Max Query Runtime'
+
+  - alert: PGQueryTime
+    expr: ccp_connection_stats_max_query_time > 86400
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} has at least one query running for over 1 day.'
+      summary: 'PGSQL Max Query Runtime'
+
+  - alert: PGConnPerc
+    expr: 100 * (ccp_connection_stats_total / ccp_connection_stats_max_connections) > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: '{{ $labels.job }} is using 75% or more of available connections ({{ $value }}%)'
+      summary: 'PGSQL Instance connections'
+
+  - alert: PGConnPerc
+    expr: 100 * (ccp_connection_stats_total / ccp_connection_stats_max_connections) > 90
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: '{{ $labels.job }} is using 90% or more of available connections ({{ $value }}%)'
+      summary: 'PGSQL Instance connections'
+
+  - alert: DiskFillPredict
+    expr: predict_linear(ccp_nodemx_data_disk_available_bytes{mount_point!~"tmpfs"}[1h], 24 * 3600) < 0 and 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 70
+    for: 5m
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      summary: 'Disk predicted to be full in 24 hours'
+      description: 'Disk on {{ $labels.pg_cluster }}:{{ $labels.kubernetes_pod_name }} is predicted to fill in 24 hrs based on current usage'
+
+  - alert: PGClusterRoleChange
+    expr: count by (pg_cluster) (ccp_is_in_recovery_status != ignoring(instance,ip,pod,role) (ccp_is_in_recovery_status offset 5m)) >= 1
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      summary: '{{ $labels.pg_cluster }} has had a switchover/failover event. Please check this cluster for more details'
+
+  - alert: PGDiskSize
+    expr: 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.deployment }} over 75% disk usage at mount point "{{ $labels.mount_point }}": {{ $value }}%'
+      summary: PGSQL Instance usage warning
+
+  - alert: PGDiskSize
+    expr: 100 * ((ccp_nodemx_data_disk_total_bytes - ccp_nodemx_data_disk_available_bytes) / ccp_nodemx_data_disk_total_bytes) > 90
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.deployment }} over 90% disk usage at mount point "{{ $labels.mount_point }}": {{ $value }}%'
+      summary: 'PGSQL Instance size critical'
+
+  - alert: PGReplicationByteLag
+    expr: ccp_replication_lag_size_bytes > 5.24288e+07
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has at least one replica lagging over 50MB behind.'
+      summary: 'PGSQL Instance replica lag warning'
+
+  - alert: PGReplicationByteLag
+    expr: ccp_replication_lag_size_bytes > 1.048576e+08
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has at least one replica lagging over 100MB behind.'
+      summary: 'PGSQL Instance replica lag warning'
+
+  - alert: PGReplicationSlotsInactive
+    expr: ccp_replication_slots_active == 0
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} has one or more inactive replication slots'
+      summary: 'PGSQL Instance inactive replication slot'
+
+  - alert: PGXIDWraparound
+    expr: ccp_transaction_wraparound_percent_towards_wraparound > 50
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 50% towards transaction id wraparound.'
+      summary: 'PGSQL Instance {{ $labels.job }} transaction id wraparound imminent'
+
+  - alert: PGXIDWraparound
+    expr: ccp_transaction_wraparound_percent_towards_wraparound > 75
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 75% towards transaction id wraparound.'
+      summary: 'PGSQL Instance transaction id wraparound imminent'
+
+  - alert: PGEmergencyVacuum
+    expr: ccp_transaction_wraparound_percent_towards_emergency_autovac > 110
+    for: 60s
+    labels:
+      service: postgresql
+      severity: warning
+      severity_num: 200
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 110% beyond autovacuum_freeze_max_age value. Autovacuum may need tuning to better keep up.'
+      summary: 'PGSQL Instance emergency vacuum imminent'
+
+  - alert: PGEmergencyVacuum
+    expr: ccp_transaction_wraparound_percent_towards_emergency_autovac > 125
+    for: 60s
+    labels:
+      service: postgresql
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'PGSQL Instance {{ $labels.job }} is over 125% beyond autovacuum_freeze_max_age value. Autovacuum needs tuning to better keep up.'
+      summary: 'PGSQL Instance emergency vacuum imminent'
+
+  - alert: PGArchiveCommandStatus
+    expr: ccp_archive_command_status_seconds_since_last_fail > 300
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'PGSQL Instance {{ $labels.job }} has a recent failing archive command'
+        summary: 'Seconds since the last recorded failure of the archive_command'
+
+  - alert: PGSequenceExhaustion
+    expr: ccp_sequence_exhaustion_count > 0
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'Count of sequences on instance {{ $labels.job }} at over 75% usage: {{ $value }}. Run following query to see full sequence status: SELECT * FROM monitor.sequence_status() WHERE percent >= 75'
+
+  - alert: PGSettingsPendingRestart
+    expr: ccp_settings_pending_restart_count > 0
+    for: 60s
+    labels:
+        service: postgresql
+        severity: critical
+        severity_num: 300
+    annotations:
+        description: 'One or more settings in the pg_settings system catalog on system {{ $labels.job }} are in a pending_restart state. Check the system catalog for which settings are pending and review postgresql.conf for changes.'
+
+########## PGBACKREST RULES ##########
+#
+# Uncomment and customize one or more of these rules to monitor your pgbackrest backups.
+# Full backups are considered the equivalent of both differentials and incrementals since both are based on the last full
+#   And differentials are considered incrementals since incrementals will be based off the last diff if one exists
+#   This avoid false alerts, for example when you don't run diff/incr backups on the days that you run a full
+# Stanza should also be set if different intervals are expected for each stanza.
+#   Otherwise rule will be applied to all stanzas returned on target system if not set.
+#
+# Relevant metric names are:
+#   ccp_backrest_last_full_backup_time_since_completion_seconds
+#   ccp_backrest_last_incr_backup_time_since_completion_seconds
+#   ccp_backrest_last_diff_backup_time_since_completion_seconds
+#
+# To avoid false positives on backup time alerts, 12 hours are added onto each threshold to allow a buffer if the backup runtime varies from day to day.
+#    Further adjustment may be needed depending on your backup runtimes/schedule.
+#
+#  - alert: PGBackRestLastCompletedFull_main
+#    expr: ccp_backrest_last_full_backup_time_since_completion_seconds{stanza="main"} > 648000
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Full backup for stanza [main] on system {{ $labels.job }} has not completed in the last week.'
+#
+#  - alert: PGBackRestLastCompletedIncr_main
+#    expr: ccp_backrest_last_incr_backup_time_since_completion_seconds{stanza="main"} > 129600
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Incremental backup for stanza [main] on system {{ $labels.job }} has not completed in the last 24 hours.'
+#
+#
+# Runtime monitoring is handled with a single metric:
+#
+#   ccp_backrest_last_info_backup_runtime_seconds
+#
+# Runtime monitoring should have the "backup_type" label set.
+#   Otherwise the rule will apply to the last run of all backup types returned (full, diff, incr)
+# Stanza should also be set if runtimes per stanza have different expected times
+#
+#  - alert: PGBackRestLastRuntimeFull_main
+#    expr: ccp_backrest_last_info_backup_runtime_seconds{backup_type="full", stanza="main"} > 14400
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Expected runtime of full backup for stanza [main] has exceeded 4 hours'
+#
+#  - alert: PGBackRestLastRuntimeDiff_main
+#    expr: ccp_backrest_last_info_backup_runtime_seconds{backup_type="diff", stanza="main"} > 3600
+#    for: 60s
+#    labels:
+#       service: postgresql
+#       severity: critical
+#       severity_num: 300
+#    annotations:
+#       summary: 'Expected runtime of diff backup for stanza [main] has exceeded 1 hour'
+##
+#
+## If the pgbackrest command fails to run, the metric disappears from the exporter output and the alert never fires.
+## An absence alert must be configured explicitly for each target (job) that backups are being monitored.
+## Checking for absence of just the full backup type should be sufficient (no need for diff/incr).
+## Note that while the backrest check command failing will likely also cause a scrape error alert, the addition of this
+## check gives a clearer answer as to what is causing it and that something is wrong with the backups.
+#
+#  - alert: PGBackrestAbsentFull_Prod
+#    expr: absent(ccp_backrest_last_full_backup_time_since_completion_seconds{job="Prod"})
+#    for: 10s
+#    labels:
+#      service: postgresql
+#      severity: critical
+#      severity_num: 300
+#    annotations:
+#      description: 'Backup Full status missing for Prod. Check that pgbackrest info command is working on target system.'