(#57) Allow gha-rs scale set to fail on all but one clusters

The effect of this patch is limited to refreshing credentials only for
namespaces that exist in the local cluster.  There is structure in place
in the CUE code to allow for namespaces bound to specific clusters, but
this is used only by the optional Vault component.

This patch was an attempt to work around
https://github.com/actions/actions-runner-controller/issues/3351 by
deploying the runner scale sets into unique namespaces.

This effort was a waste of time, only one listener pod successfully
registered for a given scale set name / group combination.

Because we have only one group named Default we can only have one
listener pod globally for a given scale set name.

Because we want our workflows to execute regardless of the availability
of a single cluster, we're going to let this fail for now.  The pod
retries every 3 seconds.  When a cluster is destroyed, another cluster
will quickly register.

A follow up patch will look to expand this retry behavior.

.github/workflows/lint.yaml

@@ -15,7 +15,7 @@ permissions:
 jobs:
   golangci:
     name: lint
-    runs-on: [self-hosted, k8s]
+    runs-on: gha-rs
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5

.github/workflows/release.yaml

@@ -2,17 +2,20 @@ name: Release
 
 on:
   push:
-    # Run only against tags
     tags:
       - '*'
+    branches:
+      - release
 
 permissions:
   contents: write
 
 jobs:
   goreleaser:
-    runs-on: [self-hosted, k8s]
+    runs-on: gha-rs
     steps:
+      - name: Provide GPG and Git
+        run: sudo apt update && sudo apt -qq -y install gnupg git
       - name: Checkout
         uses: actions/checkout@v4
         with:

.github/workflows/test.yaml

@@ -13,7 +13,7 @@ permissions:
 
 jobs:
   test:
-    runs-on: [self-hosted, k8s]
+    runs-on: gha-rs
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -23,10 +23,14 @@ jobs:
         with:
           go-version: stable
 
+      - name: Provide unzip for Helm
+        run: sudo apt update && sudo apt -qq -y install curl zip unzip tar bzip2
+
       - name: Set up Helm
-        uses: azure/setup-helm@v4.1.0
-        with:
-          version: 'latest'
+        uses: azure/setup-helm@v4
+
+      - name: Set up Kubectl
+        uses: azure/setup-kubectl@v3
 
       - name: Test
         run: ./scripts/test

cmd/holos/testdata/issue33_helm_stderr.txt

@@ -16,6 +16,7 @@ namespace: "zitadel"
 chart: {
 	name:    "zitadel"
 	version: "7.9.0"
+	release: name
 	repository: {
 		name: "zitadel"
 		url:  "https://charts.zitadel.com"

cmd/holos/testdata/issue42_kustomize_build_kind.txt

@@ -0,0 +1,33 @@
+# Kustomize is a supported holos component kind
+exec holos render --cluster-name=mycluster . --log-level=debug
+
+# Want generated output
+cmp want.yaml deploy/clusters/mycluster/components/kstest/kstest.gen.yaml
+
+-- cue.mod --
+package holos
+-- component.cue --
+package holos
+
+cluster: string @tag(cluster, string)
+
+apiVersion: "holos.run/v1alpha1"
+kind: "KustomizeBuild"
+metadata: name: "kstest"
+-- kustomization.yaml --
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: mynamespace
+resources:
+- serviceaccount.yaml
+-- serviceaccount.yaml --
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: test
+-- want.yaml --
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: test
+  namespace: mynamespace

docs/examples/cue.mod/gen/postgres-operator.crunchydata.com/pgadmin/v1beta1/types_gen.cue

@@ -0,0 +1,975 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/core2/components/prod-pgo-crds/prod-pgo-crds.gen.yaml
+
+package v1beta1
+
+import "strings"
+
+// PGAdmin is the Schema for the pgadmins API
+#PGAdmin: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "postgres-operator.crunchydata.com/v1beta1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "PGAdmin"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// PGAdminSpec defines the desired state of PGAdmin
+	spec!: #PGAdminSpec
+}
+
+// PGAdminSpec defines the desired state of PGAdmin
+#PGAdminSpec: {
+	// Scheduling constraints of the PGAdmin pod. More info:
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node
+	affinity?: {
+		// Describes node affinity scheduling rules for the pod.
+		nodeAffinity?: {
+			// The scheduler will prefer to schedule pods to nodes that
+			// satisfy the affinity expressions specified by this field, but
+			// it may choose a node that violates one or more of the
+			// expressions. The node that is most preferred is the one with
+			// the greatest sum of weights, i.e. for each node that meets all
+			// of the scheduling requirements (resource request,
+			// requiredDuringScheduling affinity expressions, etc.), compute
+			// a sum by iterating through the elements of this field and
+			// adding "weight" to the sum if the node matches the
+			// corresponding matchExpressions; the node(s) with the highest
+			// sum are the most preferred.
+			preferredDuringSchedulingIgnoredDuringExecution?: [...{
+				// A node selector term, associated with the corresponding weight.
+				preference: {
+					// A list of node selector requirements by node's labels.
+					matchExpressions?: [...{
+						// The label key that the selector applies to.
+						key: string
+
+						// Represents a key's relationship to a set of values. Valid
+						// operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+						operator: string
+
+						// An array of string values. If the operator is In or NotIn, the
+						// values array must be non-empty. If the operator is Exists or
+						// DoesNotExist, the values array must be empty. If the operator
+						// is Gt or Lt, the values array must have a single element,
+						// which will be interpreted as an integer. This array is
+						// replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// A list of node selector requirements by node's fields.
+					matchFields?: [...{
+						// The label key that the selector applies to.
+						key: string
+
+						// Represents a key's relationship to a set of values. Valid
+						// operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+						operator: string
+
+						// An array of string values. If the operator is In or NotIn, the
+						// values array must be non-empty. If the operator is Exists or
+						// DoesNotExist, the values array must be empty. If the operator
+						// is Gt or Lt, the values array must have a single element,
+						// which will be interpreted as an integer. This array is
+						// replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+				}
+
+				// Weight associated with matching the corresponding
+				// nodeSelectorTerm, in the range 1-100.
+				weight: int
+			}]
+			requiredDuringSchedulingIgnoredDuringExecution?: {
+				// Required. A list of node selector terms. The terms are ORed.
+				nodeSelectorTerms: [...{
+					// A list of node selector requirements by node's labels.
+					matchExpressions?: [...{
+						// The label key that the selector applies to.
+						key: string
+
+						// Represents a key's relationship to a set of values. Valid
+						// operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+						operator: string
+
+						// An array of string values. If the operator is In or NotIn, the
+						// values array must be non-empty. If the operator is Exists or
+						// DoesNotExist, the values array must be empty. If the operator
+						// is Gt or Lt, the values array must have a single element,
+						// which will be interpreted as an integer. This array is
+						// replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// A list of node selector requirements by node's fields.
+					matchFields?: [...{
+						// The label key that the selector applies to.
+						key: string
+
+						// Represents a key's relationship to a set of values. Valid
+						// operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+						operator: string
+
+						// An array of string values. If the operator is In or NotIn, the
+						// values array must be non-empty. If the operator is Exists or
+						// DoesNotExist, the values array must be empty. If the operator
+						// is Gt or Lt, the values array must have a single element,
+						// which will be interpreted as an integer. This array is
+						// replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+				}]
+			}
+		}
+
+		// Describes pod affinity scheduling rules (e.g. co-locate this
+		// pod in the same node, zone, etc. as some other pod(s)).
+		podAffinity?: {
+			// The scheduler will prefer to schedule pods to nodes that
+			// satisfy the affinity expressions specified by this field, but
+			// it may choose a node that violates one or more of the
+			// expressions. The node that is most preferred is the one with
+			// the greatest sum of weights, i.e. for each node that meets all
+			// of the scheduling requirements (resource request,
+			// requiredDuringScheduling affinity expressions, etc.), compute
+			// a sum by iterating through the elements of this field and
+			// adding "weight" to the sum if the node has pods which matches
+			// the corresponding podAffinityTerm; the node(s) with the
+			// highest sum are the most preferred.
+			preferredDuringSchedulingIgnoredDuringExecution?: [...{
+				// Required. A pod affinity term, associated with the
+				// corresponding weight.
+				podAffinityTerm: {
+					// A label query over a set of resources, in this case pods.
+					labelSelector?: {
+						// matchExpressions is a list of label selector requirements. The
+						// requirements are ANDed.
+						matchExpressions?: [...{
+							// key is the label key that the selector applies to.
+							key: string
+
+							// operator represents a key's relationship to a set of values.
+							// Valid operators are In, NotIn, Exists and DoesNotExist.
+							operator: string
+
+							// values is an array of string values. If the operator is In or
+							// NotIn, the values array must be non-empty. If the operator is
+							// Exists or DoesNotExist, the values array must be empty. This
+							// array is replaced during a strategic merge patch.
+							values?: [...string]
+						}]
+
+						// matchLabels is a map of {key,value} pairs. A single {key,value}
+						// in the matchLabels map is equivalent to an element of
+						// matchExpressions, whose key field is "key", the operator is
+						// "In", and the values array contains only "value". The
+						// requirements are ANDed.
+						matchLabels?: {
+							[string]: string
+						}
+					}
+
+					// A label query over the set of namespaces that the term applies
+					// to. The term is applied to the union of the namespaces
+					// selected by this field and the ones listed in the namespaces
+					// field. null selector and null or empty namespaces list means
+					// "this pod's namespace". An empty selector ({}) matches all
+					// namespaces.
+					namespaceSelector?: {
+						// matchExpressions is a list of label selector requirements. The
+						// requirements are ANDed.
+						matchExpressions?: [...{
+							// key is the label key that the selector applies to.
+							key: string
+
+							// operator represents a key's relationship to a set of values.
+							// Valid operators are In, NotIn, Exists and DoesNotExist.
+							operator: string
+
+							// values is an array of string values. If the operator is In or
+							// NotIn, the values array must be non-empty. If the operator is
+							// Exists or DoesNotExist, the values array must be empty. This
+							// array is replaced during a strategic merge patch.
+							values?: [...string]
+						}]
+
+						// matchLabels is a map of {key,value} pairs. A single {key,value}
+						// in the matchLabels map is equivalent to an element of
+						// matchExpressions, whose key field is "key", the operator is
+						// "In", and the values array contains only "value". The
+						// requirements are ANDed.
+						matchLabels?: {
+							[string]: string
+						}
+					}
+
+					// namespaces specifies a static list of namespace names that the
+					// term applies to. The term is applied to the union of the
+					// namespaces listed in this field and the ones selected by
+					// namespaceSelector. null or empty namespaces list and null
+					// namespaceSelector means "this pod's namespace".
+					namespaces?: [...string]
+
+					// This pod should be co-located (affinity) or not co-located
+					// (anti-affinity) with the pods matching the labelSelector in
+					// the specified namespaces, where co-located is defined as
+					// running on a node whose value of the label with key
+					// topologyKey matches that of any node on which any of the
+					// selected pods is running. Empty topologyKey is not allowed.
+					topologyKey: string
+				}
+
+				// weight associated with matching the corresponding
+				// podAffinityTerm, in the range 1-100.
+				weight: int
+			}]
+
+			// If the affinity requirements specified by this field are not
+			// met at scheduling time, the pod will not be scheduled onto the
+			// node. If the affinity requirements specified by this field
+			// cease to be met at some point during pod execution (e.g. due
+			// to a pod label update), the system may or may not try to
+			// eventually evict the pod from its node. When there are
+			// multiple elements, the lists of nodes corresponding to each
+			// podAffinityTerm are intersected, i.e. all terms must be
+			// satisfied.
+			requiredDuringSchedulingIgnoredDuringExecution?: [...{
+				// A label query over a set of resources, in this case pods.
+				labelSelector?: {
+					// matchExpressions is a list of label selector requirements. The
+					// requirements are ANDed.
+					matchExpressions?: [...{
+						// key is the label key that the selector applies to.
+						key: string
+
+						// operator represents a key's relationship to a set of values.
+						// Valid operators are In, NotIn, Exists and DoesNotExist.
+						operator: string
+
+						// values is an array of string values. If the operator is In or
+						// NotIn, the values array must be non-empty. If the operator is
+						// Exists or DoesNotExist, the values array must be empty. This
+						// array is replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// matchLabels is a map of {key,value} pairs. A single {key,value}
+					// in the matchLabels map is equivalent to an element of
+					// matchExpressions, whose key field is "key", the operator is
+					// "In", and the values array contains only "value". The
+					// requirements are ANDed.
+					matchLabels?: {
+						[string]: string
+					}
+				}
+
+				// A label query over the set of namespaces that the term applies
+				// to. The term is applied to the union of the namespaces
+				// selected by this field and the ones listed in the namespaces
+				// field. null selector and null or empty namespaces list means
+				// "this pod's namespace". An empty selector ({}) matches all
+				// namespaces.
+				namespaceSelector?: {
+					// matchExpressions is a list of label selector requirements. The
+					// requirements are ANDed.
+					matchExpressions?: [...{
+						// key is the label key that the selector applies to.
+						key: string
+
+						// operator represents a key's relationship to a set of values.
+						// Valid operators are In, NotIn, Exists and DoesNotExist.
+						operator: string
+
+						// values is an array of string values. If the operator is In or
+						// NotIn, the values array must be non-empty. If the operator is
+						// Exists or DoesNotExist, the values array must be empty. This
+						// array is replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// matchLabels is a map of {key,value} pairs. A single {key,value}
+					// in the matchLabels map is equivalent to an element of
+					// matchExpressions, whose key field is "key", the operator is
+					// "In", and the values array contains only "value". The
+					// requirements are ANDed.
+					matchLabels?: {
+						[string]: string
+					}
+				}
+
+				// namespaces specifies a static list of namespace names that the
+				// term applies to. The term is applied to the union of the
+				// namespaces listed in this field and the ones selected by
+				// namespaceSelector. null or empty namespaces list and null
+				// namespaceSelector means "this pod's namespace".
+				namespaces?: [...string]
+
+				// This pod should be co-located (affinity) or not co-located
+				// (anti-affinity) with the pods matching the labelSelector in
+				// the specified namespaces, where co-located is defined as
+				// running on a node whose value of the label with key
+				// topologyKey matches that of any node on which any of the
+				// selected pods is running. Empty topologyKey is not allowed.
+				topologyKey: string
+			}]
+		}
+
+		// Describes pod anti-affinity scheduling rules (e.g. avoid
+		// putting this pod in the same node, zone, etc. as some other
+		// pod(s)).
+		podAntiAffinity?: {
+			// The scheduler will prefer to schedule pods to nodes that
+			// satisfy the anti-affinity expressions specified by this field,
+			// but it may choose a node that violates one or more of the
+			// expressions. The node that is most preferred is the one with
+			// the greatest sum of weights, i.e. for each node that meets all
+			// of the scheduling requirements (resource request,
+			// requiredDuringScheduling anti-affinity expressions, etc.),
+			// compute a sum by iterating through the elements of this field
+			// and adding "weight" to the sum if the node has pods which
+			// matches the corresponding podAffinityTerm; the node(s) with
+			// the highest sum are the most preferred.
+			preferredDuringSchedulingIgnoredDuringExecution?: [...{
+				// Required. A pod affinity term, associated with the
+				// corresponding weight.
+				podAffinityTerm: {
+					// A label query over a set of resources, in this case pods.
+					labelSelector?: {
+						// matchExpressions is a list of label selector requirements. The
+						// requirements are ANDed.
+						matchExpressions?: [...{
+							// key is the label key that the selector applies to.
+							key: string
+
+							// operator represents a key's relationship to a set of values.
+							// Valid operators are In, NotIn, Exists and DoesNotExist.
+							operator: string
+
+							// values is an array of string values. If the operator is In or
+							// NotIn, the values array must be non-empty. If the operator is
+							// Exists or DoesNotExist, the values array must be empty. This
+							// array is replaced during a strategic merge patch.
+							values?: [...string]
+						}]
+
+						// matchLabels is a map of {key,value} pairs. A single {key,value}
+						// in the matchLabels map is equivalent to an element of
+						// matchExpressions, whose key field is "key", the operator is
+						// "In", and the values array contains only "value". The
+						// requirements are ANDed.
+						matchLabels?: {
+							[string]: string
+						}
+					}
+
+					// A label query over the set of namespaces that the term applies
+					// to. The term is applied to the union of the namespaces
+					// selected by this field and the ones listed in the namespaces
+					// field. null selector and null or empty namespaces list means
+					// "this pod's namespace". An empty selector ({}) matches all
+					// namespaces.
+					namespaceSelector?: {
+						// matchExpressions is a list of label selector requirements. The
+						// requirements are ANDed.
+						matchExpressions?: [...{
+							// key is the label key that the selector applies to.
+							key: string
+
+							// operator represents a key's relationship to a set of values.
+							// Valid operators are In, NotIn, Exists and DoesNotExist.
+							operator: string
+
+							// values is an array of string values. If the operator is In or
+							// NotIn, the values array must be non-empty. If the operator is
+							// Exists or DoesNotExist, the values array must be empty. This
+							// array is replaced during a strategic merge patch.
+							values?: [...string]
+						}]
+
+						// matchLabels is a map of {key,value} pairs. A single {key,value}
+						// in the matchLabels map is equivalent to an element of
+						// matchExpressions, whose key field is "key", the operator is
+						// "In", and the values array contains only "value". The
+						// requirements are ANDed.
+						matchLabels?: {
+							[string]: string
+						}
+					}
+
+					// namespaces specifies a static list of namespace names that the
+					// term applies to. The term is applied to the union of the
+					// namespaces listed in this field and the ones selected by
+					// namespaceSelector. null or empty namespaces list and null
+					// namespaceSelector means "this pod's namespace".
+					namespaces?: [...string]
+
+					// This pod should be co-located (affinity) or not co-located
+					// (anti-affinity) with the pods matching the labelSelector in
+					// the specified namespaces, where co-located is defined as
+					// running on a node whose value of the label with key
+					// topologyKey matches that of any node on which any of the
+					// selected pods is running. Empty topologyKey is not allowed.
+					topologyKey: string
+				}
+
+				// weight associated with matching the corresponding
+				// podAffinityTerm, in the range 1-100.
+				weight: int
+			}]
+
+			// If the anti-affinity requirements specified by this field are
+			// not met at scheduling time, the pod will not be scheduled onto
+			// the node. If the anti-affinity requirements specified by this
+			// field cease to be met at some point during pod execution (e.g.
+			// due to a pod label update), the system may or may not try to
+			// eventually evict the pod from its node. When there are
+			// multiple elements, the lists of nodes corresponding to each
+			// podAffinityTerm are intersected, i.e. all terms must be
+			// satisfied.
+			requiredDuringSchedulingIgnoredDuringExecution?: [...{
+				// A label query over a set of resources, in this case pods.
+				labelSelector?: {
+					// matchExpressions is a list of label selector requirements. The
+					// requirements are ANDed.
+					matchExpressions?: [...{
+						// key is the label key that the selector applies to.
+						key: string
+
+						// operator represents a key's relationship to a set of values.
+						// Valid operators are In, NotIn, Exists and DoesNotExist.
+						operator: string
+
+						// values is an array of string values. If the operator is In or
+						// NotIn, the values array must be non-empty. If the operator is
+						// Exists or DoesNotExist, the values array must be empty. This
+						// array is replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// matchLabels is a map of {key,value} pairs. A single {key,value}
+					// in the matchLabels map is equivalent to an element of
+					// matchExpressions, whose key field is "key", the operator is
+					// "In", and the values array contains only "value". The
+					// requirements are ANDed.
+					matchLabels?: {
+						[string]: string
+					}
+				}
+
+				// A label query over the set of namespaces that the term applies
+				// to. The term is applied to the union of the namespaces
+				// selected by this field and the ones listed in the namespaces
+				// field. null selector and null or empty namespaces list means
+				// "this pod's namespace". An empty selector ({}) matches all
+				// namespaces.
+				namespaceSelector?: {
+					// matchExpressions is a list of label selector requirements. The
+					// requirements are ANDed.
+					matchExpressions?: [...{
+						// key is the label key that the selector applies to.
+						key: string
+
+						// operator represents a key's relationship to a set of values.
+						// Valid operators are In, NotIn, Exists and DoesNotExist.
+						operator: string
+
+						// values is an array of string values. If the operator is In or
+						// NotIn, the values array must be non-empty. If the operator is
+						// Exists or DoesNotExist, the values array must be empty. This
+						// array is replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// matchLabels is a map of {key,value} pairs. A single {key,value}
+					// in the matchLabels map is equivalent to an element of
+					// matchExpressions, whose key field is "key", the operator is
+					// "In", and the values array contains only "value". The
+					// requirements are ANDed.
+					matchLabels?: {
+						[string]: string
+					}
+				}
+
+				// namespaces specifies a static list of namespace names that the
+				// term applies to. The term is applied to the union of the
+				// namespaces listed in this field and the ones selected by
+				// namespaceSelector. null or empty namespaces list and null
+				// namespaceSelector means "this pod's namespace".
+				namespaces?: [...string]
+
+				// This pod should be co-located (affinity) or not co-located
+				// (anti-affinity) with the pods matching the labelSelector in
+				// the specified namespaces, where co-located is defined as
+				// running on a node whose value of the label with key
+				// topologyKey matches that of any node on which any of the
+				// selected pods is running. Empty topologyKey is not allowed.
+				topologyKey: string
+			}]
+		}
+	}
+
+	// Configuration settings for the pgAdmin process. Changes to any
+	// of these values will be loaded without validation. Be careful,
+	// as you may put pgAdmin into an unusable state.
+	config?: {
+		// Files allows the user to mount projected volumes into the
+		// pgAdmin container so that files can be referenced by pgAdmin
+		// as needed.
+		files?: [...{
+			// configMap information about the configMap data to project
+			configMap?: {
+				// items if unspecified, each key-value pair in the Data field of
+				// the referenced ConfigMap will be projected into the volume as
+				// a file whose name is the key and content is the value. If
+				// specified, the listed keys will be projected into the
+				// specified paths, and unlisted keys will not be present. If a
+				// key is specified which is not present in the ConfigMap, the
+				// volume setup will error unless it is marked optional. Paths
+				// must be relative and may not contain the '..' path or start
+				// with '..'.
+				items?: [...{
+					// key is the key to project.
+					key: string
+
+					// mode is Optional: mode bits used to set permissions on this
+					// file. Must be an octal value between 0000 and 0777 or a
+					// decimal value between 0 and 511. YAML accepts both octal and
+					// decimal values, JSON requires decimal values for mode bits. If
+					// not specified, the volume defaultMode will be used. This might
+					// be in conflict with other options that affect the file mode,
+					// like fsGroup, and the result can be other mode bits set.
+					mode?: int
+
+					// path is the relative path of the file to map the key to. May
+					// not be an absolute path. May not contain the path element
+					// '..'. May not start with the string '..'.
+					path: string
+				}]
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				name?: string
+
+				// optional specify whether the ConfigMap or its keys must be
+				// defined
+				optional?: bool
+			}
+			downwardAPI?: {
+				// Items is a list of DownwardAPIVolume file
+				items?: [...{
+					// Required: Selects a field of the pod: only annotations, labels,
+					// name and namespace are supported.
+					fieldRef?: {
+						// Version of the schema the FieldPath is written in terms of,
+						// defaults to "v1".
+						apiVersion?: string
+
+						// Path of the field to select in the specified API version.
+						fieldPath: string
+					}
+
+					// Optional: mode bits used to set permissions on this file, must
+					// be an octal value between 0000 and 0777 or a decimal value
+					// between 0 and 511. YAML accepts both octal and decimal values,
+					// JSON requires decimal values for mode bits. If not specified,
+					// the volume defaultMode will be used. This might be in conflict
+					// with other options that affect the file mode, like fsGroup,
+					// and the result can be other mode bits set.
+					mode?: int
+
+					// Required: Path is the relative path name of the file to be
+					// created. Must not be absolute or contain the '..' path. Must
+					// be utf-8 encoded. The first item of the relative path must not
+					// start with '..'
+					path: string
+
+					// Selects a resource of the container: only resources limits and
+					// requests (limits.cpu, limits.memory, requests.cpu and
+					// requests.memory) are currently supported.
+					resourceFieldRef?: {
+						// Container name: required for volumes, optional for env vars
+						containerName?: string
+
+						// Specifies the output format of the exposed resources, defaults
+						// to "1"
+						divisor?: (int | string) & {
+							=~"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$"
+						}
+
+						// Required: resource to select
+						resource: string
+					}
+				}]
+			}
+
+			// secret information about the secret data to project
+			secret?: {
+				// items if unspecified, each key-value pair in the Data field of
+				// the referenced Secret will be projected into the volume as a
+				// file whose name is the key and content is the value. If
+				// specified, the listed keys will be projected into the
+				// specified paths, and unlisted keys will not be present. If a
+				// key is specified which is not present in the Secret, the
+				// volume setup will error unless it is marked optional. Paths
+				// must be relative and may not contain the '..' path or start
+				// with '..'.
+				items?: [...{
+					// key is the key to project.
+					key: string
+
+					// mode is Optional: mode bits used to set permissions on this
+					// file. Must be an octal value between 0000 and 0777 or a
+					// decimal value between 0 and 511. YAML accepts both octal and
+					// decimal values, JSON requires decimal values for mode bits. If
+					// not specified, the volume defaultMode will be used. This might
+					// be in conflict with other options that affect the file mode,
+					// like fsGroup, and the result can be other mode bits set.
+					mode?: int
+
+					// path is the relative path of the file to map the key to. May
+					// not be an absolute path. May not contain the path element
+					// '..'. May not start with the string '..'.
+					path: string
+				}]
+
+				// Name of the referent. More info:
+				// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+				name?: string
+
+				// optional field specify whether the Secret or its key must be
+				// defined
+				optional?: bool
+			}
+
+			// serviceAccountToken is information about the
+			// serviceAccountToken data to project
+			serviceAccountToken?: {
+				// audience is the intended audience of the token. A recipient of
+				// a token must identify itself with an identifier specified in
+				// the audience of the token, and otherwise should reject the
+				// token. The audience defaults to the identifier of the
+				// apiserver.
+				audience?: string
+
+				// expirationSeconds is the requested duration of validity of the
+				// service account token. As the token approaches expiration, the
+				// kubelet volume plugin will proactively rotate the service
+				// account token. The kubelet will start trying to rotate the
+				// token if the token is older than 80 percent of its time to
+				// live or if the token is older than 24 hours.Defaults to 1 hour
+				// and must be at least 10 minutes.
+				expirationSeconds?: int
+
+				// path is the path relative to the mount point of the file to
+				// project the token into.
+				path: string
+			}
+		}]
+
+		// A Secret containing the value for the LDAP_BIND_PASSWORD
+		// setting. More info:
+		// https://www.pgadmin.org/docs/pgadmin4/latest/ldap.html
+		ldapBindPassword?: {
+			// The key of the secret to select from. Must be a valid secret
+			// key.
+			key: string
+
+			// Name of the referent. More info:
+			// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+			name?: string
+
+			// Specify whether the Secret or its key must be defined
+			optional?: bool
+		}
+
+		// Settings for the pgAdmin server process. Keys should be
+		// uppercase and values must be constants. More info:
+		// https://www.pgadmin.org/docs/pgadmin4/latest/config_py.html
+		settings?: {
+			...
+		}
+	}
+
+	// Defines a PersistentVolumeClaim for pgAdmin data. More info:
+	// https://kubernetes.io/docs/concepts/storage/persistent-volumes
+	dataVolumeClaimSpec: {
+		// accessModes contains the desired access modes the volume should
+		// have. More info:
+		// https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1
+		accessModes?: [...string]
+
+		// dataSource field can be used to specify either: * An existing
+		// VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot)
+		// * An existing PVC (PersistentVolumeClaim) If the provisioner
+		// or an external controller can support the specified data
+		// source, it will create a new volume based on the contents of
+		// the specified data source. If the AnyVolumeDataSource feature
+		// gate is enabled, this field will always have the same contents
+		// as the DataSourceRef field.
+		dataSource?: {
+			// APIGroup is the group for the resource being referenced. If
+			// APIGroup is not specified, the specified Kind must be in the
+			// core API group. For any other third-party types, APIGroup is
+			// required.
+			apiGroup?: string
+
+			// Kind is the type of resource being referenced
+			kind: string
+
+			// Name is the name of resource being referenced
+			name: string
+		}
+
+		// dataSourceRef specifies the object from which to populate the
+		// volume with data, if a non-empty volume is desired. This may
+		// be any local object from a non-empty API group (non core
+		// object) or a PersistentVolumeClaim object. When this field is
+		// specified, volume binding will only succeed if the type of the
+		// specified object matches some installed volume populator or
+		// dynamic provisioner. This field will replace the functionality
+		// of the DataSource field and as such if both fields are
+		// non-empty, they must have the same value. For backwards
+		// compatibility, both fields (DataSource and DataSourceRef) will
+		// be set to the same value automatically if one of them is empty
+		// and the other is non-empty. There are two important
+		// differences between DataSource and DataSourceRef: * While
+		// DataSource only allows two specific types of objects,
+		// DataSourceRef allows any non-core object, as well as
+		// PersistentVolumeClaim objects. * While DataSource ignores
+		// disallowed values (dropping them), DataSourceRef preserves all
+		// values, and generates an error if a disallowed value is
+		// specified. (Beta) Using this field requires the
+		// AnyVolumeDataSource feature gate to be enabled.
+		dataSourceRef?: {
+			// APIGroup is the group for the resource being referenced. If
+			// APIGroup is not specified, the specified Kind must be in the
+			// core API group. For any other third-party types, APIGroup is
+			// required.
+			apiGroup?: string
+
+			// Kind is the type of resource being referenced
+			kind: string
+
+			// Name is the name of resource being referenced
+			name: string
+		}
+
+		// resources represents the minimum resources the volume should
+		// have. If RecoverVolumeExpansionFailure feature is enabled
+		// users are allowed to specify resource requirements that are
+		// lower than previous value but must still be higher than
+		// capacity recorded in the status field of the claim. More info:
+		// https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources
+		resources?: {
+			// Limits describes the maximum amount of compute resources
+			// allowed. More info:
+			// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+			limits?: {
+				[string]: (int | string) & =~"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$"
+			}
+
+			// Requests describes the minimum amount of compute resources
+			// required. If Requests is omitted for a container, it defaults
+			// to Limits if that is explicitly specified, otherwise to an
+			// implementation-defined value. More info:
+			// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+			requests?: {
+				[string]: (int | string) & =~"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$"
+			}
+		}
+
+		// selector is a label query over volumes to consider for binding.
+		selector?: {
+			// matchExpressions is a list of label selector requirements. The
+			// requirements are ANDed.
+			matchExpressions?: [...{
+				// key is the label key that the selector applies to.
+				key: string
+
+				// operator represents a key's relationship to a set of values.
+				// Valid operators are In, NotIn, Exists and DoesNotExist.
+				operator: string
+
+				// values is an array of string values. If the operator is In or
+				// NotIn, the values array must be non-empty. If the operator is
+				// Exists or DoesNotExist, the values array must be empty. This
+				// array is replaced during a strategic merge patch.
+				values?: [...string]
+			}]
+
+			// matchLabels is a map of {key,value} pairs. A single {key,value}
+			// in the matchLabels map is equivalent to an element of
+			// matchExpressions, whose key field is "key", the operator is
+			// "In", and the values array contains only "value". The
+			// requirements are ANDed.
+			matchLabels?: {
+				[string]: string
+			}
+		}
+
+		// storageClassName is the name of the StorageClass required by
+		// the claim. More info:
+		// https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1
+		storageClassName?: string
+
+		// volumeMode defines what type of volume is required by the
+		// claim. Value of Filesystem is implied when not included in
+		// claim spec.
+		volumeMode?: string
+
+		// volumeName is the binding reference to the PersistentVolume
+		// backing this claim.
+		volumeName?: string
+	}
+
+	// The image name to use for pgAdmin instance.
+	image?: string
+
+	// ImagePullPolicy is used to determine when Kubernetes will
+	// attempt to pull (download) container images. More info:
+	// https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy
+	imagePullPolicy?: "Always" | "Never" | "IfNotPresent"
+
+	// The image pull secrets used to pull from a private registry.
+	// Changing this value causes all running PGAdmin pods to
+	// restart.
+	// https://k8s.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+	imagePullSecrets?: [...{
+		// Name of the referent. More info:
+		// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+		name?: string
+	}]
+
+	// Metadata contains metadata for custom resources
+	metadata?: {
+		annotations?: {
+			[string]: string
+		}
+		labels?: {
+			[string]: string
+		}
+	}
+
+	// Priority class name for the PGAdmin pod. Changing this value
+	// causes PGAdmin pod to restart. More info:
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/
+	priorityClassName?: string
+
+	// Resource requirements for the PGAdmin container.
+	resources?: {
+		// Limits describes the maximum amount of compute resources
+		// allowed. More info:
+		// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+		limits?: {
+			[string]: (int | string) & =~"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$"
+		}
+
+		// Requests describes the minimum amount of compute resources
+		// required. If Requests is omitted for a container, it defaults
+		// to Limits if that is explicitly specified, otherwise to an
+		// implementation-defined value. More info:
+		// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+		requests?: {
+			[string]: (int | string) & =~"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$"
+		}
+	}
+
+	// ServerGroups for importing PostgresClusters to pgAdmin. To
+	// create a pgAdmin with no selectors, leave this field empty. A
+	// pgAdmin created with no `ServerGroups` will not automatically
+	// add any servers through discovery. PostgresClusters can still
+	// be added manually.
+	serverGroups?: [...{
+		// The name for the ServerGroup in pgAdmin. Must be unique in the
+		// pgAdmin's ServerGroups since it becomes the ServerGroup name
+		// in pgAdmin.
+		name: string
+
+		// PostgresClusterSelector selects clusters to dynamically add to
+		// pgAdmin by matching labels. An empty selector like `{}` will
+		// select ALL clusters in the namespace.
+		postgresClusterSelector: {
+			// matchExpressions is a list of label selector requirements. The
+			// requirements are ANDed.
+			matchExpressions?: [...{
+				// key is the label key that the selector applies to.
+				key: string
+
+				// operator represents a key's relationship to a set of values.
+				// Valid operators are In, NotIn, Exists and DoesNotExist.
+				operator: string
+
+				// values is an array of string values. If the operator is In or
+				// NotIn, the values array must be non-empty. If the operator is
+				// Exists or DoesNotExist, the values array must be empty. This
+				// array is replaced during a strategic merge patch.
+				values?: [...string]
+			}]
+
+			// matchLabels is a map of {key,value} pairs. A single {key,value}
+			// in the matchLabels map is equivalent to an element of
+			// matchExpressions, whose key field is "key", the operator is
+			// "In", and the values array contains only "value". The
+			// requirements are ANDed.
+			matchLabels?: {
+				[string]: string
+			}
+		}
+	}]
+
+	// Tolerations of the PGAdmin pod. More info:
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration
+	tolerations?: [...{
+		// Effect indicates the taint effect to match. Empty means match
+		// all taint effects. When specified, allowed values are
+		// NoSchedule, PreferNoSchedule and NoExecute.
+		effect?: string
+
+		// Key is the taint key that the toleration applies to. Empty
+		// means match all taint keys. If the key is empty, operator must
+		// be Exists; this combination means to match all values and all
+		// keys.
+		key?: string
+
+		// Operator represents a key's relationship to the value. Valid
+		// operators are Exists and Equal. Defaults to Equal. Exists is
+		// equivalent to wildcard for value, so that a pod can tolerate
+		// all taints of a particular category.
+		operator?: string
+
+		// TolerationSeconds represents the period of time the toleration
+		// (which must be of effect NoExecute, otherwise this field is
+		// ignored) tolerates the taint. By default, it is not set, which
+		// means tolerate the taint forever (do not evict). Zero and
+		// negative values will be treated as 0 (evict immediately) by
+		// the system.
+		tolerationSeconds?: int
+
+		// Value is the taint value the toleration matches to. If the
+		// operator is Exists, the value should be empty, otherwise just
+		// a regular string.
+		value?: string
+	}]
+}

docs/examples/cue.mod/gen/postgres-operator.crunchydata.com/pgupgrade/v1beta1/types_gen.cue

@@ -0,0 +1,632 @@
+// Code generated by timoni. DO NOT EDIT.
+
+//timoni:generate timoni vendor crd -f /home/jeff/workspace/holos-run/holos-infra/deploy/clusters/core2/components/prod-pgo-crds/prod-pgo-crds.gen.yaml
+
+package v1beta1
+
+import "strings"
+
+// PGUpgrade is the Schema for the pgupgrades API
+#PGUpgrade: {
+	// APIVersion defines the versioned schema of this representation
+	// of an object. Servers should convert recognized schemas to the
+	// latest internal value, and may reject unrecognized values.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+	apiVersion: "postgres-operator.crunchydata.com/v1beta1"
+
+	// Kind is a string value representing the REST resource this
+	// object represents. Servers may infer this from the endpoint
+	// the client submits requests to. Cannot be updated. In
+	// CamelCase. More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+	kind: "PGUpgrade"
+	metadata!: {
+		name!: strings.MaxRunes(253) & strings.MinRunes(1) & {
+			string
+		}
+		namespace!: strings.MaxRunes(63) & strings.MinRunes(1) & {
+			string
+		}
+		labels?: {
+			[string]: string
+		}
+		annotations?: {
+			[string]: string
+		}
+	}
+
+	// PGUpgradeSpec defines the desired state of PGUpgrade
+	spec!: #PGUpgradeSpec
+}
+
+// PGUpgradeSpec defines the desired state of PGUpgrade
+#PGUpgradeSpec: {
+	// Scheduling constraints of the PGUpgrade pod. More info:
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node
+	affinity?: {
+		// Describes node affinity scheduling rules for the pod.
+		nodeAffinity?: {
+			// The scheduler will prefer to schedule pods to nodes that
+			// satisfy the affinity expressions specified by this field, but
+			// it may choose a node that violates one or more of the
+			// expressions. The node that is most preferred is the one with
+			// the greatest sum of weights, i.e. for each node that meets all
+			// of the scheduling requirements (resource request,
+			// requiredDuringScheduling affinity expressions, etc.), compute
+			// a sum by iterating through the elements of this field and
+			// adding "weight" to the sum if the node matches the
+			// corresponding matchExpressions; the node(s) with the highest
+			// sum are the most preferred.
+			preferredDuringSchedulingIgnoredDuringExecution?: [...{
+				// A node selector term, associated with the corresponding weight.
+				preference: {
+					// A list of node selector requirements by node's labels.
+					matchExpressions?: [...{
+						// The label key that the selector applies to.
+						key: string
+
+						// Represents a key's relationship to a set of values. Valid
+						// operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+						operator: string
+
+						// An array of string values. If the operator is In or NotIn, the
+						// values array must be non-empty. If the operator is Exists or
+						// DoesNotExist, the values array must be empty. If the operator
+						// is Gt or Lt, the values array must have a single element,
+						// which will be interpreted as an integer. This array is
+						// replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// A list of node selector requirements by node's fields.
+					matchFields?: [...{
+						// The label key that the selector applies to.
+						key: string
+
+						// Represents a key's relationship to a set of values. Valid
+						// operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+						operator: string
+
+						// An array of string values. If the operator is In or NotIn, the
+						// values array must be non-empty. If the operator is Exists or
+						// DoesNotExist, the values array must be empty. If the operator
+						// is Gt or Lt, the values array must have a single element,
+						// which will be interpreted as an integer. This array is
+						// replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+				}
+
+				// Weight associated with matching the corresponding
+				// nodeSelectorTerm, in the range 1-100.
+				weight: int
+			}]
+			requiredDuringSchedulingIgnoredDuringExecution?: {
+				// Required. A list of node selector terms. The terms are ORed.
+				nodeSelectorTerms: [...{
+					// A list of node selector requirements by node's labels.
+					matchExpressions?: [...{
+						// The label key that the selector applies to.
+						key: string
+
+						// Represents a key's relationship to a set of values. Valid
+						// operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+						operator: string
+
+						// An array of string values. If the operator is In or NotIn, the
+						// values array must be non-empty. If the operator is Exists or
+						// DoesNotExist, the values array must be empty. If the operator
+						// is Gt or Lt, the values array must have a single element,
+						// which will be interpreted as an integer. This array is
+						// replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// A list of node selector requirements by node's fields.
+					matchFields?: [...{
+						// The label key that the selector applies to.
+						key: string
+
+						// Represents a key's relationship to a set of values. Valid
+						// operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+						operator: string
+
+						// An array of string values. If the operator is In or NotIn, the
+						// values array must be non-empty. If the operator is Exists or
+						// DoesNotExist, the values array must be empty. If the operator
+						// is Gt or Lt, the values array must have a single element,
+						// which will be interpreted as an integer. This array is
+						// replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+				}]
+			}
+		}
+
+		// Describes pod affinity scheduling rules (e.g. co-locate this
+		// pod in the same node, zone, etc. as some other pod(s)).
+		podAffinity?: {
+			// The scheduler will prefer to schedule pods to nodes that
+			// satisfy the affinity expressions specified by this field, but
+			// it may choose a node that violates one or more of the
+			// expressions. The node that is most preferred is the one with
+			// the greatest sum of weights, i.e. for each node that meets all
+			// of the scheduling requirements (resource request,
+			// requiredDuringScheduling affinity expressions, etc.), compute
+			// a sum by iterating through the elements of this field and
+			// adding "weight" to the sum if the node has pods which matches
+			// the corresponding podAffinityTerm; the node(s) with the
+			// highest sum are the most preferred.
+			preferredDuringSchedulingIgnoredDuringExecution?: [...{
+				// Required. A pod affinity term, associated with the
+				// corresponding weight.
+				podAffinityTerm: {
+					// A label query over a set of resources, in this case pods.
+					labelSelector?: {
+						// matchExpressions is a list of label selector requirements. The
+						// requirements are ANDed.
+						matchExpressions?: [...{
+							// key is the label key that the selector applies to.
+							key: string
+
+							// operator represents a key's relationship to a set of values.
+							// Valid operators are In, NotIn, Exists and DoesNotExist.
+							operator: string
+
+							// values is an array of string values. If the operator is In or
+							// NotIn, the values array must be non-empty. If the operator is
+							// Exists or DoesNotExist, the values array must be empty. This
+							// array is replaced during a strategic merge patch.
+							values?: [...string]
+						}]
+
+						// matchLabels is a map of {key,value} pairs. A single {key,value}
+						// in the matchLabels map is equivalent to an element of
+						// matchExpressions, whose key field is "key", the operator is
+						// "In", and the values array contains only "value". The
+						// requirements are ANDed.
+						matchLabels?: {
+							[string]: string
+						}
+					}
+
+					// A label query over the set of namespaces that the term applies
+					// to. The term is applied to the union of the namespaces
+					// selected by this field and the ones listed in the namespaces
+					// field. null selector and null or empty namespaces list means
+					// "this pod's namespace". An empty selector ({}) matches all
+					// namespaces.
+					namespaceSelector?: {
+						// matchExpressions is a list of label selector requirements. The
+						// requirements are ANDed.
+						matchExpressions?: [...{
+							// key is the label key that the selector applies to.
+							key: string
+
+							// operator represents a key's relationship to a set of values.
+							// Valid operators are In, NotIn, Exists and DoesNotExist.
+							operator: string
+
+							// values is an array of string values. If the operator is In or
+							// NotIn, the values array must be non-empty. If the operator is
+							// Exists or DoesNotExist, the values array must be empty. This
+							// array is replaced during a strategic merge patch.
+							values?: [...string]
+						}]
+
+						// matchLabels is a map of {key,value} pairs. A single {key,value}
+						// in the matchLabels map is equivalent to an element of
+						// matchExpressions, whose key field is "key", the operator is
+						// "In", and the values array contains only "value". The
+						// requirements are ANDed.
+						matchLabels?: {
+							[string]: string
+						}
+					}
+
+					// namespaces specifies a static list of namespace names that the
+					// term applies to. The term is applied to the union of the
+					// namespaces listed in this field and the ones selected by
+					// namespaceSelector. null or empty namespaces list and null
+					// namespaceSelector means "this pod's namespace".
+					namespaces?: [...string]
+
+					// This pod should be co-located (affinity) or not co-located
+					// (anti-affinity) with the pods matching the labelSelector in
+					// the specified namespaces, where co-located is defined as
+					// running on a node whose value of the label with key
+					// topologyKey matches that of any node on which any of the
+					// selected pods is running. Empty topologyKey is not allowed.
+					topologyKey: string
+				}
+
+				// weight associated with matching the corresponding
+				// podAffinityTerm, in the range 1-100.
+				weight: int
+			}]
+
+			// If the affinity requirements specified by this field are not
+			// met at scheduling time, the pod will not be scheduled onto the
+			// node. If the affinity requirements specified by this field
+			// cease to be met at some point during pod execution (e.g. due
+			// to a pod label update), the system may or may not try to
+			// eventually evict the pod from its node. When there are
+			// multiple elements, the lists of nodes corresponding to each
+			// podAffinityTerm are intersected, i.e. all terms must be
+			// satisfied.
+			requiredDuringSchedulingIgnoredDuringExecution?: [...{
+				// A label query over a set of resources, in this case pods.
+				labelSelector?: {
+					// matchExpressions is a list of label selector requirements. The
+					// requirements are ANDed.
+					matchExpressions?: [...{
+						// key is the label key that the selector applies to.
+						key: string
+
+						// operator represents a key's relationship to a set of values.
+						// Valid operators are In, NotIn, Exists and DoesNotExist.
+						operator: string
+
+						// values is an array of string values. If the operator is In or
+						// NotIn, the values array must be non-empty. If the operator is
+						// Exists or DoesNotExist, the values array must be empty. This
+						// array is replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// matchLabels is a map of {key,value} pairs. A single {key,value}
+					// in the matchLabels map is equivalent to an element of
+					// matchExpressions, whose key field is "key", the operator is
+					// "In", and the values array contains only "value". The
+					// requirements are ANDed.
+					matchLabels?: {
+						[string]: string
+					}
+				}
+
+				// A label query over the set of namespaces that the term applies
+				// to. The term is applied to the union of the namespaces
+				// selected by this field and the ones listed in the namespaces
+				// field. null selector and null or empty namespaces list means
+				// "this pod's namespace". An empty selector ({}) matches all
+				// namespaces.
+				namespaceSelector?: {
+					// matchExpressions is a list of label selector requirements. The
+					// requirements are ANDed.
+					matchExpressions?: [...{
+						// key is the label key that the selector applies to.
+						key: string
+
+						// operator represents a key's relationship to a set of values.
+						// Valid operators are In, NotIn, Exists and DoesNotExist.
+						operator: string
+
+						// values is an array of string values. If the operator is In or
+						// NotIn, the values array must be non-empty. If the operator is
+						// Exists or DoesNotExist, the values array must be empty. This
+						// array is replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// matchLabels is a map of {key,value} pairs. A single {key,value}
+					// in the matchLabels map is equivalent to an element of
+					// matchExpressions, whose key field is "key", the operator is
+					// "In", and the values array contains only "value". The
+					// requirements are ANDed.
+					matchLabels?: {
+						[string]: string
+					}
+				}
+
+				// namespaces specifies a static list of namespace names that the
+				// term applies to. The term is applied to the union of the
+				// namespaces listed in this field and the ones selected by
+				// namespaceSelector. null or empty namespaces list and null
+				// namespaceSelector means "this pod's namespace".
+				namespaces?: [...string]
+
+				// This pod should be co-located (affinity) or not co-located
+				// (anti-affinity) with the pods matching the labelSelector in
+				// the specified namespaces, where co-located is defined as
+				// running on a node whose value of the label with key
+				// topologyKey matches that of any node on which any of the
+				// selected pods is running. Empty topologyKey is not allowed.
+				topologyKey: string
+			}]
+		}
+
+		// Describes pod anti-affinity scheduling rules (e.g. avoid
+		// putting this pod in the same node, zone, etc. as some other
+		// pod(s)).
+		podAntiAffinity?: {
+			// The scheduler will prefer to schedule pods to nodes that
+			// satisfy the anti-affinity expressions specified by this field,
+			// but it may choose a node that violates one or more of the
+			// expressions. The node that is most preferred is the one with
+			// the greatest sum of weights, i.e. for each node that meets all
+			// of the scheduling requirements (resource request,
+			// requiredDuringScheduling anti-affinity expressions, etc.),
+			// compute a sum by iterating through the elements of this field
+			// and adding "weight" to the sum if the node has pods which
+			// matches the corresponding podAffinityTerm; the node(s) with
+			// the highest sum are the most preferred.
+			preferredDuringSchedulingIgnoredDuringExecution?: [...{
+				// Required. A pod affinity term, associated with the
+				// corresponding weight.
+				podAffinityTerm: {
+					// A label query over a set of resources, in this case pods.
+					labelSelector?: {
+						// matchExpressions is a list of label selector requirements. The
+						// requirements are ANDed.
+						matchExpressions?: [...{
+							// key is the label key that the selector applies to.
+							key: string
+
+							// operator represents a key's relationship to a set of values.
+							// Valid operators are In, NotIn, Exists and DoesNotExist.
+							operator: string
+
+							// values is an array of string values. If the operator is In or
+							// NotIn, the values array must be non-empty. If the operator is
+							// Exists or DoesNotExist, the values array must be empty. This
+							// array is replaced during a strategic merge patch.
+							values?: [...string]
+						}]
+
+						// matchLabels is a map of {key,value} pairs. A single {key,value}
+						// in the matchLabels map is equivalent to an element of
+						// matchExpressions, whose key field is "key", the operator is
+						// "In", and the values array contains only "value". The
+						// requirements are ANDed.
+						matchLabels?: {
+							[string]: string
+						}
+					}
+
+					// A label query over the set of namespaces that the term applies
+					// to. The term is applied to the union of the namespaces
+					// selected by this field and the ones listed in the namespaces
+					// field. null selector and null or empty namespaces list means
+					// "this pod's namespace". An empty selector ({}) matches all
+					// namespaces.
+					namespaceSelector?: {
+						// matchExpressions is a list of label selector requirements. The
+						// requirements are ANDed.
+						matchExpressions?: [...{
+							// key is the label key that the selector applies to.
+							key: string
+
+							// operator represents a key's relationship to a set of values.
+							// Valid operators are In, NotIn, Exists and DoesNotExist.
+							operator: string
+
+							// values is an array of string values. If the operator is In or
+							// NotIn, the values array must be non-empty. If the operator is
+							// Exists or DoesNotExist, the values array must be empty. This
+							// array is replaced during a strategic merge patch.
+							values?: [...string]
+						}]
+
+						// matchLabels is a map of {key,value} pairs. A single {key,value}
+						// in the matchLabels map is equivalent to an element of
+						// matchExpressions, whose key field is "key", the operator is
+						// "In", and the values array contains only "value". The
+						// requirements are ANDed.
+						matchLabels?: {
+							[string]: string
+						}
+					}
+
+					// namespaces specifies a static list of namespace names that the
+					// term applies to. The term is applied to the union of the
+					// namespaces listed in this field and the ones selected by
+					// namespaceSelector. null or empty namespaces list and null
+					// namespaceSelector means "this pod's namespace".
+					namespaces?: [...string]
+
+					// This pod should be co-located (affinity) or not co-located
+					// (anti-affinity) with the pods matching the labelSelector in
+					// the specified namespaces, where co-located is defined as
+					// running on a node whose value of the label with key
+					// topologyKey matches that of any node on which any of the
+					// selected pods is running. Empty topologyKey is not allowed.
+					topologyKey: string
+				}
+
+				// weight associated with matching the corresponding
+				// podAffinityTerm, in the range 1-100.
+				weight: int
+			}]
+
+			// If the anti-affinity requirements specified by this field are
+			// not met at scheduling time, the pod will not be scheduled onto
+			// the node. If the anti-affinity requirements specified by this
+			// field cease to be met at some point during pod execution (e.g.
+			// due to a pod label update), the system may or may not try to
+			// eventually evict the pod from its node. When there are
+			// multiple elements, the lists of nodes corresponding to each
+			// podAffinityTerm are intersected, i.e. all terms must be
+			// satisfied.
+			requiredDuringSchedulingIgnoredDuringExecution?: [...{
+				// A label query over a set of resources, in this case pods.
+				labelSelector?: {
+					// matchExpressions is a list of label selector requirements. The
+					// requirements are ANDed.
+					matchExpressions?: [...{
+						// key is the label key that the selector applies to.
+						key: string
+
+						// operator represents a key's relationship to a set of values.
+						// Valid operators are In, NotIn, Exists and DoesNotExist.
+						operator: string
+
+						// values is an array of string values. If the operator is In or
+						// NotIn, the values array must be non-empty. If the operator is
+						// Exists or DoesNotExist, the values array must be empty. This
+						// array is replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// matchLabels is a map of {key,value} pairs. A single {key,value}
+					// in the matchLabels map is equivalent to an element of
+					// matchExpressions, whose key field is "key", the operator is
+					// "In", and the values array contains only "value". The
+					// requirements are ANDed.
+					matchLabels?: {
+						[string]: string
+					}
+				}
+
+				// A label query over the set of namespaces that the term applies
+				// to. The term is applied to the union of the namespaces
+				// selected by this field and the ones listed in the namespaces
+				// field. null selector and null or empty namespaces list means
+				// "this pod's namespace". An empty selector ({}) matches all
+				// namespaces.
+				namespaceSelector?: {
+					// matchExpressions is a list of label selector requirements. The
+					// requirements are ANDed.
+					matchExpressions?: [...{
+						// key is the label key that the selector applies to.
+						key: string
+
+						// operator represents a key's relationship to a set of values.
+						// Valid operators are In, NotIn, Exists and DoesNotExist.
+						operator: string
+
+						// values is an array of string values. If the operator is In or
+						// NotIn, the values array must be non-empty. If the operator is
+						// Exists or DoesNotExist, the values array must be empty. This
+						// array is replaced during a strategic merge patch.
+						values?: [...string]
+					}]
+
+					// matchLabels is a map of {key,value} pairs. A single {key,value}
+					// in the matchLabels map is equivalent to an element of
+					// matchExpressions, whose key field is "key", the operator is
+					// "In", and the values array contains only "value". The
+					// requirements are ANDed.
+					matchLabels?: {
+						[string]: string
+					}
+				}
+
+				// namespaces specifies a static list of namespace names that the
+				// term applies to. The term is applied to the union of the
+				// namespaces listed in this field and the ones selected by
+				// namespaceSelector. null or empty namespaces list and null
+				// namespaceSelector means "this pod's namespace".
+				namespaces?: [...string]
+
+				// This pod should be co-located (affinity) or not co-located
+				// (anti-affinity) with the pods matching the labelSelector in
+				// the specified namespaces, where co-located is defined as
+				// running on a node whose value of the label with key
+				// topologyKey matches that of any node on which any of the
+				// selected pods is running. Empty topologyKey is not allowed.
+				topologyKey: string
+			}]
+		}
+	}
+
+	// The major version of PostgreSQL before the upgrade.
+	fromPostgresVersion: uint & >=10 & <=16
+
+	// The image name to use for major PostgreSQL upgrades.
+	image?: string
+
+	// ImagePullPolicy is used to determine when Kubernetes will
+	// attempt to pull (download) container images. More info:
+	// https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy
+	imagePullPolicy?: "Always" | "Never" | "IfNotPresent"
+
+	// The image pull secrets used to pull from a private registry.
+	// Changing this value causes all running PGUpgrade pods to
+	// restart.
+	// https://k8s.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+	imagePullSecrets?: [...{
+		// Name of the referent. More info:
+		// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+		name?: string
+	}]
+
+	// Metadata contains metadata for custom resources
+	metadata?: {
+		annotations?: {
+			[string]: string
+		}
+		labels?: {
+			[string]: string
+		}
+	}
+
+	// The name of the cluster to be updated
+	postgresClusterName: strings.MinRunes(1)
+
+	// Priority class name for the PGUpgrade pod. Changing this value
+	// causes PGUpgrade pod to restart. More info:
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/
+	priorityClassName?: string
+
+	// Resource requirements for the PGUpgrade container.
+	resources?: {
+		// Limits describes the maximum amount of compute resources
+		// allowed. More info:
+		// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+		limits?: {
+			[string]: (int | string) & =~"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$"
+		}
+
+		// Requests describes the minimum amount of compute resources
+		// required. If Requests is omitted for a container, it defaults
+		// to Limits if that is explicitly specified, otherwise to an
+		// implementation-defined value. More info:
+		// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+		requests?: {
+			[string]: (int | string) & =~"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$"
+		}
+	}
+
+	// The image name to use for PostgreSQL containers after upgrade.
+	// When omitted, the value comes from an operator environment
+	// variable.
+	toPostgresImage?: string
+
+	// The major version of PostgreSQL to be upgraded to.
+	toPostgresVersion: uint & >=10 & <=16
+
+	// Tolerations of the PGUpgrade pod. More info:
+	// https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration
+	tolerations?: [...{
+		// Effect indicates the taint effect to match. Empty means match
+		// all taint effects. When specified, allowed values are
+		// NoSchedule, PreferNoSchedule and NoExecute.
+		effect?: string
+
+		// Key is the taint key that the toleration applies to. Empty
+		// means match all taint keys. If the key is empty, operator must
+		// be Exists; this combination means to match all values and all
+		// keys.
+		key?: string
+
+		// Operator represents a key's relationship to the value. Valid
+		// operators are Exists and Equal. Defaults to Equal. Exists is
+		// equivalent to wildcard for value, so that a pod can tolerate
+		// all taints of a particular category.
+		operator?: string
+
+		// TolerationSeconds represents the period of time the toleration
+		// (which must be of effect NoExecute, otherwise this field is
+		// ignored) tolerates the taint. By default, it is not set, which
+		// means tolerate the taint forever (do not evict). Zero and
+		// negative values will be treated as 0 (evict immediately) by
+		// the system.
+		tolerationSeconds?: int
+
+		// Value is the taint value the toleration matches to. If the
+		// operator is Exists, the value should be empty, otherwise just
+		// a regular string.
+		value?: string
+	}]
+}

docs/examples/optionalservices.cue

@@ -0,0 +1,46 @@
+// Controls optional feature flags for services distributed across multiple holos components.
+// For example, enable issuing certificates in the provisioner cluster when an optional service is
+// enabled for a workload cluster.
+package holos
+
+import "list"
+
+#OptionalService: {
+	name:    string
+	enabled: true | *false
+	clusters: [Name=_]: #Platform.clusters[Name]
+	clusterNames: [for c in clusters {c.name}]
+
+	managedNamespaces: [Name=_]: #ManagedNamespace & {
+		namespace: metadata: name: Name
+		clusterNames: ["provisioner", for c in clusters {c.name}]
+	}
+	// servers represents istio Gateway.spec.servers.hosts entries
+	// Refer to istio/gateway/gateway.cue
+	servers: [Name=_]: {
+		hosts: [...string]
+		port: name:          Name
+		port: number:        443
+		port: protocol:      "HTTPS"
+		tls: credentialName: string
+		tls: mode:           "SIMPLE"
+	}
+	// public tls certs should align to hosts.
+	certs: [Name=_]: #Certificate & {
+		metadata: name: Name
+	}
+}
+
+#OptionalServices: {
+	[Name=_]: #OptionalService & {
+		name: Name
+	}
+}
+
+for svc in #OptionalServices {
+	for nsName, ns in svc.managedNamespaces {
+		if svc.enabled && list.Contains(ns.clusterNames, #ClusterName) {
+			#ManagedNamespaces: "\(nsName)": ns
+		}
+	}
+}

docs/examples/platforms/optional.site.cue

@@ -0,0 +1,56 @@
+package holos
+
+let CoreDomain = "core.\(#Platform.org.domain)"
+let TargetNamespace = "prod-core-vault"
+
+#OptionalServices: {
+	vault: {
+		enabled: true
+		clusters: core1: _
+		clusters: core2: _
+		managedNamespaces: "prod-core-vault": {
+			namespace: metadata: labels: "istio-injection": "enabled"
+		}
+		certs: "vault-core": #Certificate & {
+			metadata: name:      "vault-core"
+			metadata: namespace: "istio-ingress"
+			spec: {
+				commonName: "vault.\(CoreDomain)"
+				dnsNames: [commonName]
+				secretName: metadata.name
+				issuerRef: kind: "ClusterIssuer"
+				issuerRef: name: string | *"letsencrypt"
+			}
+		}
+		servers: "https-vault-core": {
+			hosts: ["\(TargetNamespace)/vault.\(CoreDomain)"]
+			tls: credentialName: certs."vault-core".spec.secretName
+		}
+		for k, v in clusters {
+			let obj = (Cert & {Name: "vault-core", Cluster: v.name}).APIObject
+			certs: "\(obj.metadata.name)": obj
+			servers: "https-\(obj.metadata.name)": {
+				hosts: [for host in obj.spec.dnsNames {"\(TargetNamespace)/\(host)"}]
+				tls: credentialName: obj.spec.secretName
+			}
+		}
+	}
+}
+
+// Cert provisions a cluster specific certificate.
+let Cert = {
+	Name:    string
+	Cluster: string
+
+	APIObject: #Certificate & {
+		metadata: name:      "\(Cluster)-\(Name)"
+		metadata: namespace: string | *"istio-ingress"
+		spec: {
+			commonName: string | *"vault.\(Cluster).\(CoreDomain)"
+			dnsNames: [commonName]
+			secretName: metadata.name
+			issuerRef: kind: "ClusterIssuer"
+			issuerRef: name: string | *"letsencrypt"
+		}
+	}
+}

docs/examples/platforms/reference/certificates.cue

@@ -44,7 +44,8 @@ package holos
 	_name:     string
 	_cluster:  string
 	_wildcard: true | *false
-	metadata: name:      string | *"\(_cluster)-\(_name)"
+	// Enforce this value
+	metadata: name:      "\(_cluster)-\(_name)"
 	metadata: namespace: string | *"istio-ingress"
 	spec: {
 		commonName: string | *"\(_name).\(_cluster).\(#Platform.org.domain)"

docs/examples/platforms/reference/clusters/accounts/iam/iam.cue

@@ -0,0 +1,7 @@
+package holos
+
+// Components under this directory are part of this collection
+#InputKeys: project: "iam"
+
+// Shared dependencies for all components in this collection.
+#DependsOn: namespaces: name: "\(#StageName)-secrets-namespaces"

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/postgres-certs/postgres-certs.cue

@@ -0,0 +1,13 @@
+package holos
+
+#InputKeys: component: "postgres-certs"
+#KubernetesObjects & {
+	apiObjects: {
+		ExternalSecret: {
+			"\(_DBName)-primary-tls": _
+			"\(_DBName)-repl-tls":    _
+			"\(_DBName)-client-tls":  _
+			"\(_DBName)-root-ca":     _
+		}
+	}
+}

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/postgres/postgres.cue

@@ -0,0 +1,174 @@
+package holos
+
+#InputKeys: component:        "postgres"
+#DependsOn: "postgres-certs": _
+
+let Cluster = #Platform.clusters[#ClusterName]
+let S3Secret = "pgo-s3-creds"
+let ZitadelUser = _DBName
+let ZitadelAdmin = "\(_DBName)-admin"
+
+// This must be an external storage bucket for our architecture.
+let BucketRepoName = "repo2"
+
+// Restore options.  Set the timestamp to a known good point in time.
+// time="2024-03-11T17:08:58Z" level=info msg="crunchy-pgbackrest ends"
+// let RestoreOptions = ["--type=time", "--target=\"2024-03-11 17:10:00+00\""]
+
+// Restore the most recent backup.
+let RestoreOptions = []
+
+#KubernetesObjects & {
+	apiObjects: {
+		ExternalSecret: "pgo-s3-creds": _
+		PostgresCluster: db: #PostgresCluster & HighlyAvailable & {
+			metadata: name:      _DBName
+			metadata: namespace: #TargetNamespace
+			spec: {
+				image:           "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.2-0"
+				postgresVersion: 16
+				// Custom certs are necessary for streaming standby replication which we use to replicate between two regions.
+				// Refer to https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/disaster-recovery#streaming-standby
+				customTLSSecret: name:            "\(_DBName)-primary-tls"
+				customReplicationTLSSecret: name: "\(_DBName)-repl-tls"
+				// Refer to https://access.crunchydata.com/documentation/postgres-operator/latest/references/crd/5.5.x/postgrescluster#postgresclusterspecusersindex
+				users: [
+					{name: ZitadelUser},
+					// NOTE: Users with SUPERUSER role cannot log in through pgbouncer.  Use options that allow zitadel admin to use pgbouncer.
+					// Refer to: https://github.com/CrunchyData/postgres-operator/issues/3095#issuecomment-1904712211
+					{name: ZitadelAdmin, options: "CREATEDB CREATEROLE", databases: [_DBName, "postgres"]},
+				]
+				users: [...{databases: [_DBName, ...]}]
+				instances: [{
+					replicas: 2
+					dataVolumeClaimSpec: {
+						accessModes: ["ReadWriteOnce"]
+						resources: requests: storage: "10Gi"
+					}
+				}]
+				standby: {
+					repoName: BucketRepoName
+					if Cluster.primary {
+						enabled: false
+					}
+					if !Cluster.primary {
+						enabled: true
+					}
+				}
+				// Restore from backup if and only if the cluster is primary
+				if Cluster.primary {
+					dataSource: pgbackrest: {
+						stanza:        "db"
+						configuration: backups.pgbackrest.configuration
+						// Restore from known good full backup taken
+						options: RestoreOptions
+						global: {
+							"\(BucketRepoName)-path":        "/pgbackrest/\(#TargetNamespace)/\(metadata.name)/\(BucketRepoName)"
+							"\(BucketRepoName)-cipher-type": "aes-256-cbc"
+						}
+						repo: {
+							name: BucketRepoName
+							s3:   backups.pgbackrest.repos[1].s3
+						}
+					}
+				}
+
+				// Refer to https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/backups
+				backups: pgbackrest: {
+					configuration: [{secret: name: S3Secret}]
+					// Defines details for manual pgBackRest backup Jobs
+					manual: {
+						// Note: the repoName value must match the config keys in the S3Secret.
+						// This must be an external repository for backup / restore / regional failovers.
+						repoName: BucketRepoName
+						options: ["--type=full", ...]
+					}
+					// Defines details for performing an in-place restore using pgBackRest
+					restore: {
+						// Enables triggering a restore by annotating the postgrescluster with postgres-operator.crunchydata.com/pgbackrest-restore="$(date)"
+						enabled:  true
+						repoName: BucketRepoName
+					}
+					global: {
+						// Store only one full backup in the PV because it's more expensive than object storage.
+						"\(repos[0].name)-retention-full": "1"
+						// Store 14 days of full backups in the bucket.
+						"\(BucketRepoName)-retention-full":      string | *"14"
+						"\(BucketRepoName)-retention-full-type": "count" | *"time" // time in days
+						// Refer to https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/backups#encryption
+						"\(BucketRepoName)-cipher-type": "aes-256-cbc"
+						// "The convention we recommend for setting this variable is /pgbackrest/$NAMESPACE/$CLUSTER_NAME/repoN"
+						// Ref: https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/backups#understanding-backup-configuration-and-basic-operations
+						"\(BucketRepoName)-path": "/pgbackrest/\(#TargetNamespace)/\(metadata.name)/\(manual.repoName)"
+					}
+					repos: [
+						{
+							name: "repo1"
+							volume: volumeClaimSpec: {
+								accessModes: ["ReadWriteOnce"]
+								resources: requests: storage: string | *"4Gi"
+							}
+						},
+						{
+							name: BucketRepoName
+							// Full backup weekly on Sunday at 1am, differntial daily at 1am every day except Sunday.
+							schedules: full:         string | *"0 1 * * 0"
+							schedules: differential: string | *"0 1 * * 1-6"
+							s3: {
+								bucket:   string | *"\(#Platform.org.name)-zitadel-backups"
+								region:   string | *#Backups.s3.region
+								endpoint: string | *"s3.dualstack.\(region).amazonaws.com"
+							}
+						},
+					]
+				}
+			}
+		}
+	}
+}
+
+// Refer to https://github.com/holos-run/postgres-operator-examples/blob/main/kustomize/high-availability/ha-postgres.yaml
+let HighlyAvailable = {
+	apiVersion: "postgres-operator.crunchydata.com/v1beta1"
+	kind:       "PostgresCluster"
+	metadata: name: string
+	spec: {
+		image:           "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.2-0"
+		postgresVersion: 16
+		instances: [{
+			name:     "pgha1"
+			replicas: 2
+			dataVolumeClaimSpec: {
+				accessModes: ["ReadWriteOnce"]
+				resources: requests: storage: string | *"10Gi"
+			}
+			affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: [{
+				weight: 1
+				podAffinityTerm: {
+					topologyKey: "kubernetes.io/hostname"
+					labelSelector: matchLabels: {
+						"postgres-operator.crunchydata.com/cluster":      metadata.name
+						"postgres-operator.crunchydata.com/instance-set": name
+					}
+				}
+			}]
+		}]
+		backups: pgbackrest: {
+			image: "registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.49-0"
+		}
+		proxy: pgBouncer: {
+			image:    "registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.21-3"
+			replicas: 2
+			affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: [{
+				weight: 1
+				podAffinityTerm: {
+					topologyKey: "kubernetes.io/hostname"
+					labelSelector: matchLabels: {
+						"postgres-operator.crunchydata.com/cluster": metadata.name
+						"postgres-operator.crunchydata.com/role":    "pgbouncer"
+					}
+				}
+			}]
+		}
+	}
+}

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel.cue

@@ -0,0 +1,10 @@
+package holos
+
+#TargetNamespace: #InstancePrefix + "-zitadel"
+
+// _DBName is the database name used across multiple holos components in this project
+_DBName: "zitadel"
+
+// The canonical login domain for the entire platform.  Zitadel will be active
+// on a single cluster at a time, but always accessible from this domain.
+#ExternalDomain: "login.\(#Platform.org.domain)"

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/values.cue

@@ -125,7 +125,7 @@ package holos
 	securityContext: {}
 
 	// Additional environment variables
-	env: []
+	env: [...]
 	// - name: ZITADEL_DATABASE_POSTGRES_HOST
 	//   valueFrom:
 	//     secretKeyRef:

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/values.holos.cue

@@ -0,0 +1,89 @@
+package holos
+
+#Values: {
+	// Database credentials
+	// Refer to https://access.crunchydata.com/documentation/postgres-operator/5.2.0/architecture/user-management/
+	// Refer to https://zitadel.com/docs/self-hosting/manage/database#postgres
+	env: [
+		// Connection
+		{
+			name: "ZITADEL_DATABASE_POSTGRES_HOST"
+			valueFrom: secretKeyRef: name: "\(_DBName)-pguser-\(_DBName)"
+			valueFrom: secretKeyRef: key:  "pgbouncer-host"
+		},
+		{
+			name: "ZITADEL_DATABASE_POSTGRES_PORT"
+			valueFrom: secretKeyRef: name: "\(_DBName)-pguser-\(_DBName)"
+			valueFrom: secretKeyRef: key:  "pgbouncer-port"
+		},
+		{
+			name: "ZITADEL_DATABASE_POSTGRES_DATABASE"
+			valueFrom: secretKeyRef: name: "\(_DBName)-pguser-\(_DBName)"
+			valueFrom: secretKeyRef: key:  "dbname"
+		},
+		// The <db>-pguser-<db> secret contains creds for the unpriviliged zitadel user
+		{
+			name: "ZITADEL_DATABASE_POSTGRES_USER_USERNAME"
+			valueFrom: secretKeyRef: name: "\(_DBName)-pguser-\(_DBName)"
+			valueFrom: secretKeyRef: key:  "user"
+		},
+		{
+			name: "ZITADEL_DATABASE_POSTGRES_USER_PASSWORD"
+			valueFrom: secretKeyRef: name: "\(_DBName)-pguser-\(_DBName)"
+			valueFrom: secretKeyRef: key:  "password"
+		},
+		// The postgres component configures privileged postgres user creds.
+		{
+			name: "ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME"
+			valueFrom: secretKeyRef: name: "\(_DBName)-pguser-\(_DBName)-admin"
+			valueFrom: secretKeyRef: key:  "user"
+		},
+		{
+			name: "ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD"
+			valueFrom: secretKeyRef: name: "\(_DBName)-pguser-\(_DBName)-admin"
+			valueFrom: secretKeyRef: key:  "password"
+		},
+		// CA Cert issued by PGO which issued the pgbouncer tls cert
+		{
+			name:  "ZITADEL_DATABASE_POSTGRES_USER_SSL_ROOTCERT"
+			value: "/\(_PGBouncer)/ca.crt"
+		},
+		{
+			name:  "ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_ROOTCERT"
+			value: "/\(_PGBouncer)/ca.crt"
+		},
+	]
+
+	// Refer to https://zitadel.com/docs/self-hosting/manage/database
+	zitadel: {
+		// Zitadel master key
+		masterkeySecretName: "zitadel-masterkey"
+		// dbSslCaCrtSecret: "pgo-root-cacert"
+
+		// All settings: https://zitadel.com/docs/self-hosting/manage/configure#runtime-configuration-file
+		// Helm interface: https://github.com/zitadel/zitadel-charts/blob/zitadel-7.4.0/charts/zitadel/values.yaml#L20-L21
+		configmapConfig: {
+			// NOTE: You can change the ExternalDomain, ExternalPort and ExternalSecure
+			// configuration options at any time. However, for ZITADEL to be able to
+			// pick up the changes, you need to rerun ZITADELs setup phase.  Do so with
+			// kubectl delete job zitadel-setup, then re-apply the new config.
+			//
+			// https://zitadel.com/docs/self-hosting/manage/custom-domain
+			ExternalSecure: true
+			ExternalDomain: #ExternalDomain
+			ExternalPort:   443
+			TLS: Enabled: false
+
+			// Database connection credentials are injected via environment variables from the db-pguser-db secret.
+			Database: postgres: {
+				MaxOpenConns:    25
+				MaxIdleConns:    10
+				MaxConnLifetime: "1h"
+				MaxConnIdleTime: "5m"
+				// verify-full verifies the host name matches cert dns names in addition to root ca signature
+				User: SSL: Mode:  "verify-full"
+				Admin: SSL: Mode: "verify-full"
+			}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/zitadel.cue

@@ -0,0 +1,130 @@
+package holos
+
+import "encoding/yaml"
+
+let Name = "zitadel"
+#InputKeys: component: Name
+#DependsOn: postgres:  _
+
+// Upstream helm chart doesn't specify the namespace field for all resources.
+#Kustomization: spec: targetNamespace: #TargetNamespace
+
+#HelmChart & {
+	namespace:   #TargetNamespace
+	enableHooks: true
+	chart: {
+		name:    Name
+		version: "7.9.0"
+		repository: {
+			name: Name
+			url:  "https://charts.zitadel.com"
+		}
+	}
+	values: #Values
+
+	apiObjects: {
+		ExternalSecret: "zitadel-masterkey": _
+		VirtualService: "\(Name)": {
+			metadata: name:      Name
+			metadata: namespace: #TargetNamespace
+			spec: hosts: ["login.\(#Platform.org.domain)"]
+			spec: gateways: ["istio-ingress/default"]
+			spec: http: [{route: [{destination: host: Name}]}]
+		}
+	}
+}
+
+// TODO: Generalize this common pattern of injecting the istio sidecar into a Deployment
+let IstioInject = [{op: "add", path: "/spec/template/metadata/labels/sidecar.istio.io~1inject", value: "true"}]
+
+_PGBouncer: "pgbouncer"
+
+let DatabaseCACertPatch = [
+	{
+		op:   "add"
+		path: "/spec/template/spec/volumes/-"
+		value: {
+			name: _PGBouncer
+			secret: {
+				secretName: "\(_DBName)-pgbouncer"
+				items: [{key: "pgbouncer-frontend.ca-roots", path: "ca.crt"}]
+			}
+		}
+	},
+	{
+		op:   "add"
+		path: "/spec/template/spec/containers/0/volumeMounts/-"
+		value: {
+			name:      _PGBouncer
+			mountPath: "/" + _PGBouncer
+		}
+	},
+]
+
+let CAPatch = #Patch & {
+	target: {
+		group:   "apps" | "batch"
+		version: "v1"
+		kind:    "Job" | "Deployment"
+		name:    string
+	}
+	patch: yaml.Marshal(DatabaseCACertPatch)
+}
+
+#KustomizePatches: {
+	mesh: {
+		target: {
+			group:   "apps"
+			version: "v1"
+			kind:    "Deployment"
+			name:    Name
+		}
+		patch: yaml.Marshal(IstioInject)
+	}
+	deploymentCA: CAPatch & {
+		target: group: "apps"
+		target: kind:  "Deployment"
+		target: name:  Name
+	}
+	initJob: CAPatch & {
+		target: group: "batch"
+		target: kind:  "Job"
+		target: name:  "\(Name)-init"
+	}
+	setupJob: CAPatch & {
+		target: group: "batch"
+		target: kind:  "Job"
+		target: name:  "\(Name)-setup"
+	}
+	if #IsPrimaryCluster == false {
+		fluxDisable: {
+			target: {
+				group:   "apps"
+				version: "v1"
+				kind:    "Deployment"
+				name:    Name
+			}
+			patch: yaml.Marshal(DisableFluxPatch)
+		}
+		initDisable: {
+			target: {
+				group:   "batch"
+				version: "v1"
+				kind:    "Job"
+				name:    "\(Name)-init"
+			}
+			patch: yaml.Marshal(DisableFluxPatch)
+		}
+		setupDisable: {
+			target: {
+				group:   "batch"
+				version: "v1"
+				kind:    "Job"
+				name:    "\(Name)-setup"
+			}
+			patch: yaml.Marshal(DisableFluxPatch)
+		}
+	}
+}
+
+let DisableFluxPatch = [{op: "replace", path: "/metadata/annotations/kustomize.toolkit.fluxcd.io~1reconcile", value: "disabled"}]

docs/examples/platforms/reference/clusters/disabled/iam/zitadel/crdb/cockroachdb.cue

@@ -0,0 +1,20 @@
+package holos
+
+#InputKeys: component: "crdb"
+
+#HelmChart & {
+	namespace: #TargetNamespace
+	chart: {
+		name:    "cockroachdb"
+		version: "11.2.3"
+		repository: {
+			name: "cockroachdb"
+			url:  "https://charts.cockroachdb.com/"
+		}
+	}
+	values: #Values
+	apiObjects: {
+		ExternalSecret: node: #ExternalSecret & {_name: "cockroachdb-node"}
+		ExternalSecret: root: #ExternalSecret & {_name: "cockroachdb-root"}
+	}
+}

docs/examples/platforms/reference/clusters/disabled/iam/zitadel/crdb/values.cue

@@ -478,7 +478,7 @@ package holos
 		copyCerts: image: "busybox"
 		certs: {
 			// Bring your own certs scenario. If provided, tls.init section will be ignored.
-			provided: false
+			provided: true | *false
 			// Secret name for the client root cert.
 			clientRootSecret: "cockroachdb-root"
 			// Secret name for node cert.
@@ -487,7 +487,7 @@ package holos
 			caSecret: "cockroach-ca"
 			// Enable if the secret is a dedicated TLS.
 			// TLS secrets are created by cert-mananger, for example.
-			tlsSecret: false
+			tlsSecret: true | *false
 			// Enable if the you want cockroach db to create its own certificates
 			selfSigner: {
 				// If set, the cockroach db will generate its own certificates

docs/examples/platforms/reference/clusters/disabled/iam/zitadel/crdb/values.holos.cue

@@ -10,11 +10,9 @@ package holos
 		certs: {
 			// https://github.com/cockroachdb/helm-charts/blob/3dcf96726ebcfe3784afb526ddcf4095a1684aea/README.md?plain=1#L204-L215
 			selfSigner: enabled: false
-			certManager: true
-			certManagerIssuer: {
-				kind: "Issuer"
-				name: #ComponentName
-			}
+			certManager: false
+			provided:    true
+			tlsSecret:   true
 		}
 	}
 

docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/arc.cue

@@ -0,0 +1,9 @@
+package holos
+
+// GitHub Actions Runner Controller
+#InputKeys: project: "github"
+#DependsOn: Namespaces: name: "prod-secrets-namespaces"
+
+#ARCSystemNamespace: "arc-system"
+#HelmChart: namespace: #TargetNamespace
+#HelmChart: chart: version: "0.8.3"

docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/runner/arc-runner.cue

@@ -0,0 +1,27 @@
+package holos
+
+#TargetNamespace: "arc-runner"
+#InputKeys: component: "arc-runner"
+#Kustomization: spec: targetNamespace: #TargetNamespace
+
+#HelmChart & {
+	values: {
+		#Values
+		controllerServiceAccount: name:      "gha-rs-controller"
+		controllerServiceAccount: namespace: "arc-system"
+		githubConfigSecret: "controller-manager"
+		githubConfigUrl:    "https://github.com/" + #Platform.org.github.orgs.primary.name
+	}
+	apiObjects: ExternalSecret: "\(values.githubConfigSecret)": _
+	chart: {
+		// Match the gha-base-name in the chart _helpers.tpl to avoid long full names.
+		// NOTE: Unfortunately the INSTALLATION_NAME is used as the helm release
+		// name and GitHub removed support for runner labels, so the only way to
+		// specify which runner a workflow runs on is using this helm release name.
+		// The quote is "Update the INSTALLATION_NAME value carefully. You will use
+		// the installation name as the value of runs-on in your workflows."  Refer to
+		// https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller
+		release: "gha-rs"
+		name:    "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set"
+	}
+}

docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/runner/values.cue

@@ -0,0 +1,192 @@
+package holos
+
+#Values: {
+	//# githubConfigUrl is the GitHub url for where you want to configure runners
+	//# ex: https://github.com/myorg/myrepo or https://github.com/myorg
+	githubConfigUrl: string | *""
+
+	//# githubConfigSecret is the k8s secrets to use when auth with GitHub API.
+	//# You can choose to use GitHub App or a PAT token
+	githubConfigSecret: string | {
+		//## GitHub Apps Configuration
+		//# NOTE: IDs MUST be strings, use quotes
+		//github_app_id: ""
+		//github_app_installation_id: ""
+		//github_app_private_key: |
+		//## GitHub PAT Configuration
+		github_token: ""
+	}
+	//# If you have a pre-define Kubernetes secret in the same namespace the gha-runner-scale-set is going to deploy,
+	//# you can also reference it via `githubConfigSecret: pre-defined-secret`.
+	//# You need to make sure your predefined secret has all the required secret data set properly.
+	//#   For a pre-defined secret using GitHub PAT, the secret needs to be created like this:
+	//#   > kubectl create secret generic pre-defined-secret --namespace=my_namespace --from-literal=github_token='ghp_your_pat'
+	//#   For a pre-defined secret using GitHub App, the secret needs to be created like this:
+	//#   > kubectl create secret generic pre-defined-secret --namespace=my_namespace --from-literal=github_app_id=123456 --from-literal=github_app_installation_id=654321 --from-literal=github_app_private_key='-----BEGIN CERTIFICATE-----*******'
+	// githubConfigSecret: pre-defined-secret
+	//# proxy can be used to define proxy settings that will be used by the
+	//# controller, the listener and the runner of this scale set.
+	//
+	// proxy:
+	//   http:
+	//     url: http://proxy.com:1234
+	//     credentialSecretRef: proxy-auth # a secret with `username` and `password` keys
+	//   https:
+	//     url: http://proxy.com:1234
+	//     credentialSecretRef: proxy-auth # a secret with `username` and `password` keys
+	//   noProxy:
+	//     - example.com
+	//     - example.org
+	//# maxRunners is the max number of runners the autoscaling runner set will scale up to.
+	// maxRunners: 5
+	//# minRunners is the min number of idle runners. The target number of runners created will be
+	//# calculated as a sum of minRunners and the number of jobs assigned to the scale set.
+	// minRunners: 0
+	// runnerGroup: "default"
+	//# name of the runner scale set to create.  Defaults to the helm release name
+	// runnerScaleSetName: ""
+	//# A self-signed CA certificate for communication with the GitHub server can be
+	//# provided using a config map key selector. If `runnerMountPath` is set, for
+	//# each runner pod ARC will:
+	//# - create a `github-server-tls-cert` volume containing the certificate
+	//#   specified in `certificateFrom`
+	//# - mount that volume on path `runnerMountPath`/{certificate name}
+	//# - set NODE_EXTRA_CA_CERTS environment variable to that same path
+	//# - set RUNNER_UPDATE_CA_CERTS environment variable to "1" (as of version
+	//#   2.303.0 this will instruct the runner to reload certificates on the host)
+	//#
+	//# If any of the above had already been set by the user in the runner pod
+	//# template, ARC will observe those and not overwrite them.
+	//# Example configuration:
+	//
+	// githubServerTLS:
+	//   certificateFrom:
+	//     configMapKeyRef:
+	//       name: config-map-name
+	//       key: ca.crt
+	//   runnerMountPath: /usr/local/share/ca-certificates/
+	//# Container mode is an object that provides out-of-box configuration
+	//# for dind and kubernetes mode. Template will be modified as documented under the
+	//# template object.
+	//#
+	//# If any customization is required for dind or kubernetes mode, containerMode should remain
+	//# empty, and configuration should be applied to the template.
+	// containerMode:
+	//   type: "dind"  ## type can be set to dind or kubernetes
+	//   ## the following is required when containerMode.type=kubernetes
+	//   kubernetesModeWorkVolumeClaim:
+	//     accessModes: ["ReadWriteOnce"]
+	//     # For local testing, use https://github.com/openebs/dynamic-localpv-provisioner/blob/develop/docs/quickstart.md to provide dynamic provision volume with storageClassName: openebs-hostpath
+	//     storageClassName: "dynamic-blob-storage"
+	//     resources:
+	//       requests:
+	//         storage: 1Gi
+	//   kubernetesModeServiceAccount:
+	//     annotations:
+	//# template is the PodSpec for each listener Pod
+	//# For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
+	// listenerTemplate:
+	//   spec:
+	//     containers:
+	//     # Use this section to append additional configuration to the listener container.
+	//     # If you change the name of the container, the configuration will not be applied to the listener,
+	//     # and it will be treated as a side-car container.
+	//     - name: listener
+	//       securityContext:
+	//         runAsUser: 1000
+	//     # Use this section to add the configuration of a side-car container.
+	//     # Comment it out or remove it if you don't need it.
+	//     # Spec for this container will be applied as is without any modifications.
+	//     - name: side-car
+	//       image: example-sidecar
+	//# template is the PodSpec for each runner Pod
+	//# For reference: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec
+	template: {
+		//# template.spec will be modified if you change the container mode
+		//# with containerMode.type=dind, we will populate the template.spec with following pod spec
+		//# template:
+		//#   spec:
+		//#     initContainers:
+		//#     - name: init-dind-externals
+		//#       image: ghcr.io/actions/actions-runner:latest
+		//#       command: ["cp", "-r", "-v", "/home/runner/externals/.", "/home/runner/tmpDir/"]
+		//#       volumeMounts:
+		//#         - name: dind-externals
+		//#           mountPath: /home/runner/tmpDir
+		//#     containers:
+		//#     - name: runner
+		//#       image: ghcr.io/actions/actions-runner:latest
+		//#       command: ["/home/runner/run.sh"]
+		//#       env:
+		//#         - name: DOCKER_HOST
+		//#           value: unix:///run/docker/docker.sock
+		//#       volumeMounts:
+		//#         - name: work
+		//#           mountPath: /home/runner/_work
+		//#         - name: dind-sock
+		//#           mountPath: /run/docker
+		//#           readOnly: true
+		//#     - name: dind
+		//#       image: docker:dind
+		//#       args:
+		//#         - dockerd
+		//#         - --host=unix:///run/docker/docker.sock
+		//#         - --group=$(DOCKER_GROUP_GID)
+		//#       env:
+		//#         - name: DOCKER_GROUP_GID
+		//#           value: "123"
+		//#       securityContext:
+		//#         privileged: true
+		//#       volumeMounts:
+		//#         - name: work
+		//#           mountPath: /home/runner/_work
+		//#         - name: dind-sock
+		//#           mountPath: /run/docker
+		//#         - name: dind-externals
+		//#           mountPath: /home/runner/externals
+		//#     volumes:
+		//#     - name: work
+		//#       emptyDir: {}
+		//#     - name: dind-sock
+		//#       emptyDir: {}
+		//#     - name: dind-externals
+		//#       emptyDir: {}
+		//#####################################################################################################
+		//# with containerMode.type=kubernetes, we will populate the template.spec with following pod spec
+		//# template:
+		//#   spec:
+		//#     containers:
+		//#     - name: runner
+		//#       image: ghcr.io/actions/actions-runner:latest
+		//#       command: ["/home/runner/run.sh"]
+		//#       env:
+		//#         - name: ACTIONS_RUNNER_CONTAINER_HOOKS
+		//#           value: /home/runner/k8s/index.js
+		//#         - name: ACTIONS_RUNNER_POD_NAME
+		//#           valueFrom:
+		//#             fieldRef:
+		//#               fieldPath: metadata.name
+		//#         - name: ACTIONS_RUNNER_REQUIRE_JOB_CONTAINER
+		//#           value: "true"
+		//#       volumeMounts:
+		//#         - name: work
+		//#           mountPath: /home/runner/_work
+		//#     volumes:
+		//#       - name: work
+		//#         ephemeral:
+		//#           volumeClaimTemplate:
+		//#             spec:
+		//#               accessModes: [ "ReadWriteOnce" ]
+		//#               storageClassName: "local-path"
+		//#               resources:
+		//#                 requests:
+		//#                   storage: 1Gi
+		spec: {
+			containers: [{
+				name:  "runner"
+				image: "ghcr.io/actions/actions-runner:latest"
+				command: ["/home/runner/run.sh"]
+			}]
+		}
+	}
+}

docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/system/arc-system.cue

@@ -0,0 +1,15 @@
+package holos
+
+#TargetNamespace: #ARCSystemNamespace
+#InputKeys: component: "arc-system"
+
+#HelmChart & {
+	values:    #Values & #DefaultSecurityContext
+	namespace: #TargetNamespace
+	chart: {
+		// Match the gha-base-name in the chart _helpers.tpl to avoid long full names.
+		release: "gha-rs-controller"
+		name:    "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller"
+		version: "0.8.3"
+	}
+}

docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/system/values.cue

@@ -0,0 +1,127 @@
+package holos
+
+#Values: {
+	// Default values for gha-runner-scale-set-controller.
+	// This is a YAML-formatted file.
+	// Declare variables to be passed into your templates.
+	labels: {}
+
+	// leaderElection will be enabled when replicaCount>1,
+	// So, only one replica will in charge of reconciliation at a given time
+	// leaderElectionId will be set to {{ define gha-runner-scale-set-controller.fullname }}.
+	replicaCount: 1
+
+	image: {
+		repository: "ghcr.io/actions/gha-runner-scale-set-controller"
+		pullPolicy: "IfNotPresent"
+		// Overrides the image tag whose default is the chart appVersion.
+		tag: ""
+	}
+
+	imagePullSecrets: []
+	nameOverride:     ""
+	fullnameOverride: ""
+
+	env: null
+	//# Define environment variables for the controller pod
+	//  - name: "ENV_VAR_NAME_1"
+	//    value: "ENV_VAR_VALUE_1"
+	//  - name: "ENV_VAR_NAME_2"
+	//    valueFrom:
+	//      secretKeyRef:
+	//        key: ENV_VAR_NAME_2
+	//        name: secret-name
+	//        optional: true
+
+	serviceAccount: {
+		// Specifies whether a service account should be created for running the controller pod
+		create: true
+		// Annotations to add to the service account
+		annotations: {}
+		// The name of the service account to use.
+		// If not set and create is true, a name is generated using the fullname template
+		// You can not use the default service account for this.
+		name: ""
+	}
+
+	podAnnotations: {}
+
+	podLabels: {}
+
+	podSecurityContext: {}
+	// fsGroup: 2000
+
+	securityContext: {...}
+	// capabilities:
+	//   drop:
+	//   - ALL
+	// readOnlyRootFilesystem: true
+	// runAsNonRoot: true
+	// runAsUser: 1000
+
+	resources: {}
+	//# We usually recommend not to specify default resources and to leave this as a conscious
+	//# choice for the user. This also increases chances charts run on environments with little
+	//# resources, such as Minikube. If you do want to specify resources, uncomment the following
+	//# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+	// limits:
+	//   cpu: 100m
+	//   memory: 128Mi
+	// requests:
+	//   cpu: 100m
+	//   memory: 128Mi
+
+	nodeSelector: {}
+
+	tolerations: []
+
+	affinity: {}
+
+	// Mount volumes in the container.
+	volumes: []
+	volumeMounts: []
+
+	// Leverage a PriorityClass to ensure your pods survive resource shortages
+	// ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+	// PriorityClass: system-cluster-critical
+	priorityClassName: ""
+
+	//# If `metrics:` object is not provided, or commented out, the following flags
+	//# will be applied the controller-manager and listener pods with empty values:
+	//# `--metrics-addr`, `--listener-metrics-addr`, `--listener-metrics-endpoint`.
+	//# This will disable metrics.
+	//#
+	//# To enable metrics, uncomment the following lines.
+	// metrics:
+	//   controllerManagerAddr: ":8080"
+	//   listenerAddr: ":8080"
+	//   listenerEndpoint: "/metrics"
+
+	flags: {
+		//# Log level can be set here with one of the following values: "debug", "info", "warn", "error".
+		//# Defaults to "debug".
+		logLevel: "debug"
+		//# Log format can be set with one of the following values: "text", "json"
+		//# Defaults to "text"
+		logFormat: "text"
+
+		//# Restricts the controller to only watch resources in the desired namespace.
+		//# Defaults to watch all namespaces when unset.
+		// watchSingleNamespace: ""
+		//# Defines how the controller should handle upgrades while having running jobs.
+		//#
+		//# The strategies available are:
+		//# - "immediate": (default) The controller will immediately apply the change causing the
+		//#   recreation of the listener and ephemeral runner set. This can lead to an
+		//#   overprovisioning of runners, if there are pending / running jobs. This should not
+		//#   be a problem at a small scale, but it could lead to a significant increase of
+		//#   resources if you have a lot of jobs running concurrently.
+		//#
+		//# - "eventual": The controller will remove the listener and ephemeral runner set
+		//#   immediately, but will not recreate them (to apply changes) until all
+		//#   pending / running jobs have completed.
+		//#   This can lead to a longer time to apply the change but it will ensure
+		//#   that you don't have any overprovisioning of runners.
+		updateStrategy: "immediate"
+	}
+}

docs/examples/platforms/reference/clusters/foundation/cloud/init/namespaces/component.cue

@@ -0,0 +1,26 @@
+package holos
+
+import "list"
+
+#TargetNamespace: "default"
+
+#InputKeys: {
+	project:   "secrets"
+	component: "namespaces"
+}
+
+#KubernetesObjects & {
+	apiObjects: {
+		// #ManagedNamespaces is the set of all namespaces across all clusters in the platform.
+		for k, ns in #ManagedNamespaces {
+			if list.Contains(ns.clusterNames, #ClusterName) {
+				Namespace: "\(k)": #Namespace & ns.namespace
+			}
+		}
+
+		// #PlatformNamespaces is deprecated in favor of #ManagedNamespaces.
+		for ns in #PlatformNamespaces {
+			Namespace: "\(ns.name)": #Namespace & {metadata: ns}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/gateway/gateway.cue

@@ -1,7 +1,8 @@
 package holos
 
-// The primary istio Gateway, named default
+import "list"
 
+// The primary istio Gateway, named default
 let Name = "gateway"
 
 #InputKeys: component: Name
@@ -31,5 +32,19 @@ let LoginCert = #PlatformCerts.login
 				},
 			]
 		}
+
+		for k, svc in #OptionalServices {
+			if svc.enabled && list.Contains(svc.clusterNames, #ClusterName) {
+				Gateway: "\(svc.name)": #Gateway & {
+					metadata: name:      svc.name
+					metadata: namespace: #TargetNamespace
+					spec: selector: istio: "ingressgateway"
+					spec: servers: [for s in svc.servers {s}]
+				}
+				for k, s in svc.servers {
+					ExternalSecret: "\(s.tls.credentialName)": _
+				}
+			}
+		}
 	}
 }

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/httpbin/httpbin.cue

@@ -18,9 +18,7 @@ let Cert = #PlatformCerts[SecretName]
 
 #KubernetesObjects & {
 	apiObjects: {
-		ExternalSecret: httpbin: #ExternalSecret & {
-			_name: Cert.spec.secretName
-		}
+		ExternalSecret: "\(Cert.spec.secretName)": _
 		Deployment: httpbin: #Deployment & {
 			metadata: Metadata
 			spec: selector: matchLabels: MatchLabels

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/ingress/ingress.cue

@@ -63,7 +63,7 @@ let RedirectMetaName = {
 // https-redirect
 _APIObjects: {
 	Gateway: {
-		httpsRedirect: #Gateway & {
+		"\(RedirectMetaName.name)": #Gateway & {
 			metadata: RedirectMetaName
 			spec: selector: GatewayLabels
 			spec: servers: [{
@@ -79,7 +79,7 @@ _APIObjects: {
 		}
 	}
 	VirtualService: {
-		httpsRedirect: #VirtualService & {
+		"\(RedirectMetaName.name)": #VirtualService & {
 			metadata: RedirectMetaName
 			spec: hosts: ["*"]
 			spec: gateways: [RedirectMetaName.name]

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/values.cue

@@ -526,7 +526,7 @@ package holos
 
 	base: {
 		// For istioctl usage to disable istio config crds in base
-		enableIstioConfigCRDs: true
+		enableIstioConfigCRDs: *true | false
 
 		// If enabled, gateway-api types will be validated using the standard upstream validation logic.
 		// This is an alternative to deploying the standalone validation server the project provides.

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/values.holos.cue

@@ -16,8 +16,8 @@ package holos
 		remotePilotAddress: ""
 	}
 	base: {
-		// Include the CRDs in the helm template output
-		enableCRDTemplates: true
+		// holos includes crd templates with the --include-crds helm flag.
+		enableCRDTemplates: false
 		// Validation webhook configuration url
 		// For example: https://$remotePilotAddress:15017/validate
 		validationURL: ""

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/controller.cue

@@ -0,0 +1,6 @@
+package holos
+
+#DependsOn: Namespaces: name: "prod-secrets-namespaces"
+#DependsOn: CRDS: name:       "\(#InstancePrefix)-crds"
+#InputKeys: component: "controller"
+{} & #KustomizeBuild

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/kustomization.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: postgres-operator
+
+labels:
+- includeTemplates: true
+  pairs:
+    app.kubernetes.io/name: pgo
+    # The version below should match the version on the PostgresCluster CRD
+    app.kubernetes.io/version: 5.5.1
+    postgres-operator.crunchydata.com/control-plane: postgres-operator
+
+resources:
+- ./rbac/cluster
+- ./manager
+
+images:
+- name: postgres-operator
+  newName: registry.developers.crunchydata.com/crunchydata/postgres-operator
+  newTag: ubi8-5.5.1-0 

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/manager/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- manager.yaml

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/manager/manager.yaml

@@ -0,0 +1,56 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pgo
+  labels:
+    postgres-operator.crunchydata.com/control-plane: postgres-operator
+spec:
+  replicas: 1
+  strategy: { type: Recreate }
+  selector:
+    matchLabels:
+      postgres-operator.crunchydata.com/control-plane: postgres-operator
+  template:
+    metadata:
+      labels:
+        postgres-operator.crunchydata.com/control-plane: postgres-operator
+    spec:
+      containers:
+      - name: operator
+        image: postgres-operator
+        env:
+        - name: PGO_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CRUNCHY_DEBUG
+          value: "true"
+        - name: RELATED_IMAGE_POSTGRES_15
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-15.6-0"
+        - name: RELATED_IMAGE_POSTGRES_15_GIS_3.3
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-15.6-3.3-0"
+        - name: RELATED_IMAGE_POSTGRES_16
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.2-0"
+        - name: RELATED_IMAGE_POSTGRES_16_GIS_3.3
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-16.2-3.3-0"
+        - name: RELATED_IMAGE_POSTGRES_16_GIS_3.4
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-16.2-3.4-0"
+        - name: RELATED_IMAGE_PGADMIN
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-22"
+        - name: RELATED_IMAGE_PGBACKREST
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.49-0"
+        - name: RELATED_IMAGE_PGBOUNCER
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.21-3"
+        - name: RELATED_IMAGE_PGEXPORTER
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-0.15.0-3"
+        - name: RELATED_IMAGE_PGUPGRADE
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-upgrade:ubi8-5.5.1-0"
+        - name: RELATED_IMAGE_STANDALONE_PGADMIN
+          value: "registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-7.8-3"
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities: { drop: [ALL] }
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+      serviceAccountName: pgo

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/rbac/cluster/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- service_account.yaml
+- role.yaml
+- role_binding.yaml

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/rbac/cluster/role.yaml

@@ -0,0 +1,146 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: postgres-operator
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  - persistentvolumeclaims
+  - secrets
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - endpoints/restricted
+  - pods/exec
+  verbs:
+  - create
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - ''
+  resources:
+  - pods
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  - statefulsets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - postgres-operator.crunchydata.com
+  resources:
+  - pgadmins
+  - pgupgrades
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - postgres-operator.crunchydata.com
+  resources:
+  - pgadmins/finalizers
+  - pgupgrades/finalizers
+  - postgresclusters/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - postgres-operator.crunchydata.com
+  resources:
+  - pgadmins/status
+  - pgupgrades/status
+  - postgresclusters/status
+  verbs:
+  - patch
+- apiGroups:
+  - postgres-operator.crunchydata.com
+  resources:
+  - postgresclusters
+  verbs:
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  - roles
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - watch

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/rbac/cluster/role_binding.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: postgres-operator
+  labels:
+    postgres-operator.crunchydata.com/control-plane: postgres-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: postgres-operator
+subjects:
+- kind: ServiceAccount
+  name: pgo

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/rbac/cluster/service_account.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pgo
+  labels:
+    postgres-operator.crunchydata.com/control-plane: postgres-operator

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/rbac/namespace/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- service_account.yaml
+- role.yaml
+- role_binding.yaml

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/rbac/namespace/role.yaml

@@ -0,0 +1,146 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: postgres-operator
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  - persistentvolumeclaims
+  - secrets
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - endpoints/restricted
+  - pods/exec
+  verbs:
+  - create
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - ''
+  resources:
+  - pods
+  verbs:
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  - statefulsets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - postgres-operator.crunchydata.com
+  resources:
+  - pgadmins
+  - pgupgrades
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - postgres-operator.crunchydata.com
+  resources:
+  - pgadmins/finalizers
+  - pgupgrades/finalizers
+  - postgresclusters/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - postgres-operator.crunchydata.com
+  resources:
+  - pgadmins/status
+  - pgupgrades/status
+  - postgresclusters/status
+  verbs:
+  - patch
+- apiGroups:
+  - postgres-operator.crunchydata.com
+  resources:
+  - postgresclusters
+  verbs:
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - rolebindings
+  - roles
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - watch

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/rbac/namespace/role_binding.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: postgres-operator
+  labels:
+    postgres-operator.crunchydata.com/control-plane: postgres-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: postgres-operator
+subjects:
+- kind: ServiceAccount
+  name: pgo

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/controller/rbac/namespace/service_account.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pgo
+  labels:
+    postgres-operator.crunchydata.com/control-plane: postgres-operator

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/crds/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- bases/postgres-operator.crunchydata.com_postgresclusters.yaml
+- bases/postgres-operator.crunchydata.com_pgupgrades.yaml
+- bases/postgres-operator.crunchydata.com_pgadmins.yaml

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/crds/pgocrds.cue

@@ -0,0 +1,6 @@
+package holos
+
+// Refer to https://github.com/CrunchyData/postgres-operator-examples/tree/main/kustomize/install/crd
+
+#InputKeys: component: "crds"
+{} & #KustomizeBuild

docs/examples/platforms/reference/clusters/foundation/cloud/pgo/pgo.cue

@@ -0,0 +1,4 @@
+package holos
+
+// Crunchy Postgres Operator
+#InputKeys: project: "pgo"

docs/examples/platforms/reference/clusters/foundation/cloud/secrets/eso-creds-refresher/component.cue

@@ -2,6 +2,15 @@ package holos
 
 import "encoding/json"
 
+#DependsOn: _ESO
+
+#InputKeys: {
+	project:   "secrets"
+	component: "eso-creds-refresher"
+}
+
+#TargetNamespace: #CredsRefresher.namespace
+
 // output kubernetes api objects for holos
 #KubernetesObjects & {
 	apiObjects: {
@@ -13,15 +22,6 @@ import "encoding/json"
 	}
 }
 
-#InputKeys: {
-	project:   "secrets"
-	component: "eso-creds-refresher"
-}
-
-#TargetNamespace: #CredsRefresher.namespace
-
-#DependsOn: Namespaces: name: #InstancePrefix + "-namespaces"
-
 let NAME = #CredsRefresher.name
 let AUD = "//iam.googleapis.com/projects/\(#InputKeys.gcpProjectNumber)/locations/global/workloadIdentityPools/holos/providers/k8s-\(#InputKeys.cluster)"
 let MOUNT = "/var/run/service-account"
@@ -93,7 +93,14 @@ provisioner get serviceaccount -A --selector=holos.run/job.name=\(NAME) --output
 
 # Create the tokens
 mkdir tokens
-jq -r '.items[].metadata | "provisioner -n \\(.namespace) create token --duration=12h \\(.name) > tokens/\\(.namespace).\\(.name).jwt"' serviceaccounts.json | bash -x
+
+kubectl get namespaces -o name > namespaces.txt
+
+# Iterate over local namespaces
+while IFS= read -r NAMESPACE; do
+  echo "Getting token for local cluster $NAMESPACE" >&2
+  jq -r '.items[] | select("namespace/"+.metadata.namespace == "'${NAMESPACE}'") | .metadata | "provisioner -n \\(.namespace) create token --duration=12h \\(.name) > tokens/\\(.namespace).\\(.name).jwt"' serviceaccounts.json | bash -x
+done < namespaces.txt
 
 # Create the secrets
 mksecret tokens/*.jwt
@@ -124,6 +131,11 @@ kubectl apply --server-side=true -f secrets.yaml
 					resources: ["secrets"]
 					verbs: ["*"]
 				},
+				{
+					apiGroups: [""]
+					resources: ["namespaces"]
+					verbs: ["list"]
+				},
 			]
 		},
 		// Bind the Role to the ServiceAccount for the Job.

docs/examples/platforms/reference/clusters/foundation/cloud/secrets/secrets.cue

@@ -1,10 +1,12 @@
 package holos
 
 // Components under this directory are part of this collection
-#InputKeys: project: "iam"
+#InputKeys: project: "secrets"
 
 // Shared dependencies for all components in this collection.
 #DependsOn: _Namespaces
 
 // Common Dependencies
 _Namespaces: Namespaces: name: "\(#StageName)-secrets-namespaces"
+_ESO: ESO: name:               "\(#InstancePrefix)-eso"
+_ESOCreds: ESOCreds: name:     "\(#InstancePrefix)-eso-creds-refresher"

docs/examples/platforms/reference/clusters/foundation/cloud/secrets/secretstores/secretstores.cue

@@ -1,10 +1,14 @@
 package holos
 
+import "list"
+
+#DependsOn: _ESOCreds
+
 #TargetNamespace: "default"
 
 #InputKeys: {
 	project:   "secrets"
-	component: "namespaces"
+	component: "stores"
 }
 
 // #PlatformNamespaceObjects defines the api objects necessary for eso SecretStores in external clusters to access secrets in a given namespace in the provisioner cluster.
@@ -12,9 +16,6 @@ package holos
 	_ns: #PlatformNamespace
 
 	objects: [
-		#Namespace & {
-			metadata: _ns
-		},
 		#SecretStore & {
 			_namespace: _ns.name
 		},
@@ -31,5 +32,12 @@ package holos
 				"\(Kind)": "\(NS)/\(Name)": obj
 			}
 		}
+
+		for nsName, ns in #ManagedNamespaces {
+			if list.Contains(ns.clusterNames, #ClusterName) {
+				let obj = #SecretStore & {_namespace: nsName}
+				SecretStore: "\(nsName)/\(obj.metadata.name)": obj
+			}
+		}
 	}
 }

docs/examples/platforms/reference/clusters/foundation/cloud/secrets/validate/component.cue

@@ -9,7 +9,7 @@ package holos
 	component: "validate"
 }
 
-#DependsOn: Namespaces: name: #InstancePrefix + "-eso"
+#DependsOn: _ESO
 
 #KubernetesObjects & {
 	apiObjects: {

docs/examples/platforms/reference/clusters/foundation/metal/ceph/values.holos.cue

@@ -48,7 +48,7 @@ package holos
 
 		// (required) Ceph pool into which the RBD image shall be created
 		// eg: pool: replicapool
-		pool: "k8s-dev"
+		pool: #Platform.clusters[#ClusterName].pool
 
 		// (optional) RBD image features, CSI creates image with image-format 2 CSI
 		// RBD currently supports `layering`, `journaling`, `exclusive-lock`,

docs/examples/platforms/reference/clusters/optional/vault/values.holos.cue

@@ -0,0 +1,146 @@
+package holos
+
+#Values: {
+
+	// Vault Helm Chart Holos Values
+	global: {
+		enabled: true
+		// Istio handles this
+		tlsDisable: true
+	}
+
+	injector: enabled: false
+
+	server: {
+		image: {
+			// repository: "hashicorp/vault"
+			repository: "quay.io/holos/hashicorp/vault"
+			tag:        "1.14.10"
+			// Overrides the default Image Pull Policy
+			pullPolicy: "IfNotPresent"
+		}
+
+		extraLabels: "sidecar.istio.io/inject": "true"
+
+		resources: requests: {
+			memory: "256Mi"
+			cpu:    "2000m"
+		}
+		// limits:
+		//   memory: 1024Mi
+		//   cpu: 2000m
+		// For HA configuration and because we need to manually init the vault,
+		// we need to define custom readiness/liveness Probe settings
+		readinessProbe: {
+			enabled: true
+			path:    "/v1/sys/health?standbyok=true&sealedcode=204&uninitcode=204"
+		}
+		livenessProbe: {
+			enabled:             true
+			path:                "/v1/sys/health?standbyok=true"
+			initialDelaySeconds: 60
+		}
+
+		// extraEnvironmentVars is a list of extra environment variables to set with
+		// the stateful set. These could be used to include variables required for
+		// auto-unseal.
+		// Vault validates an incomplete chain:
+		// https://github.com/hashicorp/vault/issues/11318
+		extraEnvironmentVars: {
+			GOMAXPROCS: "2"
+		} // Set to cpu limit, see https://github.com/uber-go/automaxprocs
+		// extraVolumes is a list of extra volumes to mount. These will be exposed
+		// to Vault in the path `/vault/userconfig/<name>/`.
+		extraVolumes: [{
+			type: "secret"
+			name: "gcpkms-creds"
+		}]
+
+		// This configures the Vault Statefulset to create a PVC for audit logs.
+		// See https://www.vaultproject.io/docs/audit/index.html to know more
+		auditStorage: {
+			enabled:   true
+			mountPath: "/var/log/vault"
+		} // for compatibility with plain debian vm location.
+
+		standalone: {
+			enabled: false
+		}
+
+		ha: {
+			enabled:  true
+			replicas: 3
+			raft: {
+				enabled:   true
+				setNodeId: true
+
+				config: """
+					ui = true
+					listener \"tcp\" {
+					  address = \"[::]:8200\"
+					  cluster_address = \"[::]:8201\"
+					  # mTLS is handled by the the istio sidecar
+					  tls_disable = \"true\"
+					  # Enable unauthenticated metrics access (necessary for Prometheus Operator)
+					  telemetry {
+					    unauthenticated_metrics_access = true
+					  }
+					}
+
+					telemetry {
+					  prometheus_retention_time = \"30s\"
+					  disable_hostname = true
+					}
+
+					seal \"gcpckms\" {
+					  credentials = \"/vault/userconfig/gcpkms-creds/credentials.json\"
+					  project     = \"v6-vault-f15f\"
+					  region      = \"us-west1\"
+					  key_ring    = \"vault-core\"
+					  crypto_key  = \"vault-core-unseal\"
+					}
+
+					# Note; the retry_join leader_api_address values come from the Stable
+					# Network ID feature of a Statefulset.  See:
+					# https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id
+					storage \"raft\" {
+					  path = \"/vault/data\"
+					  retry_join {
+					    leader_api_addr = \"http://vault-0.vault-internal:8200\"
+					    leader_tls_servername = \"vault\"
+					  }
+					  retry_join {
+					    leader_api_addr = \"http://vault-1.vault-internal:8200\"
+					    leader_tls_servername = \"vault\"
+					  }
+					  retry_join {
+					    leader_api_addr = \"http://vault-2.vault-internal:8200\"
+					    leader_tls_servername = \"vault\"
+					  }
+
+					  autopilot {
+					    cleanup_dead_servers = \"true\"
+					    last_contact_threshold = \"200ms\"
+					    last_contact_failure_threshold = \"10m\"
+					    max_trailing_logs = 250000
+					    min_quorum = 3
+					    server_stabilization_time = \"10s\"
+					  }
+
+					}
+
+					service_registration \"kubernetes\" {}
+
+					"""
+
+				// Vault UI (Will be exposed via the service mesh)
+			} // Vault UI (Will be exposed via the service mesh)
+		} // Vault UI (Will be exposed via the service mesh)
+	} // Vault UI (Will be exposed via the service mesh)// Vault UI (Will be exposed via the service mesh)
+	ui: {
+		enabled:         true
+		serviceType:     "ClusterIP"
+		serviceNodePort: null
+		externalPort:    8200
+	}
+}

docs/examples/platforms/reference/clusters/optional/vault/vault.cue

@@ -0,0 +1,75 @@
+package holos
+
+import "encoding/yaml"
+
+import "list"
+
+let Name = "vault"
+#InputKeys: component: Name
+#InputKeys: project:   "core"
+#TargetNamespace: "\(#InstancePrefix)-\(Name)"
+
+let Vault = #OptionalServices[Name]
+
+if Vault.enabled && list.Contains(Vault.clusterNames, #ClusterName) {
+	#HelmChart & {
+		namespace: #TargetNamespace
+		chart: {
+			name:    Name
+			version: "0.25.0"
+			repository: {
+				name: "hashicorp"
+				url:  "https://helm.releases.hashicorp.com"
+			}
+		}
+		values: #Values
+
+		apiObjects: {
+			ExternalSecret: "gcpkms-creds":      _
+			ExternalSecret: "vault-server-cert": _
+			VirtualService: "\(Name)": {
+				metadata: name:      Name
+				metadata: namespace: #TargetNamespace
+				spec: hosts: [for cert in Vault.certs {cert.spec.commonName}]
+				spec: gateways: ["istio-ingress/\(Name)"]
+				spec: http: [
+					{
+						route: [
+							{
+								destination: host: "\(Name)-active"
+								destination: port: number: 8200
+							},
+						]
+					},
+				]
+			}
+		}
+	}
+
+	#Kustomize: {
+		patches: [
+			{
+				target: {
+					group:   "apps"
+					version: "v1"
+					kind:    "StatefulSet"
+					name:    Name
+				}
+				patch: yaml.Marshal(EnvPatch)
+			},
+		]
+	}
+
+	let EnvPatch = [
+		{
+			op:    "test"
+			path:  "/spec/template/spec/containers/0/env/4/name"
+			value: "VAULT_ADDR"
+		},
+		{
+			op:    "replace"
+			path:  "/spec/template/spec/containers/0/env/4/value"
+			value: "http://$(VAULT_K8S_POD_NAME):8200"
+		},
+	]
+}

docs/examples/platforms/reference/clusters/provisioner/iam/zitadel/crdb/crdbissuer.cue

@@ -1,91 +0,0 @@
-package holos
-
-// Manage an Issuer for cockroachdb for zitadel.
-// For the iam login service, zitadel connects to cockroach db using tls certs for authz.
-// Upstream: "The recommended approach is to use cert-manager for certificate management. For details, refer to Deploy cert-manager for mTLS."
-// Refer to https://www.cockroachlabs.com/docs/stable/secure-cockroachdb-kubernetes#deploy-cert-manager-for-mtls
-
-#InputKeys: component: "crdb"
-
-#KubernetesObjects & {
-	apiObjects: {
-		Issuer: {
-			// https://github.com/cockroachdb/helm-charts/blob/3dcf96726ebcfe3784afb526ddcf4095a1684aea/README.md?plain=1#L196-L201
-			crdb: #Issuer & {
-				_description: "Issues the self signed root ca cert for cockroach db"
-				metadata: name:      #ComponentName
-				metadata: namespace: #TargetNamespace
-				spec: selfSigned: {}
-			}
-			"crdb-ca-issuer": #Issuer & {
-				_description: "Issues mtls certs for cockroach db"
-				metadata: name:      "crdb-ca-issuer"
-				metadata: namespace: #TargetNamespace
-				spec: ca: secretName: "cockroach-ca"
-			}
-		}
-		Certificate: {
-			"crdb-ca-cert": #Certificate & {
-				_description: "Root CA cert for cockroach db"
-				metadata: name:      "crdb-ca-cert"
-				metadata: namespace: #TargetNamespace
-				spec: {
-					commonName: "root"
-					isCA:       true
-					issuerRef: group:      "cert-manager.io"
-					issuerRef: kind:       "Issuer"
-					issuerRef: name:       "crdb"
-					privateKey: algorithm: "ECDSA"
-					privateKey: size:      256
-					secretName: "cockroach-ca"
-					subject: organizations: ["Cockroach"]
-				}
-			}
-			"crdb-node": #Certificate & {
-				metadata: name:      "crdb-node"
-				metadata: namespace: #TargetNamespace
-				spec: {
-					commonName: "node"
-					dnsNames: [
-						"localhost",
-						"127.0.0.1",
-						"crdb-public",
-						"crdb-public.\(#TargetNamespace)",
-						"crdb-public.\(#TargetNamespace).svc.cluster.local",
-						"*.crdb",
-						"*.crdb.\(#TargetNamespace)",
-						"*.crdb.\(#TargetNamespace).svc.cluster.local",
-					]
-					duration: "876h"
-					issuerRef: group:      "cert-manager.io"
-					issuerRef: kind:       "Issuer"
-					issuerRef: name:       "crdb-ca-issuer"
-					privateKey: algorithm: "RSA"
-					privateKey: size:      2048
-					renewBefore: "168h"
-					secretName:  "cockroachdb-node"
-					subject: organizations: ["Cockroach"]
-					usages: ["digital signature", "key encipherment", "server auth", "client auth"]
-				}
-			}
-			"crdb-root-client": #Certificate & {
-				metadata: name:      "crdb-root-client"
-				metadata: namespace: #TargetNamespace
-				spec: {
-					commonName: "root"
-					duration:   "672h"
-					issuerRef: group:      "cert-manager.io"
-					issuerRef: kind:       "Issuer"
-					issuerRef: name:       "crdb-ca-issuer"
-					privateKey: algorithm: "RSA"
-					privateKey: size:      2048
-					renewBefore: "48h"
-					secretName:  "cockroachdb-root"
-					subject: organizations: ["Cockroach"]
-					usages: ["digital signature", "key encipherment", "client auth"]
-				}
-			}
-
-		}
-	}
-}

docs/examples/platforms/reference/clusters/provisioner/iam/zitadel/postgres-certs/postgres-certs.cue

@@ -0,0 +1,101 @@
+package holos
+
+// Manage an Issuer for the database.
+
+// Both cockroach and postgres handle tls database connections with cert manager
+// PGO: https://github.com/CrunchyData/postgres-operator-examples/tree/main/kustomize/certmanager/certman
+// CRDB: https://github.com/cockroachdb/helm-charts/blob/3dcf96726ebcfe3784afb526ddcf4095a1684aea/README.md?plain=1#L196-L201
+
+// Refer to [Using Cert Manager to Deploy TLS for Postgres on Kubernetes](https://www.crunchydata.com/blog/using-cert-manager-to-deploy-tls-for-postgres-on-kubernetes)
+
+#InputKeys: component: "postgres-certs"
+
+let SelfSigned = "\(_DBName)-selfsigned"
+let RootCA = "\(_DBName)-root-ca"
+let Orgs = ["Database"]
+
+#KubernetesObjects & {
+	apiObjects: {
+		// Put everything in the target namespace.
+		[_]: {
+			[Name=_]: {
+				metadata: name:      Name
+				metadata: namespace: #TargetNamespace
+			}
+		}
+		Issuer: {
+			"\(SelfSigned)": #Issuer & {
+				_description: "Self signed issuer to issue ca certs"
+				metadata: name: SelfSigned
+				spec: selfSigned: {}
+			}
+			"\(RootCA)": #Issuer & {
+				_description: "Root signed intermediate ca to issue mtls database certs"
+				metadata: name: RootCA
+				spec: ca: secretName: RootCA
+			}
+		}
+		Certificate: {
+			"\(RootCA)": #Certificate & {
+				_description: "Root CA cert for database"
+				metadata: name: RootCA
+				spec: {
+					commonName: RootCA
+					isCA:       true
+					issuerRef: group:      "cert-manager.io"
+					issuerRef: kind:       "Issuer"
+					issuerRef: name:       SelfSigned
+					privateKey: algorithm: "ECDSA"
+					privateKey: size:      256
+					secretName: RootCA
+					subject: organizations: Orgs
+				}
+			}
+			"\(_DBName)-primary-tls": #DatabaseCert & {
+				// PGO managed name is "<cluster name>-cluster-cert" e.g. zitadel-cluster-cert
+				spec: {
+					commonName: "\(_DBName)-primary"
+					dnsNames: [
+						commonName,
+						"\(commonName).\(#TargetNamespace)",
+						"\(commonName).\(#TargetNamespace).svc",
+						"\(commonName).\(#TargetNamespace).svc.cluster.local",
+						"localhost",
+						"127.0.0.1",
+					]
+					usages: ["digital signature", "key encipherment"]
+				}
+			}
+			"\(_DBName)-repl-tls": #DatabaseCert & {
+				spec: {
+					commonName: "_crunchyrepl"
+					dnsNames: [commonName]
+					usages: ["digital signature", "key encipherment"]
+				}
+			}
+			"\(_DBName)-client-tls": #DatabaseCert & {
+				spec: {
+					commonName: "\(_DBName)-client"
+					dnsNames: [commonName]
+					usages: ["digital signature", "key encipherment"]
+				}
+			}
+		}
+	}
+}
+
+#DatabaseCert: #Certificate & {
+	metadata: name:      string
+	metadata: namespace: #TargetNamespace
+	spec: {
+		duration:    "2160h" // 90d
+		renewBefore: "360h"  // 15d
+		issuerRef: group:      "cert-manager.io"
+		issuerRef: kind:       "Issuer"
+		issuerRef: name:       RootCA
+		privateKey: algorithm: "ECDSA"
+		privateKey: size:      256
+		secretName: metadata.name
+		subject: organizations: Orgs
+	}
+}

docs/examples/platforms/reference/clusters/provisioner/iam/zitadel/postgres-certs/readme.md

@@ -0,0 +1,7 @@
+# Database Certs
+
+This component issues postgres certificates from the provisioner cluster using certmanager.
+
+The purpose is to define customTLSSecret and customReplicationTLSSecret to provide certs that allow the standby to authenticate to the primary. For this type of standby, you must use custom TLS.
+
+Refer to the PGO [Streaming Standby](https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/disaster-recovery#streaming-standby) tutorial.

docs/examples/platforms/reference/clusters/provisioner/iam/zitadel/zitadel.cue

@@ -2,6 +2,5 @@ package holos
 
 #TargetNamespace: #InstancePrefix + "-zitadel"
 
-#DB: {
-	Host: "crdb-public"
-}
+// _DBName is the database name used across multiple holos components in this project
+_DBName: "zitadel"

docs/examples/platforms/reference/clusters/provisioner/mesh/certificates/optional/vault.cue

@@ -0,0 +1,13 @@
+package holos
+
+let Vault = #OptionalServices.vault
+
+if Vault.enabled {
+	#KubernetesObjects & {
+		apiObjects: {
+			for k, obj in Vault.certs {
+				"\(obj.kind)": "\(obj.metadata.name)": obj
+			}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/provisioner/secrets/eso-creds-refresher/component.cue

@@ -24,6 +24,14 @@ ksObjects: []
 				"\(Kind)": "\(ns.name)/\(Name)": obj
 			}
 		}
+
+		for nsName, ns in #ManagedNamespaces {
+			for obj in (#PlatformNamespaceObjects & {_ns: ns.namespace.metadata}).objects {
+				let Kind = obj.kind
+				let Name = obj.metadata.name
+				"\(Kind)": "\(nsName)/\(Name)": obj
+			}
+		}
 	}
 }
 

docs/examples/platforms/reference/clusters/provisioner/secrets/namespaces/component.cue

@@ -7,25 +7,16 @@ package holos
 	component: "namespaces"
 }
 
-// #PlatformNamespaceObjects defines the api objects necessary for eso SecretStores in external clusters to access secrets in a given namespace in the provisioner cluster.
-#PlatformNamespaceObjects: {
-	_ns: #PlatformNamespace
-
-	objects: [
-		#Namespace & {
-			metadata: name: _ns.name
-		},
-	]
-}
-
 #KubernetesObjects & {
 	apiObjects: {
+		// #ManagedNamespaces is the set of all namespaces across all clusters in the platform.
+		for nsName, ns in #ManagedNamespaces {
+			Namespace: "\(nsName)": #Namespace & ns.namespace
+		}
+
+		// #PlatformNamespaces is deprecated in favor of #ManagedNamespaces.
 		for ns in #PlatformNamespaces {
-			for obj in (#PlatformNamespaceObjects & {_ns: ns}).objects {
-				let Kind = obj.kind
-				let Name = obj.metadata.name
-				"\(Kind)": "\(Name)": obj
-			}
+			Namespace: "\(ns.name)": #Namespace & {metadata: ns}
 		}
 	}
 }

docs/examples/platforms/reference/clusters/workload/cloud/iam/zitadel/crdb/cockroachdb.cue

@@ -1,26 +0,0 @@
-package holos
-
-#InputKeys: component: "crdb"
-
-#HelmChart & {
-	namespace: #TargetNamespace
-	chart: {
-		name:    "cockroachdb"
-		version: "11.2.3"
-		repository: {
-			name: "cockroachdb"
-			url:  "https://charts.cockroachdb.com/"
-		}
-	}
-	values: #Values
-	apiObjects: {
-		Issuer: {
-			// https://github.com/cockroachdb/helm-charts/blob/3dcf96726ebcfe3784afb526ddcf4095a1684aea/README.md?plain=1#L196-L201
-			cockroachdb: #Issuer & {
-				metadata: name:      #ComponentName
-				metadata: namespace: #TargetNamespace
-				spec: selfSigned: {}
-			}
-		}
-	}
-}

docs/examples/platforms/reference/clusters/workload/cloud/iam/zitadel/zitadel.cue

@@ -1,10 +0,0 @@
-package holos
-
-#TargetNamespace: #InstancePrefix + "-zitadel"
-
-#DB: {
-	Host: "crdb-public"
-}
-
-// The canonical login domain for the entire platform.  Zitadel will be active on a singlec cluster at a time, but always accessible from this hostname.
-#ExternalDomain: "login.\(#Platform.org.domain)"

docs/examples/platforms/reference/clusters/workload/cloud/iam/zitadel/zitadel/values.holos.cue

@@ -1,34 +0,0 @@
-package holos
-
-#Values: {
-
-	// https://raw.githubusercontent.com/zitadel/zitadel-charts/main/examples/4-cockroach-secure/zitadel-values.yaml
-	zitadel: {
-		masterkeySecretName: "zitadel-masterkey"
-		// https://github.com/zitadel/zitadel-charts/blob/zitadel-7.4.0/charts/zitadel/templates/configmap.yaml#L13
-		configmapConfig: {
-			// NOTE: You can change the ExternalDomain, ExternalPort and ExternalSecure
-			// configuration options at any time. However, for ZITADEL to be able to
-			// pick up the changes, you need to rerun ZITADELs setup phase.  Do so with
-			// kubectl delete job zitadel-setup, then re-apply the new config.
-			//
-			// https://zitadel.com/docs/self-hosting/manage/custom-domain
-			ExternalDomain: #ExternalDomain
-			ExternalPort:   443
-			ExternalSecure: true
-			TLS: Enabled: false
-			Database: Cockroach: {
-				Host: #DB.Host
-				User: SSL: Mode:  "verify-full"
-				Admin: SSL: Mode: "verify-full"
-			}
-		}
-
-		// Managed by crdb component
-		dbSslCaCrtSecret:    "cockroach-ca"
-		dbSslAdminCrtSecret: "cockroachdb-root"
-		// Managed by this component
-		dbSslUserCrtSecret: "cockroachdb-zitadel"
-	}
-
-}

docs/examples/platforms/reference/clusters/workload/cloud/iam/zitadel/zitadel/zitadel.cue

@@ -1,70 +0,0 @@
-package holos
-
-import "encoding/yaml"
-
-let Name = "zitadel"
-#InputKeys: component: Name
-
-// Upstream helm chart doesn't specify the namespace field for all resources.
-#Kustomization: spec: targetNamespace: #TargetNamespace
-
-#HelmChart & {
-	namespace: #TargetNamespace
-	chart: {
-		name:    Name
-		version: "7.9.0"
-		repository: {
-			name: Name
-			url:  "https://charts.zitadel.com"
-		}
-	}
-	values: #Values
-
-	apiObjects: {
-		ExternalSecret: masterkey: #ExternalSecret & {
-			_name: "zitadel-masterkey"
-		}
-		Certificate: zitadel: #Certificate & {
-			metadata: name:      "crdb-zitadel-client"
-			metadata: namespace: #TargetNamespace
-			spec: {
-				commonName: "zitadel"
-				issuerRef: {
-					group: "cert-manager.io"
-					kind:  "Issuer"
-					name:  "crdb-ca-issuer"
-				}
-				privateKey: algorithm: "RSA"
-				privateKey: size:      2048
-				renewBefore: "48h0m0s"
-				secretName:  "cockroachdb-zitadel"
-				subject: organizations: ["Cockroach"]
-				usages: ["digital signature", "key encipherment", "client auth"]
-			}
-		}
-		VirtualService: zitadel: #VirtualService & {
-			metadata: name:      Name
-			metadata: namespace: #TargetNamespace
-			spec: hosts: ["login.\(#Platform.org.domain)"]
-			spec: gateways: ["istio-ingress/default"]
-			spec: http: [{route: [{destination: host: Name}]}]
-		}
-	}
-}
-
-// TODO: Generalize this common pattern of injecting the istio sidecar into a Deployment
-let Patch = [{op: "add", path: "/spec/template/metadata/labels/sidecar.istio.io~1inject", value: "true"}]
-
-#Kustomize: {
-	patches: [
-		{
-			target: {
-				group:   "apps"
-				version: "v1"
-				kind:    "Deployment"
-				name:    Name
-			}
-			patch: yaml.Marshal(Patch)
-		},
-	]
-}

docs/examples/platforms/reference/namespaces.cue

@@ -12,6 +12,7 @@ let Privileged = {
 
 // #PlatformNamespaces is the union of all namespaces across all cluster types.  Namespaces are created in all clusters regardless of if they're
 // used within the cluster or not.  The is important for security and consistency with IAM, RBAC, and Secrets sync between clusters.
+// TODO: Deprecate in favor of #ManagedNamespaces because it better to add fields to an object instead adding items to a list.
 #PlatformNamespaces: [
 	{name: "external-secrets"},
 	{name: "holos-system"},
@@ -22,4 +23,8 @@ let Privileged = {
 	{name: "cert-manager"},
 	{name: "argocd"},
 	{name: "prod-iam-zitadel"},
+	{name: "arc-system"},
+	{name: "arc-runner"},
+	// https://github.com/CrunchyData/postgres-operator-examples/blob/main/kustomize/install/namespace/namespace.yaml
+	{name: "postgres-operator"},
 ]

docs/examples/schema.cue

@@ -15,6 +15,7 @@ import (
 	gw "networking.istio.io/gateway/v1beta1"
 	vs "networking.istio.io/virtualservice/v1beta1"
 	kc "sigs.k8s.io/kustomize/api/types"
+	pg "postgres-operator.crunchydata.com/postgrescluster/v1beta1"
 	"encoding/yaml"
 )
 
@@ -81,6 +82,7 @@ _apiVersion: "holos.run/v1alpha1"
 }
 
 #NamespaceObject: #ClusterObject & {
+	metadata: name:      string
 	metadata: namespace: string
 	...
 }
@@ -96,19 +98,20 @@ _apiVersion: "holos.run/v1alpha1"
 #ClusterRoleBinding: #ClusterObject & rbacv1.#ClusterRoleBinding
 #ClusterIssuer: #ClusterObject & ci.#ClusterIssuer & {...}
 
-#Issuer:         #NamespaceObject & is.#Issuer
-#Role:           #NamespaceObject & rbacv1.#Role
-#RoleBinding:    #NamespaceObject & rbacv1.#RoleBinding
-#ConfigMap:      #NamespaceObject & corev1.#ConfigMap
-#ServiceAccount: #NamespaceObject & corev1.#ServiceAccount
-#Pod:            #NamespaceObject & corev1.#Pod
-#Service:        #NamespaceObject & corev1.#Service
-#Job:            #NamespaceObject & batchv1.#Job
-#CronJob:        #NamespaceObject & batchv1.#CronJob
-#Deployment:     #NamespaceObject & appsv1.#Deployment
-#Gateway:        #NamespaceObject & gw.#Gateway
-#VirtualService: #NamespaceObject & vs.#VirtualService
-#Certificate:    #NamespaceObject & crt.#Certificate
+#Issuer:          #NamespaceObject & is.#Issuer
+#Role:            #NamespaceObject & rbacv1.#Role
+#RoleBinding:     #NamespaceObject & rbacv1.#RoleBinding
+#ConfigMap:       #NamespaceObject & corev1.#ConfigMap
+#ServiceAccount:  #NamespaceObject & corev1.#ServiceAccount
+#Pod:             #NamespaceObject & corev1.#Pod
+#Service:         #NamespaceObject & corev1.#Service
+#Job:             #NamespaceObject & batchv1.#Job
+#CronJob:         #NamespaceObject & batchv1.#CronJob
+#Deployment:      #NamespaceObject & appsv1.#Deployment
+#Gateway:         #NamespaceObject & gw.#Gateway
+#VirtualService:  #NamespaceObject & vs.#VirtualService
+#Certificate:     #NamespaceObject & crt.#Certificate
+#PostgresCluster: #NamespaceObject & pg.#PostgresCluster
 
 // #HTTP01Cert defines a http01 certificate.
 #HTTP01Cert: {
@@ -149,15 +152,16 @@ _apiVersion: "holos.run/v1alpha1"
 		suspend?:         bool
 		targetNamespace?: string
 		timeout:          string | *"3m0s"
-		wait:             bool | *true
+		// wait performs health checks for all reconciled resources. If set to true, .spec.healthChecks is ignored.
+		wait: bool | *true
 		dependsOn: [for k, v in #DependsOn {v}]
 	}
 }
 
 // #DependsOn stores all of the dependencies between components.  It's a struct to support merging across levels in the tree.
 #DependsOn: {
-	[NAME=_]: {
-		name: string
+	[Name=_]: {
+		name: string | *"\(#InstancePrefix)-\(Name)"
 	}
 	...
 }
@@ -227,18 +231,48 @@ _apiVersion: "holos.run/v1alpha1"
 	provisionerURL: string @tag(provisionerURL, type=string)
 }
 
+// #ClusterSpec is the specification of a holos platform cluster member.
+#ClusterSpec: {
+	// name is the cluster name.
+	name: string
+	// pool is the optional ceph pool of the cluster.
+	pool?: string
+	// region is the geographic region of the cluster.
+	region?: string
+	// primary is true if name matches the primaryCluster name
+	primary: bool
+}
+
 // #Platform defines the primary lookup table for the platform.  Lookup keys should be limited to those defined in #KeyTags.
 #Platform: {
 	// org holds user defined values scoped organization wide.  A platform has one and only one organization.
 	org: {
-		name:   string
+		// e.g. "example"
+		name: string
+		// e.g. "example.com"
 		domain: string
-		contact: email:    string
+		// e.g. "Example"
+		displayName: string
+		// e.g. "platform@example.com"
+		contact: email: string
+		// e.g. "platform@example.com"
 		cloudflare: email: string
+		// e.g. "example"
+		github: orgs: primary: name: string
 	}
-	clusters: [ID=_]: {
-		name:    string & ID
-		region?: string
+	// Only one cluster may be primary at a time.  All others are standby.
+	// Refer to [repo based standby](https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/disaster-recovery#repo-based-standby)
+	primaryCluster: {
+		name: string
+	}
+	clusters: [Name=_]: #ClusterSpec & {
+		name: string & Name
+		if Name == primaryCluster.name {
+			primary: true
+		}
+		if Name != primaryCluster.name {
+			primary: false
+		}
 	}
 	stages: [ID=_]: {
 		name: string & ID
@@ -252,6 +286,35 @@ _apiVersion: "holos.run/v1alpha1"
 	}
 }
 
+// ManagedNamespace is a namespace to manage across all clusters in the holos platform.
+#ManagedNamespace: {
+	namespace: {
+		metadata: {
+			name: string
+			labels: [string]: string
+		}
+	}
+	// clusterNames represents the set of clusters the namespace is managed on.  Usually all clusters.
+	clusterNames: [...string]
+}
+
+// #ManagedNamepsaces is the union of all namespaces across all cluster types and optional services.
+// Holos adopts the namespace sameness position of SIG Multicluster, refer to https://github.com/kubernetes/community/blob/dd4c8b704ef1c9c3bfd928c6fa9234276d61ad18/sig-multicluster/namespace-sameness-position-statement.md
+#ManagedNamespaces: {
+	[Name=_]: #ManagedNamespace & {
+		namespace: metadata: name: Name
+	}
+}
+
+// #Backups defines backup configuration.
+// TODO: Consider the best place for this, possibly as part of the site platform config.  This represents the primary location for backups.
+#Backups: {
+	s3: {
+		region:   string
+		endpoint: string | *"s3.dualstack.\(region).amazonaws.com"
+	}
+}
+
 // #APIObjects is the output type for api objects produced by cue.  A map is used to aid debugging and clarity.
 #APIObjects: {
 	// apiObjects holds each the api objects produced by cue.
@@ -261,6 +324,9 @@ _apiVersion: "holos.run/v1alpha1"
 				kind: Kind
 			}
 		}
+		ExternalSecret?: [Name=_]: #ExternalSecret & {_name: Name}
+		VirtualService?: [Name=_]: #VirtualService & {metadata: name: Name}
+		Issuer?: [Name=_]: #Issuer & {metadata: name: Name}
 	}
 
 	// apiObjectMap holds the marshalled representation of apiObjects
@@ -312,9 +378,10 @@ _apiVersion: "holos.run/v1alpha1"
 #Chart: {
 	name:    string
 	version: string
+	release: string | *name
 	repository: {
-		name: string
-		url:  string
+		name?: string
+		url?:  string
 	}
 }
 
@@ -347,6 +414,23 @@ _apiVersion: "holos.run/v1alpha1"
 	resourcesFile: ResourcesFile
 	// kustomizeFiles represents the files in a kustomize directory tree.
 	kustomizeFiles: #KustomizeFiles.Files
+	// enableHooks removes the --no-hooks flag from helm template
+	enableHooks: true | *false
+}
+
+// #KustomizeBuild is a holos component that uses plain yaml files as the source of api objects for a holos component.
+// Intended for upstream components like the CrunchyData Postgres Operator.  The holos cli is expected to execute kustomize build on the component directory to produce the rendered output.
+#KustomizeBuild: {
+	#OutputTypeMeta
+	#APIObjects
+	kind: "KustomizeBuild"
+	metadata: name: #InstanceName
+	// ksObjects holds the flux Kustomization objects for gitops.
+	ksObjects: [...#Kustomization] | *[#Kustomization]
+	// ksContent is the yaml representation of kustomization.
+	ksContent: yaml.MarshalStream(ksObjects)
+	// namespace defines the value passed to the helm --namespace flag
+	namespace: #TargetNamespace
 }
 
 // #PlatformSpec is the output schema of a platform specification.
@@ -387,8 +471,40 @@ _apiVersion: "holos.run/v1alpha1"
 	kind:       "Kustomization"
 	resources: [ResourcesFile]
 	...
+	if len(#KustomizePatches) > 0 {
+		patches: [for v in #KustomizePatches {v}]
+	}
 }
 
+#KustomizePatches: {
+	[_]: #Patch
+}
+
+// #Patch is a kustomize patch
+#Patch: kc.#Patch
+
+// #DefaultSecurityContext is the holos default security context to comply with the restricted namespace policy.
+// Refer to https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
+#DefaultSecurityContext: {
+	securityContext: {
+		allowPrivilegeEscalation: false
+		runAsNonRoot:             true
+		capabilities: drop: ["ALL"]
+		seccompProfile: type: "RuntimeDefault"
+	}
+	...
+}
+
+// Certificate name should always match the secret name.
+#Certificate: {
+	metadata: name:   _
+	spec: secretName: metadata.name
+}
+
+// #IsPrimaryCluster is true if the cluster being rendered is the primary cluster
+// Used by the iam project to determine where https://login.example.com is active.
+#IsPrimaryCluster: bool & #ClusterName == #Platform.primaryCluster.name
+
 // By default, render kind: Skipped so holos knows to skip over intermediate cue files.
 // This enables the use of holos render ./foo/bar/baz/... when bar contains intermediary constraints which are not complete components.
 // Holos skips over these intermediary cue instances.

docs/provisioner/readme.md

@@ -133,9 +133,12 @@ This section configured:
 
 1. Provisioner Cluster to provide secrets to workload clusters.
 2. IAM service account `eso-creds-refresher` to identify the credential refresher job.
-3. Workload identity pool to authenticate the `eso-creds-refresher` Kubernetes service account in an external cluster.
-4. IAM policy to allow `eso-creds-refresher` to authenticate to the Provisioner Cluster.
-5. RoleBinding to allow `eso-creds-refresher` to create kubernetes service account tokens representing the credentials for use by SecretStore resources in workload clusters.
+3. Workload identity pool to authenticate the `system:serviceaccount:holos-system:eso-creds-refresher` Kubernetes service account in all clusters that share the workload identity pool.
+4. IAM policy to allow the `eso-creds-refresher` IAM service account to authenticate to the Provisioner Cluster.
+5. RoleBinding to allow the `eso-creds-refresher` IAM service account to create kubernetes service account tokens representing the credentials for use by SecretStore resources in workload clusters.
+
+> [!NOTE]
+> Any cluster in the workload identity pool can impersonate the eso-creds-refresher IAM service account.
 
 ## Cluster Setup
 
@@ -150,6 +153,12 @@ HOLOS_CLUSTER_NAME=west1
 ISSUER_URL="https://example.com/clusters/${HOLOS_CLUSTER_NAME}"
 ```
 
+Alternatively:
+
+```shell
+ISSUER_URL="$(kubectl get --raw='/.well-known/openid-configuration' | jq -r .issuer)"
+```
+
 ```shell
 gcloud iam workload-identity-pools providers create-oidc \
   k8s-$HOLOS_CLUSTER_NAME \

docs/runbooks/login/failover.md

@@ -0,0 +1,305 @@
+## Overview
+
+Use this run book to move the ZITADEL primary postgres database from one cluster to another.  At the end of the process https://zitadel.example.com service will be live on the cluster which was previously the standby at the start of the process.
+
+Downtime: ~10 minutes if executed quickly.  ~60 minutes if executed slowly and deliberately.
+
+## Initial State
+1. https://login.ois.run is routed to core2.
+2. `postgrescluster/zitadel` on core2 is primary.
+3. `postgrescluster/zitadel` on core1 is standby.
+4. A recent [[#Full Backup]] has been taken and is stored in S3.  
+5. The standby cluster has been restored from the recent full backup.
+## Process
+1. [[#Edit Platform site config]]
+2. [[#Render core2]]
+3. [[#Demote core2]]  
+4. [[#Verify core2 is standby]]
+5. [[#Render core1]]
+6. [[#Promote core1]]
+7. [[#Verify core1 is primary]]
+8. [[#Reconcile Zitadel]]
+9. [[#Cut over NLB]]
+10. [[#Test Login]]
+
+## Edit Platform site config
+Change the primary cluster from core2 to core1 in `platforms/platform.site.cue`.
+```cue
+#Platform: primaryCluster: name: "core1"
+```
+## Render core2
+Configure core as a standby.
+```
+holos render --cluster-name=core2 ~/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/accounts/...
+```
+
+Expected change:
+```diff
+diff --git a/deploy/clusters/core2/components/prod-iam-postgres/prod-iam-postgres.gen.yaml b/deploy/clusters/core2/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
+index c1aa8394..45daa650 100644
+--- a/deploy/clusters/core2/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
++++ b/deploy/clusters/core2/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
+@@ -85,24 +85,6 @@ spec:
+     name: zitadel-repl-tls
+   customTLSSecret:
+     name: zitadel-primary-tls
+-  dataSource:
+-    pgbackrest:
+-      configuration:
+-        - secret:
+-            name: pgo-s3-creds
+-      global:
+-        repo2-path: /pgbackrest/prod-iam-zitadel/zitadel/repo2
+-        repo2-cipher-type: aes-256-cbc
+-      options:
+-        - --type=time
+-        - --target="2024-03-11 17:10:00+00"
+-      repo:
+-        name: repo2
+-        s3:
+-          bucket: ois-zitadel-backups
+-          endpoint: s3.dualstack.us-east-2.amazonaws.com
+-          region: us-east-2
+-      stanza: db
+   image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.2-0
+   instances:
+     - name: pgha1
+@@ -139,7 +121,7 @@ spec:
+       image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.21-3
+       replicas: 2
+   standby:
+-    enabled: false
++    enabled: true
+     repoName: repo2
+   users:
+     - databases
+```
+## Demote core2
+
+```
+10012  git add -p
+10013  git commit -m 'Make core2 a standby zitadel cluster'
+10014  git push origin HEAD
+10015  flux reconcile source git flux-system
+10016  flux get ks
+```
+
+## Verify core2 is standby
+```
+k logs $(kubectl get pods -o name -l postgres-operator.crunchydata.com/role=master) | tail -2
+```
+Expected output:
+```
+2024-03-14 21:58:26,205 INFO: no action. I am (zitadel-pgha1-smw7-0), the standby leader with the lock
+```
+## Render core1
+```
+holos render --cluster-name=core1 ~/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/accounts/...
+```
+Expected change:
+```diff
+diff --git a/deploy/clusters/core1/components/prod-iam-postgres/prod-iam-postgres.gen.yaml b/deploy/clusters/core1/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
+index aa69814e..290c0865 100644
+--- a/deploy/clusters/core1/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
++++ b/deploy/clusters/core1/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
+@@ -85,6 +85,22 @@ spec:
+     name: zitadel-repl-tls
+   customTLSSecret:
+     name: zitadel-primary-tls
++  dataSource:
++    pgbackrest:
++      configuration:
++        - secret:
++            name: pgo-s3-creds
++      global:
++        repo2-path: /pgbackrest/prod-iam-zitadel/zitadel/repo2
++        repo2-cipher-type: aes-256-cbc
++      options: []
++      repo:
++        name: repo2
++        s3:
++          bucket: ois-zitadel-backups
++          endpoint: s3.dualstack.us-east-2.amazonaws.com
++          region: us-east-2
++      stanza: db
+   image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.2-0
+   instances:
+     - name: pgha1
+@@ -121,7 +137,7 @@ spec:
+       image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.21-3
+       replicas: 2
+   standby:
+-    enabled: true
++    enabled: false
+     repoName: repo2
+   users:
+     - databases:
+```
+
+## Promote core1
+```
+flux reconcile source git flux-system
+flux reconcile ks prod-iam-postgres
+```
+## Verify core1 is primary
+```
+k logs $(kubectl get pods -o name -l postgres-operator.crunchydata.com/role=master) | tail -1
+```
+Expected output:
+```
+2024-03-14 22:05:01,159 INFO: no action. I am (zitadel-pgha1-xl2b-0), the leader with the lock
+```
+## Reconcile Zitadel
+Now that the database is back up, reconcile zitadel.
+This is pretty brutal but it works:
+```
+❯ flux delete ks prod-iam-zitadel
+Are you sure you want to delete this kustomization: y
+► deleting kustomization prod-iam-zitadel in flux-system namespace
+✔ kustomization deleted
+```
+```
+❯ k apply --server-side=true -f deploy/clusters/core1/holos/components/prod-iam-zitadel-kustomization.gen.yaml
+kustomization.kustomize.toolkit.fluxcd.io/prod-iam-zitadel serverside-applied
+```
+ZITADEL should come up READY 2/2
+```
+❯ k get pods
+NAME                               READY   STATUS      RESTARTS      AGE
+zitadel-76776f747f-94wft           2/2     Running     2 (30s ago)   34s
+zitadel-76776f747f-fl24c           2/2     Running     1 (30s ago)   34s
+zitadel-76776f747f-xx2w7           2/2     Running     2 (30s ago)   34s
+zitadel-backup-49c7-h25cj          0/1     Completed   0             3m23s
+zitadel-init-td2sh                 0/1     Completed   0             34s
+zitadel-pgbouncer-d9f8cffc-2vwk5   2/2     Running     0             22m
+zitadel-pgbouncer-d9f8cffc-rrrhd   2/2     Running     0             22m
+zitadel-pgha1-4npq-0               4/4     Running     0             3m21s
+zitadel-pgha1-xl2b-0               4/4     Running     0             3m43s
+zitadel-repo-host-0                2/2     Running     0             22m
+zitadel-setup-b7zkx                0/1     Completed   1             34s
+zitadel-test-connection            0/1     Error       0             34s
+```
+
+## Cut over NLB
+Finally, cut over the NLB config to move to core1.
+This is specific to our metal clusters.
+Apply with: `git push puppet +HEAD:production -o run=netlb`
+```diff
+diff --git a/site-modules/profile/templates/haproxy/haproxy.cfg.epp b/site-modules/profile/templates/haproxy/haproxy.cfg.epp
+index b358a70..3ee40db 100644
+--- a/site-modules/profile/templates/haproxy/haproxy.cfg.epp
++++ b/site-modules/profile/templates/haproxy/haproxy.cfg.epp
+@@ -225,7 +225,7 @@ frontend sni-ingress from tcp-ingress
+   use_backend core2-https if { req.ssl_sni -i web.holos.run }
+   use_backend core2-https if { req.ssl_sni -i -m end .web.holos.run }
+   # Identity provider is routed to one cluster at a time
+-  use_backend core2-https if { req.ssl_sni -i login.ois.run }
++  use_backend core1-https if { req.ssl_sni -i login.ois.run }
+   # Holos Dev Environments
+   use_backend k1-https if { req.ssl_sni -i holos.wtf }
+   use_backend k1-https if { req.ssl_sni -i -m end .holos.wtf }
+@@ -358,7 +358,7 @@ frontend http-ingress from http-ingress
+   bind 65.102.23.41:80
+   bind 2602:41:6617:2ec0::80:1d:80 v6only
+   # Zitadel is active on one cluster at a time
+-  use_backend core2-http if { hdr(host) -i login.ois.run }
++  use_backend core1-http if { hdr(host) -i login.ois.run }
+   # Vault is active on core1 or core2
+   use_backend core2-http if { hdr(host) -i vault.core.ois.run }
+   # Infrastructure Dex OIDC ID issuer is active on core1 or core2
+
+```
+## Test Login
+```
+kubectl oidc-login get-token \
+  --oidc-extra-scope=openid \
+  --oidc-extra-scope=email \
+  --oidc-extra-scope=profile \
+  --oidc-extra-scope=groups \
+  --oidc-extra-scope=offline_access \
+  --oidc-extra-scope=urn:zitadel:iam:org:domain:primary:openinfrastructure.co \
+  --oidc-issuer-url=https://login.ois.run \
+  --oidc-client-id=${CLIENT_ID:-257714027772314751@holos_platform} \
+  --oidc-use-pkce \
+  --force-refresh > ~/.kube/token.json
+```
+
+```
+jq -r .status.token ~/.kube/token.json | cut -d. -f2 | base64 -d | jq
+```
+
+```json
+{
+  "amr": [
+    "mfa",
+    "user"
+  ],
+  "at_hash": "5nUq6kXWT8GheCNvDNdyow",
+  "aud": [
+    "257714027772314751@holos_platform",
+    "257713952794870157"
+  ],
+  "auth_time": 1710172186,
+  "azp": "257714027772314751@holos_platform",
+  "client_id": "257714027772314751@holos_platform",
+  "email": "jeff@openinfrastructure.co",
+  "email_verified": true,
+  "exp": 1710497614,
+  "family_name": "McCune",
+  "given_name": "Jeff",
+  "groups": [
+    "prod-cluster-admin"
+  ],
+  "iat": 1710454404,
+  "iss": "https://login.ois.run",
+  "locale": null,
+  "name": "Jeff McCune",
+  "nickname": "Jeff",
+  "preferred_username": "jeff@openinfrastructure.co",
+  "sub": "257712562366383231",
+  "updated_at": 1710105084,
+  "urn:zitadel:iam:org:domain:primary": "openinfrastructure.co",
+  "urn:zitadel:iam:org:project:257713952794870157:roles": {
+    "prod-cluster-admin": {
+      "257712562366317695": "openinfrastructure.co"
+    }
+  },
+  "urn:zitadel:iam:org:project:roles": {
+    "prod-cluster-admin": {
+      "257712562366317695": "openinfrastructure.co"
+    }
+  }
+}
+```
+## Tasks
+### Identify the leader
+Same command for primary and standby clusters.
+```
+k get pods -o name -l postgres-operator.crunchydata.com/role=master
+```
+### Full Backup
+On the primary cluster:
+```
+kubectl -n prod-iam-zitadel annotate postgrescluster zitadel postgres-operator.crunchydata.com/pgbackrest-backup="$(date)" --overwrite
+```
+Watch the progress:
+```
+k -n prod-iam-zitadel logs -l postgres-operator.crunchydata.com/pgbackrest-backup=manual -f
+```
+Expected output:
+```
+time="2024-03-14T18:36:16Z" level=info msg="crunchy-pgbackrest starts"
+time="2024-03-14T18:36:16Z" level=info msg="debug flag set to false"
+time="2024-03-14T18:36:16Z" level=info msg="backrest backup command requested"
+time="2024-03-14T18:36:16Z" level=info msg="command to execute is [pgbackrest backup --stanza=db --repo=2 --type=full]"
+time="2024-03-14T18:39:11Z" level=info msg="output=[]"
+time="2024-03-14T18:39:11Z" level=info msg="stderr=[]"
+time="2024-03-14T18:39:11Z" level=info msg="crunchy-pgbackrest ends"
+```
+
+## Troubleshooting
+Full backup doesn't start
+Check the controller logs.  If you see `Switchover failed` errors, try editing the number of postgrescluster replicas down to 1 then back up to 2.  Then try a backup again.
+```
+Switchover failed, details: 503, Switchover failed\n" version=5.5.1-0-amd64
+```

pkg/cli/secret/create.go

@@ -1,6 +1,7 @@
 package secret
 
 import (
+	"bytes"
 	"fmt"
 	"github.com/holos-run/holos/pkg/cli/command"
 	"github.com/holos-run/holos/pkg/holos"
@@ -29,9 +30,10 @@ func NewCreateCmd(hc *holos.Config) *cobra.Command {
 	cfg.dryRun = flagSet.Bool("dry-run", false, "dry run")
 	cfg.appendHash = flagSet.Bool("append-hash", true, "append hash to kubernetes secret name")
 	cfg.dataStdin = flagSet.Bool("data-stdin", false, "read data field as json from stdin if")
+	cfg.trimTrailingNewlines = flagSet.Bool("trim-trailing-newlines", true, "trim trailing newlines if true")
 
 	cmd.Flags().SortFlags = false
-	cmd.Flags().AddGoFlagSet(flagSet)
+	cmd.Flags().AddFlagSet(flagSet)
 	cmd.RunE = makeCreateRunFunc(hc, cfg)
 	return cmd
 
@@ -80,7 +82,7 @@ func makeCreateRunFunc(hc *holos.Config, cfg *config) command.RunFunc {
 		}
 
 		for _, file := range cfg.files {
-			if err := filepath.WalkDir(file, makeWalkFunc(secret.Data, file)); err != nil {
+			if err := filepath.WalkDir(file, makeWalkFunc(secret.Data, file, *cfg.trimTrailingNewlines)); err != nil {
 				return wrapper.Wrap(err)
 			}
 		}
@@ -125,7 +127,7 @@ func makeCreateRunFunc(hc *holos.Config, cfg *config) command.RunFunc {
 	}
 }
 
-func makeWalkFunc(data secretData, root string) fs.WalkDirFunc {
+func makeWalkFunc(data secretData, root string, trimNewlines bool) fs.WalkDirFunc {
 	return func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return err
@@ -143,6 +145,9 @@ func makeWalkFunc(data secretData, root string) fs.WalkDirFunc {
 			if data[key], err = os.ReadFile(path); err != nil {
 				return wrapper.Wrap(err)
 			}
+			if trimNewlines {
+				data[key] = bytes.TrimRight(data[key], "\r\n")
+			}
 		}
 
 		return nil

pkg/cli/secret/get.go

@@ -31,7 +31,7 @@ func NewGetCmd(hc *holos.Config) *cobra.Command {
 	cfg.extractTo = flagSet.String("extract-to", ".", "extract to directory")
 
 	cmd.Flags().SortFlags = false
-	cmd.Flags().AddGoFlagSet(flagSet)
+	cmd.Flags().AddFlagSet(flagSet)
 	cmd.RunE = makeGetRunFunc(hc, cfg)
 	return cmd
 }

pkg/cli/secret/secret.go

@@ -1,8 +1,8 @@
 package secret
 
 import (
-	"flag"
 	"github.com/holos-run/holos/pkg/holos"
+	"github.com/spf13/pflag"
 )
 
 const NameLabel = "holos.run/secret.name"
@@ -12,21 +12,22 @@ const ClusterLabel = "holos.run/cluster.name"
 type secretData map[string][]byte
 
 type config struct {
-	files      holos.StringSlice
-	printFile  *string
-	extract    *bool
-	dryRun     *bool
-	appendHash *bool
-	dataStdin  *bool
-	cluster    *string
-	namespace  *string
-	extractTo  *string
+	files                holos.StringSlice
+	printFile            *string
+	extract              *bool
+	dryRun               *bool
+	appendHash           *bool
+	dataStdin            *bool
+	trimTrailingNewlines *bool
+	cluster              *string
+	namespace            *string
+	extractTo            *string
 }
 
-func newConfig() (*config, *flag.FlagSet) {
+func newConfig() (*config, *pflag.FlagSet) {
 	cfg := &config{}
-	flagSet := flag.NewFlagSet("", flag.ContinueOnError)
-	cfg.namespace = flagSet.String("namespace", holos.DefaultProvisionerNamespace, "namespace in the provisioner cluster")
+	flagSet := pflag.NewFlagSet("", pflag.ContinueOnError)
+	cfg.namespace = flagSet.StringP("namespace", "n", holos.DefaultProvisionerNamespace, "namespace in the provisioner cluster")
 	cfg.cluster = flagSet.String("cluster-name", "", "cluster name selector")
 	return cfg, flagSet
 }

pkg/cli/secret/testdata/create_secret_dry_run.txt

@@ -1,5 +1,5 @@
 # Create the secret
-holos create secret directory --from-file=$WORK/fixture --dry-run
+holos create secret directory --trim-trailing-newlines=false --from-file=$WORK/fixture --dry-run
 
 # Want no warnings.
 ! stderr 'WRN'

pkg/cli/secret/testdata/create_secret_from_dir.txt

@@ -1,5 +1,5 @@
 # Create the secret
-holos create secret directory --from-file=$WORK/want
+holos create secret directory --trim-trailing-newlines=false --from-file=$WORK/want
 stderr 'created: directory-..........'
 stderr 'secret=directory-..........'
 stderr 'name=directory'

pkg/cli/secret/testdata/create_secret_from_dir_strip_newlines.txt

@@ -0,0 +1,17 @@
+# Create a secret from files with trailing newlines
+holos create secret smtp --from-file=$WORK/smtp
+
+# Get the secret back expecting no trailing newlines
+mkdir have
+holos get secret smtp
+stdout '"username": "holos.run@gmail.com"'
+stdout '"password": "secret"'
+
+-- smtp/username --
+holos.run@gmail.com
+-- smtp/password --
+secret
+-- smtp/host --
+smtp.gmail.com
+-- smtp/port --
+587

pkg/cli/secret/testdata/create_secret_no_depth.txt

@@ -1,5 +1,5 @@
 # Create the secret
-holos create secret directory --from-file=$WORK/want
+holos create secret directory --trim-trailing-newlines=false --from-file=$WORK/want
 
 # Get the secret back
 mkdir have

pkg/holos/types.go

@@ -13,6 +13,11 @@ func (i *StringSlice) String() string {
 	return fmt.Sprint(*i)
 }
 
+// Type implements the pflag.Value interface and describes the type.
+func (i *StringSlice) Type() string {
+	return "strings"
+}
+
 // Set implements the flag.Value interface.
 func (i *StringSlice) Set(value string) error {
 	for _, str := range strings.Split(value, ",") {

pkg/internal/builder/builder.go

@@ -33,6 +33,8 @@ const (
 	Skip = "Skip"
 	// ChartDir is the chart cache directory name.
 	ChartDir = "vendor"
+	// KustomizeBuild is the value of the kind field of cue output indicating holos should process the component using kustomize build to render output.
+	KustomizeBuild = "KustomizeBuild"
 )
 
 // An Option configures a Builder
@@ -106,6 +108,7 @@ type Repository struct {
 type Chart struct {
 	Name       string     `json:"name"`
 	Version    string     `json:"version"`
+	Release    string     `json:"release"`
 	Repository Repository `json:"repository"`
 }
 
@@ -118,6 +121,7 @@ type HelmChart struct {
 	Namespace     string   `json:"namespace"`
 	Chart         Chart    `json:"chart"`
 	ValuesContent string   `json:"valuesContent"`
+	EnableHooks   bool     `json:"enableHooks"`
 	// APIObjectMap holds the marshalled representation of api objects.
 	APIObjectMap apiObjectMap `json:"APIObjectMap"`
 }
@@ -340,6 +344,20 @@ func (b *Builder) Run(ctx context.Context) (results []*Result, err error) {
 			if err := result.kustomize(ctx); err != nil {
 				return nil, wrapper.Wrap(fmt.Errorf("could not kustomize: %w", err))
 			}
+		case KustomizeBuild:
+			// CUE directly provides the kubernetes api objects in result.Content
+			if err := value.Decode(&result); err != nil {
+				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
+			}
+			// Run kustomize.
+			kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", instance.Dir)
+			if err != nil {
+				log.ErrorContext(ctx, kOut.Stderr.String())
+				return nil, wrapper.Wrap(err)
+			}
+			// Replace the accumulated output
+			result.accumulatedOutput = kOut.Stdout.String()
+			result.addOverlayObjects(log)
 		default:
 			return nil, wrapper.Wrap(fmt.Errorf("build kind not implemented: %v", kind))
 		}
@@ -387,21 +405,26 @@ func runHelm(ctx context.Context, hc *HelmChart, r *Result, path holos.PathCompo
 		return nil
 	}
 
-	cachedChartPath := filepath.Join(string(path), ChartDir, hc.Chart.Name)
+	cachedChartPath := filepath.Join(string(path), ChartDir, filepath.Base(hc.Chart.Name))
 	if isNotExist(cachedChartPath) {
 		// Add repositories
 		repo := hc.Chart.Repository
-		out, err := util.RunCmd(ctx, "helm", "repo", "add", repo.Name, repo.URL)
-		if err != nil {
-			log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-			return wrapper.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
-		}
-		// Update repository
-		out, err = util.RunCmd(ctx, "helm", "repo", "update", repo.Name)
-		if err != nil {
-			log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-			return wrapper.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
+		if repo.URL != "" {
+			out, err := util.RunCmd(ctx, "helm", "repo", "add", repo.Name, repo.URL)
+			if err != nil {
+				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
+				return wrapper.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
+			}
+			// Update repository
+			out, err = util.RunCmd(ctx, "helm", "repo", "update", repo.Name)
+			if err != nil {
+				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
+				return wrapper.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
+			}
+		} else {
+			log.DebugContext(ctx, "no chart repository url proceeding assuming oci chart")
 		}
+
 		// Cache the chart
 		if err := cacheChart(ctx, path, ChartDir, hc.Chart); err != nil {
 			return fmt.Errorf("could not cache chart: %w", err)
@@ -423,7 +446,12 @@ func runHelm(ctx context.Context, hc *HelmChart, r *Result, path holos.PathCompo
 
 	// Run charts
 	chart := hc.Chart
-	helmOut, err := util.RunCmd(ctx, "helm", "template", "--values", valuesPath, "--namespace", hc.Namespace, "--kubeconfig", "/dev/null", "--version", chart.Version, chart.Name, cachedChartPath)
+	args := []string{"template"}
+	if !hc.EnableHooks {
+		args = append(args, "--no-hooks")
+	}
+	args = append(args, "--include-crds", "--values", valuesPath, "--namespace", hc.Namespace, "--kubeconfig", "/dev/null", "--version", chart.Version, chart.Release, cachedChartPath)
+	helmOut, err := util.RunCmd(ctx, "helm", args...)
 	if err != nil {
 		stderr := helmOut.Stderr.String()
 		lines := strings.Split(stderr, "\n")
@@ -465,7 +493,10 @@ func cacheChart(ctx context.Context, path holos.PathComponent, chartDir string,
 	}
 	defer remove(ctx, cacheTemp)
 
-	chartName := fmt.Sprintf("%s/%s", chart.Repository.Name, chart.Name)
+	chartName := chart.Name
+	if chart.Repository.Name != "" {
+		chartName = fmt.Sprintf("%s/%s", chart.Repository.Name, chart.Name)
+	}
 	helmOut, err := util.RunCmd(ctx, "helm", "pull", "--destination", cacheTemp, "--untar=true", "--version", chart.Version, chartName)
 	if err != nil {
 		return wrapper.Wrap(fmt.Errorf("could not run helm pull: %w", err))

pkg/version/embedded/minor

@@ -1 +1 @@
-53
+58