(#178) Add BuildPlan deployFiles field

Previously, each BuildPlan has no clear way to produce an ArgoCD
Application resource.  This patch provides a general solution where each
BuildPlan can provide arbitrary files as a map[string]string where the
key is the file path relative to the gitops repository `deploy/` folder.

.github/workflows/lint.yaml

@@ -30,14 +30,15 @@ jobs:
         with:
           go-version: stable
 
-      - name: Install tools
-        run: sudo apt update && sudo apt -qq -y install curl zip unzip tar bzip2 make
+      - name: Install Packages
+        run: sudo apt update && sudo apt -qq -y install git curl zip unzip tar bzip2 make
 
-      - name: Install Deps
+      - name: Install Tools
         run: |
-          make go-deps
+          set -x
+          make tools
+          make buf
           go generate ./...
-          make frontend-deps
           make frontend
           go mod tidy
 
@@ -45,3 +46,4 @@ jobs:
         uses: golangci/golangci-lint-action@v4
         with:
           version: latest
+          skip-pkg-cache: true

.github/workflows/release.yaml

@@ -36,11 +36,12 @@ jobs:
 
       # Necessary to run these outside of goreleaser, otherwise
       # /home/runner/_work/holos/holos/internal/frontend/node_modules/.bin/protoc-gen-connect-query is not in PATH
-      - name: Install Deps
+      - name: Install Tools
         run: |
-          make go-deps
+          set -x
+          make tools
+          make buf
           go generate ./...
-          make frontend-deps
           make frontend
           go mod tidy
 
@@ -53,6 +54,9 @@ jobs:
       - name: List keys
         run: gpg -K
 
+      - name: Git diff
+        run: git diff
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v5
         with:

.github/workflows/test.yaml

@@ -28,8 +28,8 @@ jobs:
         with:
           go-version: stable
 
-      - name: Install tools
-        run: sudo apt update && sudo apt -qq -y install curl zip unzip tar bzip2 make
+      - name: Install Packages
+        run: sudo apt update && sudo apt -qq -y install git curl zip unzip tar bzip2 make
 
       - name: Set up Helm
         uses: azure/setup-helm@v4
@@ -37,11 +37,12 @@ jobs:
       - name: Set up Kubectl
         uses: azure/setup-kubectl@v3
 
-      - name: Install Deps
+      - name: Install Tools
         run: |
-          make go-deps
+          set -x
+          make tools
+          make buf
           go generate ./...
-          make frontend-deps
           make frontend
           go mod tidy
 

.gitignore

@@ -1,4 +1,4 @@
-bin/
+/bin/
 vendor/
 .idea/
 coverage.out
@@ -6,3 +6,4 @@ coverage.out
 *.hold/
 /deploy/
 .vscode/
+tmp/

Makefile

@@ -4,7 +4,7 @@ PROJ=holos
 ORG_PATH=github.com/holos-run
 REPO_PATH=$(ORG_PATH)/$(PROJ)
 
-VERSION := $(shell cat pkg/version/embedded/major pkg/version/embedded/minor pkg/version/embedded/patch | xargs printf "%s.%s.%s")
+VERSION := $(shell cat version/embedded/major version/embedded/minor version/embedded/patch | xargs printf "%s.%s.%s")
 BIN_NAME := holos
 
 DOCKER_REPO=quay.io/openinfrastructure/holos
@@ -16,10 +16,12 @@ $( shell mkdir -p bin)
 export PATH := $(PWD)/internal/frontend/holos/node_modules/.bin:$(PATH)
 
 GIT_COMMIT=$(shell git rev-parse HEAD)
+GIT_SUFFIX=$(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
+GIT_DETAIL=$(shell git describe --tags HEAD)
 GIT_TREE_STATE=$(shell test -n "`git status --porcelain`" && echo "dirty" || echo "clean")
 BUILD_DATE=$(shell date -Iseconds)
 
-LD_FLAGS="-w -X ${ORG_PATH}/${PROJ}/pkg/version.GitCommit=${GIT_COMMIT} -X ${ORG_PATH}/${PROJ}/pkg/version.GitTreeState=${GIT_TREE_STATE} -X ${ORG_PATH}/${PROJ}/pkg/version.BuildDate=${BUILD_DATE}"
+LD_FLAGS="-w -X ${ORG_PATH}/${PROJ}/version.GitDescribe=${GIT_DETAIL}${GIT_SUFFIX} -X ${ORG_PATH}/${PROJ}/version.GitCommit=${GIT_COMMIT} -X ${ORG_PATH}/${PROJ}/version.GitTreeState=${GIT_TREE_STATE} -X ${ORG_PATH}/${PROJ}/version.BuildDate=${BUILD_DATE}"
 
 .PHONY: default
 default: test
@@ -61,14 +63,26 @@ vet: ## Vet Go code.
 
 .PHONY: gencue
 gencue: ## Generate CUE definitions
-	cd docs/examples && cue get go github.com/holos-run/holos/api/...
+	cd internal/generate/platforms && cue get go github.com/holos-run/holos/api/v1alpha1/...
+
+.PHONY: rmgen
+rmgen: ## Remove generated code
+	git rm -rf service/gen/ internal/frontend/holos/src/app/gen/ || true
+	rm -rf service/gen/ internal/frontend/holos/src/app/gen/
+	git rm -rf internal/ent/
+	rm -rf internal/ent/
+	git restore --staged internal/ent/generate.go internal/ent/schema/
+	git restore internal/ent/generate.go internal/ent/schema/
+
+.PHONY: regenerate
+regenerate: generate ## Re-generate code (delete and re-create)
 
 .PHONY: generate
-generate: ## Generate code.
+generate: buf gencue ## Generate code.
 	go generate ./...
 
 .PHONY: build
-build: generate ## Build holos executable.
+build: generate frontend ## Build holos executable.
 	@echo "building ${BIN_NAME} ${VERSION}"
 	@echo "GOPATH=${GOPATH}"
 	go build -trimpath -o bin/$(BIN_NAME) -ldflags $(LD_FLAGS) $(REPO_PATH)/cmd/$(BIN_NAME)
@@ -87,6 +101,8 @@ test: ## Run tests.
 
 .PHONY: lint
 lint: ## Run linters.
+	buf lint
+	cd internal/frontend/holos && ng lint
 	golangci-lint run
 
 .PHONY: coverage
@@ -99,15 +115,18 @@ snapshot:  ## Go release snapshot
 
 .PHONY: buf
 buf: ## buf generate
-	cd service && buf mod update
+	cd service && buf dep update
 	buf generate
 
+.PHONY: tools
+tools: go-deps frontend-deps  ## install tool dependencies
+
 .PHONY: go-deps
-go-deps: ## install go executables
-	go install github.com/bufbuild/buf/cmd/buf@v1
-	go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1
-	go install google.golang.org/protobuf/cmd/protoc-gen-go@v1
-	go install connectrpc.com/connect/cmd/protoc-gen-connect-go@v1
+go-deps: ## tool versions pinned in tools.go
+	go install github.com/bufbuild/buf/cmd/buf
+	go install github.com/fullstorydev/grpcurl/cmd/grpcurl
+	go install google.golang.org/protobuf/cmd/protoc-gen-go
+	go install connectrpc.com/connect/cmd/protoc-gen-connect-go
 	go install honnef.co/go/tools/cmd/staticcheck@latest
 	# curl https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash
 
@@ -123,8 +142,8 @@ frontend-deps: ## Setup npm and vite
 
 .PHONY: frontend
 frontend: buf
+	cd internal/frontend/holos && rm -rf dist
 	mkdir -p internal/frontend/holos/dist
-	cd internal/frontend/holos/dist && rm -rf app
 	cd internal/frontend/holos && ng build
 	touch internal/frontend/frontend.go
 

Tiltfile

@@ -99,6 +99,7 @@ docker_build_with_restart(
         '--listen-port={}'.format(listen_port),
         '--oidc-issuer=https://login.ois.run',
         '--oidc-audience=262096764402729854@holos_platform',
+        '--log-level=debug',
         '--metrics-port={}'.format(metrics_port),
     ],
     dockerfile='./hack/tilt/Dockerfile',
@@ -190,7 +191,7 @@ k8s_resource(
     ],
     resource_deps=[compile_id],
     links=[
-        link('https://{}.holos.dev.k2.ois.run/app/'.format(developer), "Holos Web UI")
+        link('https://{}.app.dev.k2.holos.run/ui/'.format(developer), "Holos Web UI")
     ],
 )
 
@@ -200,11 +201,6 @@ k8s_resource(
     new_name=auth_id,
     objects=[
       '{}:virtualservice'.format(holos_server),
-      '{}-allow-groups:authorizationpolicy'.format(holos_server),
-      '{}-allow-nothing:authorizationpolicy'.format(holos_server),
-      '{}-allow-well-known-paths:authorizationpolicy'.format(holos_server),
-      '{}-auth:authorizationpolicy'.format(holos_server),
-      '{}:requestauthentication'.format(holos_server),
     ],
 )
 

api/v1alpha1/buildplan.go

@@ -16,6 +16,10 @@ type BuildPlan struct {
 type BuildPlanSpec struct {
 	Disabled   bool                `json:"disabled,omitempty" yaml:"disabled,omitempty"`
 	Components BuildPlanComponents `json:"components,omitempty" yaml:"components,omitempty"`
+	// DeployFiles keys represent file paths relative to the cluster deploy
+	// directory.  Map values represent the string encoded file contents.  Used to
+	// write the argocd Application, but may be used to render any file from CUE.
+	DeployFiles FileContentMap `json:"deployFiles,omitempty" yaml:"deployFiles,omitempty"`
 }
 
 type BuildPlanComponents struct {
@@ -38,3 +42,14 @@ func (bp *BuildPlan) Validate() error {
 	}
 	return nil
 }
+
+func (bp *BuildPlan) ResultCapacity() (count int) {
+	if bp == nil {
+		return 0
+	}
+	count = len(bp.Spec.Components.HelmChartList) +
+		len(bp.Spec.Components.KubernetesObjectsList) +
+		len(bp.Spec.Components.KustomizeBuildList) +
+		len(bp.Spec.Components.Resources)
+	return count
+}

api/v1alpha1/component.go

@@ -20,3 +20,11 @@ type HolosComponent struct {
 func (hc *HolosComponent) NewResult() *Result {
 	return &Result{HolosComponent: *hc}
 }
+
+func (hc *HolosComponent) GetAPIVersion() string {
+	return hc.APIVersion
+}
+
+func (hc *HolosComponent) GetKind() string {
+	return hc.Kind
+}

api/v1alpha1/form.go

@@ -0,0 +1,13 @@
+package v1alpha1
+
+import object "github.com/holos-run/holos/service/gen/holos/object/v1alpha1"
+
+// Form represents a collection of Formly json powered form.
+type Form struct {
+	TypeMeta `json:",inline" yaml:",inline"`
+	Spec     FormSpec `json:"spec" yaml:"spec"`
+}
+
+type FormSpec struct {
+	Form object.Form `json:"form" yaml:"form"`
+}

api/v1alpha1/helm.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 
 	"github.com/holos-run/holos"
-	"github.com/holos-run/holos/pkg/errors"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 // A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
@@ -141,9 +141,25 @@ func cacheChart(ctx context.Context, path holos.InstancePath, chartDir string, c
 	log.Debug("helm pull", "stdout", helmOut.Stdout, "stderr", helmOut.Stderr)
 
 	cachePath := filepath.Join(string(path), chartDir)
-	if err := os.Rename(cacheTemp, cachePath); err != nil {
-		return errors.Wrap(fmt.Errorf("could not rename: %w", err))
+
+	if err := os.MkdirAll(cachePath, 0777); err != nil {
+		return errors.Wrap(fmt.Errorf("could not mkdir: %w", err))
 	}
+
+	items, err := os.ReadDir(cacheTemp)
+	if err != nil {
+		return errors.Wrap(fmt.Errorf("could not read directory: %w", err))
+	}
+
+	for _, item := range items {
+		src := filepath.Join(cacheTemp, item.Name())
+		dst := filepath.Join(cachePath, item.Name())
+		log.DebugContext(ctx, "rename", "src", src, "dst", dst)
+		if err := os.Rename(src, dst); err != nil {
+			return errors.Wrap(fmt.Errorf("could not rename: %w", err))
+		}
+	}
+
 	log.InfoContext(ctx, "cached", "chart", chart.Name, "version", chart.Version, "path", cachePath)
 
 	return nil

api/v1alpha1/kustomize.go

@@ -4,9 +4,9 @@ import (
 	"context"
 
 	"github.com/holos-run/holos"
-	"github.com/holos-run/holos/pkg/errors"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 const KustomizeBuildKind = "KustomizeBuild"

api/v1alpha1/platform.go

@@ -0,0 +1,32 @@
+package v1alpha1
+
+import "google.golang.org/protobuf/types/known/structpb"
+
+// Platform represents a platform to manage.  A Platform resource informs holos
+// which components to build.  The platform resource also acts as a container
+// for the platform model form values provided by the PlatformService.  The
+// primary use case is to collect the cluster names, cluster types, platform
+// model, and holos components to build into one resource.
+type Platform struct {
+	TypeMeta `json:",inline" yaml:",inline"`
+	Metadata ObjectMeta   `json:"metadata" yaml:"metadata"`
+	Spec     PlatformSpec `json:"spec" yaml:"spec"`
+}
+
+// PlatformSpec represents the platform build plan specification.
+type PlatformSpec struct {
+	// Model represents the platform model holos gets from from the
+	// holos.platform.v1alpha1.PlatformService.GetPlatform method and provides to
+	// CUE using a tag.
+	Model      structpb.Struct         `json:"model" yaml:"model"`
+	Components []PlatformSpecComponent `json:"components" yaml:"components"`
+}
+
+// PlatformSpecComponent represents a component to build or render with flags to
+// pass, for example the cluster name.
+type PlatformSpecComponent struct {
+	// Path is the path of the component relative to the platform root.
+	Path string `json:"path" yaml:"path"`
+	// Cluster is the cluster name to use when building the component.
+	Cluster string `json:"cluster" yaml:"cluster"`
+}

api/v1alpha1/result.go

@@ -7,9 +7,9 @@ import (
 	"path/filepath"
 	"slices"
 
-	"github.com/holos-run/holos/pkg/errors"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+	"github.com/holos-run/holos/internal/util"
 )
 
 // Result is the build result for display or writing.  Holos components Render the Result as a data pipeline.
@@ -17,6 +17,18 @@ type Result struct {
 	HolosComponent
 	// accumulatedOutput accumulates rendered api objects.
 	accumulatedOutput string
+	// DeployFiles keys represent file paths relative to the cluster deploy
+	// directory.  Map values represent the string encoded file contents.  Used to
+	// write the argocd Application, but may be used to render any file from CUE.
+	DeployFiles FileContentMap `json:"deployFiles,omitempty" yaml:"deployFiles,omitempty"`
+}
+
+// Continue returns true if Skip is true indicating the result is to be skipped over.
+func (r *Result) Continue() bool {
+	if r == nil {
+		return false
+	}
+	return r.Skip
 }
 
 func (r *Result) Name() string {
@@ -32,6 +44,11 @@ func (r *Result) KustomizationFilename(writeTo string, cluster string) string {
 	return filepath.Join(writeTo, "clusters", cluster, "holos", "components", r.Metadata.Name+"-kustomization.gen.yaml")
 }
 
+// KustomizationContent returns the kustomization file contents to write.
+func (r *Result) KustomizationContent() string {
+	return r.KsContent
+}
+
 // AccumulatedOutput returns the accumulated rendered output.
 func (r *Result) AccumulatedOutput() string {
 	return r.accumulatedOutput
@@ -120,6 +137,21 @@ func (r *Result) kustomize(ctx context.Context) error {
 	return nil
 }
 
+func (r *Result) WriteDeployFiles(ctx context.Context, path string) error {
+	log := logger.FromContext(ctx)
+	if len(r.DeployFiles) == 0 {
+		return nil
+	}
+	for k, content := range r.DeployFiles {
+		path := filepath.Join(path, k)
+		if err := r.Save(ctx, path, content); err != nil {
+			return errors.Wrap(err)
+		}
+		log.InfoContext(ctx, "wrote deploy file", "path", path, "bytes", len(content))
+	}
+	return nil
+}
+
 // Save writes the content to the filesystem for git ops.
 func (r *Result) Save(ctx context.Context, path string, content string) error {
 	log := logger.FromContext(ctx)
@@ -128,7 +160,7 @@ func (r *Result) Save(ctx context.Context, path string, content string) error {
 		log.WarnContext(ctx, "could not mkdir", "path", dir, "err", err)
 		return errors.Wrap(err)
 	}
-	// Write the kube api objects
+	// Write the file content
 	if err := os.WriteFile(path, []byte(content), os.FileMode(0644)); err != nil {
 		log.WarnContext(ctx, "could not write", "path", path, "err", err)
 		return errors.Wrap(err)

api/v1alpha1/typemeta.go

@@ -8,3 +8,13 @@ type TypeMeta struct {
 func (tm *TypeMeta) GetKind() string {
 	return tm.Kind
 }
+
+func (tm *TypeMeta) GetAPIVersion() string {
+	return tm.Kind
+}
+
+// Discriminator is an interface to discriminate the kind api object.
+type Discriminator interface {
+	GetKind() string
+	GetAPIVersion() string
+}

buf.gen.yaml

@@ -11,14 +11,10 @@ plugins:
     out: service/gen
     opt: paths=source_relative
   - plugin: es
-    out: internal/frontend/holos/gen
+    out: internal/frontend/holos/src/app/gen
     opt:
       - target=ts
   - plugin: connect-es
-    out: internal/frontend/holos/gen
-    opt:
-      - target=ts
-  - plugin: connect-query
-    out: internal/frontend/holos/gen
+    out: internal/frontend/holos/src/app/gen
     opt:
       - target=ts

cmd/holos/main.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"github.com/holos-run/holos/pkg/cli"
 	"os"
+
+	"github.com/holos-run/holos/internal/cli"
 )
 
 func main() {

cmd/holos/main_test.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"github.com/holos-run/holos/pkg/cli"
-	"github.com/rogpeppe/go-internal/testscript"
 	"os"
 	"testing"
+
+	"github.com/holos-run/holos/internal/cli"
+	"github.com/rogpeppe/go-internal/testscript"
 )
 
 func TestMain(m *testing.M) {

cmd/holos/testdata/constraints.txt

@@ -2,6 +2,8 @@
 exec holos build ./foo/... --log-level debug
 stdout '^bf2bc7f9-9ba0-4f9e-9bd2-9a205627eb0b$'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- foo/constraints.cue --
@@ -20,6 +22,7 @@ spec: components: KubernetesObjectsList: [
 package holos
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 #KubernetesObjects: {
 	apiVersion: "holos.run/v1alpha1"

cmd/holos/testdata/issue15_cue_errors.txt

@@ -3,12 +3,15 @@
 stderr 'apiObjectMap.foo.bar: cannot convert incomplete value'
 stderr '/component.cue:\d+:\d+$'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
 package holos
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 apiVersion: "holos.run/v1alpha1"
 kind: "BuildPlan"

cmd/holos/testdata/issue25_apiobjects_cue.txt

@@ -3,6 +3,8 @@ exec holos build .
 stdout '^kind: SecretStore$'
 stdout '# Source: CUE apiObjects.SecretStore.default'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
@@ -13,6 +15,7 @@ kind: "BuildPlan"
 spec: components: KubernetesObjectsList: [{apiObjectMap: #APIObjects.apiObjectMap}]
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 #SecretStore: {
     kind: string

cmd/holos/testdata/issue25_apiobjects_helm.txt

@@ -4,6 +4,8 @@ stdout '^kind: SecretStore$'
 stdout '# Source: CUE apiObjects.SecretStore.default'
 stderr 'skipping helm: no chart name specified'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
@@ -14,6 +16,7 @@ kind: "BuildPlan"
 spec: components: HelmChartList: [{apiObjectMap: #APIObjects.apiObjectMap}]
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 #SecretStore: {
     kind: string

cmd/holos/testdata/issue25_show_object_names.txt

@@ -2,6 +2,8 @@
 ! exec holos build .
 stderr 'apiObjects.secretstore.default.foo: field not allowed'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
@@ -10,6 +12,7 @@ package holos
 apiVersion: "holos.run/v1alpha1"
 kind: "KubernetesObjects"
 cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 #SecretStore: {
     metadata: name: string

cmd/holos/testdata/issue33_helm_stderr.txt

@@ -2,6 +2,8 @@
 ! exec holos build .
 stderr 'Error: execution error at \(zitadel/templates/secret_zitadel-masterkey.yaml:2:4\): Either set .Values.zitadel.masterkey xor .Values.zitadel.masterkeySecretName'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- zitadel.cue --
@@ -12,6 +14,7 @@ kind: "BuildPlan"
 spec: components: HelmChartList: [_HelmChart]
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 _HelmChart: {
   apiVersion: "holos.run/v1alpha1"

cmd/holos/testdata/issue42_kustomize_build_kind.txt

@@ -1,15 +1,18 @@
 # Kustomize is a supported holos component kind
-exec holos render --cluster-name=mycluster . --log-level=debug
+exec holos render component --cluster-name=mycluster . --log-level=debug
 
 # Want generated output
 cmp want.yaml deploy/clusters/mycluster/components/kstest/kstest.gen.yaml
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
 package holos
 
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 apiVersion: "holos.run/v1alpha1"
 kind: "BuildPlan"

cmd/holos/testdata/issue72_disallow_unknown_fields.txt

@@ -3,11 +3,14 @@
 ! exec holos build .
 stderr 'unknown field \\"TypoKubernetesObjectsList\\"'
 
+-- platform.config.json --
+{}
 -- cue.mod --
 package holos
 -- component.cue --
 package holos
 _cluster: string @tag(cluster, string)
+_platform_config: string @tag(platform_config, string)
 
 apiVersion: "holos.run/v1alpha1"
 kind: "BuildPlan"

cmd/holos/testdata/version.txt

@@ -1,5 +1,3 @@
 exec holos --version
 # want version with no v on stdout
 stdout -count=1 '^\d+\.\d+\.\d+$'
-# want nothing on stderr
-! stderr .

docs/examples/api/ListPlatform/listplatform.json

@@ -0,0 +1,10 @@
+{
+  "org_id": "018f36fb-e3f7-7f7f-a1c5-c85fb735d215",
+  "field_mask": {
+    "paths": [
+      "id",
+      "name",
+      "displayName"
+    ]
+  }
+}

docs/examples/api/UpdatePlatform/clearform.json

@@ -0,0 +1,8 @@
+{
+  "update_mask": {
+    "paths": ["form"]
+  },
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94"
+  }
+}

docs/examples/api/UpdatePlatform/model.json

@@ -0,0 +1,11 @@
+{
+  "update_mask": {
+    "paths": ["model","name","display_name"]
+  },
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94",
+    "name": "bareplatform",
+    "display_name": "Bare Platform",
+    "model": {}
+  }
+}

docs/examples/api/UpdatePlatform/nomask.json

@@ -0,0 +1,6 @@
+{
+  "update": {
+    "platform_id": "018f36fb-e3ff-7f7f-a5d1-7ca2bf499e94",
+    "model": {}
+  }
+}

docs/examples/authpolicy.cue

@@ -2,7 +2,10 @@ package holos
 
 import ap "security.istio.io/authorizationpolicy/v1"
 
-// #AuthPolicyRules represents AuthorizationPolicy rules for hosts that need specialized treatment.  Entries in this struct are exclused from the blank ingressauth AuthorizationPolicy governing the ingressgateway and included in a spcialized policy
+// #AuthPolicyRules represents AuthorizationPolicy rules for hosts that need
+// specialized treatment.  Entries in this struct are excluded from
+// AuthorizationPolicy/authproxy-custom in the istio-ingress namespace.  Entries
+// are added to their own AuthorizationPolicy.
 #AuthPolicyRules: {
 	// AuthProxySpec represents the identity provider configuration
 	AuthProxySpec: #AuthProxySpec & #Platform.authproxy
@@ -14,6 +17,9 @@ import ap "security.istio.io/authorizationpolicy/v1"
 			name: Name
 			// slug is the resource name prefix
 			slug: string
+			// NoAuthorizationPolicy disables an AuthorizationPolicy for the host
+			NoAuthorizationPolicy: true | *false
+
 			// Refer to https://istio.io/latest/docs/reference/config/security/authorization-policy/#Rule
 			spec: ap.#AuthorizationPolicySpec & {
 				action: "CUSTOM"
@@ -25,11 +31,13 @@ import ap "security.istio.io/authorizationpolicy/v1"
 
 	objects: #APIObjects & {
 		for Host in hosts {
-			apiObjects: {
-				AuthorizationPolicy: "\(Host.slug)-custom": {
-					metadata: namespace: "istio-ingress"
-					metadata: name:      "\(Host.slug)-custom"
-					spec: Host.spec
+			if Host.NoAuthorizationPolicy == false {
+				apiObjects: {
+					AuthorizationPolicy: "\(Host.slug)-custom": {
+						metadata: namespace: "istio-ingress"
+						metadata: name:      "\(Host.slug)-custom"
+						spec: Host.spec
+					}
 				}
 			}
 		}

docs/examples/cue.mod/usr/k8s.io/api/apps/v1/types.cue

@@ -4,3 +4,8 @@ package v1
 	apiVersion: "apps/v1"
 	kind:       "Deployment"
 }
+
+#StatefulSet: {
+	apiVersion: "apps/v1"
+	kind:       "StatefulSet"
+}

docs/examples/helpers.cue

@@ -18,8 +18,10 @@ import "encoding/yaml"
 		Issuer?: [Name=_]: #Issuer & {metadata: name: Name}
 		Gateway?: [Name=_]: #Gateway & {metadata: name: Name}
 		ConfigMap?: [Name=_]: #ConfigMap & {metadata: name: Name}
+		ServiceAccount?: [Name=_]: #ServiceAccount & {metadata: name: Name}
 
 		Deployment?: [_]:            #Deployment
+		StatefulSet?: [_]:           #StatefulSet
 		RequestAuthentication?: [_]: #RequestAuthentication
 		AuthorizationPolicy?: [_]:   #AuthorizationPolicy
 	}

docs/examples/managednamespaces.cue

@@ -0,0 +1,26 @@
+package holos
+
+// NOTE: Beyond the base reference platform, services should typically be added to #OptionalServices instead of directly to a managed namespace.
+
+// ManagedNamespace is a namespace to manage across all clusters in the holos platform.
+#ManagedNamespace: {
+	namespace: {
+		metadata: {
+			name: string
+			labels: [string]: string
+		}
+	}
+	// clusterNames represents the set of clusters the namespace is managed on.  Usually all clusters.
+	clusterNames: [...string]
+	for cluster in clusterNames {
+		clusters: (cluster): name: cluster
+	}
+}
+
+// #ManagedNamepsaces is the union of all namespaces across all cluster types and optional services.
+// Holos adopts the namespace sameness position of SIG Multicluster, refer to https://github.com/kubernetes/community/blob/dd4c8b704ef1c9c3bfd928c6fa9234276d61ad18/sig-multicluster/namespace-sameness-position-statement.md
+#ManagedNamespaces: {
+	[Name=_]: #ManagedNamespace & {
+		namespace: metadata: name: Name
+	}
+}

docs/examples/optionalservices.cue

@@ -1,6 +1,8 @@
 // Controls optional feature flags for services distributed across multiple holos components.
 // For example, enable issuing certificates in the provisioner cluster when an optional service is
-// enabled for a workload cluster.
+// enabled for a workload cluster.  Another example is NATS, which isn't necessary on all clusters,
+// but is necessary on clusters with a project like holos which depends on NATS.
+
 package holos
 
 import "list"

docs/examples/platform.config.cue

@@ -0,0 +1,3 @@
+package holos
+
+_platform_config: string @tag(platform_config, type=string)

docs/examples/platform.config.json

@@ -0,0 +1 @@
+{}

docs/examples/platforms/holos-saas/clusters/provisioner/choria/broker/broker.cue

@@ -0,0 +1,45 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Broker = "choria-broker"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-platform-issuer": _
+
+		metadata: name: "\(Namespace)-\(Broker)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Broker
+	"app.kubernetes.io/name":     Broker
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		Certificate: "\(Broker)-tls": #Certificate & {
+			metadata: {
+				name:      "\(Broker)-tls"
+				namespace: Namespace
+				labels:    SelectorLabels
+			}
+			spec: {
+				commonName: "\(Broker).\(Namespace).svc.cluster.local"
+				dnsNames: [
+					Broker,
+					"\(Broker).\(Namespace).svc",
+					"\(Broker).\(Namespace).svc.cluster.local",
+					"*.\(Broker)",
+					"*.\(Broker).\(Namespace).svc",
+					"*.\(Broker).\(Namespace).svc.cluster.local",
+				]
+				issuerRef: kind: "ClusterIssuer"
+				issuerRef: name: "platform-issuer"
+				secretName: metadata.name
+				usages: ["signing", "key encipherment", "server auth", "client auth"]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/provisioner/choria/provisioner/provsioner.cue

@@ -0,0 +1,45 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Provisioner = "choria-provisioner"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-platform-issuer": _
+
+		metadata: name: "\(Namespace)-\(Provisioner)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Provisioner
+	"app.kubernetes.io/name":     Provisioner
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		Certificate: "\(Provisioner)-tls": #Certificate & {
+			metadata: {
+				name:      "\(Provisioner)-tls"
+				namespace: Namespace
+				labels:    SelectorLabels
+			}
+			spec: {
+				commonName: "\(Provisioner).\(Namespace).svc.cluster.local"
+				dnsNames: [
+					Provisioner,
+					"\(Provisioner).\(Namespace).svc",
+					"\(Provisioner).\(Namespace).svc.cluster.local",
+					"*.\(Provisioner)",
+					"*.\(Provisioner).\(Namespace).svc",
+					"*.\(Provisioner).\(Namespace).svc.cluster.local",
+				]
+				issuerRef: kind: "ClusterIssuer"
+				issuerRef: name: "platform-issuer"
+				secretName: metadata.name
+				usages: ["signing", "key encipherment", "server auth", "client auth"]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/broker/broker.cue

@@ -0,0 +1,177 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Broker = "choria-broker"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "\(Namespace)-\(Broker)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/part-of": "choria"
+	"app.kubernetes.io/name":    Broker
+}
+
+let Metadata = {
+	name:      Broker
+	namespace: Namespace
+	labels:    SelectorLabels
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		ExternalSecret: "\(Broker)-tls": #ExternalSecret & {
+			metadata: name:      "\(Broker)-tls"
+			metadata: namespace: Namespace
+		}
+		ExternalSecret: "\(Broker)": #ExternalSecret & {
+			metadata: name:      Broker
+			metadata: namespace: Namespace
+		}
+		StatefulSet: "\(Broker)": {
+			metadata: Metadata
+			spec: {
+				selector: matchLabels: SelectorLabels
+				serviceName: Broker
+				template: metadata: labels: SelectorLabels
+				template: spec: {
+					containers: [
+						{
+							name: Broker
+							command: ["choria", "broker", "run", "--config", "/etc/choria/broker.conf"]
+							image:           "registry.choria.io/choria/choria:0.28.0"
+							imagePullPolicy: "IfNotPresent"
+							ports: [
+								{
+									containerPort: 4222
+									name:          "tcp-nats"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 4333
+									name:          "https-wss"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 5222
+									name:          "tcp-cluster"
+									protocol:      "TCP"
+								},
+								{
+									containerPort: 8222
+									name:          "http-stats"
+									protocol:      "TCP"
+								},
+							]
+							livenessProbe: httpGet: {
+								path: "/healthz"
+								port: "http-stats"
+							}
+							readinessProbe: livenessProbe
+							resources: {}
+							securityContext: {}
+							volumeMounts: [
+								{
+									mountPath: "/etc/choria"
+									name:      Broker
+								},
+								{
+									mountPath: "/etc/choria-tls"
+									name:      "\(Broker)-tls"
+								},
+							]
+						},
+					]
+					securityContext: {}
+					serviceAccountName: Broker
+					volumes: [
+						{
+							name: Broker
+							secret: secretName: Broker
+						},
+						{
+							name: "\(Broker)-tls"
+							secret: secretName: "\(Broker)-tls"
+						},
+					]
+				}
+			}
+		}
+		ServiceAccount: "\(Broker)": #ServiceAccount & {
+			metadata: Metadata
+		}
+		Service: "\(Broker)": #Service & {
+			metadata: Metadata
+			spec: {
+				type:      "ClusterIP"
+				clusterIP: "None"
+				selector:  SelectorLabels
+				ports: [
+					{
+						name:        "tcp-nats"
+						appProtocol: "tcp"
+						port:        4222
+						protocol:    "TCP"
+						targetPort:  "tcp-nats"
+					},
+					{
+						name:        "tcp-cluster"
+						appProtocol: "tcp"
+						port:        5222
+						protocol:    "TCP"
+						targetPort:  "tcp-cluster"
+					},
+					{
+						name:        "https-wss"
+						appProtocol: "https"
+						port:        443
+						protocol:    "TCP"
+						targetPort:  "https-wss"
+					},
+				]
+			}
+		}
+		DestinationRule: "\(Broker)-wss": #DestinationRule & {
+			_decriptions: "Configures Istio to connect to Choria using a cert issued by the Platform Issuer"
+			metadata:     Metadata
+			spec: host: "\(Broker).\(Namespace).svc.cluster.local"
+			spec: trafficPolicy: tls: {
+				credentialName: "istio-ingress-mtls-cert"
+				mode:           "MUTUAL"
+				// subjectAltNames is important, otherwise istio will fail to verify the
+				// choria broker upstream server.  make sure this matches a value
+				// present in the choria broker's cert.
+				//
+				//  kubectl get secret choria-broker-tls -o json | jq --exit-status
+				//  '.data | map_values(@base64d)' | jq .\"tls.crt\" -r | openssl x509
+				//  -text -noout -in -
+				subjectAltNames: [spec.host]
+			}
+		}
+		VirtualService: "\(Broker)-wss": #VirtualService & {
+			metadata: name:      "\(Broker)-wss"
+			metadata: namespace: Namespace
+			spec: {
+				gateways: ["istio-ingress/default"]
+				hosts: ["jeff.provision.dev.\(#ClusterName).holos.run"]
+				http: [
+					{
+						route: [
+							{
+								destination: {
+									host: "\(Broker).\(Namespace).svc.cluster.local"
+									port: "number": 443
+								}
+							},
+						]
+					},
+				]
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/provisioner/Dockerfile

@@ -0,0 +1,18 @@
+FROM registry.choria.io/choria/provisioner:latest
+
+RUN curl -Lo nsc.zip https://github.com/nats-io/nsc/releases/download/v2.8.6/nsc-linux-amd64.zip &&\
+  unzip nsc.zip && \
+  mv nsc /usr/local/bin/nsc && \
+  chmod 755 /usr/local/bin/nsc && \
+  rm -f nsc.zip
+
+# TODO: Add jwt executable
+# TODO: Add helper executable
+
+USER choria
+ENV USER=choria
+
+ENTRYPOINT ["/usr/sbin/choria-provisioner"]
+
+# These two files are expected to be in the provisioner secret.
+CMD ["--config=/etc/provisioner/provisioner.yaml", "--choria-config=/etc/provisioner/choria.cfg"]

docs/examples/platforms/holos-saas/clusters/workload/choria/provisioner/provisioner.cue

@@ -0,0 +1,82 @@
+package holos
+
+let Namespace = "jeff-holos"
+let Provisioner = "choria-provisioner"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-secrets-stores": _
+
+		metadata: name: "\(Namespace)-\(Provisioner)"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelectorLabels = {
+	"app.kubernetes.io/instance": Provisioner
+	"app.kubernetes.io/name":     Provisioner
+}
+
+let Metadata = {
+	name:      Provisioner
+	namespace: Namespace
+	labels:    SelectorLabels
+}
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		ExternalSecret: "\(Provisioner)-tls": #ExternalSecret & {
+			metadata: name:      "\(Provisioner)-tls"
+			metadata: namespace: Namespace
+		}
+		ExternalSecret: "\(Provisioner)": #ExternalSecret & {
+			metadata: name:      Provisioner
+			metadata: namespace: Namespace
+		}
+		ServiceAccount: "\(Provisioner)": #ServiceAccount & {
+			metadata: Metadata
+		}
+		Deployment: "\(Provisioner)": {
+			metadata: Metadata
+			spec: {
+				selector: matchLabels: SelectorLabels
+				template: metadata: labels: SelectorLabels
+				template: spec: {
+					containers: [
+						{
+							name: Provisioner
+							command: ["bash", "/etc/provisioner/entrypoint"]
+							// skopeo inspect docker://registry.choria.io/choria/provisioner | jq .RepoTags
+							image:           "registry.choria.io/choria/provisioner:0.15.1"
+							imagePullPolicy: "IfNotPresent"
+							resources: {}
+							securityContext: {}
+							volumeMounts: [
+								{
+									mountPath: "/etc/provisioner"
+									name:      Provisioner
+								},
+								{
+									mountPath: "/etc/provisioner-tls"
+									name:      "\(Provisioner)-tls"
+								},
+							]
+						},
+					]
+					securityContext: {}
+					serviceAccountName: Provisioner
+					volumes: [
+						{
+							name: Provisioner
+							secret: secretName: name
+						},
+						{
+							name: "\(Provisioner)-tls"
+							secret: secretName: name
+						},
+					]
+				}
+			}
+		}
+	}
+}

docs/examples/platforms/holos-saas/clusters/workload/choria/readme.md

@@ -0,0 +1,8 @@
+# Machine Room Provisioner
+
+This sub-tree contains Holos Components to manage a [Choria Provisioner][1]
+system for the use case of provisioning `holos controller` instances.  These
+instances are implementations of Machine Room which are in turn implementations
+of Choria Server, hence why we use Choria Provisioner.
+
+[1]: https://choria-io.github.io/provisioner/

docs/examples/platforms/holos-saas/clusters/workload/nats/crds/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# curl -LO https://github.com/nats-io/nack/releases/latest/download/crds.yml
+resources:
+- crds.yml

docs/examples/platforms/holos-saas/clusters/workload/nats/crds/nats-crds.cue

@@ -0,0 +1,8 @@
+package holos
+
+// NATS NetStream Controller (NACK)
+spec: components: KustomizeBuildList: [
+	#KustomizeBuild & {
+		metadata: name: "prod-nack-crds"
+	},
+]

docs/examples/platforms/holos-saas/clusters/workload/nats/envs/nats.cue

@@ -0,0 +1,62 @@
+package holos
+
+// for Project in _Projects {
+// 	spec: components: resources: (#ProjectTemplate & {project: Project}).workload.resources
+// }
+
+let Namespace = "jeff-holos"
+
+#Kustomization: spec: targetNamespace: Namespace
+
+spec: components: HelmChartList: [
+	#HelmChart & {
+		metadata: name: "jeff-holos-nats"
+		namespace: Namespace
+		_dependsOn: "prod-secrets-stores": _
+		chart: {
+			name:       "nats"
+			version:    "1.1.10"
+			repository: NatsRepository
+		}
+		_values: #NatsValues & {
+			config: {
+				// https://github.com/nats-io/k8s/tree/main/helm/charts/nats#operator-mode-with-nats-resolver
+				resolver: enabled: true
+				resolver: merge: {
+					type:     "full"
+					interval: "2m"
+					timeout:  "1.9s"
+				}
+				merge: {
+					operator:       "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJUSElBTDM2NUtOS0lVVVJDMzNLNFJGQkJVRlFBSTRLS0NQTDJGVDZYVjdNQVhWU1dFNElRIiwiaWF0IjoxNzEzMjIxMzE1LCJpc3MiOiJPREtQM0RZTzc3T1NBRU5IU0FFR0s3WUNFTFBYT1FFWUI3RVFSTVBLWlBNQUxINE5BRUVLSjZDRyIsIm5hbWUiOiJIb2xvcyIsInN1YiI6Ik9ES1AzRFlPNzdPU0FFTkhTQUVHSzdZQ0VMUFhPUUVZQjdFUVJNUEtaUE1BTEg0TkFFRUtKNkNHIiwibmF0cyI6eyJ0eXBlIjoib3BlcmF0b3IiLCJ2ZXJzaW9uIjoyfX0.dQURTb-zIQMc-OYd9328oY887AEnvog6gOXY1-VCsDG3L89nq5x_ks4ME7dJ4Pn-Pvm2eyBi1Jx6ubgkthHgCQ"
+					system_account: "ADIQCYK4K3OKTPODGCLI4PDQ6XBO52MISBPTAIDESEJMLZCMNULDKCCY"
+					resolver_preload: {
+						// NOTEL: Make sure you do not include the trailing , in the SYS_ACCOUNT_JWT
+						"ADIQCYK4K3OKTPODGCLI4PDQ6XBO52MISBPTAIDESEJMLZCMNULDKCCY": "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI2SEVMNlhKSUdWUElMNFBURVI1MkUzTkFITjZLWkVUUUdFTlFVS0JWRzNUWlNLRzVLT09RIiwiaWF0IjoxNzEzMjIxMzE1LCJpc3MiOiJPREtQM0RZTzc3T1NBRU5IU0FFR0s3WUNFTFBYT1FFWUI3RVFSTVBLWlBNQUxINE5BRUVLSjZDRyIsIm5hbWUiOiJTWVMiLCJzdWIiOiJBRElRQ1lLNEszT0tUUE9ER0NMSTRQRFE2WEJPNTJNSVNCUFRBSURFU0VKTUxaQ01OVUxES0NDWSIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTF9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJhdXRob3JpemF0aW9uIjp7fSwidHlwZSI6ImFjY291bnQiLCJ2ZXJzaW9uIjoyfX0.TiGIk8XON394D9SBEowGHY_nTeOyHiM-ihyw6HZs8AngOnYPFXH9OVjsaAf8Poa2k_V84VtH7yVNgNdjBgduDA"
+					}
+				}
+				cluster: enabled:   true
+				jetstream: enabled: true
+				websocket: enabled: true
+				monitor: enabled:   true
+			}
+			promExporter: enabled: true
+			promExporter: podMonitor: enabled: true
+		}
+	},
+	#HelmChart & {
+		metadata: name: "jeff-holos-nack"
+		namespace: Namespace
+		_dependsOn: "jeff-holos-nats": _
+		chart: {
+			name:       "nack"
+			version:    "0.25.2"
+			repository: NatsRepository
+		}
+	},
+]
+
+let NatsRepository = {
+	name: "nats"
+	url:  "https://nats-io.github.io/k8s/helm/charts/"
+}

docs/examples/platforms/holos-saas/clusters/workload/nats/envs/nats.values.cue

@@ -0,0 +1,722 @@
+package holos
+
+#NatsValues: {
+	//###############################################################################
+	// Global options
+	//###############################################################################
+	global: {
+		image: {
+			// global image pull policy to use for all container images in the chart
+			// can be overridden by individual image pullPolicy
+			pullPolicy: null
+			// global list of secret names to use as image pull secrets for all pod specs in the chart
+			// secrets must exist in the same namespace
+			// https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+			pullSecretNames: []
+			// global registry to use for all container images in the chart
+			// can be overridden by individual image registry
+			registry: null
+		}
+
+		// global labels will be applied to all resources deployed by the chart
+		labels: {}
+	}
+
+	//###############################################################################
+	// Common options
+	//###############################################################################
+	// override name of the chart
+	nameOverride: null
+	// override full name of the chart+release
+	fullnameOverride: null
+	// override the namespace that resources are installed into
+	namespaceOverride: null
+
+	// reference a common CA Certificate or Bundle in all nats config `tls` blocks and nats-box contexts
+	// note: `tls.verify` still must be set in the appropriate nats config `tls` blocks to require mTLS
+	tlsCA: {
+		enabled: false
+		// set configMapName in order to mount an existing configMap to dir
+		configMapName: null
+		// set secretName in order to mount an existing secretName to dir
+		secretName: null
+		// directory to mount the configMap or secret to
+		dir: "/etc/nats-ca-cert"
+		// key in the configMap or secret that contains the CA Certificate or Bundle
+		key: "ca.crt"
+	}
+
+	//###############################################################################
+	// NATS Stateful Set and associated resources
+	//###############################################################################
+	//###########################################################
+	// NATS config
+	//###########################################################
+	config: {
+		cluster: {
+			enabled: true | *false
+			port:    6222
+			// must be 2 or higher when jetstream is enabled
+			replicas: 3
+
+			// apply to generated route URLs that connect to other pods in the StatefulSet
+			routeURLs: {
+				// if both user and password are set, they will be added to route URLs
+				// and the cluster authorization block
+				user:     null
+				password: null
+				// set to true to use FQDN in route URLs
+				useFQDN:          false
+				k8sClusterDomain: "cluster.local"
+			}
+
+			tls: {
+				enabled: true | *false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/cluster"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the cluster config
+			// https://docs.nats.io/running-a-nats-service/configuration/clustering/cluster_config
+			merge: {}
+			patch: []
+		}
+
+		jetstream: {
+			enabled: true | *false
+
+			fileStore: {
+				enabled: true
+				dir:     "/data"
+
+				//###########################################################
+				// stateful set -> volume claim templates -> jetstream pvc
+				//###########################################################
+				pvc: {
+					enabled:          true
+					size:             "10Gi"
+					storageClassName: null
+
+					// merge or patch the jetstream pvc
+					// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core
+					merge: {}
+					patch: []
+					// defaults to "{{ include "nats.fullname" $ }}-js"
+					name: null
+				}
+
+				// defaults to the PVC size
+				maxSize: null
+			}
+
+			memoryStore: {
+				enabled: false
+				// ensure that container has a sufficient memory limit greater than maxSize
+				maxSize: "1Gi"
+			}
+
+			// merge or patch the jetstream config
+			// https://docs.nats.io/running-a-nats-service/configuration#jetstream
+			merge: {}
+			patch: []
+		}
+
+		nats: {
+			port: 4222
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/nats"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+		}
+
+		leafnodes: {
+			enabled: false
+			port:    7422
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/leafnodes"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the leafnodes config
+			// https://docs.nats.io/running-a-nats-service/configuration/leafnodes/leafnode_conf
+			merge: {}
+			patch: []
+		}
+
+		websocket: {
+			enabled: true | *false
+			port:    8080
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/websocket"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			//###########################################################
+			// ingress
+			//###########################################################
+			// service must be enabled also
+			ingress: {
+				enabled: false
+				// must contain at least 1 host otherwise ingress will not be created
+				hosts: []
+				path:     "/"
+				pathType: "Exact"
+				// sets to the ingress class name
+				className: null
+				// set to an existing secret name to enable TLS on the ingress; applies to all hosts
+				tlsSecretName: null
+
+				// merge or patch the ingress
+				// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#ingress-v1-networking-k8s-io
+				merge: {}
+				patch: []
+				// defaults to "{{ include "nats.fullname" $ }}-ws"
+				name: null
+			}
+
+			// merge or patch the websocket config
+			// https://docs.nats.io/running-a-nats-service/configuration/websocket/websocket_conf
+			merge: {}
+			patch: []
+		}
+
+		mqtt: {
+			enabled: false
+			port:    1883
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/mqtt"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the mqtt config
+			// https://docs.nats.io/running-a-nats-service/configuration/mqtt/mqtt_config
+			merge: {}
+			patch: []
+		}
+
+		gateway: {
+			enabled: false
+			port:    7222
+			tls: {
+				enabled: false
+				// set secretName in order to mount an existing secret to dir
+				secretName: null
+				dir:        "/etc/nats-certs/gateway"
+				cert:       "tls.crt"
+				key:        "tls.key"
+				// merge or patch the tls config
+				// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+				merge: {}
+				patch: []
+			}
+
+			// merge or patch the gateway config
+			// https://docs.nats.io/running-a-nats-service/configuration/gateways/gateway#gateway-configuration-block
+			merge: {}
+			patch: []
+		}
+
+		monitor: {
+			enabled: true
+			port:    8222
+			tls: {
+				// config.nats.tls must be enabled also
+				// when enabled, monitoring port will use HTTPS with the options from config.nats.tls
+				enabled: false
+			}
+		}
+
+		profiling: {
+			enabled: false
+			port:    65432
+		}
+
+		resolver: {
+			enabled: true | *false
+			dir:     "/data/resolver"
+
+			//###########################################################
+			// stateful set -> volume claim templates -> resolver pvc
+			//###########################################################
+			pvc: {
+				enabled:          true
+				size:             "1Gi"
+				storageClassName: null
+
+				// merge or patch the pvc
+				// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaim-v1-core
+				merge: {}
+				patch: []
+				// defaults to "{{ include "nats.fullname" $ }}-resolver"
+				name: null
+			}
+
+			// merge or patch the resolver
+			// https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/jwt/resolver
+			merge: {
+				type?:     string
+				interval?: string
+				timeout?:  string
+			}
+			patch: []
+		}
+
+		// adds a prefix to the server name, which defaults to the pod name
+		// helpful for ensuring server name is unique in a super cluster
+		serverNamePrefix: ""
+
+		// merge or patch the nats config
+		// https://docs.nats.io/running-a-nats-service/configuration
+		// following special rules apply
+		//  1. strings that start with << and end with >> will be unquoted
+		//     use this for variables and numbers with units
+		//  2. keys ending in $include will be switched to include directives
+		//     keys are sorted alphabetically, use prefix before $includes to control includes ordering
+		//     paths should be relative to /etc/nats-config/nats.conf
+		// example:
+		//
+		//   merge:
+		//     $include: ./my-config.conf
+		//     zzz$include: ./my-config-last.conf
+		//     server_name: nats
+		//     authorization:
+		//       token: << $TOKEN >>
+		//     jetstream:
+		//       max_memory_store: << 1GB >>
+		//
+		// will yield the config:
+		// {
+		//   include ./my-config.conf;
+		//   "authorization": {
+		//     "token": $TOKEN
+		//   },
+		//   "jetstream": {
+		//     "max_memory_store": 1GB
+		//   },
+		//   "server_name": "nats",
+		//   include ./my-config-last.conf;
+		// }
+		merge: {
+			operator?:       string
+			system_account?: string
+			resolver_preload?: [string]: string
+		}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> nats container
+	//###########################################################
+	container: {
+		image: {
+			repository: "nats"
+			tag:        "2.10.12-alpine"
+			pullPolicy: null
+			registry:   null
+		}
+
+		// container port options
+		// must be enabled in the config section also
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#containerport-v1-core
+		ports: {
+			nats: {}
+			leafnodes: {}
+			websocket: {}
+			mqtt: {}
+			cluster: {}
+			gateway: {}
+			monitor: {}
+			profiling: {}
+		}
+
+		// map with key as env var name, value can be string or map
+		// example:
+		//
+		//   env:
+		//     GOMEMLIMIT: 7GiB
+		//     TOKEN:
+		//       valueFrom:
+		//         secretKeyRef:
+		//           name: nats-auth
+		//           key: token
+		env: {}
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> reloader container
+	//###########################################################
+	reloader: {
+		enabled: true
+		image: {
+			repository: "natsio/nats-server-config-reloader"
+			tag:        "0.14.1"
+			pullPolicy: null
+			registry:   null
+		}
+
+		// env var map, see nats.env for an example
+		env: {}
+
+		// all nats container volume mounts with the following prefixes
+		// will be mounted into the reloader container
+		natsVolumeMountPrefixes: ["/etc/"]
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+	}
+
+	//###########################################################
+	// stateful set -> pod template -> prom-exporter container
+	//###########################################################
+	// config.monitor must be enabled
+	promExporter: {
+		enabled: true | *false
+		image: {
+			repository: "natsio/prometheus-nats-exporter"
+			tag:        "0.14.0"
+			pullPolicy: null
+			registry:   null
+		}
+
+		port: 7777
+		// env var map, see nats.env for an example
+		env: {}
+
+		// merge or patch the container
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+		merge: {}
+		patch: []
+
+		//###########################################################
+		// prometheus pod monitor
+		//###########################################################
+		podMonitor: {
+			enabled: true | *false
+
+			// merge or patch the pod monitor
+			// https://prometheus-operator.dev/docs/operator/api/#monitoring.coreos.com/v1.PodMonitor
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}"
+			name: null
+		}
+	}
+
+	//###########################################################
+	// service
+	//###########################################################
+	service: {
+		enabled: true
+
+		// service port options
+		// additional boolean field enable to control whether port is exposed in the service
+		// must be enabled in the config section also
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceport-v1-core
+		ports: {
+			nats: enabled:      true
+			leafnodes: enabled: true
+			websocket: enabled: true
+			mqtt: enabled:      true
+			cluster: enabled:   false
+			gateway: enabled:   false
+			monitor: enabled:   false
+			profiling: enabled: false
+		}
+
+		// merge or patch the service
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	//###########################################################
+	// other nats extension points
+	//###########################################################
+	// stateful set
+	statefulSet: {
+		// merge or patch the stateful set
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#statefulset-v1-apps
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	// stateful set -> pod template
+	podTemplate: {
+		// adds a hash of the ConfigMap as a pod annotation
+		// this will cause the StatefulSet to roll when the ConfigMap is updated
+		configChecksumAnnotation: true
+
+		// map of topologyKey: topologySpreadConstraint
+		// labelSelector will be added to match StatefulSet pods
+		//
+		// topologySpreadConstraints:
+		//   kubernetes.io/hostname:
+		//     maxSkew: 1
+		//
+		topologySpreadConstraints: {}
+
+		// merge or patch the pod template
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core
+		merge: {}
+		patch: []
+	}
+
+	// headless service
+	headlessService: {
+		// merge or patch the headless service
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#service-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}-headless"
+		name: null
+	}
+
+	// config map
+	configMap: {
+		// merge or patch the config map
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#configmap-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}-config"
+		name: null
+	}
+
+	// pod disruption budget
+	podDisruptionBudget: {
+		enabled: true
+		// merge or patch the pod disruption budget
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#poddisruptionbudget-v1-policy
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	// service account
+	serviceAccount: {
+		enabled: false
+		// merge or patch the service account
+		// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceaccount-v1-core
+		merge: {}
+		patch: []
+		// defaults to "{{ include "nats.fullname" $ }}"
+		name: null
+	}
+
+	//###########################################################
+	// natsBox
+	//
+	// NATS Box Deployment and associated resources
+	//###########################################################
+	natsBox: {
+		enabled: true
+
+		//###########################################################
+		// NATS contexts
+		//###########################################################
+		contexts: {
+			default: {
+				creds: {
+					// set contents in order to create a secret with the creds file contents
+					contents: null
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-creds/<context-name>
+					dir: null
+					key: "nats.creds"
+				}
+				nkey: {
+					// set contents in order to create a secret with the nkey file contents
+					contents: null
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-nkeys/<context-name>
+					dir: null
+					key: "nats.nk"
+				}
+				// used to connect with client certificates
+				tls: {
+					// set secretName in order to mount an existing secret to dir
+					secretName: null
+					// defaults to /etc/nats-certs/<context-name>
+					dir:  null
+					cert: "tls.crt"
+					key:  "tls.key"
+				}
+
+				// merge or patch the context
+				// https://docs.nats.io/using-nats/nats-tools/nats_cli#nats-contexts
+				merge: {}
+				patch: []
+			}
+		}
+
+		// name of context to select by default
+		defaultContextName: "default"
+
+		//###########################################################
+		// deployment -> pod template -> nats-box container
+		//###########################################################
+		container: {
+			image: {
+				repository: "natsio/nats-box"
+				tag:        "0.14.2"
+				pullPolicy: null
+				registry:   null
+			}
+
+			// env var map, see nats.env for an example
+			env: {}
+
+			// merge or patch the container
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core
+			merge: {}
+			patch: []
+		}
+
+		//###########################################################
+		// other nats-box extension points
+		//###########################################################
+		// deployment
+		deployment: {
+			// merge or patch the deployment
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#deployment-v1-apps
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box"
+			name: null
+		}
+
+		// deployment -> pod template
+		podTemplate: {
+			// merge or patch the pod template
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#pod-v1-core
+			merge: {}
+			patch: []
+		}
+
+		// contexts secret
+		contextsSecret: {
+			// merge or patch the context secret
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box-contexts"
+			name: null
+		}
+
+		// contents secret
+		contentsSecret: {
+			// merge or patch the contents secret
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secret-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box-contents"
+			name: null
+		}
+
+		// service account
+		serviceAccount: {
+			enabled: false
+			// merge or patch the service account
+			// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#serviceaccount-v1-core
+			merge: {}
+			patch: []
+			// defaults to "{{ include "nats.fullname" $ }}-box"
+			name: null
+		}
+	}
+
+	//###############################################################################
+	// Extra user-defined resources
+	//###############################################################################
+	//
+	// add arbitrary user-generated resources
+	// example:
+	//
+	// config:
+	//   websocket:
+	//     enabled: true
+	// extraResources:
+	// - apiVersion: networking.istio.io/v1beta1
+	//   kind: VirtualService
+	//   metadata:
+	//     name:
+	//       $tplYaml: >
+	//         {{ include "nats.fullname" $ | quote }}
+	//     labels:
+	//       $tplYaml: |
+	//         {{ include "nats.labels" $ }}
+	//   spec:
+	//     hosts:
+	//     - demo.nats.io
+	//     gateways:
+	//     - my-gateway
+	//     http:
+	//     - name: default
+	//       match:
+	//       - name: root
+	//         uri:
+	//           exact: /
+	//       route:
+	//       - destination:
+	//           host:
+	//             $tplYaml: >
+	//               {{ .Values.service.name | quote }}
+	//           port:
+	//             number:
+	//               $tplYaml: >
+	//                 {{ .Values.config.websocket.port }}
+	//
+	extraResources: []
+}

docs/examples/platforms/holos-saas/clusters/workload/nats/scripts/initialize-auth

@@ -0,0 +1,81 @@
+#! /bin/bash
+#
+# This script initializes authorization for a nats cluster.  The process is:
+#
+# Locally:
+# 1. Generate the nats operator jwt.
+# 2. Generate a SYS account jwt issued by the operator.
+# 3. Store both into vault
+#
+# When nats is deployed an ExternalSecret populates auth.conf which is included
+# into nats.conf.  This approach allows helm values to be used for most things
+# except for secrets.
+#
+# Clean up by removing the nsc directory.
+
+set -euo pipefail
+
+tmpdir="$(mktemp -d)"
+finish() {
+  [[ -d "$tmpdir" ]] && rm -rf "$tmpdir"
+}
+trap finish EXIT
+
+PARENT="$(cd "$(dirname $0)" && pwd)"
+: "${OPERATOR_NAME:="Holos"}"
+
+: "${OIX_NAMESPACE:=$(kubectl config view --minify --flatten -ojsonpath='{.contexts[0].context.namespace}')}"
+nsc="${HOME}/.bin/nsc"
+
+ROOT="${PARENT}/${OIX_NAMESPACE}/nsc"
+export NKEYS_PATH="${ROOT}/nkeys"
+export NSC_HOME="${ROOT}/accounts"
+
+mkdir -p "$NKEYS_PATH"
+mkdir -p "$NSC_HOME"
+
+# Install nsc if not already installed
+if ! [[ -x $nsc ]]; then
+  platform="$(kubectl version --output=json | jq .clientVersion.platform -r)"
+  platform="${platform//\//-}"
+  curl -fSLo "${tmpdir}/nsc.zip" "https://github.com/nats-io/nsc/releases/download/v2.8.6/nsc-${platform}.zip"
+  (cd "${tmpdir}" && unzip nsc.zip)
+  sudo install -o 0 -g 0 -m 0755 "${tmpdir}/nsc" $nsc
+fi
+
+echo "export NKEYS_PATH='${NKEYS_PATH}'" > "${ROOT}/nsc.env"
+echo "export NSC_HOME='${NSC_HOME}'" >> "${ROOT}/nsc.env"
+# use kubectl port-forward nats-headless 4222
+echo "export NATS_URL='nats://localhost:4222'" >> "${ROOT}/nsc.env"
+echo "export NATS_CREDS='${ROOT}/nkeys/creds/${OPERATOR_NAME}/SYS/sys.creds'" >> "${ROOT}/nsc.env"
+
+echo "export NATS_CA='${ROOT}/ca.crt'" >> "${ROOT}/nsc.env"
+echo "export NATS_CERT='${ROOT}/tls.crt'" >> "${ROOT}/nsc.env"
+echo "export NATS_KEY='${ROOT}/tls.key'" >> "${ROOT}/nsc.env"
+
+$nsc --data-dir="${ROOT}/stores" list operators
+
+# Create operator
+$nsc add operator --name "${OPERATOR_NAME}"
+# Create system account
+$nsc add account --name SYS
+$nsc add user    --name sys
+# Create account for STAN purposes.
+$nsc add account --name STAN
+$nsc add user    --name stan
+
+# Generate an auth config compatible with the StatefulSet mounting the
+# nats-jwt-pvc PersistentVolumeClaim at path /data/accounts
+$nsc generate config --sys-account SYS --nats-resolver \
+  | sed "s,dir.*jwt',dir: '/data/accounts'" \
+  > "${ROOT}/auth.conf"
+
+# Store the auth config in vault.
+# vault kv put kv/${OIX_CLUSTER_NAME}/kube-namespace/holos-dev/nats-auth-config "auth.conf=@${tmpdir}/auth.conf"
+# Store the SYS creds in vault for use by the nack controller.
+# vault kv put kv/${OIX_CLUSTER_NAME}/kube-namespace/holos-dev/nats-sys-creds "sys.creds=@${OIX_CLUSTER_NAME}/nsc/nkeys/creds/${OPERATOR_NAME}/SYS/sys.creds"
+
+echo "After deploying the nats component, use the get-cert command to fetch the client cert."
+
+echo "Use kubectl port-forward svc/nats-headless 4222" >&2
+echo "source ${ROOT}/nsc.env to make it all work." >&2

docs/examples/platforms/holos-saas/readme.md

@@ -0,0 +1,5 @@
+# Holos
+
+This subtree contains holos components for holos itself.  We strive for minimal dependencies, so this is likely going to contain NATS and/or Postgres resources.
+
+Components depend on the holos project and may iterate over the defined environments in the project stages.

docs/examples/platforms/reference/clusters/foundation/cloud/argocd/argocd.cue

@@ -71,14 +71,15 @@ let IstioInject = [{op: "add", path: "/spec/template/metadata/labels/sidecar.ist
 	}
 }
 
-// Probably shouldn't use the authproxy struct and should instead define an identity provider struct.
-let AuthProxySpec = #AuthProxySpec & #Platform.authproxy
+let OAuthClient = #Platform.oauthClients.argocd.spec
 
 let OIDCConfig = {
-	name:     "Holos Platform"
-	issuer:   AuthProxySpec.issuer
-	clientID: #Platform.argocd.clientID
-	requestedIDTokenClaims: groups: essential: true
-	requestedScopes: ["openid", "profile", "email", "groups", "urn:zitadel:iam:org:domain:primary:\(AuthProxySpec.orgDomain)"]
+	name:            "Holos Platform"
+	issuer:          OAuthClient.issuer
+	clientID:        OAuthClient.clientID
+	requestedScopes: OAuthClient.scopesList
+	// Set redirect uri to https://argocd.example.com/pkce/verify
 	enablePKCEAuthentication: true
+
+	requestedIDTokenClaims: groups: essential: true
 }

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/gateway/gateway.cue

@@ -21,12 +21,12 @@ spec: components: KubernetesObjectsList: [
 // GatewayServers represents all hosts for all VirtualServices in the cluster attached to Gateway/default
 // NOTE: This is a critical structure because the default Gateway should be used in most cases.
 let GatewayServers = {
+	// Critical Feature: Map all Project hosts to the default Gateway.
 	for Project in _Projects {
-		for server in (#ProjectTemplate & {project: Project}).ClusterGatewayServers {
-			(server.port.name): server
-		}
+		(#ProjectTemplate & {project: Project}).ClusterDefaultGatewayServers
 	}
 
+	// TODO: Refactor to use FQDN as key
 	for k, svc in #OptionalServices {
 		if svc.enabled && list.Contains(svc.clusterNames, #ClusterName) {
 			for server in svc.servers {
@@ -35,6 +35,7 @@ let GatewayServers = {
 		}
 	}
 
+	// TODO: Remove?  Why aren't these part of the platform project?
 	if #PlatformServers[#ClusterName] != _|_ {
 		for server in #PlatformServers[#ClusterName] {
 			(server.port.name): server
@@ -52,6 +53,11 @@ let OBJECTS = #APIObjects & {
 			spec: servers: [for x in GatewayServers {x}]
 		}
 
+		// Manage an ExternalSecret for each server defined in the default Gateway to sync the cert.
+		for Server in Gateway.default.spec.servers {
+			ExternalSecret: "\(Server.tls.credentialName)": metadata: namespace: "istio-ingress"
+		}
+
 		for k, svc in #OptionalServices {
 			if svc.enabled && list.Contains(svc.clusterNames, #ClusterName) {
 				for k, s in svc.servers {

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/ingress/ingress.cue

@@ -8,7 +8,7 @@ let ComponentName = "\(#InstancePrefix)-ingress"
 
 spec: components: HelmChartList: [
 	#HelmChart & {
-		_dependsOn: "prod-secrets-namespaces":       _
+		_dependsOn: "prod-secrets-stores":           _
 		_dependsOn: "\(#InstancePrefix)-istio-base": _
 		_dependsOn: "\(#InstancePrefix)-istiod":     _
 
@@ -76,6 +76,10 @@ let RedirectMetaName = {
 
 let OBJECTS = #APIObjects & {
 	apiObjects: {
+		ExternalSecret: "istio-ingress-mtls-cert": #ExternalSecret & {
+			metadata: name:      "istio-ingress-mtls-cert"
+			metadata: namespace: #TargetNamespace
+		}
 		Gateway: {
 			"\(RedirectMetaName.name)": #Gateway & {
 				metadata: RedirectMetaName

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/mesh.cue

@@ -18,6 +18,10 @@ _IngressAuthProxy: {
 	Domains: (#Platform.org.domain):                    _
 	Domains: "\(#ClusterName).\(#Platform.org.domain)": _
 
+	// TODO: This should be generated from ProjectHosts
+	Domains: "holos.run":                 _
+	Domains: "\(#ClusterName).holos.run": _
+
 	let Metadata = {
 		name:      string
 		namespace: Namespace
@@ -85,7 +89,8 @@ _IngressAuthProxy: {
 						spec: {
 							securityContext: seccompProfile: type: "RuntimeDefault"
 							containers: [{
-								image:           "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0"
+								// image: "quay.io/oauth3-proxy/oauth2-proxy:v7.6.0"
+								image:           "quay.io/holos/oauth2-proxy:v7.6.0-1-g77a03ae2"
 								imagePullPolicy: "IfNotPresent"
 								name:            "oauth2-proxy"
 								volumeMounts: [{
@@ -271,11 +276,14 @@ _IngressAuthProxy: {
 					rules: [
 						{
 							to: [{
+								// Refer to https://istio.io/latest/docs/ops/best-practices/security/#writing-host-match-policies
 								operation: notHosts: [
 									// Never send requests for the login service through the authorizer, would block login.
 									AuthProxySpec.issuerHost,
+									"\(AuthProxySpec.issuerHost):*",
 									// Exclude hosts with specialized rules from the catch-all.
 									for x in _AuthPolicyRules.hosts {x.name},
+									for x in _AuthPolicyRules.hosts {"\(x.name):*"},
 								]
 							}]
 							when: [
@@ -298,7 +306,7 @@ _IngressAuthProxy: {
 _AuthPolicyRules: #AuthPolicyRules & {
 	hosts: {
 		let Vault = "vault.core.ois.run"
-		(Vault): {
+		"\(Vault)": {
 			slug: "vault"
 			// Rules for when to route requests through the auth proxy
 			spec: rules: [
@@ -321,3 +329,20 @@ _AuthPolicyRules: #AuthPolicyRules & {
 		}
 	}
 }
+
+// Exclude project hosts from the auth proxy if configured to do so.  The
+// intended effect is to exclude the host from the blanket `authproxy-custom`
+// AuthorizationPolicy rule _without_ adding a specialized AuthorizationPolicy
+// for the same host.  This has the effect of completely excluding the host from
+// authorization policy.
+for Project in _Projects {
+	let ProjectHosts = (#ProjectHosts & {project: Project}).Hosts
+
+	for FQDN, Host in ProjectHosts {
+		if Host.NoAuthorizationPolicy {
+			if Host.clusters[#ClusterName] != _|_ {
+				_AuthPolicyRules: hosts: "\(Host.fqdn)": NoAuthorizationPolicy: true
+			}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/foundation/cloud/obs/obs.cue

@@ -4,7 +4,7 @@ package holos
 
 let Namespace = "prod-platform"
 
-// FYI: kube-prometheus-stack is a large umbrella chart what brings in other large charts like
+// FYI: kube-prometheus-stack is a large umbrella chart that brings in other large charts like
 // [grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana).
 // This may make affect maintainability.  Consider breaking the integration down into
 // constituent charts represented as holos component instances.
@@ -77,7 +77,7 @@ spec: components: HelmChartList: [
 						token_url:           OIDC.token_endpoint
 						api_url:             OIDC.userinfo_endpoint
 						use_pkce:            true
-						name_attribute_path: name
+						name_attribute_path: "name"
 						// TODO: Lift the admin, editor, and viewer group names up to the plaform config struct.
 						role_attribute_path: "contains(groups[*], 'prod-cluster-admin') && 'Admin' || contains(groups[*], 'prod-cluster-editor') && 'Editor' || 'Viewer'"
 					}

docs/examples/platforms/reference/clusters/platform-projects.cue

@@ -4,15 +4,23 @@ package holos
 
 let ZitadelProjectID = 257713952794870157
 
+let AllClusters = {
+	// platform level services typically run in the core cluster pair.
+	core1: _
+	core2: _
+	// for development, probably wouldn't run these services in the workload clusters.
+	k1: _
+	k2: _
+	k3: _
+	k4: _
+	k5: _
+}
+
 _Projects: #Projects & {
 	// The platform project is required and where platform services reside.  ArgoCD, Grafana, Prometheus, etc...
 	platform: {
 		resourceId: ZitadelProjectID
-		// platform level services typically run in the core cluster pair.
-		clusters: core1: _
-		clusters: core2: _
-		// for development, probably wouldn't run these services in the workload clusters.
-		clusters: k2: _
+		clusters:   AllClusters
 		// Services hosted in the platform project
 		hosts: argocd:     _
 		hosts: grafana:    _
@@ -21,8 +29,9 @@ _Projects: #Projects & {
 
 	holos: {
 		resourceId: ZitadelProjectID
-		clusters: k1: _
-		clusters: k2: _
+		domain:     "holos.run"
+		clusters:   AllClusters
+
 		environments: {
 			prod: stage: "prod"
 			dev: stage:  "dev"
@@ -30,6 +39,13 @@ _Projects: #Projects & {
 			gary: stage: dev.stage
 			nate: stage: dev.stage
 		}
+
+		// app is the holos web app and grpc api.
+		hosts: app: _
+		// provision is the choria broker provisioning system.
+		hosts: provision: NoAuthorizationPolicy: true
+		// nats is the nats service holos controller machine room agents connect after provisioning.
+		hosts: nats: NoAuthorizationPolicy: true
 	}
 
 	iam: {

docs/examples/platforms/reference/clusters/provisioner/mesh/istio/ingress/ingress.cue

@@ -0,0 +1,40 @@
+package holos
+
+// Certificate used by the ingress to connect to services using a platform
+// issued certificate but which are not using istio sidecar injection.
+// Examples are keycloak, vault, nats, choria, etc...
+
+let Namespace = "istio-ingress"
+let CertName = "istio-ingress-mtls-cert"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		_dependsOn: "prod-platform-issuer": _
+
+		metadata: name: CertName
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		Certificate: "\(CertName)": #Certificate & {
+			metadata: {
+				name:      CertName
+				namespace: Namespace
+			}
+			spec: {
+				secretName: metadata.name
+				issuerRef: kind: "ClusterIssuer"
+				issuerRef: name: "platform-issuer"
+				commonName: "istio-ingress"
+				dnsNames: [
+					"istio-ingress",
+					"istio-ingress.\(Namespace)",
+					"istio-ingress.\(Namespace).svc",
+					"istio-ingress.\(Namespace).svc.cluster.local",
+				]
+			}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/provisioner/platform-issuer/platform-issuer.cue

@@ -0,0 +1,53 @@
+package holos
+
+// Refer to https://cert-manager.io/docs/configuration/selfsigned/#bootstrapping-ca-issuers
+
+let Namespace = "cert-manager"
+
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		metadata: name: "prod-platform-issuer"
+
+		_dependsOn: "prod-mesh-certmanager": _
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let SelfSigned = "platform-selfsigned"
+let PlatformIssuer = "platform-issuer"
+
+let OBJECTS = #APIObjects & {
+	apiObjects: {
+		ClusterIssuer: {
+			"\(SelfSigned)": #ClusterIssuer & {
+				metadata: name: SelfSigned
+				spec: selfSigned: {}
+			}
+		}
+		Certificate: {
+			"\(PlatformIssuer)": #Certificate & {
+				metadata: name:      PlatformIssuer
+				metadata: namespace: Namespace
+				spec: {
+					duration:   "999999h"
+					isCA:       true
+					commonName: PlatformIssuer
+					secretName: PlatformIssuer
+					privateKey: algorithm: "ECDSA"
+					privateKey: size:      256
+					issuerRef: {
+						name:  SelfSigned
+						kind:  "ClusterIssuer"
+						group: "cert-manager.io"
+					}
+				}
+			}
+		}
+		ClusterIssuer: {
+			"\(PlatformIssuer)": #ClusterIssuer & {
+				metadata: name: PlatformIssuer
+				spec: ca: secretName: PlatformIssuer
+			}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/provisioner/platform-issuer/readme.md

@@ -0,0 +1,5 @@
+# Platform Issuer
+
+The platform issuer is a self signed root certificate authority which acts as a private pki for the platform.  Used to issue certificates for use internally within the platform in a way that supports multi-cluster communication.
+
+Refer to [Bootstrapping CA Issuers](https://cert-manager.io/docs/configuration/selfsigned/#bootstrapping-ca-issuers)

docs/examples/platforms/reference/clusters/provisioner/projects/projects.cue

@@ -1,5 +1,11 @@
 package holos
 
 for Project in _Projects {
-	spec: components: resources: (#ProjectTemplate & {project: Project}).provisioner.resources
+
+	// Debugging variable to enable inspecting the project host data:
+	// cue eval --out json -t cluster=provisioner ./platforms/reference/clusters/provisioner/projects/... -e _ProjectHosts.holos  > hosts.json
+	let ProjectData = (#ProjectTemplate & {project: Project})
+	_ProjectHosts: "\(Project.name)": ProjectData.ProjectHosts
+
+	spec: components: resources: ProjectData.provisioner.resources
 }

docs/examples/platforms/reference/clusters/workload/holos-run/dev/app/holos-run-app.cue

@@ -0,0 +1,73 @@
+package holos
+
+let Namespace = "dev-holos"
+let Holos = "holos"
+
+// spec represents the output provided to holos
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		metadata: name: "dev-holos-app"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+// OBJECTS represents the kubernetes api objects to manage.
+let OBJECTS = #APIObjects & {
+	apiObjects: Deployment: holos: {
+		metadata: {
+			name:      Holos
+			namespace: Namespace
+			labels: app: Holos
+		}
+		spec: {
+			selector: matchLabels: app: Holos
+			template: metadata: labels: {
+				app:                       Holos
+				"sidecar.istio.io/inject": "true"
+			}
+			strategy: rollingUpdate: maxSurge:       1
+			strategy: rollingUpdate: maxUnavailable: 0
+			template: {
+				spec: {
+					serviceAccountName: Holos
+					securityContext: seccompProfile: type: "RuntimeDefault"
+					containers: [
+						{
+							name:            Holos
+							image:           "271053619184.dkr.ecr.us-east-2.amazonaws.com/holos-run/holos-server/holos:v0.79.0"
+							imagePullPolicy: "Always"
+							env: [
+								{
+									name:  "TZ"
+									value: "America/Los_Angeles"
+								},
+								{
+									name: "DATABASE_URL"
+									valueFrom: secretKeyRef: {
+										key:  "uri"
+										name: "holos-pguser-holos"
+									}
+								},
+							]
+							ports: [
+								{
+									containerPort: 3000
+									name:          "http"
+									protocol:      "TCP"
+								},
+							]
+							securityContext: capabilities: drop: ["ALL"]
+							securityContext: allowPrivilegeEscalation: false
+							securityContext: runAsNonRoot:             true
+							resources: limits: {
+								cpu:    "0.25"
+								memory: "256Mi"
+							}
+							resources: requests: resources.limits
+						},
+					]
+				}
+			}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/workload/holos-run/dev/infra/holos-run-infra.cue

@@ -0,0 +1,129 @@
+package holos
+
+let Namespace = "dev-holos"
+let Holos = "holos"
+
+// spec represents the output provided to holos
+spec: components: KubernetesObjectsList: [
+	#KubernetesObjects & {
+		metadata: name: "dev-holos-infra"
+		apiObjectMap: OBJECTS.apiObjectMap
+	},
+]
+
+let Metadata = {
+	name:      Holos
+	namespace: Namespace
+	labels: app: Holos
+}
+
+// OBJECTS represents the kubernetes api objects to manage.
+let OBJECTS = #APIObjects & {
+	// Postgres
+	// Deployment
+	// VirtualService
+
+	apiObjects: ServiceAccount: holos: {
+		metadata: Metadata
+		imagePullSecrets: [{name: "kube-system-ecr-image-pull-creds"}]
+	}
+
+	apiObjects: PostgresCluster: holos: {
+		apiVersion: "postgres-operator.crunchydata.com/v1beta1"
+		metadata:   Metadata
+		spec: {
+			image: "registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.1-0"
+			instances: [{
+				affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: [{
+					podAffinityTerm: {
+						labelSelector: matchLabels: "postgres-operator.crunchydata.com/cluster": "holos"
+						topologyKey: "kubernetes.io/hostname"
+					}
+					weight: 1
+				}]
+				dataVolumeClaimSpec: {
+					accessModes: ["ReadWriteOnce"]
+					resources: requests: storage: "1Gi"
+				}
+				name:     "db"
+				replicas: 1
+			}]
+			port:            5432
+			postgresVersion: 16
+			users: [{
+				databases: ["holos"]
+				name:    "holos"
+				options: "SUPERUSER"
+			}]
+			backups: pgbackrest: {
+				global: {
+					"archive-async":          "y"
+					"archive-push-queue-max": "100MiB"
+					"spool-path":             "/pgdata/backups"
+				}
+				image: "registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.47-2"
+				repos: [{
+					name: "repo1"
+					volume: volumeClaimSpec: {
+						accessModes: ["ReadWriteOnce"]
+						resources: requests: storage: "1Gi"
+					}
+				}]
+			}
+		}
+	}
+
+	apiObjects: Service: holos: {
+		apiVersion: "v1"
+		metadata:   Metadata
+		spec: {
+			type: "ClusterIP"
+			selector: app: "holos"
+			ports: [{
+				appProtocol: "http2"
+				name:        "http"
+				port:        3000
+				protocol:    "TCP"
+				targetPort:  3000
+			}, {
+				appProtocol: "http"
+				name:        "metrics"
+				port:        9090
+				protocol:    "TCP"
+				targetPort:  9090
+			}]
+		}
+	}
+
+	apiObjects: VirtualService: holos: {
+		apiVersion: "networking.istio.io/v1beta1"
+		metadata:   Metadata
+		spec: {
+			gateways: ["istio-ingress/default"]
+			hosts: [
+				"app.dev.holos.run",
+				"app.dev.\(#ClusterName).holos.run",
+			]
+			http: [{
+				match: [{
+					uri: prefix: "/ui"
+				}]
+				name: "ui"
+				route: [{
+					destination: {
+						host: "holos"
+						port: number: 3000
+					}
+				}]
+			}, {
+				name: "api"
+				route: [{
+					destination: {
+						host: "holos"
+						port: number: 3000
+					}
+				}]
+			}]
+		}
+	}
+}

docs/examples/platforms/reference/platform-project-schema.cue

@@ -0,0 +1,25 @@
+package holos
+
+// Platform level definition of a project.
+#Project: {
+	name: string
+
+	// All projects have at least a prod and dev environment and stage.
+
+	// Omit the prod stage segment from hostnames.  foo.holos.run not foo.prod.holos.run
+	stages: prod: stageSegments: []
+	environments: prod: stage: "prod"
+	// Omit the prod env segment from hostnames.  foo.holos.run not prod.foo.holos.run
+	environments: prod: envSegments: []
+
+	stages: dev: _
+	environments: dev: stage: "dev"
+	// Omit the dev env segment from hostnames.  foo.dev.holos.run not dev.foo.dev.holos.run
+	environments: dev: envSegments: []
+
+	// environments share the stage segments of their stage.
+	environments: [_]: {
+		stage:         string
+		stageSegments: stages[stage].stageSegments
+	}
+}

docs/examples/platforms/reference/project_constraints.cue

@@ -1 +0,0 @@
-package holos

docs/examples/project-hosts.cue

@@ -0,0 +1,143 @@
+package holos
+
+import "strings"
+
+// #ProjectHosts represents all of the hosts associated with the project
+// organized for use in Certificates, Gateways, VirtualServices.
+#ProjectHosts: {
+	project: #Project
+
+	// Hosts map key fqdn to host to reduce into structs organized by stage,
+	// canonical name, etc...  The flat nature and long list of properties is
+	// intended to make it straight forward to derive another struct for Gateways,
+	// VirtualServices, Certificates, AuthProxy cookie domains, etc...
+	Hosts: {
+		for Env in project.environments {
+			for Host in project.hosts {
+				// Global hostname, e.g. app.holos.run
+				let CertInfo = (#MakeCertInfo & {
+					host:       Host
+					env:        Env
+					domain:     project.domain
+					clusterMap: project.clusters
+				}).CertInfo
+
+				"\(CertInfo.fqdn)": CertInfo
+
+				// Cluster hostname, e.g. app.east1.holos.run, app.west1.holos.run
+				for Cluster in project.clusters {
+					let CertInfo = (#MakeCertInfo & {
+						host:       Host
+						env:        Env
+						domain:     project.domain
+						cluster:    Cluster.name
+						clusterMap: project.clusters
+					}).CertInfo
+
+					"\(CertInfo.fqdn)": CertInfo
+				}
+			}
+		}
+	}
+}
+
+// #MakeCertInfo provides dns info for a certificate
+// Refer to: https://github.com/holos-run/holos/issues/66#issuecomment-2027562626
+#MakeCertInfo: {
+	host:       #Host
+	env:        #Environment
+	domain:     string
+	cluster:    string
+	clusterMap: #ClusterMap
+
+	let Stage = #StageInfo & {name: env.stage, project: env.project}
+	let Env = env
+
+	// DNS segments from left to right.
+	let EnvSegments = env.envSegments
+
+	WildcardSegments: [...string]
+	if len(env.envSegments) > 0 {
+		WildcardSegments: ["*"]
+	}
+
+	let HostSegments = [host.name]
+
+	let StageSegments = env.stageSegments
+
+	ClusterSegments: [...string]
+	if cluster != _|_ {
+		ClusterSegments: [cluster]
+	}
+
+	let DomainSegments = [domain]
+
+	// Assemble the segments
+
+	let FQDN = EnvSegments + HostSegments + StageSegments + ClusterSegments + DomainSegments
+	let WILDCARD = WildcardSegments + HostSegments + StageSegments + ClusterSegments + DomainSegments
+	let CANONICAL = HostSegments + StageSegments + DomainSegments
+
+	CertInfo: #CertInfo & {
+		fqdn:      strings.Join(FQDN, ".")
+		wildcard:  strings.Join(WILDCARD, ".")
+		canonical: strings.Join(CANONICAL, ".")
+
+		project: name: Env.project
+		stage: #StageOrEnvRef & {
+			name:      Stage.name
+			slug:      Stage.slug
+			namespace: Stage.namespace
+		}
+		env: #StageOrEnvRef & {
+			name:      Env.name
+			slug:      Env.slug
+			namespace: Env.namespace
+		}
+
+		if cluster != _|_ {
+			// Host is valid on a single cluster.
+			clusters: "\(cluster)": _
+		}
+
+		if cluster == _|_ {
+			// Host is valid on all project clusters.
+			clusters: clusterMap
+		}
+
+		NoAuthorizationPolicy: host.NoAuthorizationPolicy
+	}
+}
+
+// #CertInfo defines the attributes associated with a fully qualfied domain name
+#CertInfo: {
+	// fqdn is the fully qualified domain name, never a wildcard.
+	fqdn: string
+	// canonical is the canonical name this name may be an alternate name for.
+	canonical: string
+	// wildcard may replace the left most segment fqdn with a wildcard to consolidate cert dnsNames.  If not a wildcad, must be fqdn
+	wildcard: string
+
+	// Project, stage and env attributes for mapping and collecting.
+	project: name: string
+
+	stage: #StageOrEnvRef
+	env:   #StageOrEnvRef
+
+	// clusters represents the cluster names the fqdn is valid on.
+	clusters: #ClusterMap
+	// hosts are always valid on the provisioner cluster
+	clusters: provisioner: _
+
+	// NoAuthorizationPolicy excludes the host from the auth proxy integrated with
+	// the default ingress Gateway.
+	NoAuthorizationPolicy: true | *false
+}
+
+#ClusterMap: [Name=string]: #Cluster & {name: Name}
+
+#StageOrEnvRef: {
+	name:      string
+	slug:      string
+	namespace: string
+}

docs/examples/project-template-provisioner-certs.cue

@@ -0,0 +1,45 @@
+package holos
+
+#ProjectTemplate: {
+	project:        _
+	GatewayServers: _
+
+	// Sort GatewayServers by the tls credentialName to issue wildcards
+	let GatewayCerts = {
+		for FQDN, Server in GatewayServers {
+			let CertInfo = Server._CertInfo
+
+			// Sort into stage for the holos components, e.g. prod-iam-certs, dev-iam-certs
+			"\(CertInfo.stage.slug)": {
+				"\(Server.tls.credentialName)": #Certificate & {
+
+					// Store the dnsNames in a struct so they can be collected into a list
+					_dnsNames: "\(CertInfo.wildcard)": CertInfo.wildcard
+
+					metadata: name:      CertInfo.canonical & Server.tls.credentialName
+					metadata: namespace: "istio-ingress"
+					spec: {
+						commonName: CertInfo.canonical
+						secretName: CertInfo.canonical & Server.tls.credentialName
+						dnsNames: [for x in _dnsNames {x}]
+						issuerRef: {
+							kind: "ClusterIssuer"
+							name: "letsencrypt"
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// Resources to be managed on the provisioner cluster.
+	provisioner: resources: {
+		for stage in project.stages {
+			"\(stage.slug)-certs": #KubernetesObjects & {
+				apiObjectMap: (#APIObjects & {
+					apiObjects: Certificate: GatewayCerts[stage.slug]
+				}).apiObjectMap
+			}
+		}
+	}
+}

docs/examples/project-template.cue

@@ -1,87 +1,77 @@
 package holos
 
-import "encoding/yaml"
+import (
+	h "github.com/holos-run/holos/api/v1alpha1"
+	"encoding/yaml"
+	"strings"
+)
 
-// Platform level definition of a project.
-#Project: {
-	name: string
+// let SourceLoc = "project-template.cue"
 
-	// All projects have at least a prod environment and stage.
-	stages: prod: stageSegments: []
-	environments: prod: stage: "prod"
-	environments: prod: envSegments: []
-	stages: dev: _
-	environments: dev: stage: "dev"
-	environments: dev: envSegments: []
-	// Ensure at least the project name is a short hostname.  Additional may be added.
-	hosts: (name): _
+#ProjectTemplate: {
+	project: #Project
 
-	// environments share the stage segments of their stage.
-	environments: [_]: {
-		stage:         string
-		stageSegments: stages[stage].stageSegments
+	// workload cluster resources
+	workload: resources: [Name=_]: h.#KubernetesObjects & {
+		metadata: name: Name
+	}
+
+	// provisioner cluster resources
+	provisioner: resources: [Name=_]: h.#KubernetesObjects & {
+		metadata: name: Name
 	}
 }
 
+// Reference Platform Project Template
 #ProjectTemplate: {
 	project: #Project
 	let Project = project
 
-	// GatewayServers maps Gateway spec.servers #GatewayServer values indexed by stage then name.
-	let GatewayServers = {
-		// Initialize all stages, even if they have no environments.
-		for stage in project.stages {
-			(stage.name): {}
-		}
+	// ProjectHosts represents all of the hosts associated with a project indexed
+	// by FQDN with #CertInfo values.  Slice and dice this struct as needed to
+	// work with hosts in the platform.
+	ProjectHosts: (#ProjectHosts & {project: Project}).Hosts
 
-		// For each stage, construct entries for the Gateway spec.servers.hosts field.
-		for env in project.environments {
-			(env.stage): {
-				let Env = env
-				let Stage = project.stages[env.stage]
-				for host in (#EnvHosts & {project: Project, env: Env}).hosts {
-					(host.name): #GatewayServer & {
-						hosts: [
-							"\(env.namespace)/\(host.name)",
-							// Allow the authproxy VirtualService to match the project.authProxyPrefix path.
-							"\(Stage.namespace)/\(host.name)",
-						]
-						port: host.port
-						tls: credentialName: host.name
-						tls: mode:           "SIMPLE"
+	// GatewayServers maps Gateway spec.servers #GatewayServer values indexed by stage then name.
+	GatewayServers: {
+		for FQDN, Host in ProjectHosts {
+			// If the host is valid on the cluster being rendered
+			if Host.clusters[#ClusterName] != _|_ {
+				"\(FQDN)": #GatewayServer & {
+					_CertInfo: Host
+					hosts: [
+						"\(Host.env.namespace)/\(FQDN)",
+						// Allow the authproxy VirtualService to match the project.authProxyPrefix path.
+						"\(Host.stage.namespace)/\(FQDN)",
+					]
+					port: {
+						// NOTE: port names in servers must be unique: duplicate name https
+						name:     "https-" + strings.Replace(FQDN, ".", "-", -1)
+						number:   443
+						protocol: "HTTPS"
 					}
+					tls: credentialName: Host.canonical
+					tls: mode:           "SIMPLE"
 				}
 			}
 		}
 	}
 
-	// ClusterGatewayServers provides a struct of Gateway servers for the current cluster.
+	// ClusterDefaultGatewayServers provides a struct of Gateway servers for the current cluster.
 	// This is intended for Gateway/default to add all servers to the default gateway.
-	ClusterGatewayServers: {
+	ClusterDefaultGatewayServers: {
 		if project.clusters[#ClusterName] != _|_ {
-			for Stage in project.stages {
-				for server in GatewayServers[Stage.name] {
-					(server.port.name): server
-				}
-			}
+			GatewayServers
 		}
 	}
 
 	workload: resources: {
-		// Provide resources only if the project is managed on --cluster-name
+		// Provide resources only if the project is managed on the cluster specified
+		// by --cluster-name
 		if project.clusters[#ClusterName] != _|_ {
 			for stage in project.stages {
 				let Stage = stage
 
-				// Istio Gateway
-				"\(stage.slug)-gateway": #KubernetesObjects & {
-					apiObjectMap: (#APIObjects & {
-						for host in GatewayServers[stage.name] {
-							apiObjects: ExternalSecret: (host.tls.credentialName): metadata: namespace: "istio-ingress"
-						}
-					}).apiObjectMap
-				}
-
 				// Manage auth-proxy in each stage
 				if project.features.authproxy.enabled {
 					"\(stage.slug)-authproxy": #KubernetesObjects & {
@@ -114,90 +104,6 @@ import "encoding/yaml"
 			}
 		}
 	}
-
-	provisioner: resources: {
-		for stage in project.stages {
-			"\(stage.slug)-certs": #KubernetesObjects & {
-				apiObjectMap: (#APIObjects & {
-					for host in GatewayServers[stage.name] {
-						let CN = host.tls.credentialName
-						apiObjects: Certificate: (CN): #Certificate & {
-							metadata: name:      CN
-							metadata: namespace: "istio-ingress"
-							spec: {
-								commonName: CN
-								dnsNames: [CN]
-								secretName: CN
-								issuerRef: {
-									kind: "ClusterIssuer"
-									name: "letsencrypt"
-								}
-							}
-						}
-					}
-				}).apiObjectMap
-			}
-		}
-	}
-}
-
-let HTTPBIN = {
-	name:    string | *"httpbin"
-	project: #Project
-	env:     #Environment
-	let Name = name
-	let Stage = project.stages[env.stage]
-
-	let Metadata = {
-		name:      Name
-		namespace: env.namespace
-		labels: app: name
-	}
-	let Labels = {
-		"app.kubernetes.io/name":       Name
-		"app.kubernetes.io/instance":   env.slug
-		"app.kubernetes.io/part-of":    env.project
-		"security.holos.run/authproxy": Stage.extAuthzProviderName
-	}
-
-	apiObjects: {
-		Deployment: (Name): #Deployment & {
-			metadata: Metadata
-
-			spec: selector: matchLabels: Metadata.labels
-			spec: template: {
-				metadata: labels: Metadata.labels & #IstioSidecar & Labels
-				spec: securityContext: seccompProfile: type: "RuntimeDefault"
-				spec: containers: [{
-					name:  Name
-					image: "quay.io/holos/mccutchen/go-httpbin"
-					ports: [{containerPort: 8080}]
-					securityContext: {
-						seccompProfile: type: "RuntimeDefault"
-						allowPrivilegeEscalation: false
-						runAsNonRoot:             true
-						runAsUser:                8192
-						runAsGroup:               8192
-						capabilities: drop: ["ALL"]
-					}}]
-			}
-		}
-		Service: (Name): #Service & {
-			metadata: Metadata
-			spec: selector: Metadata.labels
-			spec: ports: [
-				{port: 80, targetPort: 8080, protocol: "TCP", name: "http"},
-			]
-		}
-		VirtualService: (Name): #VirtualService & {
-			metadata: Metadata
-			let Project = project
-			let Env = env
-			spec: hosts: [for host in (#EnvHosts & {project: Project, env: Env}).hosts {host.name}]
-			spec: gateways: ["istio-ingress/default"]
-			spec: http: [{route: [{destination: host: Name}]}]
-		}
-	}
 }
 
 // AUTHPROXY configures one oauth2-proxy deployment for each host in each stage of a project.  Multiple deployments per stage are used to narrow down the cookie domain.
@@ -462,6 +368,65 @@ let AUTHPROXY = {
 	}
 }
 
+let HTTPBIN = {
+	name:    string | *"httpbin"
+	project: #Project
+	env:     #Environment
+	let Name = name
+	let Stage = project.stages[env.stage]
+
+	let Metadata = {
+		name:      Name
+		namespace: env.namespace
+		labels: app: name
+	}
+	let Labels = {
+		"app.kubernetes.io/name":       Name
+		"app.kubernetes.io/instance":   env.slug
+		"app.kubernetes.io/part-of":    env.project
+		"security.holos.run/authproxy": Stage.extAuthzProviderName
+	}
+
+	apiObjects: {
+		Deployment: (Name): #Deployment & {
+			metadata: Metadata
+
+			spec: selector: matchLabels: Metadata.labels
+			spec: template: {
+				metadata: labels: Metadata.labels & #IstioSidecar & Labels
+				spec: securityContext: seccompProfile: type: "RuntimeDefault"
+				spec: containers: [{
+					name:  Name
+					image: "quay.io/holos/mccutchen/go-httpbin"
+					ports: [{containerPort: 8080}]
+					securityContext: {
+						seccompProfile: type: "RuntimeDefault"
+						allowPrivilegeEscalation: false
+						runAsNonRoot:             true
+						runAsUser:                8192
+						runAsGroup:               8192
+						capabilities: drop: ["ALL"]
+					}}]
+			}
+		}
+		Service: (Name): #Service & {
+			metadata: Metadata
+			spec: selector: Metadata.labels
+			spec: ports: [
+				{port: 80, targetPort: 8080, protocol: "TCP", name: "http"},
+			]
+		}
+		VirtualService: (Name): #VirtualService & {
+			metadata: Metadata
+			let Project = project
+			let Env = env
+			spec: hosts: [for host in (#EnvHosts & {project: Project, env: Env}).hosts {host.name}]
+			spec: gateways: ["istio-ingress/default"]
+			spec: http: [{route: [{destination: host: Name}]}]
+		}
+	}
+}
+
 // AUTHPOLICY configures the baseline AuthorizationPolicy and RequestAuthentication policy for each stage of each project.
 let AUTHPOLICY = {
 	project: #Project

docs/examples/project_types.cue

@@ -1,12 +1,12 @@
 package holos
 
-import h "github.com/holos-run/holos/api/v1alpha1"
-
 import "strings"
 
 // #Projects is a map of all the projects in the platform.
 #Projects: [Name=_]: #Project & {name: Name}
 
+_Projects: #Projects
+
 // The platform project is required and where platform services reside.  ArgoCD, Grafana, Prometheus, etc...
 #Projects: platform: _
 
@@ -59,7 +59,7 @@ import "strings"
 		}
 	}
 
-	// features is YAGNI maybe? 
+	// Thes are useful to enable / disable.
 	features: [Name=string]: #Feature & {name: Name}
 	features: authproxy: _
 	features: httpbin:   _
@@ -68,8 +68,13 @@ import "strings"
 // #Cluster defines a cluster
 #Cluster: name: string
 
-// #Host defines a short hostname
-#Host: name: string
+#Host: {
+	// #Host defines a short hostname
+	name: string
+	// NoAuthorizationPolicy excludes the host from the auth proxy integrated with
+	// the default ingress Gateway.
+	NoAuthorizationPolicy: true | *false
+}
 
 #Environment: {
 	// name uniquely identifies the environment within the scope of the project.
@@ -91,15 +96,25 @@ import "strings"
 		name:     string
 		cluster?: string
 		clusterSegments: [...string]
+		wildcard: true | *false
 		if cluster != _|_ {
 			clusterSegments: [cluster]
 		}
-		let SEGMENTS = envSegments + [name] + stageSegments + clusterSegments + [#Platform.org.domain]
+		_EnvSegments: [...string]
+		if wildcard {
+			if len(envSegments) > 0 {
+				_EnvSegments: ["*"]
+			}
+		}
+		if !wildcard {
+			_EnvSegments: envSegments
+		}
+		let SEGMENTS = _EnvSegments + [name] + stageSegments + clusterSegments + [_Projects[project].domain]
 		let NAMESEGMENTS = ["https"] + SEGMENTS
 		host: {
 			name: strings.Join(SEGMENTS, ".")
 			port: {
-				name:     strings.Replace(strings.Join(NAMESEGMENTS, "-"), ".", "-", -1)
+				name:     strings.Replace(strings.Replace(strings.Join(NAMESEGMENTS, "-"), ".", "-", -1), "*", "wildcard", -1)
 				number:   443
 				protocol: "HTTPS"
 			}
@@ -107,17 +122,26 @@ import "strings"
 	}
 }
 
-#Stage: {
+#StageInfo: {
 	name:    string
 	project: string
 	slug:    "\(name)-\(project)"
 	// namespace is the system namespace for the project stage
 	namespace: "\(name)-\(project)-system"
+}
+
+#Stage: {
+	#StageInfo
+	name:      string
+	project:   string
+	namespace: string
+	slug:      string
+
 	// Manage a system namespace for each stage
 	namespaces: [Name=_]: name: Name
-	namespaces: (namespace): _
+	namespaces: "\(namespace)": _
 	// stageSegments are the stage portion of the dns segments
-	stageSegments: [...string] | *[name]
+	stageSegments: [] | *[name]
 	// authProxyClientID is the ClientID registered with the oidc issuer.
 	authProxyClientID: string
 	// extAuthzProviderName is the provider name in the mesh config
@@ -130,20 +154,6 @@ import "strings"
 	enabled:     true | *false
 }
 
-#ProjectTemplate: {
-	project: #Project
-
-	// workload cluster resources
-	workload: resources: [Name=_]: h.#KubernetesObjects & {
-		metadata: name: Name
-	}
-
-	// provisioner cluster resources
-	provisioner: resources: [Name=_]: h.#KubernetesObjects & {
-		metadata: name: Name
-	}
-}
-
 // #EnvHosts provides hostnames given a project and environment.
 // Refer to https://github.com/holos-run/holos/issues/66#issuecomment-2027562626
 #EnvHosts: {
@@ -166,7 +176,7 @@ import "strings"
 }
 
 // #StageDomains provides hostnames given a project and stage.  Primarily for the
-// auth proxy.
+// auth proxy cookie domains.
 // Refer to https://github.com/holos-run/holos/issues/66#issuecomment-2027562626
 #StageDomains: {
 	// names are the leading prefix names to create hostnames for.

docs/examples/schema.cue

@@ -15,6 +15,7 @@ import (
 	crt "cert-manager.io/certificate/v1"
 	gw "networking.istio.io/gateway/v1beta1"
 	vs "networking.istio.io/virtualservice/v1beta1"
+	dr "networking.istio.io/destinationrule/v1beta1"
 	ra "security.istio.io/requestauthentication/v1"
 	ap "security.istio.io/authorizationpolicy/v1"
 	pg "postgres-operator.crunchydata.com/postgrescluster/v1beta1"
@@ -77,7 +78,9 @@ _apiVersion: "holos.run/v1alpha1"
 #Job:                   #NamespaceObject & batchv1.#Job
 #CronJob:               #NamespaceObject & batchv1.#CronJob
 #Deployment:            #NamespaceObject & appsv1.#Deployment
+#StatefulSet:           #NamespaceObject & appsv1.#StatefulSet
 #VirtualService:        #NamespaceObject & vs.#VirtualService
+#DestinationRule:       #NamespaceObject & dr.#DestinationRule
 #RequestAuthentication: #NamespaceObject & ra.#RequestAuthentication
 #AuthorizationPolicy:   #NamespaceObject & ap.#AuthorizationPolicy
 #Certificate:           #NamespaceObject & crt.#Certificate
@@ -182,7 +185,7 @@ _apiVersion: "holos.run/v1alpha1"
 	pool?: string
 	// region is the geographic region of the cluster.
 	region?: string
-	// primary is true if name matches the primaryCluster name
+	// primary is true if the cluster is the primary cluster among a group of related clusters.
 	primary: bool
 }
 
@@ -219,6 +222,7 @@ _apiVersion: "holos.run/v1alpha1"
 			primary: false
 		}
 	}
+	// TODO: Remove stages, they're in the subdomain of projects.
 	stages: [ID=_]: {
 		name: string & ID
 		environments: [...{name: string}]
@@ -226,9 +230,11 @@ _apiVersion: "holos.run/v1alpha1"
 	projects: [ID=_]: {
 		name: string & ID
 	}
+	// TODO: Remove services, they're in the subdomain of projects.
 	services: [ID=_]: {
 		name: string & ID
 	}
+
 	// authproxy configures the auth proxy attached to the default ingress gateway in the istio-ingress namespace.
 	authproxy: #AuthProxySpec & {
 		namespace: "istio-ingress"
@@ -277,29 +283,6 @@ _apiVersion: "holos.run/v1alpha1"
 	idTokenHeader: string | *"x-oidc-id-token"
 }
 
-// ManagedNamespace is a namespace to manage across all clusters in the holos platform.
-#ManagedNamespace: {
-	namespace: {
-		metadata: {
-			name: string
-			labels: [string]: string
-		}
-	}
-	// clusterNames represents the set of clusters the namespace is managed on.  Usually all clusters.
-	clusterNames: [...string]
-	for cluster in clusterNames {
-		clusters: (cluster): name: cluster
-	}
-}
-
-// #ManagedNamepsaces is the union of all namespaces across all cluster types and optional services.
-// Holos adopts the namespace sameness position of SIG Multicluster, refer to https://github.com/kubernetes/community/blob/dd4c8b704ef1c9c3bfd928c6fa9234276d61ad18/sig-multicluster/namespace-sameness-position-statement.md
-#ManagedNamespaces: {
-	[Name=_]: #ManagedNamespace & {
-		namespace: metadata: name: Name
-	}
-}
-
 // #Backups defines backup configuration.
 // TODO: Consider the best place for this, possibly as part of the site platform config.  This represents the primary location for backups.
 #Backups: {

docs/runbooks/argocd.md

@@ -0,0 +1,14 @@
+# ArgoCD
+
+Create the deploy key secret in the management cluster.
+
+```bash
+tmp="$(mktemp -d)"
+(cd $tmp && ssh-keygen -t ed25519 -f sshPrivateKey -m pem -C argocd -N '')
+echo git@github.com:holos-run/holos-infra.git > "${tmp}/url"
+holos create secret -n argocd --append-hash=false creds-holos-infra --from-file $tmp
+rm -rf "$tmp"
+```
+
+When syncing the secret, the ExternalSecret needs to set the label
+`argocd.argoproj.io/secret-type: repo-creds`.

docs/runbooks/login/backups.md

@@ -0,0 +1,97 @@
+# PostgresCluster Backups
+
+This document describes how the S3 bucket for `PostgresCluster` backups is configured.  These buckets are configured both for ZITADEL and for Holos 
+Server and are applicable to any service in Holos that stores data in a pgo `PostgresCluster` resource.
+
+## Create the Bucket
+Name: `holos-zitadel-backups` for `zitadel`
+Name: `holos-server-backups` for `holos server`
+> [!NOTE]
+> The settings below match the default settings recommended by AWS.
+
+Object Ownership: `ACLs disabled` (recommended) Checked.
+Block Public Access settings for this bucket: **`Block all public access`** Checked.
+Bucket Versioning: `Disable`
+Default encryption: `Server-side encryption with Amazon S3 managed keys (SSE-S3)`
+Bucket Key: `Enable`
+Object Lock: `Disable`
+
+## Create an IAM Policy
+Create one IAM Policy for each bucket to grant full access to the bucket.  Replace the resource with each bucket name.
+Name: `holos-zitadel-backups` for `zitadel`
+Name: `holos-server-backups` for `holos server`
+Description: `Read and write access to a specific bucket for pgrest operating within a pgo PostgresCluster.`
+
+Policy JSON:
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:GetBucketLocation",
+                "s3:ListAllMyBuckets"
+            ],
+            "Resource": "arn:aws:s3:::*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": [
+                "arn:aws:s3:::holos-zitadel-backups",
+                "arn:aws:s3:::holos-zitadel-backups/*"
+            ]
+        }
+    ]
+}
+```
+## Create an IAM Group
+Create an IAM Group to attach the policy granting access to the bucket.
+Name: `holos-zitadel-backups` for `zitadel`
+Attach permission policies: `holos-zitadel-backups`
+
+Name: `holos-server-backups` for `holos server`
+Attach permission policies: `holos-server-backups`
+## Create the IAM User
+Create an IAM User entity for each PostgresCluster.  Do not provide user access to the AWS Management Console.
+Name: `holos-zitadel-backups` for `zitadel`
+Group: `holos-zitadel-backups`
+
+Name: `holos-server-backups` for `holos server`
+Group: `holos-server-backups`
+
+## Create an Access Key
+Create an access key for `pgbackrest` associated with the `PostgresCluster`.
+
+Description: 
+> Used by pgbackrest associated with the PostgresCluster resource.  Refer to the PostgresCluster resource pgbackrest.cofiguration.secret.name for the stored location of the access key.  Synced from the Management Cluster using an ExternalSecret.
+## Create the Secret
+Create a `Secret` in the holos management cluster usable by pgbackrest.  This is a secret with a single key, `s3.conf` with the following format:
+```
+[global]
+repo2-cipher-pass=
+repo2-s3-key=
+repo2-s3-key-secret=
+repo3-cipher-pass=
+repo3-s3-key=
+repo3-s3-key-secret=
+```
+> [!NOTE]
+> Use the same values for repo2 and repo3.  The purpose is to make space for migrating if need be in the future.
+
+Generate the cipher pass using.  This password is used to encrypt all backups using client side before the backup is written to the bucket.
+```
+tr -dc A-Za-z0-9 </dev/urandom | head -c 64
+```
+
+Store the secret into the management cluster:
+```
+holos create secret --namespace zitadel holos-zitadel-backups \
+  --append-hash=false --from-file .
+```
+
+```
+holos create secret --namespace holos holos-server-backups \
+  --append-hash=false --from-file .
+```

docs/runbooks/login/pgbouncer.md

@@ -0,0 +1,92 @@
+# PG Bouncer
+
+Every few days ZITADEL fails.  The problem seems to be related to pgbouncer not
+being able to resolve DNS.  Restarting the pgbouncer pod fixes the issue.
+
+See [How to load-balance queries between several servers?](https://www.pgbouncer.org/faq.html#how-to-load-balance-queries-between-several-servers)
+
+> [!NOTE]
+> DNS round-robin. Use several IPs behind one DNS name. PgBouncer does not look up DNS each time a new connection is launched. Instead, it caches all IPs and does round-robin internally. Note: if there are more than 8 IPs behind one name, the DNS backend must support the EDNS0 protocol. See README for details.
+
+## Workaround
+
+```sh
+# Get the tls based creds to bypass oidc
+(cd ~/.kube && holos get secret core2-kubeconfig-admin --print-key kubeconfig.admin > core2.admin)
+export KUBECONFIG=$HOME/.kube/core2.admin
+# Restart pgbouncer
+kubectl -n prod-iam rollout restart deployment zitadel-pgbouncer
+```
+
+## Symptom logs
+
+```sh
+kubectl -n prod-iam logs -c pgbouncer -l postgres-operator.crunchydata.com/role=pgbouncer
+```
+
+```txt
+2024-05-08 17:56:11.424 UTC [7] LOG S-0x559b03f90ff0: zitadel/zitadel@10.110.109.110:5432 SSL established: TLSv1.3/TLS_AES_256_GCM_SHA384/ECDH=prime256v1
+2024-05-08 17:56:11.429 UTC [7] LOG S-0x559b03f92820: zitadel/zitadel@10.110.109.110:5432 new connection to server (from 10.244.5.38:53658)
+2024-05-08 17:56:11.435 UTC [7] LOG S-0x559b03f92820: zitadel/zitadel@10.110.109.110:5432 SSL established: TLSv1.3/TLS_AES_256_GCM_SHA384/ECDH=prime256v1
+2024-05-08 17:56:11.476 UTC [7] LOG C-0x559b03f7a610: zitadel/zitadel@10.244.2.89:34932 closing because: client close request (age=440s)
+2024-05-08 17:56:19.708 UTC [7] LOG stats: 15 xacts/s, 42 queries/s, 0 client parses/s, 0 server parses/s, 0 binds/s, in 6159 B/s, out 6124 B/s, xact 3930 us, query 869 us, wait 490 us
+[msg] Nameserver 10.96.0.10:53 is back up
+2024-05-08 17:57:09.366 UTC [7] LOG C-0x559b03f7a610: zitadel/zitadel@10.244.3.187:58674 login attempt: db=zitadel user=zitadel tls=TLSv1.3/TLS_AES_256_GCM_SHA384
+2024-05-08 17:57:09.391 UTC [7] LOG C-0x559b03f7a610: zitadel/zitadel@10.244.3.187:58674 closing because: client close request (age=0s)
+2024-05-08 17:57:19.709 UTC [7] LOG stats: 9 xacts/s, 24 queries/s, 0 client parses/s, 0 server parses/s, 0 binds/s, in 2870 B/s, out 3018 B/s, xact 4147 us, query 958 us, wait 23 us
+2024-05-08 17:58:19.708 UTC [7] LOG stats: 12 xacts/s, 32 queries/s, 0 client parses/s, 0 server parses/s, 0 binds/s, in 3861 B/s, out 3533 B/s, xact 3843 us, query 853 us, wait 0 us
+2024-05-08 17:56:11.411 UTC [8] LOG S-0x55a894e36650: zitadel/_crunchypgbouncer@10.110.109.110:5432 new connection to server (from 10.244.3.227:58984)
+2024-05-08 17:56:11.411 UTC [8] LOG S-0x55a894e37920: zitadel/zitadel@10.110.109.110:5432 new connection to server (from 10.244.3.227:58992)
+2024-05-08 17:56:11.418 UTC [8] LOG S-0x55a894e37920: zitadel/zitadel@10.110.109.110:5432 SSL established: TLSv1.3/TLS_AES_256_GCM_SHA384/ECDH=prime256v1
+2024-05-08 17:56:11.420 UTC [8] LOG S-0x55a894e36650: zitadel/_crunchypgbouncer@10.110.109.110:5432 SSL established: TLSv1.3/TLS_AES_256_GCM_SHA384/ECDH=prime256v1
+2024-05-08 17:56:11.438 UTC [8] LOG S-0x55a894e35b90: zitadel/zitadel@10.110.109.110:5432 new connection to server (from 10.244.3.227:59004)
+2024-05-08 17:56:11.445 UTC [8] LOG S-0x55a894e35b90: zitadel/zitadel@10.110.109.110:5432 SSL established: TLSv1.3/TLS_AES_256_GCM_SHA384/ECDH=prime256v1
+2024-05-08 17:56:17.148 UTC [8] LOG stats: 9 xacts/s, 27 queries/s, 0 client parses/s, 0 server parses/s, 0 binds/s, in 3236 B/s, out 2826 B/s, xact 5224 us, query 910 us, wait 1182 us
+[msg] Nameserver 10.96.0.10:53 is back up
+2024-05-08 17:57:17.145 UTC [8] LOG stats: 10 xacts/s, 31 queries/s, 0 client parses/s, 0 server parses/s, 0 binds/s, in 4342 B/s, out 4305 B/s, xact 4536 us, query 776 us, wait 0 us
+2024-05-08 17:58:17.149 UTC [8] LOG stats: 5 xacts/s, 15 queries/s, 0 client parses/s, 0 server parses/s, 0 binds/s, in 1641 B/s, out 1582 B/s, xact 7819 us, query 1426 us, wait 0 us
+```
+
+## Relevant Configuration
+
+`/etc/pgbouncer/pgbouncer.ini` is empty.
+
+```
+bash-4.4$ cat /etc/pgbouncer/~postgres-operator.ini
+# Generated by postgres-operator. DO NOT EDIT.
+# Your changes will not be saved.
+
+[pgbouncer]
+%include /etc/pgbouncer/pgbouncer.ini
+
+[pgbouncer]
+auth_file = /etc/pgbouncer/~postgres-operator/users.txt
+auth_query = SELECT username, password from pgbouncer.get_auth($1)
+auth_user = _crunchypgbouncer
+client_tls_ca_file = /etc/pgbouncer/~postgres-operator/frontend-ca.crt
+client_tls_cert_file = /etc/pgbouncer/~postgres-operator/frontend-tls.crt
+client_tls_key_file = /etc/pgbouncer/~postgres-operator/frontend-tls.key
+client_tls_sslmode = require
+conffile = /etc/pgbouncer/~postgres-operator.ini
+ignore_startup_parameters = extra_float_digits
+listen_addr = *
+listen_port = 5432
+server_tls_ca_file = /etc/pgbouncer/~postgres-operator/backend-ca.crt
+server_tls_sslmode = verify-full
+unix_socket_dir =
+
+[databases]
+* = host=zitadel-primary port=5432
+```
+
+### [host](https://www.pgbouncer.org/config.html#host)
+
+> Host name or IP address to connect to. Host names are resolved at connection time, the result is cached per dns_max_ttl parameter. When a host name’s resolution changes, existing server connections are automatically closed when they are released (according to the pooling mode), and new server connections immediately use the new resolution. If DNS returns several results, they are used in a round-robin manner.
+
+### dns_max_ttl
+
+[dns_max_ttl](https://www.pgbouncer.org/config.html#dns_max_ttl)
+
+How long DNS lookups can be cached. The actual DNS TTL is ignored.
+
+Default: 15.0 (seconds)

docs/runbooks/namespace.md

@@ -0,0 +1,25 @@
+# Namespaces
+
+Holos follows the [Namespace Sameness - Sig Multicluster Position][1].  A
+namespace is the same on all clusters within the scope of a platform.
+
+Namespaces are also security boundaries for role based access control.  As such,
+permission to read a secret in a namespace means the secret is readable on all
+clusters in the platform.
+
+When adding a component to a platform, create a namespace using the following
+process.  This ensures a namespace scoped `SecretStore` is created to sync
+`ExternalSecret` resources from the management cluster.
+
+ 1. Add a new project to the `_Projects` struct in `platform.cue`.
+ 2. Add the namespace to the `spec.namespaces` field of the project.
+ 3. Render the platform
+ 4. Apply the `namespaces` component to the management cluster
+ 5. Apply the `eso-creds-manager` component to the management cluster to create the `eso-reader` ksa for the namespace `SecretStore`
+ 6. Apply the `namespaces` component to the workload clusters
+ 7. On the workload cluster, run the job to fetch the eso-reader creds: `kubectl create job -n holos-system --from=cronjob/eso-creds-refresher eso-creds-refresher-$(date +%s)`
+ 8. Apply the secretstores component to the workload cluster.
+
+Your namespace is created and you have the ability to create secrets in the management cluster and pull them using ExternalSecret resources. (edited)  
+
+[1]: https://github.com/kubernetes/community/blob/dd4c8b704ef1c9c3bfd928c6fa9234276d61ad18/sig-multicluster/namespace-sameness-position-statement.md

docs/runbooks/workload-identity.md

@@ -0,0 +1,31 @@
+# Workload Identity
+
+When a new workload cluster is provisioned, allow it to access the Management
+Cluster using workload identity.  This is necessary for the
+`eso-creds-refresher` component and `Job` that executes in each workload
+cluster, which in turn enables the `SecretStore` in each namespace to sync
+secrets.
+
+Build the cluster with Cluster API.
+See https://github.com/holos-run/holos-infra/blob/main/hack/capi/eks/aws2/aws2-managedmachinepool.yaml#L81-L84
+
+## Workload Identity Provider
+Add the Cluster as a workload identity provider to the `holos-ops` gcp project.
+
+Pool: [holos](https://console.cloud.google.com/iam-admin/workload-identity-pools/pool/holos?organizationId=358674006047&project=holos-ops)
+Name: `k8s-aws1`, `k8s-aws2`, etc...
+### Issuer URL:
+```
+kubectl create -n default token default | cut -d. -f2 | base64 -d | jq -r .iss
+```
+
+### Audience
+Use the default audience.
+### Attribute Mapping
+
+| Google                           | OIDC                                                   |
+| -------------------------------- | ------------------------------------------------------ |
+| `google.subject`                 | `assertion.sub`                                        |
+| `attribute.service_account_name` | `assertion['kubernetes.io']['serviceaccount']['name']` |
+| `attribute.uid`                  | `assertion['kubernetes.io']['serviceaccount']['uid']`  |
+| `attribute.pod`                  | `assertion['kubernetes.io']['pod']['name']`            |

go.mod

@@ -5,27 +5,35 @@ go 1.21.5
 require (
 	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1
 	connectrpc.com/connect v1.16.0
+	connectrpc.com/grpcreflect v1.2.0
+	connectrpc.com/otelconnect v0.7.0
 	connectrpc.com/validate v0.1.0
 	cuelang.org/go v0.8.0
 	entgo.io/ent v0.13.1
+	github.com/bufbuild/buf v1.30.1
+	github.com/choria-io/machine-room v0.0.0-20240417064836-c604da2f005e
 	github.com/coreos/go-oidc/v3 v3.10.0
+	github.com/fullstorydev/grpcurl v1.9.1
 	github.com/go-jose/go-jose/v3 v3.0.3
 	github.com/gofrs/uuid v4.4.0+incompatible
-	github.com/google/uuid v1.5.0
+	github.com/int128/kubelogin v1.28.0
 	github.com/jackc/pgx/v5 v5.5.5
 	github.com/lmittmann/tint v1.0.4
 	github.com/mattn/go-isatty v0.0.20
-	github.com/mattn/go-runewidth v0.0.9
+	github.com/mattn/go-runewidth v0.0.15
+	github.com/mennanov/fieldmask-utils v1.1.2
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/prometheus/client_golang v1.19.0
 	github.com/rogpeppe/go-internal v1.12.0
 	github.com/sethvargo/go-retry v0.2.4
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.8.4
-	golang.org/x/net v0.22.0
-	golang.org/x/tools v0.19.0
-	google.golang.org/protobuf v1.33.0
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/net v0.24.0
+	golang.org/x/tools v0.20.0
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa
+	google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002
+	honnef.co/go/tools v0.4.7
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2
 	k8s.io/client-go v0.29.2
@@ -36,72 +44,212 @@ require (
 
 require (
 	ariga.io/atlas v0.19.1-0.20240203083654-5948b60a8e43 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cuelabs.dev/go/oci/ociregistry v0.0.0-20240314152124-224736b49f2e // indirect
+	github.com/AlecAivazis/survey/v2 v2.3.7 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/Freman/eventloghook v0.0.0-20191003051739-e4d803b6b48b // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/OneOfOne/xxhash v1.2.8 // indirect
+	github.com/achanda/go-sysctl v0.0.0-20160222034550-6be7678c45d2 // indirect
 	github.com/agext/levenshtein v1.2.1 // indirect
-	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect
+	github.com/agnivade/levenshtein v1.1.1 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bufbuild/protovalidate-go v0.3.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/bufbuild/protocompile v0.10.0 // indirect
+	github.com/bufbuild/protovalidate-go v0.6.0 // indirect
+	github.com/bufbuild/protoyaml-go v0.1.8 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/choria-io/fisk v0.6.2 // indirect
+	github.com/choria-io/go-choria v0.28.1-0.20240416190746-b3bf9c7d5a45 // indirect
+	github.com/choria-io/go-updater v0.1.0 // indirect
+	github.com/choria-io/stream-replicator v0.8.3-0.20230503130504-86152f798aec // indirect
+	github.com/choria-io/tokens v0.0.4-0.20240316144214-a929d9325d48 // indirect
+	github.com/cloudevents/sdk-go/v2 v2.15.2 // indirect
+	github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect
+	github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect
 	github.com/cockroachdb/apd/v3 v3.2.1 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/cli v26.0.0+incompatible // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v26.0.2+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.8.1 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/emicklei/proto v1.10.0 // indirect
+	github.com/envoyproxy/go-control-plane v0.12.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
+	github.com/expr-lang/expr v1.16.4 // indirect
+	github.com/fatih/color v1.16.0 // indirect
+	github.com/felixge/fgprof v0.9.4 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-chi/chi/v5 v5.0.12 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-openapi/inflect v0.19.0 // indirect
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gofrs/uuid/v5 v5.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/cel-go v0.17.4 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/cel-go v0.20.1 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-containerregistry v0.19.1 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20240416155748-26353dc0451f // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/google/wire v0.5.0 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
+	github.com/goss-org/GOnetstat v0.0.0-20230101144325-22be0bd9e64d // indirect
+	github.com/goss-org/go-ps v0.0.0-20230609005227-7b318e6a56e5 // indirect
+	github.com/goss-org/goss v0.4.6 // indirect
+	github.com/gosuri/uilive v0.0.4 // indirect
+	github.com/gosuri/uiprogress v0.0.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
+	github.com/guptarohit/asciigraph v0.7.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/hcl/v2 v2.13.0 // indirect
+	github.com/hashicorp/logutils v1.0.0 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/int128/listener v1.1.0 // indirect
+	github.com/int128/oauth2cli v1.14.0 // indirect
+	github.com/int128/oauth2dev v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jdx/go-netrc v1.0.0 // indirect
+	github.com/jhump/protoreflect v1.16.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/lib/pq v1.10.9 // indirect
+	github.com/looplab/fsm v1.0.1 // indirect
+	github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/miekg/dns v1.1.58 // indirect
+	github.com/miekg/pkcs11 v1.1.1 // indirect
+	github.com/minio/highwayhash v1.0.2 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.1 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/nats-io/jsm.go v0.1.1 // indirect
+	github.com/nats-io/jwt/v2 v2.5.5 // indirect
+	github.com/nats-io/nats-server/v2 v2.10.14 // indirect
+	github.com/nats-io/nats.go v1.34.1 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
-	github.com/onsi/ginkgo/v2 v2.15.0 // indirect
-	github.com/onsi/gomega v1.31.1 // indirect
+	github.com/oleiade/reflections v1.0.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.17.1 // indirect
+	github.com/onsi/gomega v1.32.0 // indirect
+	github.com/open-policy-agent/opa v0.63.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.48.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/pkg/profile v1.7.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.52.3 // indirect
+	github.com/prometheus/procfs v0.13.0 // indirect
 	github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/robfig/cron v1.2.0 // indirect
+	github.com/rs/cors v1.10.1 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/samber/lo v1.39.0 // indirect
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
+	github.com/segmentio/ksuid v1.0.4 // indirect
+	github.com/shirou/gopsutil/v3 v3.24.3 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
+	github.com/tidwall/gjson v1.17.1 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.13 // indirect
+	github.com/tklauser/numcpus v0.7.0 // indirect
+	github.com/vbatts/tar-split v0.11.5 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xlab/tablewriter v0.0.0-20160610135559-80b567a11ad5 // indirect
+	github.com/yashtewari/glob-intersection v0.2.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zclconf/go-cty v1.8.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect
-	golang.org/x/mod v0.16.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.25.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 // indirect
+	go.opentelemetry.io/otel/metric v1.25.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.25.0 // indirect
+	go.opentelemetry.io/otel/trace v1.25.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
+	golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
+	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240325203815-454cdb8f5daa // indirect
+	google.golang.org/grpc v1.62.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

hack/choria/gen-machine-signer

@@ -0,0 +1,22 @@
+#! /bin/bash
+#
+# build github.com/choria-io/go-choria with go build -trimpath -o choria -ldflags "-w" ./
+# Refer to https://github.com/ripienaar/choria-compose/blob/main/setup.sh#L41
+# Refer to https://github.com/holos-run/holos-infra/blob/v0.60.4/experiments/components/holos-saas/initialize/setup
+# choria jwt keys machine-signer.seed machine-signer.public
+
+set -euo pipefail
+
+PARENT="$(cd "$(dirname "$0")" && pwd)"
+
+tmpdir="$(mktemp -d)"
+finish() {
+  [[ -d "$tmpdir" ]] && rm -rf "$tmpdir"
+}
+trap finish EXIT
+cd "$tmpdir"
+
+mkdir machine-signer
+cd machine-signer
+choria jwt keys machine-signer.seed machine-signer.public
+holos create secret machine-signer --from-file .

hack/choria/initialize/.gitignore

@@ -0,0 +1,5 @@
+/issuer/
+/provisioner/
+/broker/
+/customers/
+/agents/

hack/choria/initialize/README.md

@@ -0,0 +1,20 @@
+Initialize machine room provisioning credentials
+
+When you want the holos controller to provision while operating in the current
+working directory, run:
+
+ 1. `init-choria-provisioner-creds` to populate secrets in the Holos
+    Provisioner Cluster (not to be confused with the Choria Provisioner).
+ 2. `make-provisioning-jwt` to issue a `provisioning.jwt` file for `holos
+    controller` to use.
+ 3. `holos controller --config=agent.cfg` to read `provisioning.jwt` and write
+    the provisioned config file and credentials to the current directory.
+
+Expect the controller to provision.
+
+Setup Notes:
+
+The holos server flag `--provisioner-seed` must match the issuer.seed value.
+To get the correct value to configure for holos server:
+
+    holos get secret choria-issuer --print-key=issuer.seed --namespace $NAMESPACE

hack/choria/initialize/init-choria-provisioner-creds

@@ -0,0 +1,64 @@
+#! /bin/bash
+#
+
+export BROKER_PASSWORD="$(LC_ALL=C tr -dc "[:alpha:]" </dev/random | tr '[:upper:]' '[:lower:]' | head -c 32)"
+export PROVISIONER_TOKEN="$(LC_ALL=C tr -dc "[:alpha:]" </dev/random | tr '[:upper:]' '[:lower:]' | head -c 32)"
+
+set -xeuo pipefail
+
+# Make sure gomplate is available
+gomplate --version
+
+PARENT="$(cd $(dirname "$0") && pwd)"
+TOPLEVEL="$(cd "${PARENT}" && git rev-parse --show-toplevel)"
+: "${NAMESPACE:=jeff-holos}"
+export NAMESPACE
+
+tmpdir="$(mktemp -d)"
+finish() {
+  [[ -d "$tmpdir" ]] && rm -rf "$tmpdir"
+}
+trap finish EXIT
+cd "$tmpdir"
+
+# Generate Secrets
+
+# Create organization issuer
+mkdir issuer
+choria jwt keys "./issuer/issuer.seed" "./issuer/issuer.public"
+ISSUER="$(<issuer/issuer.public)"
+export ISSUER
+
+# Provisioner token used for ???
+mkdir provisioner
+echo -n "${PROVISIONER_TOKEN}" > ./provisioner/token
+
+# Provisioner signer
+choria jwt keys ./provisioner/signer.seed ./provisioner/signer.public
+choria jwt client ./provisioner/signer.jwt provisioner_signer ./issuer/issuer.seed \
+  --public-key "$(<provisioner/signer.public)" --server-provisioner --validity $((100*365))d --issuer
+
+# Provisioner Secret
+mkdir -p provisioner/secret
+gomplate --input-dir "${PARENT}/templates/provisioner" --output-dir ./provisioner/secret/
+cp ./provisioner/signer.seed ./provisioner/secret/signer.seed
+cp ./provisioner/signer.jwt ./provisioner/secret/signer.jwt
+
+# Provisioner Broker
+mkdir broker
+choria jwt keys ./broker/broker.seed ./broker/broker.public
+choria jwt server ./broker/broker.jwt broker.holos.local "$(<broker/broker.public)" ./issuer/issuer.seed \
+  --org choria \
+  --collectives choria \
+  --subjects 'choria.node_metadata.>'
+gomplate --input-dir "${PARENT}/templates/broker/" --output-dir ./broker/
+echo -n "${BROKER_PASSWORD}" > ./broker/password
+
+mkdir agents
+choria jwt keys ./agents/signer.seed ./agents/signer.public
+
+# Now save the secrets
+holos create secret --append-hash=false --namespace $NAMESPACE choria-issuer --from-file=issuer
+holos create secret --append-hash=false --namespace $NAMESPACE choria-broker --from-file=broker
+holos create secret --append-hash=false --namespace $NAMESPACE choria-provisioner --from-file=provisioner/secret
+holos create secret --append-hash=false --namespace $NAMESPACE choria-agents --from-file=agents

hack/choria/initialize/make-provisioning-jwt

@@ -0,0 +1,50 @@
+#! /bin/bash
+#
+# Make a provisioner.jwt and put it in the current directory.
+#
+# Use the provisioner.jwt with `holos controller --config=controller.cfg` which
+# will read the jwt from the same directory as the config file.
+#
+# Refer to Arri's
+# [setup.sh](https://github.com/ripienaar/machine-room-mvp/blob/main/example/setup/setup.sh#L41)
+# And our own nites at https://github.com/holos-run/holos/issues/142
+
+PARENT="$(cd $(dirname "$0") && pwd)"
+OUTDIR="$(pwd)"
+
+: "${NAMESPACE:=jeff-holos}"
+
+tmpdir="$(mktemp -d)"
+finish() {
+  [[ -d "$tmpdir" ]] && rm -rvf "$tmpdir"
+}
+trap finish EXIT
+cd "$tmpdir"
+
+set -xeuo pipefail
+
+# e.g. jeff.provision.dev.k2.holos.run
+#
+kubectl -n $NAMESPACE get virtualservice choria-broker-wss -o json > vs.json
+jq --exit-status -r '.spec.hosts[0]' vs.json > host
+
+# Get the issuer.seed
+holos -n $NAMESPACE get secret choria-issuer --to-file issuer.seed
+
+# Get the provisioner token to embed in the provisioning.jwt file.
+holos -n $NAMESPACE get secret choria-provisioner --to-file token
+
+# The --token flag value must be the same value set in the token field of provisioner.yaml
+# Refer to https://github.com/ripienaar/machine-room-mvp/blob/main/example/setup/setup.sh#L41
+# Refer to https://github.com/ripienaar/machine-room-mvp/blob/main/example/setup/templates/provisioner/provisioner.yaml#L6
+choria jwt prov provisioning.jwt "issuer.seed" \
+  --token "$(<token)" \
+  --urls wss://$(<host):443 \
+  --default \
+  --protocol-v2 \
+  --insecure \
+  --update \
+  --validity 30d \
+  --extensions '{}'
+
+cp provisioning.jwt "${OUTDIR}/"

hack/choria/initialize/reset-choria-config

@@ -0,0 +1,23 @@
+#! /bin/bash
+#
+# This script resets the choria config for a Namespace
+
+PARENT="$(cd $(dirname "$0") && pwd)"
+: "${NAMESPACE:=jeff-holos}"
+export NAMESPACE
+
+set -xeuo pipefail
+
+KUBECONFIG=$HOME/.holos/kubeconfig.provisioner kubectl delete secret -n jeff-holos choria-agents choria-broker choria-provisioner choria-issuer
+
+"${PARENT}/init-choria-provisioner-creds"
+
+stamp="$(date)"
+
+kubectl -n $NAMESPACE annotate externalsecret choria-broker secret.holos.run/refresh="$stamp" --overwrite
+kubectl -n $NAMESPACE annotate externalsecret choria-provisioner secret.holos.run/refresh="$stamp" --overwrite
+
+kubectl -n $NAMESPACE wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' externalsecret choria-provisioner choria-broker
+
+kubectl -n $NAMESPACE rollout restart statefulset choria-broker
+kubectl -n $NAMESPACE rollout restart deployment choria-provisioner

hack/choria/initialize/templates/broker/broker.conf

@@ -0,0 +1,23 @@
+loglevel = info
+plugin.choria.stats_address = 0.0.0.0
+plugin.choria.stats_port = 8222
+plugin.choria.broker_network = true
+plugin.choria.network.client_port = 4222
+plugin.choria.network.peer_port = 5222
+plugin.choria.network.system.user = system
+plugin.choria.network.system.password = system
+plugin.choria.network.peers = nats://choria-broker-0.choria-broker:5222,nats://choria-broker-1.choria-broker:5222,nats://choria-broker-2.choria-broker:5222
+plugin.choria.use_srv = false
+plugin.choria.network.websocket_port = 4333
+
+plugin.security.provider = choria
+# NOTE: plugin.security.choria.ca must not be set or provisioning will fail
+# with a unhandled choria_provisioning purpose token error
+plugin.security.choria.certificate = /etc/choria-tls/tls.crt
+plugin.security.choria.key = /etc/choria-tls/tls.key
+plugin.security.choria.token_file = /etc/choria/broker.jwt
+plugin.security.choria.seed_file = /etc/choria/broker.seed
+plugin.choria.network.provisioning.client_password = {{ .Env.BROKER_PASSWORD }}
+
+plugin.security.issuer.names = choria
+plugin.security.issuer.choria.public = {{ .Env.ISSUER }}

hack/choria/initialize/templates/provisioner/ISSUER

@@ -0,0 +1 @@
+{{ .Env.ISSUER -}}

hack/choria/initialize/templates/provisioner/choria.cfg

@@ -0,0 +1,7 @@
+plugin.security.provider = choria
+plugin.security.choria.token_file = /etc/provisioner/signer.jwt
+plugin.security.choria.seed_file = /etc/provisioner/signer.seed
+
+identity = provisioner_signer
+
+plugin.choria.middleware_hosts = choria-broker-0.choria-broker:4222,choria-broker-1.choria-broker:4222,choria-broker-2.choria-broker:4222

hack/choria/initialize/templates/provisioner/entrypoint

@@ -0,0 +1,9 @@
+#! /bin/bash
+#
+
+set -xeuo pipefail
+
+mkdir -p /home/choria/bin
+install -m 0755 /etc/provisioner/helper.rb /home/choria/bin/helper.rb
+
+exec /usr/sbin/choria-provisioner --config=/etc/provisioner/provisioner.yaml --choria-config=/etc/provisioner/choria.cfg

hack/choria/initialize/templates/provisioner/helper.rb

@@ -0,0 +1,134 @@
+#!/usr/bin/ruby
+
+require "json"
+require "yaml"
+require "base64"
+require "net/http"
+require "openssl"
+
+def parse_input
+  input = STDIN.read
+
+  begin
+    File.open("/tmp/request.json", "w") {|f| f.write(input)}
+  rescue Exception
+  end
+
+  request = JSON.parse(input)
+  request["inventory"] = JSON.parse(request["inventory"])
+
+  request
+end
+
+def validate!(request, reply)
+  if request["identity"] && request["identity"].length == 0
+    reply["msg"] = "No identity received in request"
+    reply["defer"] = true
+    return false
+  end
+
+  unless request["ed25519_pubkey"]
+    reply["msg"] = "No ed15519 public key received"
+    reply["defer"] = true
+    return false
+  end
+
+  unless request["ed25519_pubkey"]
+    reply["msg"] = "No ed15519 directory received"
+    reply["defer"] = true
+    return false
+  end
+
+  if request["ed25519_pubkey"]["directory"].length == 0
+    reply["msg"] = "No ed15519 directory received"
+    reply["defer"] = true
+    return false
+  end
+
+  true
+end
+
+def publish_reply(reply)
+  begin
+    File.open("/tmp/reply.json", "w") {|f| f.write(reply.to_json)}
+  rescue Exception
+  end
+
+  puts reply.to_json
+end
+
+def publish_reply!(reply)
+  publish_reply(reply)
+  exit
+end
+
+def set_config!(request, reply)
+  # stub data the helper will fetch from the saas
+  customers = {
+        "one" => {
+            :brokers => "nats://managed.example.net:9222", # whoever is the leader for this site
+            :site => "customer_one",
+            :source => {
+                :host => "nats://cust_one:s3cret@saas-nats.choria.local",
+            }
+        }
+    }
+
+
+  customer = request["jwt"]["extensions"]["customer"]
+  brokers = customers[customer][:brokers]
+  source = customers[customer][:source]
+
+  reply["configuration"].merge!(
+    "identity" => request["identity"],
+    "loglevel" => "warn",
+    "plugin.choria.server.provision" => "false",
+    "plugin.choria.middleware_hosts" => brokers,
+    "plugin.security.issuer.names" => "choria",
+    "plugin.security.issuer.choria.public" => "{{ .Env.ISSUER }}",
+    "plugin.security.provider" => "choria",
+    "plugin.security.choria.token_file" => File.join(request["ed25519_pubkey"]["directory"], "server.jwt"),
+    "plugin.security.choria.seed_file" => File.join(request["ed25519_pubkey"]["directory"], "server.seed"),
+    "machine_room.role" => "leader",
+    "machine_room.site" => customers[customer][:site],
+    "machine_room.source.host" => source[:host],
+  )
+
+  reply["server_claims"].merge!(
+    "exp" => 5*60*60*24*365,
+    "pub_subjects" => [">"],
+    "permissions" => {
+      "streams" => true,
+      "submission" => true,
+      "service_host" => true,
+    }
+  )
+end
+
+reply = {
+  "defer" => false,
+  "msg" => "",
+  "certificate" => "",
+  "ca" => "",
+  "configuration" => {},
+  "server_claims" => {}
+}
+
+begin
+  request = parse_input
+
+  reply["msg"] = "Validating"
+  unless validate!(request, reply)
+    publish_reply!(reply)
+  end
+
+  set_config!(request, reply)
+
+  reply["msg"] = "Done"
+  publish_reply!(reply)
+rescue SystemExit
+rescue Exception
+  reply["msg"] = "Unexpected failure during provisioning: %s: %s" % [$!.class, $!.to_s]
+  reply["defer"] = true
+  publish_reply!(reply)
+end

hack/choria/initialize/templates/provisioner/provisioner.yaml

@@ -0,0 +1,17 @@
+workers: 4
+interval: 1m
+logfile: /dev/stdout
+loglevel: info
+# The entrypoint script installs this helper script.
+helper: /home/choria/bin/helper.rb
+token: "{{ .Env.PROVISIONER_TOKEN }}"
+choria_insecure: false
+site: holos
+broker_provisioning_password: "{{ .Env.BROKER_PASSWORD }}"
+jwt_verify_cert: "{{ .Env.ISSUER }}"
+jwt_signing_key: /etc/provisioner/signer.seed
+jwt_signing_token: /etc/provisioner/signer.jwt
+
+features:
+  jwt: true
+  ed25519: true

hack/choria/initialize/templates/provisioner/token

@@ -0,0 +1 @@
+{{ .Env.PROVISIONER_TOKEN -}}

hack/setup/bare

@@ -0,0 +1,16 @@
+#! /bin/bash
+
+set -euo pipefail
+TOPLEVEL="$(cd $(dirname "$0") && git rev-parse --show-toplevel)"
+
+host="jeff.app.dev.k2.holos.run:443"
+
+read -p "Reset all data in $host? " choice
+case "$choice" in
+  y|Y) echo "proceeding...";;
+  *) exit 1;;
+esac
+
+
+grpcurl -H "x-oidc-id-token: $(holos token)" $host holos.v1alpha1.SystemService.DropTables
+grpcurl -H "x-oidc-id-token: $(holos token)" $host holos.v1alpha1.SystemService.SeedDatabase

hack/tilt/Dockerfile

@@ -3,5 +3,6 @@ USER root
 WORKDIR /app
 ADD bin bin
 RUN chown -R app: /app
-USER app
+# Kubernetes requires the user to be numeric
+USER 8192
 ENTRYPOINT bin/holos server

hack/tilt/bin/tilt

@@ -0,0 +1,9 @@
+#! /bin/bash
+# Override kubeconfig so we can create it with local()
+set -euo pipefail
+TOPLEVEL="$(cd $(dirname "$0")/.. && pwd)"
+export NAMESPACE="${USER}-holos"
+export KUBECONFIG="${TOPLEVEL}/kubeconfig"
+envsubst < "${KUBECONFIG}.template" > "${KUBECONFIG}"
+export TILT_WRAPPER=1
+exec tilt "$@"

hack/tilt/k8s.yaml

@@ -101,15 +101,44 @@ spec:
   gateways:
     - istio-ingress/default
   hosts:
-    - '{developer}.holos.dev.k2.ois.run'
+    - '{developer}.app.dev.k2.holos.run'
   http:
-    - route:
+    - name: "coffee-ui"
+      match:
+      - uri:
+          prefix: "/ui"
+      route:
+      - destination:
+          host: coffee
+          port:
+            number: 4200
+    - name: "holos-api"
+      route:
         - destination:
             host: '{name}'
             port:
               number: {listen_port}
 ---
 apiVersion: v1
+kind: Service
+metadata:
+  name: coffee
+spec:
+  ports:
+  - protocol: TCP
+    port: 4200
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: coffee
+subsets:
+  - addresses:
+    - ip: 192.168.2.21
+    ports:
+    - port: 4200
+---
+apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: holos
@@ -120,117 +149,6 @@ metadata:
 imagePullSecrets:
   - name: kube-system-ecr-image-pull-creds
 ---
-apiVersion: security.istio.io/v1beta1
-kind: AuthorizationPolicy
-metadata:
-  labels:
-    app: '{name}'
-    holos.run/developer: '{developer}'
-  name: '{name}-allow-groups'
-  namespace: '{namespace}'
-spec:
-  action: ALLOW
-  rules:
-    - when:
-        - key: request.auth.claims[groups]
-          values:
-            - holos-developer
-            - holos-developer@openinfrastructure.co
-  selector:
-    matchLabels:
-      holos.run/authz: dev-holos-sso
----
-apiVersion: security.istio.io/v1beta1
-kind: AuthorizationPolicy
-metadata:
-  name: '{name}-allow-nothing'
-  namespace: '{namespace}'
-  labels:
-    app: '{name}'
-    holos.run/developer: '{developer}'
-spec:
-  action: ALLOW
-  selector:
-    matchLabels:
-      holos.run/authz: dev-holos-sso
----
-apiVersion: security.istio.io/v1beta1
-kind: AuthorizationPolicy
-metadata:
-  name: '{name}-allow-well-known-paths'
-  namespace: '{namespace}'
-  labels:
-    app: '{name}'
-    holos.run/developer: '{developer}'
-spec:
-  action: ALLOW
-  rules:
-    - to:
-        - operation:
-            paths:
-              - /healthz
-              - /metrics
-              - /callbacks/github
-  selector:
-    matchLabels:
-      holos.run/authz: dev-holos-sso
----
-apiVersion: security.istio.io/v1beta1
-kind: AuthorizationPolicy
-metadata:
-  name: '{name}-auth'
-  namespace: '{namespace}'
-  labels:
-    app: '{name}'
-    holos.run/developer: '{developer}'
-spec:
-  action: CUSTOM
-  provider:
-    name: dev-holos-sso
-  rules:
-    - to:
-        - operation:
-            notPaths:
-              - /healthz
-              - /metrics
-              - /callbacks/github
-      when:
-        - key: request.headers[Authorization]
-          notValues:
-            - Bearer *
-  selector:
-    matchLabels:
-      holos.run/authz: dev-holos-sso
----
-apiVersion: security.istio.io/v1beta1
-kind: RequestAuthentication
-metadata:
-  name: '{name}'
-  namespace: '{namespace}'
-  labels:
-    app: '{name}'
-    holos.run/developer: '{developer}'
-spec:
-  jwtRules:
-    - audiences:
-        - https://sso.dev.holos.run
-      forwardOriginalToken: true
-      fromHeaders:
-        - name: x-auth-request-access-token
-      issuer: https://idex.core.ois.run
-      jwksUri: https://idex.core.ois.run/keys
-    - audiences:
-        - holos-cli
-      forwardOriginalToken: true
-      fromHeaders:
-        - name: authorization
-          prefix: 'Bearer '
-      issuer: https://idex.core.ois.run
-      jwksUri: https://idex.core.ois.run/keys
-  selector:
-    matchLabels:
-      holos.run/authz: dev-holos-sso
----
 apiVersion: postgres-operator.crunchydata.com/v1beta1
 kind: PGAdmin
 metadata:
@@ -266,19 +184,9 @@ spec:
       databases:
         - holos
       options: 'SUPERUSER'
-    - name: kratos
-      databases:
-        - kratos
-      options: 'SUPERUSER'
-    - name: hydra
-      databases:
-        - hydra
-      options: 'SUPERUSER'
     - name: '{developer}'
       databases:
         - holos
-        - kratos
-        - hydra
         - '{developer}'
       options: 'SUPERUSER'
   # https://access.crunchydata.com/documentation/postgres-operator/latest/architecture/user-management

holos.go

@@ -1,10 +1,10 @@
 // Package holos defines types for the rest of the system.
 package holos
 
-// A PathCueMod is a string representing the filesystem path of a cue module.
-// It is given a unique type so the API is clear.
+// A PathCueMod is a string representing the absolute filesystem path of a cue
+// module.  It is given a unique type so the API is clear.
 type PathCueMod string
 
-// A InstancePath is a string representing the filesystem path of a holos instance.
-// It is given a unique type so the API is clear.
+// A InstancePath is a string representing the absolute filesystem path of a
+// holos instance.  It is given a unique type so the API is clear.
 type InstancePath string

internal/builder/builder.go

@@ -0,0 +1,302 @@
+// Package builder is responsible for building fully rendered kubernetes api
+// objects from various input directories. A directory may contain a platform
+// spec or a component spec.
+package builder
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"cuelang.org/go/cue/build"
+	"cuelang.org/go/cue/cuecontext"
+	"cuelang.org/go/cue/load"
+	"github.com/holos-run/holos/api/v1alpha1"
+
+	"github.com/holos-run/holos"
+	"github.com/holos-run/holos/internal/client"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+)
+
+const (
+	KubernetesObjects = v1alpha1.KubernetesObjectsKind
+	// Helm is the value of the kind field of holos build output indicating helm
+	// values and helm command information.
+	Helm = v1alpha1.HelmChartKind
+	// Skip is the value when the instance should be skipped
+	Skip = "Skip"
+	// KustomizeBuild is the value of the kind field of cue output indicating holos should process the component using kustomize build to render output.
+	KustomizeBuild = v1alpha1.KustomizeBuildKind
+)
+
+// An Option configures a Builder
+type Option func(*config)
+
+type config struct {
+	args    []string
+	cluster string
+}
+
+type Builder struct {
+	cfg config
+}
+
+// New returns a new *Builder configured by opts Option.
+func New(opts ...Option) *Builder {
+	var cfg config
+	for _, f := range opts {
+		f(&cfg)
+	}
+	b := &Builder{cfg: cfg}
+	return b
+}
+
+// Entrypoints configures the leaf directories Builder builds.
+func Entrypoints(args []string) Option {
+	return func(cfg *config) { cfg.args = args }
+}
+
+// Cluster configures the cluster name for the holos component instance.
+func Cluster(name string) Option {
+	return func(cfg *config) { cfg.cluster = name }
+}
+
+// Cluster returns the cluster name of the component instance being built.
+func (b *Builder) Cluster() string {
+	return b.cfg.cluster
+}
+
+// Instances returns the cue build instances being built.
+func (b *Builder) Instances(ctx context.Context, cfg *client.Config) ([]*build.Instance, error) {
+	log := logger.FromContext(ctx)
+
+	mod, err := b.findCueMod()
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+	dir := string(mod)
+
+	cueConfig := load.Config{Dir: dir}
+
+	// Get the platform model from the PlatformConfig
+	pc, err := client.LoadPlatformConfig(ctx, dir)
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+	data, err := json.Marshal(pc)
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+
+	// Refer to https://github.com/cue-lang/cue/blob/v0.7.0/cmd/cue/cmd/common.go#L429
+	cueConfig.Tags = append(cueConfig.Tags, "platform_config="+string(data))
+	if b.Cluster() != "" {
+		cueConfig.Tags = append(cueConfig.Tags, "cluster="+b.Cluster())
+	}
+	log.DebugContext(ctx, fmt.Sprintf("cue: tags %v", cueConfig.Tags))
+
+	prefix := []string{"cue", "export", "--out", "yaml"}
+	for _, tag := range cueConfig.Tags {
+		prefix = append(prefix, "-t", fmt.Sprintf("'%s'", tag))
+	}
+
+	// Make args relative to the module directory
+	args := make([]string, len(b.cfg.args))
+	for idx, path := range b.cfg.args {
+		target, err := filepath.Abs(path)
+		if err != nil {
+			return nil, errors.Wrap(fmt.Errorf("could not find absolute path: %w", err))
+		}
+		relPath, err := filepath.Rel(dir, target)
+		if err != nil {
+			return nil, errors.Wrap(fmt.Errorf("invalid argument, must be relative to cue.mod: %w", err))
+		}
+		relPath = "./" + relPath
+		args[idx] = relPath
+
+		equiv := make([]string, len(prefix), 1+len(prefix))
+		copy(equiv, prefix)
+		equiv = append(equiv, relPath)
+		log.Debug(strings.Join(equiv, " "), "comment", "cue equivalent command")
+	}
+
+	return load.Instances(args, &cueConfig), nil
+}
+
+func (b *Builder) Run(ctx context.Context, cfg *client.Config) (results []*v1alpha1.Result, err error) {
+	log := logger.FromContext(ctx)
+	log.DebugContext(ctx, "cue: building instances")
+	instances, err := b.Instances(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+	results = make([]*v1alpha1.Result, 0, len(instances)*8)
+
+	// Each CUE instance provides a BuildPlan
+	for idx, instance := range instances {
+		log.DebugContext(ctx, "cue: building instance", "idx", idx, "dir", instance.Dir)
+		r, err := b.runInstance(ctx, instance)
+		if err != nil {
+			return nil, errors.Wrap(fmt.Errorf("could not run: %w", err))
+		}
+		results = append(results, r...)
+	}
+
+	return results, nil
+}
+
+func (b Builder) runInstance(ctx context.Context, instance *build.Instance) (results []*v1alpha1.Result, err error) {
+	path := holos.InstancePath(instance.Dir)
+	log := logger.FromContext(ctx).With("dir", path)
+
+	if err := instance.Err; err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not load: %w", err))
+	}
+	cueCtx := cuecontext.New()
+	value := cueCtx.BuildInstance(instance)
+	if err := value.Err(); err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not build %s: %w", instance.Dir, err))
+	}
+	log.DebugContext(ctx, "cue: validating instance")
+	if err := value.Validate(); err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not validate: %w", err))
+	}
+
+	log.DebugContext(ctx, "cue: decoding holos build plan")
+	jsonBytes, err := value.MarshalJSON()
+	if err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not marshal cue instance %s: %w", instance.Dir, err))
+	}
+	decoder := json.NewDecoder(bytes.NewReader(jsonBytes))
+	// Discriminate the type of build plan.
+	tm := &v1alpha1.TypeMeta{}
+	err = decoder.Decode(tm)
+	if err != nil {
+		return nil, errors.Wrap(fmt.Errorf("invalid BuildPlan: %s: %w", instance.Dir, err))
+	}
+
+	log.DebugContext(ctx, "cue: discriminated build kind: "+tm.Kind, "kind", tm.Kind, "apiVersion", tm.APIVersion)
+
+	// New decoder for the full object
+	decoder = json.NewDecoder(bytes.NewReader(jsonBytes))
+
+	// TODO: When we release v1, explicitly allow unknown fields so we can add
+	// fields without needing to bump the major version.  Disallow until we reach
+	// v1 for clear error reporting.
+	decoder.DisallowUnknownFields()
+
+	switch tm.Kind {
+	// TODO(jeff) Process a v1alpha1.Result here, the result is tightly coupled to a BuildPlan.
+	case "BuildPlan":
+		var bp v1alpha1.BuildPlan
+		if err = decoder.Decode(&bp); err != nil {
+			err = errors.Wrap(fmt.Errorf("could not decode BuildPlan %s: %w", instance.Dir, err))
+			return
+		}
+		results, err = b.buildPlan(ctx, &bp, path)
+		if err != nil {
+			return results, err
+		}
+	default:
+		err = errors.Wrap(fmt.Errorf("unknown kind: %v", tm.Kind))
+	}
+
+	return results, err
+}
+
+func (b *Builder) buildPlan(ctx context.Context, buildPlan *v1alpha1.BuildPlan, path holos.InstancePath) (results []*v1alpha1.Result, err error) {
+	log := logger.FromContext(ctx)
+
+	if err := buildPlan.Validate(); err != nil {
+		log.WarnContext(ctx, "could not validate", "skipped", true, "err", err)
+		return nil, errors.Wrap(fmt.Errorf("could not validate %w", err))
+	}
+
+	if buildPlan.Spec.Disabled {
+		log.DebugContext(ctx, "skipped: spec.disabled is true", "skipped", true)
+		return
+	}
+
+	// TODO: concurrent renders
+	results = make([]*v1alpha1.Result, 0, buildPlan.ResultCapacity())
+	log.DebugContext(ctx, "allocated results slice", "cap", buildPlan.ResultCapacity())
+	for _, component := range buildPlan.Spec.Components.Resources {
+		if result, err := component.Render(ctx, path); err != nil {
+			return nil, errors.Wrap(fmt.Errorf("could not render: %w", err))
+		} else {
+			results = append(results, result)
+		}
+	}
+
+	for _, component := range buildPlan.Spec.Components.KubernetesObjectsList {
+		if result, err := component.Render(ctx, path); err != nil {
+			return nil, errors.Wrap(fmt.Errorf("could not render: %w", err))
+		} else {
+			results = append(results, result)
+		}
+	}
+	for _, component := range buildPlan.Spec.Components.HelmChartList {
+		if result, err := component.Render(ctx, path); err != nil {
+			return nil, errors.Wrap(fmt.Errorf("could not render: %w", err))
+		} else {
+			results = append(results, result)
+		}
+	}
+	for _, component := range buildPlan.Spec.Components.KustomizeBuildList {
+		if result, err := component.Render(ctx, path); err != nil {
+			return nil, errors.Wrap(fmt.Errorf("could not render: %w", err))
+		} else {
+			results = append(results, result)
+		}
+	}
+
+	// Add a separate Result if there are DeployFiles from the BuildPlan.
+	if len(buildPlan.Spec.DeployFiles) > 0 {
+		results = append(results, &v1alpha1.Result{
+			HolosComponent: v1alpha1.HolosComponent{
+				TypeMeta: buildPlan.TypeMeta,
+				Metadata: buildPlan.Metadata,
+			},
+			DeployFiles: buildPlan.Spec.DeployFiles,
+		})
+	}
+
+	log.DebugContext(ctx, "returning results", "len", len(results))
+
+	return results, nil
+}
+
+// findCueMod returns the root module location containing the cue.mod file or
+// directory or an error if the builder arguments do not share a common root
+// module.
+func (b *Builder) findCueMod() (dir holos.PathCueMod, err error) {
+	for _, origPath := range b.cfg.args {
+		absPath, err := filepath.Abs(origPath)
+		if err != nil {
+			return "", err
+		}
+		path := holos.PathCueMod(absPath)
+		for {
+			if _, err := os.Stat(filepath.Join(string(path), "cue.mod")); err == nil {
+				if dir != "" && dir != path {
+					return "", fmt.Errorf("multiple modules not supported: %v is not %v", dir, path)
+				}
+				dir = path
+				break
+			} else if !os.IsNotExist(err) {
+				return "", err
+			}
+			parentPath := holos.PathCueMod(filepath.Dir(string(path)))
+			if parentPath == path {
+				return "", fmt.Errorf("no cue.mod from root to leaf: %v", origPath)
+			}
+			path = parentPath
+		}
+	}
+	return dir, nil
+}

internal/builder/platform.go

@@ -0,0 +1,90 @@
+package builder
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"cuelang.org/go/cue/build"
+	"cuelang.org/go/cue/cuecontext"
+	"github.com/holos-run/holos"
+	"github.com/holos-run/holos/api/v1alpha1"
+	"github.com/holos-run/holos/internal/client"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/logger"
+)
+
+// Platform builds a platform
+func (b *Builder) Platform(ctx context.Context, cfg *client.Config) (*v1alpha1.Platform, error) {
+	log := logger.FromContext(ctx)
+	log.DebugContext(ctx, "cue: building platform instance")
+	instances, err := b.Instances(ctx, cfg)
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+
+	if len(instances) != 1 {
+		return nil, errors.Wrap(errors.New(fmt.Sprintf("instances length %d must be exactly 1", len(instances))))
+	}
+
+	// We only process the first instance, assume the render platform subcommand enforces this.
+	instance := instances[0]
+	log.DebugContext(ctx, "cue: building instance", "dir", instance.Dir)
+	p, err := b.runPlatform(ctx, instance)
+	if err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not build platform: %w", err))
+	}
+	return p, nil
+}
+
+func (b Builder) runPlatform(ctx context.Context, instance *build.Instance) (*v1alpha1.Platform, error) {
+	path := holos.InstancePath(instance.Dir)
+	log := logger.FromContext(ctx).With("dir", path)
+
+	if err := instance.Err; err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not load: %w", err))
+	}
+	cueCtx := cuecontext.New()
+	value := cueCtx.BuildInstance(instance)
+	if err := value.Err(); err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not build %s: %w", instance.Dir, err))
+	}
+	log.DebugContext(ctx, "cue: validating instance")
+	if err := value.Validate(); err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not validate: %w", err))
+	}
+
+	log.DebugContext(ctx, "cue: decoding holos platform")
+	jsonBytes, err := value.MarshalJSON()
+	if err != nil {
+		return nil, errors.Wrap(fmt.Errorf("could not marshal cue instance %s: %w", instance.Dir, err))
+	}
+	decoder := json.NewDecoder(bytes.NewReader(jsonBytes))
+	// Discriminate the type of build plan.
+	tm := &v1alpha1.TypeMeta{}
+	err = decoder.Decode(tm)
+	if err != nil {
+		return nil, errors.Wrap(fmt.Errorf("invalid platform: %s: %w", instance.Dir, err))
+	}
+
+	log.DebugContext(ctx, "cue: discriminated build kind: "+tm.Kind, "kind", tm.Kind, "apiVersion", tm.APIVersion)
+
+	// New decoder for the full object
+	decoder = json.NewDecoder(bytes.NewReader(jsonBytes))
+	decoder.DisallowUnknownFields()
+
+	var pf v1alpha1.Platform
+	switch tm.Kind {
+	case "Platform":
+		if err = decoder.Decode(&pf); err != nil {
+			err = errors.Wrap(fmt.Errorf("could not decode platform %s: %w", instance.Dir, err))
+			return nil, err
+		}
+		return &pf, nil
+	default:
+		err = errors.Wrap(fmt.Errorf("unknown kind: %v", tm.Kind))
+	}
+
+	return nil, err
+}

internal/cli/build/build.go

@@ -0,0 +1,57 @@
+package build
+
+import (
+	"fmt"
+	"log/slog"
+	"strings"
+
+	"github.com/holos-run/holos/internal/builder"
+	"github.com/holos-run/holos/internal/cli/command"
+	"github.com/holos-run/holos/internal/client"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/holos"
+	"github.com/holos-run/holos/internal/server/middleware/logger"
+	"github.com/spf13/cobra"
+)
+
+// makeBuildRunFunc returns the internal implementation of the build cli command
+func makeBuildRunFunc(cfg *client.Config) command.RunFunc {
+	return func(cmd *cobra.Command, args []string) error {
+		ctx := cmd.Root().Context()
+		logger.FromContext(ctx).DebugContext(ctx, "RunE", "args", args)
+		build := builder.New(builder.Entrypoints(args), builder.Cluster(cfg.Holos().ClusterName()))
+		results, err := build.Run(ctx, cfg)
+		if err != nil {
+			return err
+		}
+		outs := make([]string, 0, len(results))
+		for idx, result := range results {
+			if result == nil || result.Skip {
+				slog.Debug("skip result", "idx", idx, "result", result)
+				continue
+			}
+			slog.Debug("append result", "idx", idx, "result.kind", result.Kind)
+			outs = append(outs, result.AccumulatedOutput())
+		}
+		out := strings.Join(outs, "---\n")
+		if _, err := fmt.Fprintln(cmd.OutOrStdout(), out); err != nil {
+			return errors.Wrap(err)
+		}
+		return nil
+	}
+}
+
+// New returns the build subcommand for the root command
+func New(cfg *holos.Config) *cobra.Command {
+	cmd := command.New("build [directory...]")
+	cmd.Args = cobra.MinimumNArgs(1)
+	cmd.Short = "build kubernetes api objects from a directory"
+
+	cmd.Flags().AddGoFlagSet(cfg.ClusterFlagSet())
+	config := client.NewConfig(cfg)
+	cmd.PersistentFlags().AddGoFlagSet(config.ClientFlagSet())
+	cmd.PersistentFlags().AddGoFlagSet(config.TokenFlagSet())
+
+	cmd.RunE = makeBuildRunFunc(config)
+	return cmd
+}

internal/cli/command/cmd.go

@@ -1,10 +1,7 @@
 package command
 
 import (
-	"fmt"
-
-	"github.com/holos-run/holos/pkg/errors"
-	"github.com/holos-run/holos/pkg/version"
+	"github.com/holos-run/holos/version"
 	"github.com/spf13/cobra"
 )
 
@@ -20,11 +17,9 @@ func New(name string) *cobra.Command {
 		CompletionOptions: cobra.CompletionOptions{
 			HiddenDefaultCmd: true,
 		},
-		RunE: func(c *cobra.Command, args []string) error {
-			return errors.Wrap(fmt.Errorf("could not run %v: not implemented", c.Name()))
-		},
 		SilenceUsage:  true,
 		SilenceErrors: true,
 	}
+	cmd.Flags().SortFlags = false
 	return cmd
 }

internal/cli/controller/controller.go

@@ -0,0 +1,50 @@
+// Package controller integrates Choria Machine Room into Holos for cluster management.
+package controller
+
+import (
+	"context"
+	"fmt"
+
+	mr "github.com/choria-io/machine-room"
+	"github.com/holos-run/holos/internal/cli/command"
+	"github.com/holos-run/holos/internal/errors"
+	"github.com/holos-run/holos/internal/holos"
+	"github.com/holos-run/holos/version"
+	"github.com/spf13/cobra"
+)
+
+var (
+	// SigningKey is the public key from choria jwt keys machine-signer.seed machine-signer.public, refer to gen-machine-signer.
+	SigningKey = "2a136e3875f4375968ae8e8d400ba24864d3ed7c4109675f357d32cc3ca1d5a7"
+)
+
+func New(cfg *holos.Config) *cobra.Command {
+	cmd := command.New("controller")
+	cmd.Args = cobra.ArbitraryArgs
+	cmd.DisableFlagParsing = true
+	cmd.RunE = func(c *cobra.Command, args []string) error {
+		if SigningKey == "" {
+			return errors.Wrap(fmt.Errorf("could not run: controller.SigningKey not set from build variables"))
+		}
+
+		ctx := c.Context()
+		if ctx == nil {
+			ctx = context.Background()
+		}
+
+		app, err := mr.New(mr.Options{
+			Name:              "controller",
+			Contact:           "jeff@openinfrastructure.co",
+			Version:           version.Version,
+			Help:              "Holos Controller",
+			MachineSigningKey: SigningKey,
+			Args:              args,
+		})
+		if err != nil {
+			return errors.Wrap(fmt.Errorf("could not make machine room app: %w", err))
+		}
+
+		return app.Run(ctx)
+	}
+	return cmd
+}