(#71) Add HelmChart BuildPlan support

This patch refactors the #HelmChart definition to a BuildPlan.HelmCharts,
which executes a collection of HelmCharts.  The same behavior is
preserved, helm template executes then a kustomize post processor
executes.

```
❯ holos render --cluster-name=k2 ~/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/... --log-level=debug
9:53PM DBG config.go:150 finalized config from flags version=0.60.1 state=finalized
9:53PM DBG builder.go:108 cue: building instances version=0.60.1
9:53PM DBG builder.go:95 cue: equivalent command: cue export --out yaml -t cluster=k2 ./platforms/reference/clusters/foundation/cloud/github/arc/... version=0.60.1
9:53PM DBG builder.go:100 cue: tags [cluster=k2] version=0.60.1
9:53PM DBG builder.go:122 cue: building instance version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc
9:53PM DBG builder.go:127 cue: validating instance version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc
9:53PM DBG builder.go:131 cue: decoding holos build plan version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc
9:53PM DBG builder.go:122 cue: building instance version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/runner
9:53PM DBG builder.go:127 cue: validating instance version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/runner
9:53PM DBG builder.go:131 cue: decoding holos build plan version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/runner
9:53PM DBG result.go:61 ExternalSecret/controller-manager version=0.60.1 kind=ExternalSecret name=controller-manager
9:53PM DBG builder.go:122 cue: building instance version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/system
9:53PM DBG builder.go:127 cue: validating instance version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/system
9:53PM DBG builder.go:131 cue: decoding holos build plan version=0.60.1 dir=/home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/system
9:53PM DBG helm.go:95 helm: wrote values version=0.60.1 chart=oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller path=/tmp/holos1163326896/values.yaml bytes=653
9:53PM DBG run.go:40 running: helm version=0.60.1 name=helm args="[template --no-hooks --include-crds --values /tmp/holos1163326896/values.yaml --namespace arc-system --kubeconfig /dev/null --version 0.8.3 gha-rs-controller /home/jeff/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/system/vendor/gha-runner-scale-set-controller]"
9:53PM DBG remove.go:15 tmp: removed version=0.60.1 path=/tmp/holos1163326896
9:53PM DBG result.go:95 wrote: /tmp/holos.kustomize3569816247/resources.yaml version=0.60.1 op=write path=/tmp/holos.kustomize3569816247/resources.yaml bytes=2019229
9:53PM DBG result.go:108 wrote: /tmp/holos.kustomize3569816247/kustomization.yaml version=0.60.1 op=write path=/tmp/holos.kustomize3569816247/kustomization.yaml bytes=94
9:53PM DBG run.go:40 running: kubectl version=0.60.1 name=kubectl args="[kustomize /tmp/holos.kustomize3569816247]"
9:53PM DBG remove.go:15 tmp: removed version=0.60.1 path=/tmp/holos.kustomize3569816247
9:53PM DBG result.go:135 out: wrote deploy/clusters/k2/components/prod-github-arc-runner/prod-github-arc-runner.gen.yaml version=0.60.1 action=write path=deploy/clusters/k2/components/prod-github-arc-runner/prod-github-arc-runner.gen.yaml status=ok
9:53PM DBG result.go:135 out: wrote deploy/clusters/k2/holos/components/prod-github-arc-runner-kustomization.gen.yaml version=0.60.1 action=write path=deploy/clusters/k2/holos/components/prod-github-arc-runner-kustomization.gen.yaml status=ok
9:53PM INF render.go:43 rendered prod-github-arc-runner version=0.60.1 status=ok action=rendered name=prod-github-arc-runner
9:53PM DBG result.go:135 out: wrote deploy/clusters/k2/components/prod-github-arc-system/prod-github-arc-system.gen.yaml version=0.60.1 action=write path=deploy/clusters/k2/components/prod-github-arc-system/prod-github-arc-system.gen.yaml status=ok
9:53PM DBG result.go:135 out: wrote deploy/clusters/k2/holos/components/prod-github-arc-system-kustomization.gen.yaml version=0.60.1 action=write path=deploy/clusters/k2/holos/components/prod-github-arc-system-kustomization.gen.yaml status=ok
9:53PM INF render.go:43 rendered prod-github-arc-system version=0.60.1 status=ok action=rendered name=prod-github-arc-system
```

Makefile

@@ -44,13 +44,18 @@ tidy: ## Tidy go module.
 	go mod tidy
 
 .PHONY: fmt
-fmt: ## Format Go code.
+fmt: ## Format code.
+	cd docs/examples && cue fmt ./...
 	go fmt ./...
 
 .PHONY: vet
 vet: ## Vet Go code.
 	go vet ./...
 
+.PHONY: gencue
+gencue: ## Generate CUE definitions
+	cd docs/examples && cue get go github.com/holos-run/holos/api/...
+
 .PHONY: generate
 generate: ## Generate code.
 	go generate ./...

api/v1alpha1/buildplan.go

@@ -0,0 +1,39 @@
+package v1alpha1
+
+import (
+	"fmt"
+	"strings"
+)
+
+// BuildPlan is the primary interface between CUE and the Holos cli.
+type BuildPlan struct {
+	TypeMeta `json:",inline" yaml:",inline"`
+	// Metadata represents the holos component name
+	Metadata ObjectMeta    `json:"metadata,omitempty" yaml:"metadata,omitempty"`
+	Spec     BuildPlanSpec `json:"spec,omitempty" yaml:"spec,omitempty"`
+}
+
+type BuildPlanSpec struct {
+	Disabled   bool                `json:"disabled,omitempty" yaml:"disabled,omitempty"`
+	Components BuildPlanComponents `json:"components,omitempty" yaml:"components,omitempty"`
+}
+
+type BuildPlanComponents struct {
+	HelmCharts        []HelmChart         `json:"helmCharts,omitempty" yaml:"helmCharts,omitempty"`
+	KubernetesObjects []KubernetesObjects `json:"kubernetesObjects,omitempty" yaml:"kubernetesObjects,omitempty"`
+	KustomizeBuilds   []KustomizeBuild    `json:"kustomizeBuilds,omitempty" yaml:"kustomizeBuilds,omitempty"`
+}
+
+func (bp *BuildPlan) Validate() error {
+	errs := make([]string, 0, 2)
+	if bp.Kind != BuildPlanKind {
+		errs = append(errs, fmt.Sprintf("kind invalid: want: %s have: %s", BuildPlanKind, bp.Kind))
+	}
+	if bp.APIVersion != APIVersion {
+		errs = append(errs, fmt.Sprintf("apiVersion invalid: want: %s have: %s", APIVersion, bp.APIVersion))
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf("invalid BuildPlan: " + strings.Join(errs, ", "))
+	}
+	return nil
+}

api/v1alpha1/component.go

@@ -0,0 +1,22 @@
+package v1alpha1
+
+// HolosComponent defines the fields common to all holos component kinds including the Render Result.
+type HolosComponent struct {
+	TypeMeta `json:",inline" yaml:",inline"`
+	// Metadata represents the holos component name
+	Metadata ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
+	// APIObjectMap holds the marshalled representation of api objects. Think of
+	// these as resources overlaid at the back of the render pipeline.
+	APIObjectMap APIObjectMap `json:"apiObjectMap,omitempty" yaml:"apiObjectMap,omitempty"`
+	// Kustomization holds the marshalled representation of the flux kustomization
+	// which reconciles resources in git with the api server.
+	Kustomization `json:",inline" yaml:",inline"`
+	// Kustomize represents a kubectl kustomize build post-processing step.
+	Kustomize `json:",inline" yaml:",inline"`
+	// Skip causes holos to take no action regarding the component.
+	Skip bool
+}
+
+func (hc *HolosComponent) NewResult() *Result {
+	return &Result{HolosComponent: *hc}
+}

api/v1alpha1/constants.go

@@ -0,0 +1,11 @@
+package v1alpha1
+
+const (
+	APIVersion    = "holos.run/v1alpha1"
+	BuildPlanKind = "BuildPlan"
+	HelmChartKind = "HelmChart"
+	// ChartDir is the directory name created in the holos component directory to cache a chart.
+	ChartDir = "vendor"
+	// ResourcesFile is the file name used to store component output when post-processing with kustomize.
+	ResourcesFile = "resources.yaml"
+)

api/v1alpha1/doc.go

@@ -0,0 +1,2 @@
+// Package v1alpha1 defines the api boundary between CUE and Holos.
+package v1alpha1

api/v1alpha1/helm.go

@@ -0,0 +1,154 @@
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/holos-run/holos"
+	"github.com/holos-run/holos/pkg/logger"
+	"github.com/holos-run/holos/pkg/util"
+	"github.com/holos-run/holos/pkg/wrapper"
+)
+
+// A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
+type HelmChart struct {
+	HolosComponent `json:",inline" yaml:",inline"`
+	// Namespace is the namespace to install into.  TODO: Use metadata.namespace instead.
+	Namespace     string `json:"namespace"`
+	Chart         Chart  `json:"chart"`
+	ValuesContent string `json:"valuesContent"`
+	EnableHooks   bool   `json:"enableHooks"`
+}
+
+type Chart struct {
+	Name       string     `json:"name"`
+	Version    string     `json:"version"`
+	Release    string     `json:"release"`
+	Repository Repository `json:"repository,omitempty"`
+}
+
+type Repository struct {
+	Name string `json:"name"`
+	URL  string `json:"url"`
+}
+
+func (hc *HelmChart) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+	result := Result{HolosComponent: hc.HolosComponent}
+	if err := hc.helm(ctx, &result, path); err != nil {
+		return nil, err
+	}
+	result.addObjectMap(ctx, hc.APIObjectMap)
+	if err := result.kustomize(ctx); err != nil {
+		return nil, wrapper.Wrap(fmt.Errorf("could not kustomize: %w", err))
+	}
+	return &result, nil
+}
+
+// runHelm provides the values produced by CUE to helm template and returns
+// the rendered kubernetes api objects in the result.
+func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.PathComponent) error {
+	log := logger.FromContext(ctx).With("chart", hc.Chart.Name)
+	if hc.Chart.Name == "" {
+		log.WarnContext(ctx, "skipping helm: no chart name specified, use a different component type")
+		return nil
+	}
+
+	cachedChartPath := filepath.Join(string(path), ChartDir, filepath.Base(hc.Chart.Name))
+	if isNotExist(cachedChartPath) {
+		// Add repositories
+		repo := hc.Chart.Repository
+		if repo.URL != "" {
+			out, err := util.RunCmd(ctx, "helm", "repo", "add", repo.Name, repo.URL)
+			if err != nil {
+				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
+				return wrapper.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
+			}
+			// Update repository
+			out, err = util.RunCmd(ctx, "helm", "repo", "update", repo.Name)
+			if err != nil {
+				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
+				return wrapper.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
+			}
+		} else {
+			log.DebugContext(ctx, "no chart repository url proceeding assuming oci chart")
+		}
+
+		// Cache the chart
+		if err := cacheChart(ctx, path, ChartDir, hc.Chart); err != nil {
+			return fmt.Errorf("could not cache chart: %w", err)
+		}
+	}
+
+	// Write values file
+	tempDir, err := os.MkdirTemp("", "holos")
+	if err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+	}
+	defer util.Remove(ctx, tempDir)
+
+	valuesPath := filepath.Join(tempDir, "values.yaml")
+	if err := os.WriteFile(valuesPath, []byte(hc.ValuesContent), 0644); err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not write values: %w", err))
+	}
+	log.DebugContext(ctx, "helm: wrote values", "path", valuesPath, "bytes", len(hc.ValuesContent))
+
+	// Run charts
+	chart := hc.Chart
+	args := []string{"template"}
+	if !hc.EnableHooks {
+		args = append(args, "--no-hooks")
+	}
+	namespace := hc.Namespace
+	args = append(args, "--include-crds", "--values", valuesPath, "--namespace", namespace, "--kubeconfig", "/dev/null", "--version", chart.Version, chart.Release, cachedChartPath)
+	helmOut, err := util.RunCmd(ctx, "helm", args...)
+	if err != nil {
+		stderr := helmOut.Stderr.String()
+		lines := strings.Split(stderr, "\n")
+		for _, line := range lines {
+			if strings.HasPrefix(line, "Error:") {
+				err = fmt.Errorf("%s: %w", line, err)
+			}
+		}
+		return wrapper.Wrap(fmt.Errorf("could not run helm template: %w", err))
+	}
+
+	r.accumulatedOutput = helmOut.Stdout.String()
+
+	return nil
+}
+
+// cacheChart stores a cached copy of Chart in the chart subdirectory of path.
+func cacheChart(ctx context.Context, path holos.PathComponent, chartDir string, chart Chart) error {
+	log := logger.FromContext(ctx)
+
+	cacheTemp, err := os.MkdirTemp(string(path), chartDir)
+	if err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+	}
+	defer util.Remove(ctx, cacheTemp)
+
+	chartName := chart.Name
+	if chart.Repository.Name != "" {
+		chartName = fmt.Sprintf("%s/%s", chart.Repository.Name, chart.Name)
+	}
+	helmOut, err := util.RunCmd(ctx, "helm", "pull", "--destination", cacheTemp, "--untar=true", "--version", chart.Version, chartName)
+	if err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not run helm pull: %w", err))
+	}
+	log.Debug("helm pull", "stdout", helmOut.Stdout, "stderr", helmOut.Stderr)
+
+	cachePath := filepath.Join(string(path), chartDir)
+	if err := os.Rename(cacheTemp, cachePath); err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not rename: %w", err))
+	}
+	log.InfoContext(ctx, "cached", "chart", chart.Name, "version", chart.Version, "path", cachePath)
+
+	return nil
+}
+func isNotExist(path string) bool {
+	_, err := os.Stat(path)
+	return os.IsNotExist(err)
+}

api/v1alpha1/kubernetesobjects.go

@@ -0,0 +1,20 @@
+package v1alpha1
+
+import (
+	"context"
+	"github.com/holos-run/holos"
+)
+
+const KubernetesObjectsKind = "KubernetesObjects"
+
+// KubernetesObjects represents CUE output which directly provides Kubernetes api objects to holos.
+type KubernetesObjects struct {
+	HolosComponent `json:",inline" yaml:",inline"`
+}
+
+// Render produces kubernetes api objects from the APIObjectMap
+func (o *KubernetesObjects) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+	result := Result{HolosComponent: o.HolosComponent}
+	result.addObjectMap(ctx, o.APIObjectMap)
+	return &result, nil
+}

api/v1alpha1/kustomization.go

@@ -0,0 +1,7 @@
+package v1alpha1
+
+// Kustomization holds the rendered flux kustomization api object content for git ops.
+type Kustomization struct {
+	// KsContent is the yaml representation of the flux kustomization for gitops.
+	KsContent string `json:"ksContent,omitempty" yaml:"ksContent,omitempty"`
+}

api/v1alpha1/kustomize.go

@@ -0,0 +1,46 @@
+package v1alpha1
+
+import (
+	"context"
+	"github.com/holos-run/holos"
+	"github.com/holos-run/holos/pkg/logger"
+	"github.com/holos-run/holos/pkg/util"
+	"github.com/holos-run/holos/pkg/wrapper"
+)
+
+const KustomizeBuildKind = "KustomizeBuild"
+
+// Kustomize represents resources necessary to execute a kustomize build.
+// Intended for at least two use cases:
+//
+//  1. Process raw yaml file resources in a holos component directory.
+//  2. Post process a HelmChart to inject istio, add custom labels, etc...
+type Kustomize struct {
+	// KustomizeFiles holds file contents for kustomize, e.g. patch files.
+	KustomizeFiles FileContentMap `json:"kustomizeFiles,omitempty" yaml:"kustomizeFiles,omitempty"`
+	// ResourcesFile is the file name used for api objects in kustomization.yaml
+	ResourcesFile string `json:"resourcesFile,omitempty" yaml:"resourcesFile,omitempty"`
+}
+
+// KustomizeBuild renders plain yaml files in the holos component directory using kubectl kustomize build.
+type KustomizeBuild struct {
+	HolosComponent `json:",inline" yaml:",inline"`
+}
+
+// Render produces a Result by executing kubectl kustomize on the holos
+// component path. Useful for processing raw yaml files.
+func (kb *KustomizeBuild) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+	log := logger.FromContext(ctx)
+	result := Result{HolosComponent: kb.HolosComponent}
+	// Run kustomize.
+	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", string(path))
+	if err != nil {
+		log.ErrorContext(ctx, kOut.Stderr.String())
+		return nil, wrapper.Wrap(err)
+	}
+	// Replace the accumulated output
+	result.accumulatedOutput = kOut.Stdout.String()
+	// Add CUE based api objects.
+	result.addObjectMap(ctx, kb.APIObjectMap)
+	return &result, nil
+}

api/v1alpha1/objectmap.go

@@ -0,0 +1,8 @@
+package v1alpha1
+
+// APIObjectMap is the shape of marshalled api objects returned from cue to the
+// holos cli. A map is used to improve the clarity of error messages from cue.
+type APIObjectMap map[string]map[string]string
+
+// FileContentMap is a map of file names to file contents.
+type FileContentMap map[string]string

api/v1alpha1/objectmeta.go

@@ -0,0 +1,15 @@
+package v1alpha1
+
+// ObjectMeta represents metadata of a holos component object. The fields are a
+// copy of upstream kubernetes api machinery but are by holos objects distinct
+// from kubernetes api objects.
+type ObjectMeta struct {
+	// Name uniquely identifies the holos component instance and must be suitable as a file name.
+	Name string `json:"name,omitempty" yaml:"name,omitempty"`
+	// Namespace confines a holos component to a single namespace via kustomize if set.
+	Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
+	// Labels are not used but are copied from api machinery ObjectMeta for completeness.
+	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
+	// Annotations are not used but are copied from api machinery ObjectMeta for completeness.
+	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
+}

api/v1alpha1/render.go

@@ -0,0 +1,21 @@
+package v1alpha1
+
+import (
+	"context"
+	"github.com/holos-run/holos"
+)
+
+type Renderer interface {
+	GetKind() string
+	Render(ctx context.Context, path holos.PathComponent) (*Result, error)
+}
+
+// Render produces a Result representing the kubernetes api objects to
+// configure. Each of the various holos component types, e.g. Helm, Kustomize,
+// et al, should implement the Renderer interface. This process is best
+// conceptualized as a data pipeline, for example a component may render a
+// result by first calling helm template, then passing the result through
+// kustomize, then mixing in overlay api objects.
+func Render(ctx context.Context, r Renderer, path holos.PathComponent) (*Result, error) {
+	return r.Render(ctx, path)
+}

api/v1alpha1/result.go

@@ -0,0 +1,137 @@
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+	"github.com/holos-run/holos/pkg/logger"
+	"github.com/holos-run/holos/pkg/util"
+	"github.com/holos-run/holos/pkg/wrapper"
+	"os"
+	"path/filepath"
+	"slices"
+)
+
+// Result is the build result for display or writing.  Holos components Render the Result as a data pipeline.
+type Result struct {
+	HolosComponent
+	// accumulatedOutput accumulates rendered api objects.
+	accumulatedOutput string
+}
+
+func (r *Result) Name() string {
+	return r.Metadata.Name
+}
+
+func (r *Result) Filename(writeTo string, cluster string) string {
+	name := r.Metadata.Name
+	return filepath.Join(writeTo, "clusters", cluster, "components", name, name+".gen.yaml")
+}
+
+func (r *Result) KustomizationFilename(writeTo string, cluster string) string {
+	return filepath.Join(writeTo, "clusters", cluster, "holos", "components", r.Metadata.Name+"-kustomization.gen.yaml")
+}
+
+// AccumulatedOutput returns the accumulated rendered output.
+func (r *Result) AccumulatedOutput() string {
+	return r.accumulatedOutput
+}
+
+// addObjectMap renders the provided APIObjectMap into the accumulated output.
+func (r *Result) addObjectMap(ctx context.Context, objectMap APIObjectMap) {
+	log := logger.FromContext(ctx)
+	b := []byte(r.AccumulatedOutput())
+	kinds := make([]string, 0, len(objectMap))
+	// Sort the keys
+	for kind := range objectMap {
+		kinds = append(kinds, kind)
+	}
+	slices.Sort(kinds)
+
+	for _, kind := range kinds {
+		v := objectMap[kind]
+		// Sort the keys
+		names := make([]string, 0, len(v))
+		for name := range v {
+			names = append(names, name)
+		}
+		slices.Sort(names)
+
+		for _, name := range names {
+			yamlString := v[name]
+			log.Debug(fmt.Sprintf("%s/%s", kind, name), "kind", kind, "name", name)
+			b = util.EnsureNewline(b)
+			header := fmt.Sprintf("---\n# Source: CUE apiObjects.%s.%s\n", kind, name)
+			b = append(b, []byte(header+yamlString)...)
+			b = util.EnsureNewline(b)
+		}
+	}
+	r.accumulatedOutput = string(b)
+}
+
+// kustomize replaces the accumulated output with the output of kustomize build
+func (r *Result) kustomize(ctx context.Context) error {
+	log := logger.FromContext(ctx)
+	if r.ResourcesFile == "" {
+		log.DebugContext(ctx, "skipping kustomize: no resourcesFile")
+		return nil
+	}
+	if len(r.KustomizeFiles) < 1 {
+		log.DebugContext(ctx, "skipping kustomize: no kustomizeFiles")
+		return nil
+	}
+	tempDir, err := os.MkdirTemp("", "holos.kustomize")
+	if err != nil {
+		return wrapper.Wrap(err)
+	}
+	defer util.Remove(ctx, tempDir)
+
+	// Write the main api object resources file for kustomize.
+	target := filepath.Join(tempDir, r.ResourcesFile)
+	b := []byte(r.AccumulatedOutput())
+	b = util.EnsureNewline(b)
+	if err := os.WriteFile(target, b, 0644); err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not write resources: %w", err))
+	}
+	log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
+
+	// Write the kustomization tree, kustomization.yaml must be in this map for kustomize to work.
+	for file, content := range r.KustomizeFiles {
+		target := filepath.Join(tempDir, file)
+		if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+			return wrapper.Wrap(err)
+		}
+		b := []byte(content)
+		b = util.EnsureNewline(b)
+		if err := os.WriteFile(target, b, 0644); err != nil {
+			return wrapper.Wrap(fmt.Errorf("could not write: %w", err))
+		}
+		log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
+	}
+
+	// Run kustomize.
+	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", tempDir)
+	if err != nil {
+		log.ErrorContext(ctx, kOut.Stderr.String())
+		return wrapper.Wrap(err)
+	}
+	// Replace the accumulated output
+	r.accumulatedOutput = kOut.Stdout.String()
+	return nil
+}
+
+// Save writes the content to the filesystem for git ops.
+func (r *Result) Save(ctx context.Context, path string, content string) error {
+	log := logger.FromContext(ctx)
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
+		log.WarnContext(ctx, "could not mkdir", "path", dir, "err", err)
+		return wrapper.Wrap(err)
+	}
+	// Write the kube api objects
+	if err := os.WriteFile(path, []byte(content), os.FileMode(0644)); err != nil {
+		log.WarnContext(ctx, "could not write", "path", path, "err", err)
+		return wrapper.Wrap(err)
+	}
+	log.DebugContext(ctx, "out: wrote "+path, "action", "write", "path", path, "status", "ok")
+	return nil
+}

api/v1alpha1/typemeta.go

@@ -0,0 +1,10 @@
+package v1alpha1
+
+type TypeMeta struct {
+	Kind       string `json:"kind,omitempty" yaml:"kind,omitempty"`
+	APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty"`
+}
+
+func (tm *TypeMeta) GetKind() string {
+	return tm.Kind
+}

cmd/holos/testdata/constraints.txt

@@ -1,7 +1,6 @@
 # Want support for intermediary constraints
 exec holos build ./foo/... --log-level debug
 stdout '^bf2bc7f9-9ba0-4f9e-9bd2-9a205627eb0b$'
-stderr 'processing holos component kind Skip'
 
 -- cue.mod --
 package holos
@@ -12,31 +11,21 @@ metadata: name: "jeff"
 -- foo/bar/bar.cue --
 package holos
 
-#KubernetesObjects & {
-  apiObjectMap: foo: bar: "bf2bc7f9-9ba0-4f9e-9bd2-9a205627eb0b"
-}
+spec: components: KubernetesObjects: [
+  #KubernetesObjects & {
+    apiObjectMap: foo: bar: "bf2bc7f9-9ba0-4f9e-9bd2-9a205627eb0b"
+  }
+]
 -- schema.cue --
 package holos
 
 cluster: string @tag(cluster, string)
 
-// #OutputTypeMeta is shared among all output types
-#OutputTypeMeta: {
-	apiVersion: "holos.run/v1alpha1"
-	kind: #KubernetesObjects.kind | #NoOutput.kind
-	metadata: name: string
-}
-
 #KubernetesObjects: {
-	#OutputTypeMeta
+	apiVersion: "holos.run/v1alpha1"
 	kind: "KubernetesObjects"
 	apiObjectMap: {...}
 }
 
-#NoOutput: {
-	#OutputTypeMeta
-	kind: string | *"Skip"
-	metadata: name: string | *"skipped"
-}
-
-#NoOutput & {}
+apiVersion: "holos.run/v1alpha1"
+kind: "BuildPlan"

cmd/holos/testdata/issue15_cue_errors.txt

@@ -1,7 +1,7 @@
 # Want cue errors to show files and lines
 ! exec holos build .
-stderr '^apiObjectMap.foo.bar: cannot convert non-concrete value string'
-stderr '/component.cue:7:20$'
+stderr 'apiObjectMap.foo.bar: cannot convert non-concrete value string:'
+stderr '/component.cue:\d+:\d+$'
 
 -- cue.mod --
 package holos
@@ -9,8 +9,8 @@ package holos
 package holos
 
 apiVersion: "holos.run/v1alpha1"
-kind: "KubernetesObjects"
+kind: "BuildPlan"
 cluster: string @tag(cluster, string)
 
-apiObjectMap: foo: bar: baz
 baz: string
+spec: components: KubernetesObjects: [{apiObjectMap: foo: bar: baz}]

cmd/holos/testdata/issue25_apiobjects_cue.txt

@@ -9,7 +9,9 @@ package holos
 package holos
 
 apiVersion: "holos.run/v1alpha1"
-kind: "KubernetesObjects"
+kind: "BuildPlan"
+spec: components: KubernetesObjects: [{apiObjectMap: #APIObjects.apiObjectMap}]
+
 cluster: string @tag(cluster, string)
 
 #SecretStore: {
@@ -17,7 +19,7 @@ cluster: string @tag(cluster, string)
     metadata: name: string
 }
 
-#APIObjects & {
+#APIObjects: {
   apiObjects: {
     SecretStore: {
       default: #SecretStore & { metadata: name: "default" }
@@ -54,4 +56,3 @@ import "encoding/yaml"
 		}
 	}
 }
-

cmd/holos/testdata/issue25_apiobjects_helm.txt

@@ -10,7 +10,9 @@ package holos
 package holos
 
 apiVersion: "holos.run/v1alpha1"
-kind: "HelmChart"
+kind: "BuildPlan"
+spec: components: HelmCharts: [{apiObjectMap: #APIObjects.apiObjectMap}]
+
 cluster: string @tag(cluster, string)
 
 #SecretStore: {
@@ -18,7 +20,7 @@ cluster: string @tag(cluster, string)
     metadata: name: string
 }
 
-#APIObjects & {
+#APIObjects: {
   apiObjects: {
     SecretStore: {
       default: #SecretStore & { metadata: name: "default" }
@@ -55,4 +57,3 @@ import "encoding/yaml"
 		}
 	}
 }
-

cmd/holos/testdata/issue33_helm_stderr.txt

@@ -7,23 +7,28 @@ package holos
 -- zitadel.cue --
 package holos
 
+apiVersion: "holos.run/v1alpha1"
+kind: "BuildPlan"
+spec: components: HelmCharts: [_HelmChart]
+
 cluster: string @tag(cluster, string)
 
-apiVersion: "holos.run/v1alpha1"
-kind: "HelmChart"
-metadata: name: "zitadel"
-namespace: "zitadel"
-chart: {
-	name:    "zitadel"
-	version: "7.9.0"
-	release: name
-	repository: {
-		name: "zitadel"
-		url:  "https://charts.zitadel.com"
-	}
+_HelmChart: {
+  apiVersion: "holos.run/v1alpha1"
+  kind: "HelmChart"
+  metadata: name: "zitadel"
+  namespace: "zitadel"
+  chart: {
+    name:    "zitadel"
+    version: "7.9.0"
+    release: name
+    repository: {
+      name: "zitadel"
+      url:  "https://charts.zitadel.com"
+    }
+  }
 }
 
-
 -- vendor/zitadel/templates/secret_zitadel-masterkey.yaml --
 {{- if (or (and .Values.zitadel.masterkey .Values.zitadel.masterkeySecretName) (and (not .Values.zitadel.masterkey) (not .Values.zitadel.masterkeySecretName)) ) }}
 {{- fail "Either set .Values.zitadel.masterkey xor .Values.zitadel.masterkeySecretName" }}

cmd/holos/testdata/issue42_kustomize_build_kind.txt

@@ -12,19 +12,22 @@ package holos
 cluster: string @tag(cluster, string)
 
 apiVersion: "holos.run/v1alpha1"
-kind: "KustomizeBuild"
-metadata: name: "kstest"
+kind: "BuildPlan"
+spec: components: KustomizeBuilds: [{metadata: name: "kstest"}]
+
 -- kustomization.yaml --
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: mynamespace
 resources:
 - serviceaccount.yaml
+
 -- serviceaccount.yaml --
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: test
+
 -- want.yaml --
 apiVersion: v1
 kind: ServiceAccount

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/buildplan_go_gen.cue

@@ -0,0 +1,25 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+// BuildPlan is the primary interface between CUE and the Holos cli.
+#BuildPlan: {
+	#TypeMeta
+
+	// Metadata represents the holos component name
+	metadata?: #ObjectMeta    @go(Metadata)
+	spec?:     #BuildPlanSpec @go(Spec)
+}
+
+#BuildPlanSpec: {
+	disabled?:   bool                 @go(Disabled)
+	components?: #BuildPlanComponents @go(Components)
+}
+
+#BuildPlanComponents: {
+	helmCharts?: [...#HelmChart] @go(HelmCharts,[]HelmChart)
+	kubernetesObjects?: [...#KubernetesObjects] @go(KubernetesObjects,[]KubernetesObjects)
+	kustomizeBuilds?: [...#KustomizeBuild] @go(KustomizeBuilds,[]KustomizeBuild)
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/component_go_gen.cue

@@ -0,0 +1,24 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+// HolosComponent defines the fields common to all holos component kinds including the Render Result.
+#HolosComponent: {
+	#TypeMeta
+
+	// Metadata represents the holos component name
+	metadata?: #ObjectMeta @go(Metadata)
+
+	// APIObjectMap holds the marshalled representation of api objects. Think of
+	// these as resources overlaid at the back of the render pipeline.
+	apiObjectMap?: #APIObjectMap @go(APIObjectMap)
+
+	#Kustomization
+
+	#Kustomize
+
+	// Skip causes holos to take no action regarding the component.
+	Skip: bool
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/constants_go_gen.cue

@@ -0,0 +1,15 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+#APIVersion:    "holos.run/v1alpha1"
+#BuildPlanKind: "BuildPlan"
+#HelmChartKind: "HelmChart"
+
+// ChartDir is the directory name created in the holos component directory to cache a chart.
+#ChartDir: "vendor"
+
+// ResourcesFile is the file name used to store component output when post processing with kustomize.
+#ResourcesFile: "resources.yaml"

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/doc_go_gen.cue

@@ -0,0 +1,6 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+// Package v1alpha1 defines the api boundary between CUE and Holos.
+package v1alpha1

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/helm_go_gen.cue

@@ -0,0 +1,28 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+// A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
+#HelmChart: {
+	#HolosComponent
+
+	// Namespace is the namespace to install into.  TODO: Use metadata.namespace instead.
+	namespace:     string @go(Namespace)
+	chart:         #Chart @go(Chart)
+	valuesContent: string @go(ValuesContent)
+	enableHooks:   bool   @go(EnableHooks)
+}
+
+#Chart: {
+	name:        string      @go(Name)
+	version:     string      @go(Version)
+	release:     string      @go(Release)
+	repository?: #Repository @go(Repository)
+}
+
+#Repository: {
+	name: string @go(Name)
+	url:  string @go(URL)
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/kubernetesobjects_go_gen.cue

@@ -0,0 +1,12 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+#KubernetesObjectsKind: "KubernetesObjects"
+
+// KubernetesObjects represents CUE output which directly provides Kubernetes api objects to holos.
+#KubernetesObjects: {
+	#HolosComponent
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/kustomization_go_gen.cue

@@ -0,0 +1,11 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+// Kustomization holds the rendered flux kustomization api object content for git ops.
+#Kustomization: {
+	// KsContent is the yaml representation of the flux kustomization for gitops.
+	ksContent?: string @go(KsContent)
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/kustomize_go_gen.cue

@@ -0,0 +1,25 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+#KustomizeBuildKind: "KustomizeBuild"
+
+// Kustomize represents resources necessary to execute a kustomize build.
+// Intended for at least two use cases:
+//
+//  1. Process raw yaml file resources in a holos component directory.
+//  2. Post process a HelmChart to inject istio, add custom labels, etc...
+#Kustomize: {
+	// KustomizeFiles holds file contents for kustomize, e.g. patch files.
+	kustomizeFiles?: #FileContentMap @go(KustomizeFiles)
+
+	// ResourcesFile is the file name used for api objects in kustomization.yaml
+	resourcesFile?: string @go(ResourcesFile)
+}
+
+// KustomizeBuild renders plain yaml files in the holos component directory using kubectl kustomize build.
+#KustomizeBuild: {
+	#HolosComponent
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/kustomizebuild_go_gen.cue

@@ -0,0 +1,12 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+#KustomizeBuildKind: "KustomizeBuild"
+
+// KustomizeBuild
+#KustomizeBuild: {
+	#HolosComponent
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/objectmap_go_gen.cue

@@ -0,0 +1,12 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+// APIObjectMap is the shape of marshalled api objects returned from cue to the
+// holos cli. A map is used to improve the clarity of error messages from cue.
+#APIObjectMap: {[string]: [string]: string}
+
+// FileContentMap is a map of file names to file contents.
+#FileContentMap: {[string]: string}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/objectmeta_go_gen.cue

@@ -0,0 +1,22 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+// ObjectMeta represents metadata of a holos component object. The fields are a
+// copy of upstream kubernetes api machinery but are by holos objects distinct
+// from kubernetes api objects.
+#ObjectMeta: {
+	// Name uniquely identifies the holos component instance and must be suitable as a file name.
+	name?: string @go(Name)
+
+	// Namespace confines a holos component to a single namespace via kustomize if set.
+	namespace?: string @go(Namespace)
+
+	// Labels are not used but are copied from api machinery ObjectMeta for completeness.
+	labels?: {[string]: string} @go(Labels,map[string]string)
+
+	// Annotations are not used but are copied from api machinery ObjectMeta for completeness.
+	annotations?: {[string]: string} @go(Annotations,map[string]string)
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/render_go_gen.cue

@@ -0,0 +1,7 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+#Renderer: _

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/result_go_gen.cue

@@ -0,0 +1,10 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+// Result is the build result for display or writing.  Holos components Render the Result as a data pipeline.
+#Result: {
+	HolosComponent: #HolosComponent
+}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/typemeta_go_gen.cue

@@ -0,0 +1,10 @@
+// Code generated by cue get go. DO NOT EDIT.
+
+//cue:generate cue get go github.com/holos-run/holos/api/v1alpha1
+
+package v1alpha1
+
+#TypeMeta: {
+	kind?:       string @go(Kind)
+	apiVersion?: string @go(APIVersion)
+}

docs/examples/cue.mod/usr/github.com/holos-run/holos/api/v1alpha1/buildplan.cue

@@ -0,0 +1,8 @@
+package v1alpha1
+
+// #BuildPlan is the API contract between CUE and the Holos cli.
+// Holos requires CUE to evaluate and provide a valid #BuildPlan.
+#BuildPlan: {
+	kind:       #BuildPlanKind
+	apiVersion: #APIVersion
+}

docs/examples/cue.mod/usr/github.com/holos-run/holos/api/v1alpha1/constraints.cue

@@ -0,0 +1,3 @@
+package v1alpha1
+
+#HolosComponent: metadata: name: string

docs/examples/cue.mod/usr/github.com/holos-run/holos/api/v1alpha1/defaults.cue

@@ -0,0 +1,5 @@
+package v1alpha1
+
+#HolosComponent: Skip: true | *false
+
+#HelmChart: enableHooks: true | *false

docs/examples/helpers.cue

@@ -0,0 +1,31 @@
+package holos
+
+import "encoding/yaml"
+
+// #APIObjects is the output type for api objects produced by cue.
+#APIObjects: {
+	// apiObjects holds each the api objects produced by cue.
+	apiObjects: {
+		[Kind=_]: {
+			[Name=_]: {
+				kind: Kind
+				...
+			}
+		}
+		ExternalSecret?: [Name=_]: #ExternalSecret & {_name: Name}
+		VirtualService?: [Name=_]: #VirtualService & {metadata: name: Name}
+		Issuer?: [Name=_]: #Issuer & {metadata: name: Name}
+	}
+
+	// apiObjectMap holds the marshalled representation of apiObjects
+	apiObjectMap: {
+		for kind, v in apiObjects {
+			"\(kind)": {
+				for name, obj in v {
+					"\(name)": yaml.Marshal(obj)
+				}
+			}
+		}
+		...
+	}
+}

docs/examples/holos.cue

@@ -0,0 +1,111 @@
+package holos
+
+import (
+	"encoding/yaml"
+	h "github.com/holos-run/holos/api/v1alpha1"
+	kc "sigs.k8s.io/kustomize/api/types"
+	ksv1 "kustomize.toolkit.fluxcd.io/kustomization/v1"
+)
+
+// The overall structure of the data is:
+// 1 CUE Instance => 1 BuildPlan => 0..N HolosComponents
+
+// Holos requires CUE to evaluate and provide a valid BuildPlan.
+// Constrain each CUE instance to output a BuildPlan.
+{} & h.#BuildPlan
+
+let DependsOn = {[Name=_]: name: string & Name}
+
+// #HolosComponent defines struct fields common to all holos component types.
+#HolosComponent: {
+	h.#HolosComponent
+	_dependsOn: DependsOn
+	let DEPENDS_ON = _dependsOn
+	metadata: name: string
+	#namelen: len(metadata.name) & >=1
+	let Name = metadata.name
+	ksContent: yaml.Marshal(#Kustomization & {
+		_dependsOn: DEPENDS_ON
+		metadata: name: Name
+	})
+	// Leave the HolosComponent open for components with additional fields like HelmChart.
+	// Refer to https://cuelang.org/docs/tour/types/closed/
+	...
+}
+
+//#KustomizeFiles represents resources for holos to write into files for kustomize post-processing.
+#KustomizeFiles: {
+	// Objects collects files for Holos to write for kustomize post-processing.
+	Objects: "kustomization.yaml": #Kustomize
+	// Files holds the marshaled output of Objects holos writes to the filesystem before calling the kustomize post-processor.
+	Files: {
+		for filename, obj in Objects {
+			"\(filename)": yaml.Marshal(obj)
+		}
+	}
+}
+
+// Holos component types.
+#HelmChart: #HolosComponent & h.#HelmChart & {
+	_values:         _
+	_kustomizeFiles: #KustomizeFiles
+
+	// Render the values to yaml for holos to provide to helm.
+	valuesContent: yaml.Marshal(_values)
+	// Kustomize post-processor
+	// resources is the intermediate file name for api objects.
+	resourcesFile: h.#ResourcesFile
+	// kustomizeFiles represents the files in a kustomize directory tree.
+	kustomizeFiles: _kustomizeFiles.Files
+}
+#KubernetesObjects: #HolosComponent & h.#KubernetesObjects
+#KustomizeBuild:    #HolosComponent & h.#KustomizeBuild
+
+// #ClusterName is the cluster name for cluster scoped resources.
+#ClusterName: #InputKeys.cluster
+
+// Flux Kustomization CRDs
+#Kustomization: #NamespaceObject & ksv1.#Kustomization & {
+	_dependsOn: DependsOn
+
+	metadata: {
+		name:      string
+		namespace: string | *"flux-system"
+	}
+	spec: ksv1.#KustomizationSpec & {
+		interval:      string | *"30m0s"
+		path:          string | *"deploy/clusters/\(#InputKeys.cluster)/components/\(metadata.name)"
+		prune:         bool | *true
+		retryInterval: string | *"2m0s"
+		sourceRef: {
+			kind: string | *"GitRepository"
+			name: string | *"flux-system"
+		}
+		suspend?:         bool
+		targetNamespace?: string
+		timeout:          string | *"3m0s"
+		// wait performs health checks for all reconciled resources. If set to true, .spec.healthChecks is ignored.
+		// Setting this to true for all components generates considerable load on the api server from watches.
+		// Operations are additionally more complicated when all resources are watched.  Consider setting wait true for
+		// relatively simple components, otherwise target specific resources with spec.healthChecks.
+		wait: true | *false
+		dependsOn: [for k, v in _dependsOn {v}, ...]
+	}
+}
+
+// #KustomizeTree represents a kustomize build.
+#KustomizeFiles: {
+}
+
+// #Kustomize represents the kustomize post processor.
+#Kustomize: kc.#Kustomization & {
+	_patches: {[_]: kc.#Patch}
+	apiVersion: "kustomize.config.k8s.io/v1beta1"
+	kind:       "Kustomization"
+	// resources are file names holos will use to store intermediate component output for kustomize to post-process (i.e. helm template | kubectl kustomize)
+	// See the related resourcesFile field of the holos component.
+	resources: [h.#ResourcesFile]
+	if len(_patches) > 0 {
+		patches: [for v in _patches {v}]
+	}
+}

docs/examples/optionalservices.cue

@@ -1,16 +1,19 @@
-package holos
-
 // Controls optional feature flags for services distributed across multiple holos components.
 // For example, enable issuing certificates in the provisioner cluster when an optional service is
 // enabled for a workload cluster.
+package holos
+
+import "list"
 
 #OptionalService: {
 	name:    string
 	enabled: true | *false
 	clusters: [Name=_]: #Platform.clusters[Name]
-	clusterNames: [for k, v in clusters {k}]
-	namespaces: [Name=_]: #ManagedNamespace & {
-		name: Name
+	clusterNames: [for c in clusters {c.name}]
+
+	managedNamespaces: [Name=_]: #ManagedNamespace & {
+		namespace: metadata: name: Name
+		clusterNames: ["provisioner", for c in clusters {c.name}]
 	}
 	// servers represents istio Gateway.spec.servers.hosts entries
 	// Refer to istio/gateway/gateway.cue
@@ -34,6 +37,10 @@ package holos
 	}
 }
 
-for k, v in #OptionalServices {
-	#ManagedNamespaces: v.namespaces
+for svc in #OptionalServices {
+	for nsName, ns in svc.managedNamespaces {
+		if svc.enabled && list.Contains(ns.clusterNames, #ClusterName) {
+			#ManagedNamespaces: "\(nsName)": ns
+		}
+	}
 }

docs/examples/platforms/optional.site.cue

@@ -8,7 +8,9 @@ let TargetNamespace = "prod-core-vault"
 		enabled: true
 		clusters: core1: _
 		clusters: core2: _
-		namespaces: "prod-core-vault": labels: "istio-injection": "enabled"
+		managedNamespaces: "prod-core-vault": {
+			namespace: metadata: labels: "istio-injection": "enabled"
+		}
 		certs: "vault-core": #Certificate & {
 			metadata: name:      "vault-core"
 			metadata: namespace: "istio-ingress"

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/postgres-certs/postgres-certs.cue

@@ -1,13 +1,29 @@
 package holos
 
 #InputKeys: component: "postgres-certs"
+
+let SecretNames = {
+	[Name=_]: {name: Name}
+	"\(_DBName)-primary-tls": _
+	"\(_DBName)-repl-tls":    _
+	"\(_DBName)-client-tls":  _
+	"\(_DBName)-root-ca":     _
+}
+
+#Kustomization: spec: targetNamespace: #TargetNamespace
+#Kustomization: spec: healthChecks: [
+	for s in SecretNames {
+		apiVersion: "external-secrets.io/v1beta1"
+		kind:       "ExternalSecret"
+		name:       s.name
+		namespace:  #TargetNamespace
+	},
+]
+
 #KubernetesObjects & {
 	apiObjects: {
-		ExternalSecret: {
-			"\(_DBName)-primary-tls": _
-			"\(_DBName)-repl-tls":    _
-			"\(_DBName)-client-tls":  _
-			"\(_DBName)-root-ca":     _
+		for s in SecretNames {
+			ExternalSecret: "\(s.name)": _
 		}
 	}
 }

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/postgres/postgres.cue

@@ -13,11 +13,29 @@ let BucketRepoName = "repo2"
 
 // Restore options.  Set the timestamp to a known good point in time.
 // time="2024-03-11T17:08:58Z" level=info msg="crunchy-pgbackrest ends"
-let RestoreOptions = ["--type=time", "--target=\"2024-03-11 17:10:00+00\""]
+// let RestoreOptions = ["--type=time", "--target=\"2024-03-11 17:10:00+00\""]
+
+// Restore the most recent backup.
+let RestoreOptions = []
+
+#Kustomization: spec: healthChecks: [
+	{
+		apiVersion: "external-secrets.io/v1beta1"
+		kind:       "ExternalSecret"
+		name:       S3Secret
+		namespace:  #TargetNamespace
+	},
+	{
+		apiVersion: "postgres-operator.crunchydata.com/v1beta1"
+		kind:       "PostgresCluster"
+		name:       _DBName
+		namespace:  #TargetNamespace
+	},
+]
 
 #KubernetesObjects & {
 	apiObjects: {
-		ExternalSecret: "pgo-s3-creds": _
+		ExternalSecret: "\(S3Secret)": _
 		PostgresCluster: db: #PostgresCluster & HighlyAvailable & {
 			metadata: name:      _DBName
 			metadata: namespace: #TargetNamespace
@@ -40,7 +58,7 @@ let RestoreOptions = ["--type=time", "--target=\"2024-03-11 17:10:00+00\""]
 					replicas: 2
 					dataVolumeClaimSpec: {
 						accessModes: ["ReadWriteOnce"]
-						resources: requests: storage: string | *"1Gi"
+						resources: requests: storage: "10Gi"
 					}
 				}]
 				standby: {
@@ -103,7 +121,7 @@ let RestoreOptions = ["--type=time", "--target=\"2024-03-11 17:10:00+00\""]
 							name: "repo1"
 							volume: volumeClaimSpec: {
 								accessModes: ["ReadWriteOnce"]
-								resources: requests: storage: string | *"1Gi"
+								resources: requests: storage: string | *"4Gi"
 							}
 						},
 						{
@@ -137,7 +155,7 @@ let HighlyAvailable = {
 			replicas: 2
 			dataVolumeClaimSpec: {
 				accessModes: ["ReadWriteOnce"]
-				resources: requests: storage: "1Gi"
+				resources: requests: storage: string | *"10Gi"
 			}
 			affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: [{
 				weight: 1

docs/examples/platforms/reference/clusters/accounts/iam/zitadel/zitadel/zitadel.cue

@@ -7,7 +7,33 @@ let Name = "zitadel"
 #DependsOn: postgres:  _
 
 // Upstream helm chart doesn't specify the namespace field for all resources.
-#Kustomization: spec: targetNamespace: #TargetNamespace
+#Kustomization: spec: {
+	targetNamespace: #TargetNamespace
+	wait:            false
+}
+
+if #IsPrimaryCluster == true {
+	#Kustomization: spec: healthChecks: [
+		{
+			apiVersion: "apps/v1"
+			kind:       "Deployment"
+			name:       Name
+			namespace:  #TargetNamespace
+		},
+		{
+			apiVersion: "batch/v1"
+			kind:       "Job"
+			name:       "\(Name)-init"
+			namespace:  #TargetNamespace
+		},
+		{
+			apiVersion: "batch/v1"
+			kind:       "Job"
+			name:       "\(Name)-setup"
+			namespace:  #TargetNamespace
+		},
+	]
+}
 
 #HelmChart & {
 	namespace:   #TargetNamespace
@@ -61,43 +87,79 @@ let DatabaseCACertPatch = [
 	},
 ]
 
-#Kustomize: {
-	patches: [
-		{
+let CAPatch = #Patch & {
+	target: {
+		group:   "apps" | "batch"
+		version: "v1"
+		kind:    "Job" | "Deployment"
+		name:    string
+	}
+	patch: yaml.Marshal(DatabaseCACertPatch)
+}
+
+// TODO: Replace with #Kustomize & { _patches: foo: {} }
+#KustomizePatches: {
+	mesh: {
+		target: {
+			group:   "apps"
+			version: "v1"
+			kind:    "Deployment"
+			name:    Name
+		}
+		patch: yaml.Marshal(IstioInject)
+	}
+	deploymentCA: CAPatch & {
+		target: group: "apps"
+		target: kind:  "Deployment"
+		target: name:  Name
+	}
+	initJob: CAPatch & {
+		target: group: "batch"
+		target: kind:  "Job"
+		target: name:  "\(Name)-init"
+	}
+	setupJob: CAPatch & {
+		target: group: "batch"
+		target: kind:  "Job"
+		target: name:  "\(Name)-setup"
+	}
+	testDisable: {
+		target: {
+			version: "v1"
+			kind:    "Pod"
+			name:    "\(Name)-test-connection"
+		}
+		patch: yaml.Marshal(DisableFluxPatch)
+	}
+	if #IsPrimaryCluster == false {
+		fluxDisable: {
 			target: {
 				group:   "apps"
 				version: "v1"
 				kind:    "Deployment"
 				name:    Name
 			}
-			patch: yaml.Marshal(IstioInject)
-		},
-		{
-			target: {
-				group:   "apps"
-				version: "v1"
-				kind:    "Deployment"
-				name:    Name
-			}
-			patch: yaml.Marshal(DatabaseCACertPatch)
-		},
-		{
+			patch: yaml.Marshal(DisableFluxPatch)
+		}
+		initDisable: {
 			target: {
 				group:   "batch"
 				version: "v1"
 				kind:    "Job"
 				name:    "\(Name)-init"
 			}
-			patch: yaml.Marshal(DatabaseCACertPatch)
-		},
-		{
+			patch: yaml.Marshal(DisableFluxPatch)
+		}
+		setupDisable: {
 			target: {
 				group:   "batch"
 				version: "v1"
 				kind:    "Job"
 				name:    "\(Name)-setup"
 			}
-			patch: yaml.Marshal(DatabaseCACertPatch)
-		},
-	]
+			patch: yaml.Marshal(DisableFluxPatch)
+		}
+	}
 }
+
+let DisableFluxPatch = [{op: "replace", path: "/metadata/annotations/kustomize.toolkit.fluxcd.io~1reconcile", value: "disabled"}]

docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/arc.cue

@@ -4,6 +4,6 @@ package holos
 #InputKeys: project: "github"
 #DependsOn: Namespaces: name: "prod-secrets-namespaces"
 
-#TargetNamespace: #InputKeys.component
+#ARCSystemNamespace: "arc-system"
 #HelmChart: namespace: #TargetNamespace
 #HelmChart: chart: version: "0.8.3"

docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/runner/arc-runner.cue

@@ -1,26 +1,51 @@
 package holos
 
+#TargetNamespace: "arc-runner"
 #InputKeys: component: "arc-runner"
 #Kustomization: spec: targetNamespace: #TargetNamespace
 
-#HelmChart & {
-	values: {
-		#Values
-		controllerServiceAccount: name:      "gha-rs-controller"
-		controllerServiceAccount: namespace: "arc-system"
-		githubConfigSecret: "controller-manager"
-		githubConfigUrl:    "https://github.com/" + #Platform.org.github.orgs.primary.name
-	}
-	apiObjects: ExternalSecret: "\(values.githubConfigSecret)": _
-	chart: {
-		// Match the gha-base-name in the chart _helpers.tpl to avoid long full names.
-		// NOTE: Unfortunately the INSTALLATION_NAME is used as the helm release
-		// name and GitHub removed support for runner labels, so the only way to
-		// specify which runner a workflow runs on is using this helm release name.
-		// The quote is "Update the INSTALLATION_NAME value carefully. You will use
-		// the installation name as the value of runs-on in your workflows."  Refer to
-		// https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller
-		release: "gha-rs"
-		name:    "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set"
-	}
+let GitHubConfigSecret = "controller-manager"
+
+// Just sync the external secret, don't configure the scale set
+// Work around https://github.com/actions/actions-runner-controller/issues/3351
+if #IsPrimaryCluster == false {
+	spec: components: KubernetesObjects: [
+		#KubernetesObjects & {
+			metadata: name:                        "prod-github-arc-runner"
+			_dependsOn: "prod-secrets-namespaces": _
+
+			apiObjectMap: (#APIObjects & {
+				apiObjects: ExternalSecret: "\(GitHubConfigSecret)": _
+			}).apiObjectMap
+		},
+	]
+}
+
+// Put the scale set on the primary cluster.
+if #IsPrimaryCluster == true {
+	spec: components: HelmCharts: [
+		#HelmChart & {
+			_dependsOn: "prod-secrets-namespaces": _
+			metadata: name:                        "prod-github-arc-runner"
+			_values: {
+				#Values
+				controllerServiceAccount: name:      "gha-rs-controller"
+				controllerServiceAccount: namespace: "arc-system"
+				githubConfigSecret: GitHubConfigSecret
+				githubConfigUrl:    "https://github.com/" + #Platform.org.github.orgs.primary.name
+			}
+			apiObjectMap: (#APIObjects & {apiObjects: ExternalSecret: "\(_values.githubConfigSecret)": _}).apiObjectMap
+			chart: {
+				// Match the gha-base-name in the chart _helpers.tpl to avoid long full names.
+				// NOTE: Unfortunately the INSTALLATION_NAME is used as the helm release
+				// name and GitHub removed support for runner labels, so the only way to
+				// specify which runner a workflow runs on is using this helm release name.
+				// The quote is "Update the INSTALLATION_NAME value carefully. You will use
+				// the installation name as the value of runs-on in your workflows."  Refer to
+				// https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller
+				release: "gha-rs"
+				name:    "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set"
+			}
+		},
+	]
 }

docs/examples/platforms/reference/clusters/foundation/cloud/github/arc/system/arc-system.cue

@@ -1,15 +1,20 @@
 package holos
 
-#TargetNamespace: "arc-system"
+#TargetNamespace: #ARCSystemNamespace
 #InputKeys: component: "arc-system"
 
-#HelmChart & {
-	values:    #Values & #DefaultSecurityContext
-	namespace: #TargetNamespace
-	chart: {
-		// Match the gha-base-name in the chart _helpers.tpl to avoid long full names.
-		release: "gha-rs-controller"
-		name:    "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller"
-		version: "0.8.3"
-	}
-}
+spec: components: HelmCharts: [
+	#HelmChart & {
+		metadata: name: "prod-github-arc-system"
+
+		_dependsOn: "prod-secrets-namespaces": _
+		_values:   #Values & #DefaultSecurityContext
+		namespace: #TargetNamespace
+		chart: {
+			// Match the gha-base-name in the chart _helpers.tpl to avoid long full names.
+			release: "gha-rs-controller"
+			name:    "oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller"
+			version: "0.8.3"
+		}
+	},
+]

docs/examples/platforms/reference/clusters/foundation/cloud/init/namespaces/component.cue

@@ -1,22 +1,24 @@
 package holos
 
-#TargetNamespace: "default"
+import "list"
 
-#InputKeys: {
-	project:   "secrets"
-	component: "namespaces"
-}
+spec: components: KubernetesObjects: [
+	#KubernetesObjects & {
+		metadata: name: "prod-secrets-namespaces"
+		apiObjectMap: (#APIObjects & {
+			apiObjects: {
+				// #ManagedNamespaces is the set of all namespaces across all clusters in the platform.
+				for k, ns in #ManagedNamespaces {
+					if list.Contains(ns.clusterNames, #ClusterName) {
+						Namespace: "\(k)": #Namespace & ns.namespace
+					}
+				}
 
-#KubernetesObjects & {
-	apiObjects: {
-		// #ManagedNamespaces is the set of all namespaces across all clusters in the platform.
-		for k, ns in #ManagedNamespaces {
-			Namespace: "\(ns.name)": #Namespace & {metadata: ns}
-		}
-
-		// #PlatformNamespaces is deprecated in favor of #ManagedNamespaces.
-		for ns in #PlatformNamespaces {
-			Namespace: "\(ns.name)": #Namespace & {metadata: ns}
-		}
-	}
-}
+				// #PlatformNamespaces is deprecated in favor of #ManagedNamespaces.
+				for ns in #PlatformNamespaces {
+					Namespace: "\(ns.name)": #Namespace & {metadata: ns}
+				}
+			}
+		}).apiObjectMap
+	},
+]

docs/examples/platforms/reference/clusters/foundation/cloud/mesh/istio/gateway/gateway.cue

@@ -34,17 +34,15 @@ let LoginCert = #PlatformCerts.login
 		}
 
 		for k, svc in #OptionalServices {
-			if svc.enabled {
-				if list.Contains(svc.clusterNames, #ClusterName) {
-					Gateway: "\(svc.name)": #Gateway & {
-						metadata: name:      svc.name
-						metadata: namespace: #TargetNamespace
-						spec: selector: istio: "ingressgateway"
-						spec: servers: [for s in svc.servers {s}]
-					}
-					for k, s in svc.servers {
-						ExternalSecret: "\(s.tls.credentialName)": _
-					}
+			if svc.enabled && list.Contains(svc.clusterNames, #ClusterName) {
+				Gateway: "\(svc.name)": #Gateway & {
+					metadata: name:      svc.name
+					metadata: namespace: #TargetNamespace
+					spec: selector: istio: "ingressgateway"
+					spec: servers: [for s in svc.servers {s}]
+				}
+				for k, s in svc.servers {
+					ExternalSecret: "\(s.tls.credentialName)": _
 				}
 			}
 		}

docs/examples/platforms/reference/clusters/foundation/cloud/secrets/eso-creds-refresher/component.cue

@@ -93,7 +93,14 @@ provisioner get serviceaccount -A --selector=holos.run/job.name=\(NAME) --output
 
 # Create the tokens
 mkdir tokens
-jq -r '.items[].metadata | "provisioner -n \\(.namespace) create token --duration=12h \\(.name) > tokens/\\(.namespace).\\(.name).jwt"' serviceaccounts.json | bash -x
+
+kubectl get namespaces -o name > namespaces.txt
+
+# Iterate over local namespaces
+while IFS= read -r NAMESPACE; do
+  echo "Getting token for local cluster $NAMESPACE" >&2
+  jq -r '.items[] | select("namespace/"+.metadata.namespace == "'${NAMESPACE}'") | .metadata | "provisioner -n \\(.namespace) create token --duration=12h \\(.name) > tokens/\\(.namespace).\\(.name).jwt"' serviceaccounts.json | bash -x
+done < namespaces.txt
 
 # Create the secrets
 mksecret tokens/*.jwt
@@ -124,6 +131,11 @@ kubectl apply --server-side=true -f secrets.yaml
 					resources: ["secrets"]
 					verbs: ["*"]
 				},
+				{
+					apiGroups: [""]
+					resources: ["namespaces"]
+					verbs: ["list"]
+				},
 			]
 		},
 		// Bind the Role to the ServiceAccount for the Job.

docs/examples/platforms/reference/clusters/foundation/cloud/secrets/secretstores/secretstores.cue

@@ -1,5 +1,7 @@
 package holos
 
+import "list"
+
 #DependsOn: _ESOCreds
 
 #TargetNamespace: "default"
@@ -31,9 +33,11 @@ package holos
 			}
 		}
 
-		for k, ns in #ManagedNamespaces {
-			let obj = #SecretStore & {_namespace: ns.name}
-			SecretStore: "\(ns.name)/\(obj.metadata.name)": obj
+		for nsName, ns in #ManagedNamespaces {
+			if list.Contains(ns.clusterNames, #ClusterName) {
+				let obj = #SecretStore & {_namespace: nsName}
+				SecretStore: "\(nsName)/\(obj.metadata.name)": obj
+			}
 		}
 	}
 }

docs/examples/platforms/reference/clusters/optional/vault/vault.cue

@@ -11,67 +11,65 @@ let Name = "vault"
 
 let Vault = #OptionalServices[Name]
 
-if Vault.enabled {
-	if list.Contains(Vault.clusterNames, #ClusterName) {
-		#HelmChart & {
-			namespace: #TargetNamespace
-			chart: {
-				name:    Name
-				version: "0.25.0"
-				repository: {
-					name: "hashicorp"
-					url:  "https://helm.releases.hashicorp.com"
-				}
-			}
-			values: #Values
-
-			apiObjects: {
-				ExternalSecret: "gcpkms-creds":      _
-				ExternalSecret: "vault-server-cert": _
-				VirtualService: "\(Name)": {
-					metadata: name:      Name
-					metadata: namespace: #TargetNamespace
-					spec: hosts: [for cert in Vault.certs {cert.spec.commonName}]
-					spec: gateways: ["istio-ingress/\(Name)"]
-					spec: http: [
-						{
-							route: [
-								{
-									destination: host: "\(Name)-active"
-									destination: port: number: 8200
-								},
-							]
-						},
-					]
-				}
+if Vault.enabled && list.Contains(Vault.clusterNames, #ClusterName) {
+	#HelmChart & {
+		namespace: #TargetNamespace
+		chart: {
+			name:    Name
+			version: "0.25.0"
+			repository: {
+				name: "hashicorp"
+				url:  "https://helm.releases.hashicorp.com"
 			}
 		}
+		values: #Values
 
-		#Kustomize: {
-			patches: [
-				{
-					target: {
-						group:   "apps"
-						version: "v1"
-						kind:    "StatefulSet"
-						name:    Name
-					}
-					patch: yaml.Marshal(EnvPatch)
-				},
-			]
+		apiObjects: {
+			ExternalSecret: "gcpkms-creds":      _
+			ExternalSecret: "vault-server-cert": _
+			VirtualService: "\(Name)": {
+				metadata: name:      Name
+				metadata: namespace: #TargetNamespace
+				spec: hosts: [for cert in Vault.certs {cert.spec.commonName}]
+				spec: gateways: ["istio-ingress/\(Name)"]
+				spec: http: [
+					{
+						route: [
+							{
+								destination: host: "\(Name)-active"
+								destination: port: number: 8200
+							},
+						]
+					},
+				]
+			}
 		}
+	}
 
-		let EnvPatch = [
+	#Kustomize: {
+		patches: [
 			{
-				op:    "test"
-				path:  "/spec/template/spec/containers/0/env/4/name"
-				value: "VAULT_ADDR"
-			},
-			{
-				op:    "replace"
-				path:  "/spec/template/spec/containers/0/env/4/value"
-				value: "http://$(VAULT_K8S_POD_NAME):8200"
+				target: {
+					group:   "apps"
+					version: "v1"
+					kind:    "StatefulSet"
+					name:    Name
+				}
+				patch: yaml.Marshal(EnvPatch)
 			},
 		]
 	}
+
+	let EnvPatch = [
+		{
+			op:    "test"
+			path:  "/spec/template/spec/containers/0/env/4/name"
+			value: "VAULT_ADDR"
+		},
+		{
+			op:    "replace"
+			path:  "/spec/template/spec/containers/0/env/4/value"
+			value: "http://$(VAULT_K8S_POD_NAME):8200"
+		},
+	]
 }

docs/examples/platforms/reference/clusters/projects/projects.cue

@@ -0,0 +1,3 @@
+package holos
+
+#InputKeys: project: "projects"

docs/examples/platforms/reference/clusters/provisioner/secrets/eso-creds-refresher/component.cue

@@ -25,11 +25,11 @@ ksObjects: []
 			}
 		}
 
-		for k, ns in #ManagedNamespaces {
-			for obj in (#PlatformNamespaceObjects & {_ns: ns}).objects {
+		for nsName, ns in #ManagedNamespaces {
+			for obj in (#PlatformNamespaceObjects & {_ns: ns.namespace.metadata}).objects {
 				let Kind = obj.kind
 				let Name = obj.metadata.name
-				"\(Kind)": "\(ns.name)/\(Name)": obj
+				"\(Kind)": "\(nsName)/\(Name)": obj
 			}
 		}
 	}

docs/examples/platforms/reference/clusters/provisioner/secrets/namespaces/component.cue

@@ -10,8 +10,8 @@ package holos
 #KubernetesObjects & {
 	apiObjects: {
 		// #ManagedNamespaces is the set of all namespaces across all clusters in the platform.
-		for k, ns in #ManagedNamespaces {
-			Namespace: "\(ns.name)": #Namespace & {metadata: ns}
+		for nsName, ns in #ManagedNamespaces {
+			Namespace: "\(nsName)": #Namespace & ns.namespace
 		}
 
 		// #PlatformNamespaces is deprecated in favor of #ManagedNamespaces.

docs/examples/platforms/reference/projects/secrets/components/namespaces/inputkeys.cue

@@ -1,8 +0,0 @@
-package holos
-
-#TargetNamespace: "external-secrets"
-
-#InputKeys: {
-	project: "secrets"
-	service: "eso"
-}

docs/examples/platforms/reference/projects/secrets/components/namespaces/namespaces.cue

@@ -1,10 +0,0 @@
-package holos
-
-#InputKeys: component: "namespaces"
-
-metadata: name: #InstanceName
-objects: [
-	#Namespace & {
-		metadata: name: #TargetNamespace
-	},
-]

docs/examples/platforms/reference/projects/secrets/components/namespaces/output.cue

@@ -1,6 +0,0 @@
-package holos
-
-// Output schema
-{} & #KubernetesObjects & {
-	ksObjects: [#Kustomization]
-}

docs/examples/project_types.cue

@@ -0,0 +1,11 @@
+package holos
+
+#Project: {
+	name: string
+}
+
+#Projects: {
+	[Name=_]: #Project & {
+		name: Name
+	}
+}

docs/examples/schema.cue

@@ -2,7 +2,6 @@ package holos
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	ksv1 "kustomize.toolkit.fluxcd.io/kustomization/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -14,59 +13,26 @@ import (
 	crt "cert-manager.io/certificate/v1"
 	gw "networking.istio.io/gateway/v1beta1"
 	vs "networking.istio.io/virtualservice/v1beta1"
-	kc "sigs.k8s.io/kustomize/api/types"
 	pg "postgres-operator.crunchydata.com/postgrescluster/v1beta1"
-	"encoding/yaml"
 )
 
-let ResourcesFile = "resources.yaml"
-
 // _apiVersion is the version of this schema.  Defines the interface between CUE output and the holos cli.
 _apiVersion: "holos.run/v1alpha1"
 
-// #ClusterName is the cluster name for cluster scoped resources.
-#ClusterName: #InputKeys.cluster
+// #ComponentName is the name of the holos component.
+// TODO: Refactor to support multiple components per BuildPlan
+#ComponentName: #InputKeys.component
 
 // #StageName is prod, dev, stage, etc...  Usually prod for platform components.
 #StageName: #InputKeys.stage
 
-// #CollectionName is the preferred handle to the collection element of the instance name.  A collection name mapes to an "application name" as described in the kubernetes recommended labels documentation.  Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
-#CollectionName: #InputKeys.project
-
-// #ComponentName is the name of the holos component.
-#ComponentName: #InputKeys.component
-
-// #InstanceName is the name of the holos component instance being managed varying by stage, project, and component names.
-#InstanceName: "\(#StageName)-\(#CollectionName)-\(#ComponentName)"
-
-// #InstancePrefix is the stage and project without the component name.  Useful for dependency management among multiple components for a project stage.
-#InstancePrefix: "\(#StageName)-\(#CollectionName)"
-
 // #TargetNamespace is the target namespace for a holos component.
 #TargetNamespace: string
 
-// #SelectorLabels are mixed into selectors.
-#SelectorLabels: {
-	"holos.run/stage.name":     #StageName
-	"holos.run/project.name":   #CollectionName
-	"holos.run/component.name": #ComponentName
-	...
-}
-
-// #CommonLabels are mixed into every kubernetes api object.
-#CommonLabels: {
-	#SelectorLabels
-	"app.kubernetes.io/part-of":   #StageName
-	"app.kubernetes.io/name":      #CollectionName
-	"app.kubernetes.io/component": #ComponentName
-	"app.kubernetes.io/instance":  #InstanceName
-	...
-}
-
 #ClusterObject: {
 	_description: string | *""
 	metadata: metav1.#ObjectMeta & {
-		labels: #CommonLabels
+		// labels: #CommonLabels
 		annotations: #Description & {
 			_Description: _description
 			...
@@ -88,12 +54,8 @@ _apiVersion: "holos.run/v1alpha1"
 }
 
 // Kubernetes API Objects
-#Namespace: corev1.#Namespace & #ClusterObject & {
-	metadata: {
-		name: string
-		labels: "kubernetes.io/metadata.name": name
-	}
-}
+#Namespace: corev1.#Namespace
+
 #ClusterRole:        #ClusterObject & rbacv1.#ClusterRole
 #ClusterRoleBinding: #ClusterObject & rbacv1.#ClusterRoleBinding
 #ClusterIssuer: #ClusterObject & ci.#ClusterIssuer & {...}
@@ -134,37 +96,6 @@ _apiVersion: "holos.run/v1alpha1"
 	}
 }
 
-// Flux Kustomization CRDs
-#Kustomization: #NamespaceObject & ksv1.#Kustomization & {
-	metadata: {
-		name:      #InstanceName
-		namespace: string | *"flux-system"
-	}
-	spec: ksv1.#KustomizationSpec & {
-		interval:      string | *"30m0s"
-		path:          string | *"deploy/clusters/\(#InputKeys.cluster)/components/\(#InstanceName)"
-		prune:         bool | *true
-		retryInterval: string | *"2m0s"
-		sourceRef: {
-			kind: string | *"GitRepository"
-			name: string | *"flux-system"
-		}
-		suspend?:         bool
-		targetNamespace?: string
-		timeout:          string | *"3m0s"
-		wait:             bool | *true
-		dependsOn: [for k, v in #DependsOn {v}]
-	}
-}
-
-// #DependsOn stores all of the dependencies between components.  It's a struct to support merging across levels in the tree.
-#DependsOn: {
-	[Name=_]: {
-		name: string | *"\(#InstancePrefix)-\(Name)"
-	}
-	...
-}
-
 // External Secrets CRDs
 #ExternalSecret: #NamespaceObject & es.#ExternalSecret & {
 	_name: string
@@ -287,16 +218,21 @@ _apiVersion: "holos.run/v1alpha1"
 
 // ManagedNamespace is a namespace to manage across all clusters in the holos platform.
 #ManagedNamespace: {
-	// TODO metadata labels and annotations
-	name: string
-	labels: [string]: string
+	namespace: {
+		metadata: {
+			name: string
+			labels: [string]: string
+		}
+	}
+	// clusterNames represents the set of clusters the namespace is managed on.  Usually all clusters.
+	clusterNames: [...string]
 }
 
 // #ManagedNamepsaces is the union of all namespaces across all cluster types and optional services.
 // Holos adopts the namespace sameness position of SIG Multicluster, refer to https://github.com/kubernetes/community/blob/dd4c8b704ef1c9c3bfd928c6fa9234276d61ad18/sig-multicluster/namespace-sameness-position-statement.md
 #ManagedNamespaces: {
-	[Name=_]: {
-		name: Name
+	[Name=_]: #ManagedNamespace & {
+		namespace: metadata: name: Name
 	}
 }
 
@@ -309,65 +245,6 @@ _apiVersion: "holos.run/v1alpha1"
 	}
 }
 
-// #APIObjects is the output type for api objects produced by cue.  A map is used to aid debugging and clarity.
-#APIObjects: {
-	// apiObjects holds each the api objects produced by cue.
-	apiObjects: {
-		[Kind=_]: {
-			[Name=_]: metav1.#TypeMeta & {
-				kind: Kind
-			}
-		}
-		ExternalSecret?: [Name=_]: #ExternalSecret & {_name: Name}
-		VirtualService?: [Name=_]: #VirtualService & {metadata: name: Name}
-		Issuer?: [Name=_]: #Issuer & {metadata: name: Name}
-	}
-
-	// apiObjectMap holds the marshalled representation of apiObjects
-	apiObjectMap: {
-		for kind, v in apiObjects {
-			"\(kind)": {
-				for name, obj in v {
-					"\(name)": yaml.Marshal(obj)
-				}
-			}
-		}
-		...
-	}
-}
-
-// #OutputTypeMeta is shared among all output types
-#OutputTypeMeta: {
-	// apiVersion is the output api version
-	apiVersion: _apiVersion
-	// kind is a discriminator of the type of output
-	kind: #PlatformSpec.kind | #KubernetesObjects.kind | #HelmChart.kind | #NoOutput.kind
-	// name holds a unique name suitable for a filename
-	metadata: name: string
-	// debug returns arbitrary debug output.
-	debug?: _
-}
-
-#NoOutput: {
-	#OutputTypeMeta
-	kind: string | *"Skip"
-	metadata: name: string | *"skipped"
-}
-
-// #KubernetesObjectOutput is the output schema of a single component.
-#KubernetesObjects: {
-	#OutputTypeMeta
-	#APIObjects
-	kind: "KubernetesObjects"
-	metadata: name: #InstanceName
-	// ksObjects holds the flux Kustomization objects for gitops
-	ksObjects: [...#Kustomization] | *[#Kustomization]
-	// ksContent is the yaml representation of kustomization
-	ksContent: yaml.Marshal(#Kustomization)
-	// platform returns the platform data structure for visibility / troubleshooting.
-	platform: #Platform
-}
-
 // #Chart defines an upstream helm chart
 #Chart: {
 	name:    string
@@ -382,57 +259,6 @@ _apiVersion: "holos.run/v1alpha1"
 // #ChartValues represent the values provided to a helm chart.  Existing values may be imorted using cue import values.yaml -p holos then wrapping the values.cue content in #Values: {}
 #ChartValues: {...}
 
-// #HelmChart is a holos component which produces kubernetes api objects from cue values provided to the helm template command.
-#HelmChart: {
-	#OutputTypeMeta
-	#APIObjects
-	kind: "HelmChart"
-	metadata: name: #InstanceName
-	// ksObjects holds the flux Kustomization objects for gitops.
-	ksObjects: [...#Kustomization] | *[#Kustomization]
-	// ksContent is the yaml representation of kustomization.
-	ksContent: yaml.MarshalStream(ksObjects)
-	// namespace defines the value passed to the helm --namespace flag
-	namespace: #TargetNamespace
-	// chart defines the upstream helm chart to process.
-	chart: #Chart
-	// values represents the helm values to provide to the chart.
-	values: #ChartValues
-	// valuesContent holds the values yaml
-	valuesContent: yaml.Marshal(values)
-	// platform returns the platform data structure for visibility / troubleshooting.
-	platform: #Platform
-	// instance returns the key values of the holos component instance.
-	instance: #InputKeys
-	// resources is the intermediate file name for api objects.
-	resourcesFile: ResourcesFile
-	// kustomizeFiles represents the files in a kustomize directory tree.
-	kustomizeFiles: #KustomizeFiles.Files
-	// enableHooks removes the --no-hooks flag from helm template
-	enableHooks: true | *false
-}
-
-// #KustomizeBuild is a holos component that uses plain yaml files as the source of api objects for a holos component.
-// Intended for upstream components like the CrunchyData Postgres Operator.  The holos cli is expected to execute kustomize build on the component directory to produce the rendered output.
-#KustomizeBuild: {
-	#OutputTypeMeta
-	#APIObjects
-	kind: "KustomizeBuild"
-	metadata: name: #InstanceName
-	// ksObjects holds the flux Kustomization objects for gitops.
-	ksObjects: [...#Kustomization] | *[#Kustomization]
-	// ksContent is the yaml representation of kustomization.
-	ksContent: yaml.MarshalStream(ksObjects)
-	// namespace defines the value passed to the helm --namespace flag
-	namespace: #TargetNamespace
-}
-
-// #PlatformSpec is the output schema of a platform specification.
-#PlatformSpec: {
-	#OutputTypeMeta
-	kind: "PlatformSpec"
-}
-
 // #SecretName is the name of a Secret, ususally coupling a Deployment to an ExternalSecret
 #SecretName: string
 
@@ -445,28 +271,6 @@ _apiVersion: "holos.run/v1alpha1"
 	...
 }
 
-// #KustomizeTree represents a kustomize build.
-#KustomizeFiles: {
-	Objects: {
-		"kustomization.yaml": #Kustomize
-	}
-	// Files holds the marshaled output holos writes to the filesystem
-	Files: {
-		for filename, obj in Objects {
-			"\(filename)": yaml.Marshal(obj)
-		}
-		...
-	}
-}
-
-// kustomization.yaml
-#Kustomize: kc.#Kustomization & {
-	apiVersion: "kustomize.config.k8s.io/v1beta1"
-	kind:       "Kustomization"
-	resources: [ResourcesFile]
-	...
-}
-
 // #DefaultSecurityContext is the holos default security context to comply with the restricted namespace policy.
 // Refer to https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
 #DefaultSecurityContext: {
@@ -485,7 +289,6 @@ _apiVersion: "holos.run/v1alpha1"
 	spec: secretName: metadata.name
 }
 
-// By default, render kind: Skipped so holos knows to skip over intermediate cue files.
-// This enables the use of holos render ./foo/bar/baz/... when bar contains intermediary constraints which are not complete components.
-// Holos skips over these intermediary cue instances.
-{} & #NoOutput
+// #IsPrimaryCluster is true if the cluster being rendered is the primary cluster
+// Used by the iam project to determine where https://login.example.com is active.
+#IsPrimaryCluster: bool & #ClusterName == #Platform.primaryCluster.name

docs/runbooks/login/failover.md

@@ -0,0 +1,305 @@
+## Overview
+
+Use this run book to move the ZITADEL primary postgres database from one cluster to another.  At the end of the process https://zitadel.example.com service will be live on the cluster which was previously the standby at the start of the process.
+
+Downtime: ~10 minutes if executed quickly.  ~60 minutes if executed slowly and deliberately.
+
+## Initial State
+1. https://login.ois.run is routed to core2.
+2. `postgrescluster/zitadel` on core2 is primary.
+3. `postgrescluster/zitadel` on core1 is standby.
+4. A recent [[#Full Backup]] has been taken and is stored in S3.  
+5. The standby cluster has been restored from the recent full backup.
+## Process
+1. [[#Edit Platform site config]]
+2. [[#Render core2]]
+3. [[#Demote core2]]  
+4. [[#Verify core2 is standby]]
+5. [[#Render core1]]
+6. [[#Promote core1]]
+7. [[#Verify core1 is primary]]
+8. [[#Reconcile Zitadel]]
+9. [[#Cut over NLB]]
+10. [[#Test Login]]
+
+## Edit Platform site config
+Change the primary cluster from core2 to core1 in `platforms/platform.site.cue`.
+```cue
+#Platform: primaryCluster: name: "core1"
+```
+## Render core2
+Configure core as a standby.
+```
+holos render --cluster-name=core2 ~/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/accounts/...
+```
+
+Expected change:
+```diff
+diff --git a/deploy/clusters/core2/components/prod-iam-postgres/prod-iam-postgres.gen.yaml b/deploy/clusters/core2/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
+index c1aa8394..45daa650 100644
+--- a/deploy/clusters/core2/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
++++ b/deploy/clusters/core2/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
+@@ -85,24 +85,6 @@ spec:
+     name: zitadel-repl-tls
+   customTLSSecret:
+     name: zitadel-primary-tls
+-  dataSource:
+-    pgbackrest:
+-      configuration:
+-        - secret:
+-            name: pgo-s3-creds
+-      global:
+-        repo2-path: /pgbackrest/prod-iam-zitadel/zitadel/repo2
+-        repo2-cipher-type: aes-256-cbc
+-      options:
+-        - --type=time
+-        - --target="2024-03-11 17:10:00+00"
+-      repo:
+-        name: repo2
+-        s3:
+-          bucket: ois-zitadel-backups
+-          endpoint: s3.dualstack.us-east-2.amazonaws.com
+-          region: us-east-2
+-      stanza: db
+   image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.2-0
+   instances:
+     - name: pgha1
+@@ -139,7 +121,7 @@ spec:
+       image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.21-3
+       replicas: 2
+   standby:
+-    enabled: false
++    enabled: true
+     repoName: repo2
+   users:
+     - databases
+```
+## Demote core2
+
+```
+10012  git add -p
+10013  git commit -m 'Make core2 a standby zitadel cluster'
+10014  git push origin HEAD
+10015  flux reconcile source git flux-system
+10016  flux get ks
+```
+
+## Verify core2 is standby
+```
+k logs $(kubectl get pods -o name -l postgres-operator.crunchydata.com/role=master) | tail -2
+```
+Expected output:
+```
+2024-03-14 21:58:26,205 INFO: no action. I am (zitadel-pgha1-smw7-0), the standby leader with the lock
+```
+## Render core1
+```
+holos render --cluster-name=core1 ~/workspace/holos-run/holos/docs/examples/platforms/reference/clusters/accounts/...
+```
+Expected change:
+```diff
+diff --git a/deploy/clusters/core1/components/prod-iam-postgres/prod-iam-postgres.gen.yaml b/deploy/clusters/core1/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
+index aa69814e..290c0865 100644
+--- a/deploy/clusters/core1/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
++++ b/deploy/clusters/core1/components/prod-iam-postgres/prod-iam-postgres.gen.yaml
+@@ -85,6 +85,22 @@ spec:
+     name: zitadel-repl-tls
+   customTLSSecret:
+     name: zitadel-primary-tls
++  dataSource:
++    pgbackrest:
++      configuration:
++        - secret:
++            name: pgo-s3-creds
++      global:
++        repo2-path: /pgbackrest/prod-iam-zitadel/zitadel/repo2
++        repo2-cipher-type: aes-256-cbc
++      options: []
++      repo:
++        name: repo2
++        s3:
++          bucket: ois-zitadel-backups
++          endpoint: s3.dualstack.us-east-2.amazonaws.com
++          region: us-east-2
++      stanza: db
+   image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.2-0
+   instances:
+     - name: pgha1
+@@ -121,7 +137,7 @@ spec:
+       image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.21-3
+       replicas: 2
+   standby:
+-    enabled: true
++    enabled: false
+     repoName: repo2
+   users:
+     - databases:
+```
+
+## Promote core1
+```
+flux reconcile source git flux-system
+flux reconcile ks prod-iam-postgres
+```
+## Verify core1 is primary
+```
+k logs $(kubectl get pods -o name -l postgres-operator.crunchydata.com/role=master) | tail -1
+```
+Expected output:
+```
+2024-03-14 22:05:01,159 INFO: no action. I am (zitadel-pgha1-xl2b-0), the leader with the lock
+```
+## Reconcile Zitadel
+Now that the database is back up, reconcile zitadel.
+This is pretty brutal but it works:
+```
+❯ flux delete ks prod-iam-zitadel
+Are you sure you want to delete this kustomization: y
+► deleting kustomization prod-iam-zitadel in flux-system namespace
+✔ kustomization deleted
+```
+```
+❯ k apply --server-side=true -f deploy/clusters/core1/holos/components/prod-iam-zitadel-kustomization.gen.yaml
+kustomization.kustomize.toolkit.fluxcd.io/prod-iam-zitadel serverside-applied
+```
+ZITADEL should come up READY 2/2
+```
+❯ k get pods
+NAME                               READY   STATUS      RESTARTS      AGE
+zitadel-76776f747f-94wft           2/2     Running     2 (30s ago)   34s
+zitadel-76776f747f-fl24c           2/2     Running     1 (30s ago)   34s
+zitadel-76776f747f-xx2w7           2/2     Running     2 (30s ago)   34s
+zitadel-backup-49c7-h25cj          0/1     Completed   0             3m23s
+zitadel-init-td2sh                 0/1     Completed   0             34s
+zitadel-pgbouncer-d9f8cffc-2vwk5   2/2     Running     0             22m
+zitadel-pgbouncer-d9f8cffc-rrrhd   2/2     Running     0             22m
+zitadel-pgha1-4npq-0               4/4     Running     0             3m21s
+zitadel-pgha1-xl2b-0               4/4     Running     0             3m43s
+zitadel-repo-host-0                2/2     Running     0             22m
+zitadel-setup-b7zkx                0/1     Completed   1             34s
+zitadel-test-connection            0/1     Error       0             34s
+```
+
+## Cut over NLB
+Finally, cut over the NLB config to move to core1.
+This is specific to our metal clusters.
+Apply with: `git push puppet +HEAD:production -o run=netlb`
+```diff
+diff --git a/site-modules/profile/templates/haproxy/haproxy.cfg.epp b/site-modules/profile/templates/haproxy/haproxy.cfg.epp
+index b358a70..3ee40db 100644
+--- a/site-modules/profile/templates/haproxy/haproxy.cfg.epp
++++ b/site-modules/profile/templates/haproxy/haproxy.cfg.epp
+@@ -225,7 +225,7 @@ frontend sni-ingress from tcp-ingress
+   use_backend core2-https if { req.ssl_sni -i web.holos.run }
+   use_backend core2-https if { req.ssl_sni -i -m end .web.holos.run }
+   # Identity provider is routed to one cluster at a time
+-  use_backend core2-https if { req.ssl_sni -i login.ois.run }
++  use_backend core1-https if { req.ssl_sni -i login.ois.run }
+   # Holos Dev Environments
+   use_backend k1-https if { req.ssl_sni -i holos.wtf }
+   use_backend k1-https if { req.ssl_sni -i -m end .holos.wtf }
+@@ -358,7 +358,7 @@ frontend http-ingress from http-ingress
+   bind 65.102.23.41:80
+   bind 2602:41:6617:2ec0::80:1d:80 v6only
+   # Zitadel is active on one cluster at a time
+-  use_backend core2-http if { hdr(host) -i login.ois.run }
++  use_backend core1-http if { hdr(host) -i login.ois.run }
+   # Vault is active on core1 or core2
+   use_backend core2-http if { hdr(host) -i vault.core.ois.run }
+   # Infrastructure Dex OIDC ID issuer is active on core1 or core2
+
+```
+## Test Login
+```
+kubectl oidc-login get-token \
+  --oidc-extra-scope=openid \
+  --oidc-extra-scope=email \
+  --oidc-extra-scope=profile \
+  --oidc-extra-scope=groups \
+  --oidc-extra-scope=offline_access \
+  --oidc-extra-scope=urn:zitadel:iam:org:domain:primary:openinfrastructure.co \
+  --oidc-issuer-url=https://login.ois.run \
+  --oidc-client-id=${CLIENT_ID:-257714027772314751@holos_platform} \
+  --oidc-use-pkce \
+  --force-refresh > ~/.kube/token.json
+```
+
+```
+jq -r .status.token ~/.kube/token.json | cut -d. -f2 | base64 -d | jq
+```
+
+```json
+{
+  "amr": [
+    "mfa",
+    "user"
+  ],
+  "at_hash": "5nUq6kXWT8GheCNvDNdyow",
+  "aud": [
+    "257714027772314751@holos_platform",
+    "257713952794870157"
+  ],
+  "auth_time": 1710172186,
+  "azp": "257714027772314751@holos_platform",
+  "client_id": "257714027772314751@holos_platform",
+  "email": "jeff@openinfrastructure.co",
+  "email_verified": true,
+  "exp": 1710497614,
+  "family_name": "McCune",
+  "given_name": "Jeff",
+  "groups": [
+    "prod-cluster-admin"
+  ],
+  "iat": 1710454404,
+  "iss": "https://login.ois.run",
+  "locale": null,
+  "name": "Jeff McCune",
+  "nickname": "Jeff",
+  "preferred_username": "jeff@openinfrastructure.co",
+  "sub": "257712562366383231",
+  "updated_at": 1710105084,
+  "urn:zitadel:iam:org:domain:primary": "openinfrastructure.co",
+  "urn:zitadel:iam:org:project:257713952794870157:roles": {
+    "prod-cluster-admin": {
+      "257712562366317695": "openinfrastructure.co"
+    }
+  },
+  "urn:zitadel:iam:org:project:roles": {
+    "prod-cluster-admin": {
+      "257712562366317695": "openinfrastructure.co"
+    }
+  }
+}
+```
+## Tasks
+### Identify the leader
+Same command for primary and standby clusters.
+```
+k get pods -o name -l postgres-operator.crunchydata.com/role=master
+```
+### Full Backup
+On the primary cluster:
+```
+kubectl -n prod-iam-zitadel annotate postgrescluster zitadel postgres-operator.crunchydata.com/pgbackrest-backup="$(date)" --overwrite
+```
+Watch the progress:
+```
+k -n prod-iam-zitadel logs -l postgres-operator.crunchydata.com/pgbackrest-backup=manual -f
+```
+Expected output:
+```
+time="2024-03-14T18:36:16Z" level=info msg="crunchy-pgbackrest starts"
+time="2024-03-14T18:36:16Z" level=info msg="debug flag set to false"
+time="2024-03-14T18:36:16Z" level=info msg="backrest backup command requested"
+time="2024-03-14T18:36:16Z" level=info msg="command to execute is [pgbackrest backup --stanza=db --repo=2 --type=full]"
+time="2024-03-14T18:39:11Z" level=info msg="output=[]"
+time="2024-03-14T18:39:11Z" level=info msg="stderr=[]"
+time="2024-03-14T18:39:11Z" level=info msg="crunchy-pgbackrest ends"
+```
+
+## Troubleshooting
+Full backup doesn't start
+Check the controller logs.  If you see `Switchover failed` errors, try editing the number of postgrescluster replicas down to 1 then back up to 2.  Then try a backup again.
+```
+Switchover failed, details: 503, Switchover failed\n" version=5.5.1-0-amd64
+```

go.mod

@@ -3,11 +3,12 @@ module github.com/holos-run/holos
 go 1.21.5
 
 require (
-	cuelang.org/go v0.7.0
+	cuelang.org/go v0.8.0
 	github.com/mattn/go-isatty v0.0.20
 	github.com/rogpeppe/go-internal v1.12.0
-	github.com/spf13/cobra v1.7.0
-	golang.org/x/tools v0.18.0
+	github.com/spf13/cobra v1.8.0
+	github.com/spf13/pflag v1.0.5
+	golang.org/x/tools v0.19.0
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2
 	k8s.io/client-go v0.29.2
@@ -16,50 +17,51 @@ require (
 )
 
 require (
-	cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13 // indirect
+	cuelabs.dev/go/oci/ociregistry v0.0.0-20240314152124-224736b49f2e // indirect
 	github.com/cockroachdb/apd/v3 v3.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/emicklei/proto v1.10.0 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/onsi/ginkgo/v2 v2.15.0 // indirect
+	github.com/onsi/gomega v1.31.1 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc4 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/net v0.21.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/oauth2 v0.18.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	k8s.io/kube-openapi v0.0.0-20231206194836-bf4651e18aa8 // indirect
+	k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

go.sum

@@ -1,10 +1,10 @@
-cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13 h1:zkiIe8AxZ/kDjqQN+mDKc5BxoVJOqioSdqApjc+eB1I=
-cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13/go.mod h1:XGKYSMtsJWfqQYPwq51ZygxAPqpEUj/9bdg16iDPTAA=
-cuelang.org/go v0.7.0 h1:gMztinxuKfJwMIxtboFsNc6s8AxwJGgsJV+3CuLffHI=
-cuelang.org/go v0.7.0/go.mod h1:ix+3dM/bSpdG9xg6qpCgnJnpeLtciZu+O/rDbywoMII=
+cuelabs.dev/go/oci/ociregistry v0.0.0-20240314152124-224736b49f2e h1:GwCVItFUPxwdsEYnlUcJ6PJxOjTeFFCKOh6QWg4oAzQ=
+cuelabs.dev/go/oci/ociregistry v0.0.0-20240314152124-224736b49f2e/go.mod h1:ApHceQLLwcOkCEXM1+DyCXTHEJhNGDpJ2kmV6axsx24=
+cuelang.org/go v0.8.0 h1:fO1XPe/SUGtc7dhnGnTPbpIDoQm/XxhDtoSF7jzO01c=
+cuelang.org/go v0.8.0/go.mod h1:CoDbYolfMms4BhWUlhD+t5ORnihR7wvjcfgyO9lL5FI=
 github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
 github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -13,24 +13,26 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw=
 github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
-github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
-github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
+github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
 github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ=
+github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA=
 github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
 github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
-github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
 github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
 github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
@@ -44,10 +46,10 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
-github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
+github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -65,8 +67,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
-github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -78,18 +80,16 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto=
-github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
-github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
-github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
-github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
+github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
+github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo=
+github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0=
-github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
+github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
+github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -99,8 +99,8 @@ github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0/go.mod h1
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -114,54 +114,65 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
-golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
+golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
+golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
-golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
+golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
@@ -175,7 +186,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A=
@@ -186,12 +196,12 @@ k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg=
 k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA=
 k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
-k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/kube-openapi v0.0.0-20231206194836-bf4651e18aa8 h1:vzKzxN5uyJZLY8HL1/OovW7BJefnsBIWt8T7Gjh2boQ=
+k8s.io/kube-openapi v0.0.0-20231206194836-bf4651e18aa8/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
 k8s.io/kubectl v0.29.2 h1:uaDYaBhumvkwz0S2XHt36fK0v5IdNgL7HyUniwb2IUo=
 k8s.io/kubectl v0.29.2/go.mod h1:BhizuYBGcKaHWyq+G7txGw2fXg576QbPrrnQdQDZgqI=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI=
+k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=

pkg/internal/builder/builder.go

@@ -6,35 +6,28 @@ package builder
 import (
 	"context"
 	"fmt"
-	"log/slog"
 	"os"
 	"path/filepath"
-	"slices"
-	"strings"
 
 	"cuelang.org/go/cue/build"
 	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/load"
+	"github.com/holos-run/holos/api/v1alpha1"
 
 	"github.com/holos-run/holos"
 	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
 	"github.com/holos-run/holos/pkg/wrapper"
 )
 
 const (
-	// Kube is the value of the kind field of holos build output indicating
-	// kubernetes api objects.
-	Kube = "KubernetesObjects"
+	KubernetesObjects = v1alpha1.KubernetesObjectsKind
 	// Helm is the value of the kind field of holos build output indicating helm
 	// values and helm command information.
-	Helm = "HelmChart"
+	Helm = v1alpha1.HelmChartKind
 	// Skip is the value when the instance should be skipped
 	Skip = "Skip"
-	// ChartDir is the chart cache directory name.
-	ChartDir = "vendor"
 	// KustomizeBuild is the value of the kind field of cue output indicating holos should process the component using kustomize build to render output.
-	KustomizeBuild = "KustomizeBuild"
+	KustomizeBuild = v1alpha1.KustomizeBuildKind
 )
 
 // An Option configures a Builder
@@ -69,181 +62,6 @@ func Cluster(name string) Option {
 	return func(cfg *config) { cfg.cluster = name }
 }
 
-type buildInfo struct {
-	APIVersion string `json:"apiVersion,omitempty"`
-	Kind       string `json:"kind,omitempty"`
-}
-
-// Metadata represents the standard metadata fields of the cue output
-type Metadata struct {
-	Name string `json:"name,omitempty"`
-}
-
-// apiObjectMap is the shape of marshalled api objects returned from cue to the
-// holos cli. A map is used to improve the clarity of error messages from cue.
-type apiObjectMap map[string]map[string]string
-
-// fileContentMap is a map of file names to file contents.
-type fileContentMap map[string]string
-
-// Result is the build result for display or writing.
-type Result struct {
-	Metadata  Metadata `json:"metadata,omitempty"`
-	KsContent string   `json:"ksContent,omitempty"`
-	// APIObjectMap holds the marshalled representation of api objects.
-	APIObjectMap      apiObjectMap `json:"apiObjectMap,omitempty"`
-	accumulatedOutput string
-	Skip              bool
-	// KustomizeFiles holds the files for a kustomize kustomization directory.
-	KustomizeFiles fileContentMap `json:"kustomizeFiles"`
-	// ResourcesFile is the file name used for api objects in kustomization.yaml
-	ResourcesFile string `json:"resourcesFile,omitempty"`
-}
-
-type Repository struct {
-	Name string `json:"name"`
-	URL  string `json:"url"`
-}
-
-type Chart struct {
-	Name       string     `json:"name"`
-	Version    string     `json:"version"`
-	Release    string     `json:"release"`
-	Repository Repository `json:"repository"`
-}
-
-// A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
-type HelmChart struct {
-	APIVersion    string   `json:"apiVersion"`
-	Kind          string   `json:"kind"`
-	Metadata      Metadata `json:"metadata"`
-	KsContent     string   `json:"ksContent"`
-	Namespace     string   `json:"namespace"`
-	Chart         Chart    `json:"chart"`
-	ValuesContent string   `json:"valuesContent"`
-	EnableHooks   bool     `json:"enableHooks"`
-	// APIObjectMap holds the marshalled representation of api objects.
-	APIObjectMap apiObjectMap `json:"APIObjectMap"`
-}
-
-// Name returns the metadata name of the result. Equivalent to the
-// OrderedComponent name specified in platform.yaml in the holos prototype.
-func (r *Result) Name() string {
-	return r.Metadata.Name
-}
-
-func (r *Result) Filename(writeTo string, cluster string) string {
-	return filepath.Join(writeTo, "clusters", cluster, "components", r.Name(), r.Name()+".gen.yaml")
-}
-
-func (r *Result) KustomizationFilename(writeTo string, cluster string) string {
-	return filepath.Join(writeTo, "clusters", cluster, "holos", "components", r.Name()+"-kustomization.gen.yaml")
-}
-
-// AccumulatedOutput returns the accumulated rendered output.
-func (r *Result) AccumulatedOutput() string {
-	return r.accumulatedOutput
-}
-
-// addAPIObjects adds the overlay api objects to accumulatedOutput.
-func (r *Result) addOverlayObjects(log *slog.Logger) {
-	b := []byte(r.AccumulatedOutput())
-	kinds := make([]string, 0, len(r.APIObjectMap))
-	// Sort the keys
-	for kind := range r.APIObjectMap {
-		kinds = append(kinds, kind)
-	}
-	slices.Sort(kinds)
-
-	for _, kind := range kinds {
-		v := r.APIObjectMap[kind]
-		// Sort the keys
-		names := make([]string, 0, len(v))
-		for name := range v {
-			names = append(names, name)
-		}
-		slices.Sort(names)
-
-		for _, name := range names {
-			yamlString := v[name]
-			log.Debug(fmt.Sprintf("%s/%s", kind, name), "kind", kind, "name", name)
-			b = util.EnsureNewline(b)
-			header := fmt.Sprintf("---\n# Source: CUE apiObjects.%s.%s\n", kind, name)
-			b = append(b, []byte(header+yamlString)...)
-			b = util.EnsureNewline(b)
-		}
-	}
-	r.accumulatedOutput = string(b)
-}
-
-// kustomize replaces the final output with the output of kustomize build if the
-func (r *Result) kustomize(ctx context.Context) error {
-	log := logger.FromContext(ctx)
-	if r.ResourcesFile == "" {
-		log.DebugContext(ctx, "skipping kustomize: no resourcesFile")
-		return nil
-	}
-	if len(r.KustomizeFiles) < 1 {
-		log.DebugContext(ctx, "skipping kustomize: no kustomizeFiles")
-		return nil
-	}
-	tempDir, err := os.MkdirTemp("", "holos.kustomize")
-	if err != nil {
-		return wrapper.Wrap(err)
-	}
-	defer remove(ctx, tempDir)
-
-	// Write the main api object resources file for kustomize.
-	target := filepath.Join(tempDir, r.ResourcesFile)
-	b := []byte(r.AccumulatedOutput())
-	b = util.EnsureNewline(b)
-	if err := os.WriteFile(target, b, 0644); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not write resources: %w", err))
-	}
-	log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
-
-	// Write the kustomization tree, kustomization.yaml must be in this map for kustomize to work.
-	for file, content := range r.KustomizeFiles {
-		target := filepath.Join(tempDir, file)
-		if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
-			return wrapper.Wrap(err)
-		}
-		b := []byte(content)
-		b = util.EnsureNewline(b)
-		if err := os.WriteFile(target, b, 0644); err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not write: %w", err))
-		}
-		log.DebugContext(ctx, "wrote: "+target, "op", "write", "path", target, "bytes", len(b))
-	}
-
-	// Run kustomize.
-	kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", tempDir)
-	if err != nil {
-		log.ErrorContext(ctx, kOut.Stderr.String())
-		return wrapper.Wrap(err)
-	}
-	// Replace the accumulated output
-	r.accumulatedOutput = kOut.Stdout.String()
-	return nil
-}
-
-// Save writes the content to the filesystem for git ops.
-func (r *Result) Save(ctx context.Context, path string, content string) error {
-	log := logger.FromContext(ctx)
-	dir := filepath.Dir(path)
-	if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
-		log.WarnContext(ctx, "could not mkdir", "path", dir, "err", err)
-		return wrapper.Wrap(err)
-	}
-	// Write the kube api objects
-	if err := os.WriteFile(path, []byte(content), os.FileMode(0644)); err != nil {
-		log.WarnContext(ctx, "could not write", "path", path, "err", err)
-		return wrapper.Wrap(err)
-	}
-	log.DebugContext(ctx, "out: wrote "+path, "action", "write", "path", path, "status", "ok")
-	return nil
-}
-
 // Cluster returns the cluster name of the component instance being built.
 func (b *Builder) Cluster() string {
 	return b.cfg.cluster
@@ -285,8 +103,8 @@ func (b *Builder) Instances(ctx context.Context) ([]*build.Instance, error) {
 	return load.Instances(args, &cfg), nil
 }
 
-func (b *Builder) Run(ctx context.Context) (results []*Result, err error) {
-	results = make([]*Result, 0, len(b.cfg.args))
+func (b *Builder) Run(ctx context.Context) (results []*v1alpha1.Result, err error) {
+	results = make([]*v1alpha1.Result, 0, len(b.cfg.args))
 	cueCtx := cuecontext.New()
 	logger.FromContext(ctx).DebugContext(ctx, "cue: building instances")
 	instances, err := b.Instances(ctx)
@@ -294,72 +112,57 @@ func (b *Builder) Run(ctx context.Context) (results []*Result, err error) {
 		return results, err
 	}
 
+	// Each CUE instance provides a BuildPlan
 	for _, instance := range instances {
-		var info buildInfo
-		var result Result
+		var buildPlan v1alpha1.BuildPlan
+
 		log := logger.FromContext(ctx).With("dir", instance.Dir)
-		results = append(results, &result)
 		if err := instance.Err; err != nil {
 			return nil, wrapper.Wrap(fmt.Errorf("could not load: %w", err))
 		}
 		log.DebugContext(ctx, "cue: building instance")
 		value := cueCtx.BuildInstance(instance)
 		if err := value.Err(); err != nil {
-			return nil, wrapper.Wrap(fmt.Errorf("could not build: %w", err))
+			return nil, wrapper.Wrap(fmt.Errorf("could not build %s: %w", instance.Dir, err))
 		}
 		log.DebugContext(ctx, "cue: validating instance")
 		if err := value.Validate(); err != nil {
 			return nil, wrapper.Wrap(fmt.Errorf("could not validate: %w", err))
 		}
-		log.DebugContext(ctx, "cue: decoding holos component build info")
-		if err := value.Decode(&info); err != nil {
+		log.DebugContext(ctx, "cue: decoding holos build plan")
+		if err := value.Decode(&buildPlan); err != nil {
 			return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
 		}
 
-		log.DebugContext(ctx, "cue: processing holos component kind "+info.Kind)
-		switch kind := info.Kind; kind {
-		case Skip:
-			result.Skip = true
-		case Kube:
-			// CUE directly provides the kubernetes api objects in result.Content
-			if err := value.Decode(&result); err != nil {
-				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
+		if err := buildPlan.Validate(); err != nil {
+			return nil, wrapper.Wrap(fmt.Errorf("could not render: %w", err))
+		}
+
+		if buildPlan.Spec.Disabled {
+			log.DebugContext(ctx, "skipped: spec.disabled is true", "skipped", true)
+			continue
+		}
+
+		for _, component := range buildPlan.Spec.Components.KubernetesObjects {
+			if result, err := component.Render(ctx, holos.PathComponent(instance.Dir)); err != nil {
+				return nil, wrapper.Wrap(fmt.Errorf("could not render: %w", err))
+			} else {
+				results = append(results, result)
 			}
-			result.addOverlayObjects(log)
-		case Helm:
-			var helmChart HelmChart
-			// First decode into the result.  Helm will populate the api objects later.
-			if err := value.Decode(&result); err != nil {
-				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
+		}
+		for _, component := range buildPlan.Spec.Components.HelmCharts {
+			if result, err := component.Render(ctx, holos.PathComponent(instance.Dir)); err != nil {
+				return nil, wrapper.Wrap(fmt.Errorf("could not render: %w", err))
+			} else {
+				results = append(results, result)
 			}
-			// Decode again into the helm chart struct to get the values content to provide to helm.
-			if err := value.Decode(&helmChart); err != nil {
-				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
+		}
+		for _, component := range buildPlan.Spec.Components.KustomizeBuilds {
+			if result, err := component.Render(ctx, holos.PathComponent(instance.Dir)); err != nil {
+				return nil, wrapper.Wrap(fmt.Errorf("could not render: %w", err))
+			} else {
+				results = append(results, result)
 			}
-			// runHelm populates result.Content from helm template output.
-			if err := runHelm(ctx, &helmChart, &result, holos.PathComponent(instance.Dir)); err != nil {
-				return nil, err
-			}
-			result.addOverlayObjects(log)
-			if err := result.kustomize(ctx); err != nil {
-				return nil, wrapper.Wrap(fmt.Errorf("could not kustomize: %w", err))
-			}
-		case KustomizeBuild:
-			// CUE directly provides the kubernetes api objects in result.Content
-			if err := value.Decode(&result); err != nil {
-				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
-			}
-			// Run kustomize.
-			kOut, err := util.RunCmd(ctx, "kubectl", "kustomize", instance.Dir)
-			if err != nil {
-				log.ErrorContext(ctx, kOut.Stderr.String())
-				return nil, wrapper.Wrap(err)
-			}
-			// Replace the accumulated output
-			result.accumulatedOutput = kOut.Stdout.String()
-			result.addOverlayObjects(log)
-		default:
-			return nil, wrapper.Wrap(fmt.Errorf("build kind not implemented: %v", kind))
 		}
 	}
 
@@ -395,119 +198,3 @@ func (b *Builder) findCueMod() (dir holos.PathCueMod, err error) {
 	}
 	return dir, nil
 }
-
-// runHelm provides the values produced by CUE to helm template and returns
-// the rendered kubernetes api objects in the result.
-func runHelm(ctx context.Context, hc *HelmChart, r *Result, path holos.PathComponent) error {
-	log := logger.FromContext(ctx).With("chart", hc.Chart.Name)
-	if hc.Chart.Name == "" {
-		log.WarnContext(ctx, "skipping helm: no chart name specified, use a different component type")
-		return nil
-	}
-
-	cachedChartPath := filepath.Join(string(path), ChartDir, filepath.Base(hc.Chart.Name))
-	if isNotExist(cachedChartPath) {
-		// Add repositories
-		repo := hc.Chart.Repository
-		if repo.URL != "" {
-			out, err := util.RunCmd(ctx, "helm", "repo", "add", repo.Name, repo.URL)
-			if err != nil {
-				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-				return wrapper.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
-			}
-			// Update repository
-			out, err = util.RunCmd(ctx, "helm", "repo", "update", repo.Name)
-			if err != nil {
-				log.ErrorContext(ctx, "could not run helm", "stderr", out.Stderr.String(), "stdout", out.Stdout.String())
-				return wrapper.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
-			}
-		} else {
-			log.DebugContext(ctx, "no chart repository url proceeding assuming oci chart")
-		}
-
-		// Cache the chart
-		if err := cacheChart(ctx, path, ChartDir, hc.Chart); err != nil {
-			return fmt.Errorf("could not cache chart: %w", err)
-		}
-	}
-
-	// Write values file
-	tempDir, err := os.MkdirTemp("", "holos")
-	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
-	}
-	defer remove(ctx, tempDir)
-
-	valuesPath := filepath.Join(tempDir, "values.yaml")
-	if err := os.WriteFile(valuesPath, []byte(hc.ValuesContent), 0644); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not write values: %w", err))
-	}
-	log.DebugContext(ctx, "helm: wrote values", "path", valuesPath, "bytes", len(hc.ValuesContent))
-
-	// Run charts
-	chart := hc.Chart
-	args := []string{"template"}
-	if !hc.EnableHooks {
-		args = append(args, "--no-hooks")
-	}
-	args = append(args, "--include-crds", "--values", valuesPath, "--namespace", hc.Namespace, "--kubeconfig", "/dev/null", "--version", chart.Version, chart.Release, cachedChartPath)
-	helmOut, err := util.RunCmd(ctx, "helm", args...)
-	if err != nil {
-		stderr := helmOut.Stderr.String()
-		lines := strings.Split(stderr, "\n")
-		for _, line := range lines {
-			if strings.HasPrefix(line, "Error:") {
-				err = fmt.Errorf("%s: %w", line, err)
-			}
-		}
-		return wrapper.Wrap(fmt.Errorf("could not run helm template: %w", err))
-	}
-
-	r.accumulatedOutput = helmOut.Stdout.String()
-
-	return nil
-}
-
-// remove cleans up path, useful for temporary directories.
-func remove(ctx context.Context, path string) {
-	log := logger.FromContext(ctx)
-	if err := os.RemoveAll(path); err != nil {
-		log.WarnContext(ctx, "tmp: could not remove", "err", err, "path", path)
-	} else {
-		log.DebugContext(ctx, "tmp: removed", "path", path)
-	}
-}
-
-func isNotExist(path string) bool {
-	_, err := os.Stat(path)
-	return os.IsNotExist(err)
-}
-
-// cacheChart stores a cached copy of Chart in the chart subdirectory of path.
-func cacheChart(ctx context.Context, path holos.PathComponent, chartDir string, chart Chart) error {
-	log := logger.FromContext(ctx)
-
-	cacheTemp, err := os.MkdirTemp(string(path), chartDir)
-	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
-	}
-	defer remove(ctx, cacheTemp)
-
-	chartName := chart.Name
-	if chart.Repository.Name != "" {
-		chartName = fmt.Sprintf("%s/%s", chart.Repository.Name, chart.Name)
-	}
-	helmOut, err := util.RunCmd(ctx, "helm", "pull", "--destination", cacheTemp, "--untar=true", "--version", chart.Version, chartName)
-	if err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not run helm pull: %w", err))
-	}
-	log.Debug("helm pull", "stdout", helmOut.Stdout, "stderr", helmOut.Stderr)
-
-	cachePath := filepath.Join(string(path), chartDir)
-	if err := os.Rename(cacheTemp, cachePath); err != nil {
-		return wrapper.Wrap(fmt.Errorf("could not rename: %w", err))
-	}
-	log.InfoContext(ctx, "cached", "chart", chart.Name, "version", chart.Version, "path", cachePath)
-
-	return nil
-}

pkg/util/remove.go

@@ -0,0 +1,17 @@
+package util
+
+import (
+	"context"
+	"github.com/holos-run/holos/pkg/logger"
+	"os"
+)
+
+// Remove cleans up path, useful for temporary directories.
+func Remove(ctx context.Context, path string) {
+	log := logger.FromContext(ctx)
+	if err := os.RemoveAll(path); err != nil {
+		log.WarnContext(ctx, "tmp: could not Remove", "err", err, "path", path)
+	} else {
+		log.DebugContext(ctx, "tmp: removed", "path", path)
+	}
+}

pkg/version/embedded/minor

@@ -1 +1 @@
-57
+60

pkg/version/embedded/patch

@@ -1 +1 @@
-0
+1