builder: protect LoadInstance with a mutex

CUE is not safe for concurrent access so we protect the main
LoadInstance function with a mutex lock.

.cspell.json

@@ -37,6 +37,7 @@
     "blackbox",
     "buildplan",
     "buildplans",
+    "Buildx",
     "builtinpluginloadingoptions",
     "cachedir",
     "cadvisor",
@@ -74,6 +75,7 @@
     "creds",
     "crossplane",
     "crunchydata",
+    "ctxt",
     "cuecontext",
     "cuelang",
     "customresourcedefinition",
@@ -83,6 +85,7 @@
     "destinationrules",
     "devel",
     "devicecode",
+    "distroless",
     "dnsmasq",
     "dscacheutil",
     "ecrauthorizationtoken",
@@ -280,6 +283,7 @@
     "serviceentries",
     "serviceentry",
     "servicemonitor",
+    "sigstore",
     "somevalue",
     "SOMEVAR",
     "sortoptions",
@@ -320,6 +324,7 @@
     "udev",
     "uibutton",
     "Unmarshal",
+    "unshallow",
     "unstage",
     "untar",
     "upbound",

.github/workflows/container.yaml

@@ -0,0 +1,143 @@
+name: Container
+
+# Only allow actors with write permission to the repository to trigger this
+# workflow.
+permissions:
+  contents: write
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      git_ref:
+        description: 'Git ref to build (e.g., refs/tags/v1.2.3, refs/heads/main)'
+        required: true
+        type: string
+
+jobs:
+  buildx:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    steps:
+      - name: Set tag from trigger event
+        id: opts
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "ref=${{ inputs.git_ref }}" >> $GITHUB_OUTPUT
+          else
+            echo "ref=${GITHUB_REF}" >> $GITHUB_OUTPUT
+          fi
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.opts.outputs.ref }}
+      - name: SHA
+        id: sha
+        run: echo "sha=$(/usr/bin/git log -1 --format='%H')" >> $GITHUB_OUTPUT
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Fetch tags
+        run: git fetch --prune --unshallow --tags
+      - name: Set Tags
+        id: tags
+        run: |
+          echo "detail=$(/usr/bin/git describe --tags HEAD)" >> $GITHUB_OUTPUT
+          echo "suffix=$(test -n "$(git status --porcelain)" && echo '-dirty' || echo '')" >> $GITHUB_OUTPUT
+          echo "tag=$(/usr/bin/git describe --tags HEAD)$(test -n "$(git status --porcelain)" && echo '-dirty' || echo '')" >> $GITHUB_OUTPUT
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push container images
+        id: build-and-push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ghcr.io/holos-run/holos:${{ steps.tags.outputs.tag }}
+            ghcr.io/holos-run/holos:${{ steps.sha.outputs.sha }}${{ steps.tags.outputs.suffix }}
+      - name: Setup Cosign to sign container images
+        uses: sigstore/cosign-installer@v3.7.0
+      - name: Sign with GitHub OIDC Token
+        env:
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        run: |
+          cosign sign --yes ghcr.io/holos-run/holos:${{ steps.tags.outputs.tag }}@${DIGEST}
+          cosign sign --yes ghcr.io/holos-run/holos:${{ steps.sha.outputs.sha }}${{ steps.tags.outputs.suffix }}@${DIGEST}
+
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          owner: ${{ github.repository_owner }}
+          app-id: ${{ vars.GORELEASER_APP_ID }}
+          private-key: ${{ secrets.GORELEASER_APP_PRIVATE_KEY }}
+      - name: Get GitHub App User ID
+        id: get-user-id
+        run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+      - run: |
+          git config --global user.name '${{ steps.app-token.outputs.app-slug }}[bot]'
+          git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com'
+      - name: Update holos-run/holos-action
+        env:
+          IMAGE: ghcr.io/holos-run/holos:v0.102.1
+          VERSION: ${{ steps.tags.outputs.tag }}
+          USER_ID: ${{ steps.get-user-id.outputs.user-id }}
+          TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          set -euo pipefail
+          git clone "https://github.com/holos-run/holos-action"
+          cd holos-action
+          git remote set-url origin https://${USER_ID}:${TOKEN}@github.com/holos-run/holos-action
+          docker pull --quiet "${IMAGE}"
+          docker run -v $(pwd):/app --workdir /app --rm "${IMAGE}"  \
+            holos cue export --out yaml action.cue -t "version=${VERSION}" > action.yml
+          git add action.yml
+          git commit -m "ci: update holos to ${VERSION} - https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" || (echo "No changes to commit"; exit 0)
+          git push origin HEAD:main HEAD:v0 HEAD:v1
+
+      - name: Login to quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USER }}
+          password: ${{ secrets.QUAY_TOKEN }}
+      - name: Push to quay.io
+        env:
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        run: |
+          # docker push quay.io/holos-run/holos:${{ steps.tags.outputs.tag }}
+          docker pull --quiet ghcr.io/holos-run/holos:${{ steps.tags.outputs.tag }}@${DIGEST}
+          docker tag ghcr.io/holos-run/holos:${{ steps.tags.outputs.tag }}@${DIGEST} \
+                     quay.io/holos-run/holos:${{ steps.tags.outputs.tag }}
+          docker push quay.io/holos-run/holos:${{ steps.tags.outputs.tag }}
+
+          docker pull --quiet ghcr.io/holos-run/holos:${{ steps.sha.outputs.sha }}${{ steps.tags.outputs.suffix }}@${DIGEST}
+          docker tag ghcr.io/holos-run/holos:${{ steps.sha.outputs.sha }}${{ steps.tags.outputs.suffix }}@${DIGEST} \
+                     quay.io/holos-run/holos:${{ steps.sha.outputs.sha }}${{ steps.tags.outputs.suffix }}
+          docker push quay.io/holos-run/holos:${{ steps.sha.outputs.sha }}${{ steps.tags.outputs.suffix }}
+      - name: Sign quay.io image
+        env:
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        run: |
+          cosign sign --yes quay.io/holos-run/holos:${{ steps.tags.outputs.tag }}@${DIGEST}
+          cosign sign --yes quay.io/holos-run/holos:${{ steps.sha.outputs.sha }}${{ steps.tags.outputs.suffix }}@${DIGEST}
+
+    outputs:
+      tag: ${{ steps.tags.outputs.tag }}
+      detail: ${{ steps.tags.outputs.detail }}

.github/workflows/lint.yaml

@@ -1,6 +1,5 @@
 ---
-# https://github.com/golangci/golangci-lint-action?tab=readme-ov-file#how-to-use
-name: Lint
+name: Spelling
 "on":
   push:
     branches:
@@ -8,35 +7,11 @@ name: Lint
       - test
   pull_request:
     types: [opened, synchronize]
-
 permissions:
   contents: read
-
 jobs:
-  lint:
-    name: lint
+  cspell:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-
-      ## Not needed on ubuntu-latest
-      # - name: Install Packages
-      #   run: sudo apt update && sudo apt -qq -y install git curl zip unzip tar bzip2 make
-
-      - name: Install Tools
-        run: make tools
-
-      - name: Lint
-        # golangci-lint runs in a separate workflow.
-        run: make lint -o golangci-lint
+      - uses: actions/checkout@v4
+      - run: ./hack/cspell

.github/workflows/test.yaml

@@ -28,19 +28,11 @@ jobs:
         with:
           go-version: stable
 
-      - name: Install Packages
-        run: sudo apt update && sudo apt -qq -y install git curl zip unzip tar bzip2 make
-
       - name: Set up Helm
         uses: azure/setup-helm@v4
 
       - name: Set up Kubectl
         uses: azure/setup-kubectl@v4
 
-      - name: Install Tools
-        run: |
-          set -x
-          make tools
-
       - name: Test
         run: ./scripts/test

Dockerfile

@@ -1,8 +1,31 @@
-FROM quay.io/holos-run/debian:bullseye AS final
-USER root
-WORKDIR /app
-ADD bin bin
-RUN chown -R app: /app
-# Kubernetes requires the user to be numeric
-USER 8192
-ENTRYPOINT bin/holos server
+FROM registry.k8s.io/kubectl:v1.31.0 AS kubectl
+# https://github.com/GoogleContainerTools/distroless
+FROM golang:1.23 AS build
+
+WORKDIR /go/src/app
+COPY . .
+
+RUN CGO_ENABLED=0 make install
+RUN CGO_ENABLED=0 go install sigs.k8s.io/kustomize/kustomize/v5
+
+# Install helm to /usr/local/bin/helm
+# https://helm.sh/docs/intro/install/#from-script
+# https://holos.run/docs/v1alpha5/tutorial/setup/#dependencies
+RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 \
+  && chmod 700 get_helm.sh \
+  && DESIRED_VERSION=v3.16.2 ./get_helm.sh \
+  && rm -f get_helm.sh
+
+COPY --from=kubectl /bin/kubectl /usr/local/bin/
+
+# distroless
+FROM gcr.io/distroless/static-debian12 AS final
+COPY --from=build \
+     /go/bin/holos \
+     /go/bin/kustomize \
+     /usr/local/bin/kubectl \
+     /usr/local/bin/helm \
+     /bin/
+
+# Usage: docker run -v $(pwd):/app --workdir /app --rm -it quay.io/holos-run/holos holos render platform
+CMD ["/bin/holos"]

README.md

@@ -119,12 +119,12 @@ here to help.
 
 Holos is licensed under Apache 2.0 as found in the [LICENSE file](LICENSE).
 
-[Holos]: https://holos.run
+[Holos]: https://holos.run/docs/overview/
 [rendered manifests pattern]: https://akuity.io/blog/the-rendered-manifests-pattern
 [CUE]: https://cuelang.org/
 [Discord]: https://discord.gg/JgDVbNpye7
 [GitHub discussions]: https://github.com/holos-run/holos/discussions
 [Why CUE for Configuration]: https://holos.run/blog/why-cue-for-configuration/
-[topics]: https://holos.run/docs/topics/
+[tutorial]: https://holos.run/docs/overview/
 [setup]: https://holos.run/docs/setup/
-[tutorial]: https://holos.run/docs/tutorial/
+[topics]: https://holos.run/docs/topics/

api/core/v1alpha5/types.go

@@ -263,7 +263,7 @@ type Metadata struct {
 	// Labels represents a resource selector.
 	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 	// Annotations represents arbitrary non-identifying metadata.  For example
-	// holos uses the `cli.holos.run/description` annotation to log resources in a
+	// holos uses the `app.holos.run/description` annotation to log resources in a
 	// user customized way.
 	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
 }
@@ -303,6 +303,10 @@ type Component struct {
 	// Path represents the path of the component relative to the platform root.
 	// Injected as the tag variable "holos_component_path".
 	Path string `json:"path" yaml:"path"`
+	// Instances represents additional cue instance paths to unify with Path.
+	// Useful to unify data files into a component BuildPlan.  Added in holos
+	// 0.101.7.
+	Instances []Instance `json:"instances,omitempty" yaml:"instances,omitempty"`
 	// WriteTo represents the holos render component --write-to flag.  If empty,
 	// the default value for the --write-to flag is used.
 	WriteTo string `json:"writeTo,omitempty" yaml:"writeTo,omitempty"`
@@ -316,6 +320,30 @@ type Component struct {
 	// resulting BuildPlan.
 	Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
 	// Annotations represents arbitrary non-identifying metadata.  Use the
-	// `cli.holos.run/description` to customize the log message of each BuildPlan.
+	// `app.holos.run/description` to customize the log message of each BuildPlan.
 	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
 }
+
+// Instance represents a data instance to unify with the configuration.
+//
+// Useful to unify json and yaml files with cue configuration files for
+// integration with other tools.  For example, executing holos render platform
+// from a pull request workflow after [Kargo] executes the [yaml update] and
+// [git wait for pr] promotion steps.
+//
+// [Kargo]: https://docs.kargo.io/
+// [yaml update]: https://docs.kargo.io/references/promotion-steps#yaml-update
+// [git wait for pr]: https://docs.kargo.io/references/promotion-steps#git-wait-for-pr
+type Instance struct {
+	// Kind is a discriminator.
+	Kind string `json:"kind" yaml:"kind" cue:"\"ExtractYAML\""`
+	// Ignored unless kind is ExtractYAML.
+	ExtractYAML ExtractYAML `json:"extractYAML,omitempty" yaml:"extractYAML,omitempty"`
+}
+
+// ExtractYAML represents a cue data instance encoded as yaml or json. If Path
+// refers to a directory all files in the directory are extracted
+// non-recursively.  Otherwise, path must refer to a file.
+type ExtractYAML struct {
+	Path string `json:"path" yaml:"path"`
+}

cmd/cmd.go

@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"runtime/pprof"
+	"runtime/trace"
+
+	"github.com/holos-run/holos/internal/cli"
+	"github.com/holos-run/holos/internal/holos"
+)
+
+// MakeMain makes a main function for the cli or tests.
+func MakeMain(options ...holos.Option) func() int {
+	return func() (exitCode int) {
+		cfg := holos.New(options...)
+		slog.SetDefault(cfg.Logger())
+		ctx := context.Background()
+
+		if format := os.Getenv("HOLOS_CPU_PROFILE"); format != "" {
+			f, _ := os.Create(fmt.Sprintf(format, os.Getppid(), os.Getpid()))
+			err := pprof.StartCPUProfile(f)
+			defer func() {
+				pprof.StopCPUProfile()
+				f.Close()
+			}()
+			if err != nil {
+				return cli.HandleError(ctx, err, cfg)
+			}
+		}
+		defer memProfile(ctx, cfg)
+
+		if format := os.Getenv("HOLOS_TRACE"); format != "" {
+			f, _ := os.Create(fmt.Sprintf(format, os.Getppid(), os.Getpid()))
+			err := trace.Start(f)
+			defer func() {
+				trace.Stop()
+				f.Close()
+			}()
+			if err != nil {
+				return cli.HandleError(ctx, err, cfg)
+			}
+		}
+
+		feature := &holos.EnvFlagger{}
+		if err := cli.New(cfg, feature).ExecuteContext(ctx); err != nil {
+			return cli.HandleError(ctx, err, cfg)
+		}
+		return 0
+	}
+}
+
+func memProfile(ctx context.Context, cfg *holos.Config) {
+	if format := os.Getenv("HOLOS_MEM_PROFILE"); format != "" {
+		f, _ := os.Create(fmt.Sprintf(format, os.Getppid(), os.Getpid()))
+		defer f.Close()
+		if err := pprof.WriteHeapProfile(f); err != nil {
+			_ = cli.HandleError(ctx, err, cfg)
+		}
+	}
+}

cmd/holos/main.go

@@ -3,9 +3,9 @@ package main
 import (
 	"os"
 
-	"github.com/holos-run/holos/internal/cli"
+	"github.com/holos-run/holos/cmd"
 )
 
 func main() {
-	os.Exit(cli.MakeMain()())
+	os.Exit(cmd.MakeMain()())
 }

cmd/holos/main_test.go

@@ -6,13 +6,13 @@ import (
 	"testing"
 
 	cue "cuelang.org/go/cmd/cue/cmd"
-	"github.com/holos-run/holos/internal/cli"
+	"github.com/holos-run/holos/cmd"
 	"github.com/rogpeppe/go-internal/testscript"
 )
 
 func TestMain(m *testing.M) {
 	os.Exit(testscript.RunMain(m, map[string]func() int{
-		"holos": cli.MakeMain(),
+		"holos": cmd.MakeMain(),
 		"cue":   cue.Main,
 	}))
 }

doc/md/api/core.md

@@ -22,12 +22,14 @@ Package core contains schemas for a [Platform](<#Platform>) and [BuildPlan](<#Bu
 - [type Chart](<#Chart>)
 - [type Command](<#Command>)
 - [type Component](<#Component>)
+- [type ExtractYAML](<#ExtractYAML>)
 - [type File](<#File>)
 - [type FileContent](<#FileContent>)
 - [type FileContentMap](<#FileContentMap>)
 - [type FilePath](<#FilePath>)
 - [type Generator](<#Generator>)
 - [type Helm](<#Helm>)
+- [type Instance](<#Instance>)
 - [type InternalLabel](<#InternalLabel>)
 - [type Join](<#Join>)
 - [type Kind](<#Kind>)
@@ -169,6 +171,10 @@ type Component struct {
     // Path represents the path of the component relative to the platform root.
     // Injected as the tag variable "holos_component_path".
     Path string `json:"path" yaml:"path"`
+    // Instances represents additional cue instance paths to unify with Path.
+    // Useful to unify data files into a component BuildPlan.  Added in holos
+    // 0.101.7.
+    Instances []Instance `json:"instances,omitempty" yaml:"instances,omitempty"`
     // WriteTo represents the holos render component --write-to flag.  If empty,
     // the default value for the --write-to flag is used.
     WriteTo string `json:"writeTo,omitempty" yaml:"writeTo,omitempty"`
@@ -182,11 +188,22 @@ type Component struct {
     // resulting BuildPlan.
     Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
     // Annotations represents arbitrary non-identifying metadata.  Use the
-    // `cli.holos.run/description` to customize the log message of each BuildPlan.
+    // `app.holos.run/description` to customize the log message of each BuildPlan.
     Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
 }
 ```
 
+<a name="ExtractYAML"></a>
+## type ExtractYAML {#ExtractYAML}
+
+ExtractYAML represents a cue data instance encoded as yaml or json. If Path refers to a directory all files in the directory are extracted non\-recursively. Otherwise, path must refer to a file.
+
+```go
+type ExtractYAML struct {
+    Path string `json:"path" yaml:"path"`
+}
+```
+
 <a name="File"></a>
 ## type File {#File}
 
@@ -279,6 +296,22 @@ type Helm struct {
 }
 ```
 
+<a name="Instance"></a>
+## type Instance {#Instance}
+
+Instance represents a data instance to unify with the configuration.
+
+Useful to unify json and yaml files with cue configuration files for integration with other tools. For example, executing holos render platform from a pull request workflow after [Kargo](<https://docs.kargo.io/>) executes the [yaml update](<https://docs.kargo.io/references/promotion-steps#yaml-update>) and [git wait for pr](<https://docs.kargo.io/references/promotion-steps#git-wait-for-pr>) promotion steps.
+
+```go
+type Instance struct {
+    // Kind is a discriminator.
+    Kind string `json:"kind" yaml:"kind" cue:"\"ExtractYAML\""`
+    // Ignored unless kind is ExtractYAML.
+    ExtractYAML ExtractYAML `json:"extractYAML,omitempty" yaml:"extractYAML,omitempty"`
+}
+```
+
 <a name="InternalLabel"></a>
 ## type InternalLabel {#InternalLabel}
 
@@ -343,7 +376,7 @@ type Metadata struct {
     // Labels represents a resource selector.
     Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
     // Annotations represents arbitrary non-identifying metadata.  For example
-    // holos uses the `cli.holos.run/description` annotation to log resources in a
+    // holos uses the `app.holos.run/description` annotation to log resources in a
     // user customized way.
     Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
 }

doc/md/topics/comparison.mdx

@@ -0,0 +1,57 @@
+---
+description: Holos compared to other tools
+sidebar_label: Comparison
+slug: comparison
+sidebar_position: 40
+---
+
+{/* cspell:ignore Prodan, rollouts */}
+
+# Holos compared to other tools
+
+## Timoni
+
+Holos and Timoni both aim to solve similar problems but approach them at
+different levels of the stack.
+
+Timoni focuses on managing applications by evaluating [CUE] stored in OCI
+containers. Its creator, Stephan Prodan, envisions a controller that applies the
+resulting manifests. In this process, Timoni defers to [Flux] for managing Helm
+charts within the cluster.
+
+In contrast, Holos implements the [Rendered Manifests Pattern] and takes a
+different approach, particularly in how it handles [Helm] charts. Like
+[ArgoCD], Holos renders Helm charts into manifests using the `helm template`
+command in its rendering pipeline. Holos differs from Timoni in several important
+ways:
+
+1. **Separation of Responsibilities:**  Holos stops short of applying
+rendered manifests to a cluster, leaving that task to existing tools like
+[ArgoCD], [Flux], or even basic `kubectl apply` commands.
+
+2. **Ecosystem Integration:**  By focusing solely on rendering Kubernetes
+manifests, Holos creates space for other tools to handle deployment and
+management. For instance, Holos integrates seamlessly with [Kargo] for
+progressive rollouts, as [Kargo] operates between Holos and the Kubernetes API.
+This approach ensures that you're not locked into any specific tool and can
+choose the best solution for each task.
+
+3. **Platform Integration:** Holos focuses on integrating multiple Components
+into a larger Platform. In Holos terminology, a Component refers to a wrapper
+for [Helm] charts, [Kustomize] bases, or raw YAML files, integrated into the
+rendering pipeline through [CUE].  A Platform represents the full combination of
+these components.
+
+4. **Explicit Rendering Pipeline:** Holos emphasizes flexibility in its
+rendering pipeline. The system allows any tool that generates Kubernetes
+manifests to be wrapped in a Generator, which can then feed into existing
+transformers like [Kustomize]. This explicit separation makes Holos highly
+adaptable for different workflows.
+
+[Kargo]: https://kargo.io/
+[Flux]: https://fluxcd.io
+[Helm]: https://helm.sh
+[ArgoCD]: https://argoproj.github.io/cd/
+[Kustomize]: https://kustomize.io/
+[CUE]: https://cuelang.org/
+[Rendered Manifests Pattern]: https://akuity.io/blog/the-rendered-manifests-pattern

doc/website/static/_redirects

@@ -1,16 +1,22 @@
 /docs                /docs/v1alpha5/ 301
 /docs/               /docs/v1alpha5/ 301
-/docs/tutorial       /docs/v1alpha5/tutorial/ 301
-/docs/tutorial/      /docs/v1alpha5/tutorial/ 301
+/docs/tutorial       /docs/v1alpha5/tutorial/overview/ 301
+/docs/tutorial/      /docs/v1alpha5/tutorial/overview/ 301
 /docs/quickstart     /docs/v1alpha5/tutorial/overview/ 301
 /docs/quickstart/    /docs/v1alpha5/tutorial/overview/ 301
 /docs/overview       /docs/v1alpha5/tutorial/overview/ 301
 /docs/overview/      /docs/v1alpha5/tutorial/overview/ 301
-/docs/topics         /docs/v1alpha5/topics/structures/ 301
-/docs/topics/        /docs/v1alpha5/topics/structures/ 301
+/docs/topics         /docs/v1alpha5/topics/ 301
+/docs/topics/        /docs/v1alpha5/topics/ 301
 /docs/setup          /docs/v1alpha5/tutorial/setup/ 301
 /docs/setup/         /docs/v1alpha5/tutorial/setup/ 301
 /docs/local-cluster  /docs/v1alpha5/topics/local-cluster/ 301
 /docs/local-cluster/ /docs/v1alpha5/topics/local-cluster/ 301
 /docs/guides/helm    /docs/v1alpha5/tutorial/helm-values/ 301
 /docs/guides/helm/   /docs/v1alpha5/tutorial/helm-values/ 301
+/docs/kargo          /docs/v1alpha5/topics/kargo/ 301
+/docs/kargo/         /docs/v1alpha5/topics/kargo/ 301
+/docs/comparison     /docs/v1alpha5/topics/comparison/ 301
+/docs/comparison/    /docs/v1alpha5/topics/comparison/ 301
+/docs/support        /docs/v1alpha5/tutorial/overview/#getting-help 301
+/docs/support/       /docs/v1alpha5/tutorial/overview/#getting-help 301

go.mod

@@ -10,7 +10,7 @@ require (
 	connectrpc.com/grpcreflect v1.2.0
 	connectrpc.com/otelconnect v0.7.0
 	connectrpc.com/validate v0.1.0
-	cuelang.org/go v0.11.0
+	cuelang.org/go v0.11.1
 	entgo.io/ent v0.13.1
 	github.com/bufbuild/buf v1.35.1
 	github.com/choria-io/machine-room v0.0.0-20240417064836-c604da2f005e
@@ -46,6 +46,7 @@ require (
 	k8s.io/client-go v0.31.1
 	k8s.io/kubectl v0.31.1
 	modernc.org/sqlite v1.29.6
+	sigs.k8s.io/kustomize/kustomize/v5 v5.5.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -354,7 +355,6 @@ require (
 	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
-	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/automaxprocs v1.5.3 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
@@ -394,7 +394,8 @@ require (
 	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kind v0.23.0 // indirect
-	sigs.k8s.io/kustomize/api v0.17.2 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect
+	sigs.k8s.io/kustomize/api v0.18.0 // indirect
+	sigs.k8s.io/kustomize/cmd/config v0.15.0 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

go.sum

@@ -52,8 +52,8 @@ connectrpc.com/validate v0.1.0 h1:r55jirxMK7HO/xZwVHj3w2XkVFarsUM77ZDy367NtH4=
 connectrpc.com/validate v0.1.0/go.mod h1:GU47c9/x/gd+u9wRSPkrQOP46gx2rMN+Wo37EHgI3Ow=
 cuelabs.dev/go/oci/ociregistry v0.0.0-20240906074133-82eb438dd565 h1:R5wwEcbEZSBmeyg91MJZTxfd7WpBo2jPof3AYjRbxwY=
 cuelabs.dev/go/oci/ociregistry v0.0.0-20240906074133-82eb438dd565/go.mod h1:5A4xfTzHTXfeVJBU6RAUf+QrlfTCW+017q/QiW+sMLg=
-cuelang.org/go v0.11.0 h1:2af2nhipqlUHtXk2dtOP5xnMm1ObGvKqIsJUJL1sRE4=
-cuelang.org/go v0.11.0/go.mod h1:PBY6XvPUswPPJ2inpvUozP9mebDVTXaeehQikhZPBz0=
+cuelang.org/go v0.11.1 h1:pV+49MX1mmvDm8Qh3Za3M786cty8VKPWzQ1Ho4gZRP0=
+cuelang.org/go v0.11.1/go.mod h1:PBY6XvPUswPPJ2inpvUozP9mebDVTXaeehQikhZPBz0=
 dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
 dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -1077,8 +1077,6 @@ go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVf
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
 go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
-go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
-go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
@@ -1282,7 +1280,6 @@ golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
 golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
@@ -1557,10 +1554,14 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kind v0.23.0 h1:8fyDGWbWTeCcCTwA04v4Nfr45KKxbSPH1WO9K+jVrBg=
 sigs.k8s.io/kind v0.23.0/go.mod h1:ZQ1iZuJLh3T+O8fzhdi3VWcFTzsdXtNv2ppsHc8JQ7s=
-sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g=
-sigs.k8s.io/kustomize/api v0.17.2/go.mod h1:UWTz9Ct+MvoeQsHcJ5e+vziRRkwimm3HytpZgIYqye0=
-sigs.k8s.io/kustomize/kyaml v0.17.1 h1:TnxYQxFXzbmNG6gOINgGWQt09GghzgTP6mIurOgrLCQ=
-sigs.k8s.io/kustomize/kyaml v0.17.1/go.mod h1:9V0mCjIEYjlXuCdYsSXvyoy2BTsLESH7TlGV81S282U=
+sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo=
+sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U=
+sigs.k8s.io/kustomize/cmd/config v0.15.0 h1:WkdY8V2+8J+W00YbImXa2ke9oegfrHH79e+kywW7EdU=
+sigs.k8s.io/kustomize/cmd/config v0.15.0/go.mod h1:Jq57b0nPaoYUlOqg//0JtAh6iibboqMcfbtCYoWPM00=
+sigs.k8s.io/kustomize/kustomize/v5 v5.5.0 h1:o1mtt6vpxsxDYaZKrw3BnEtc+pAjLz7UffnIvHNbvW0=
+sigs.k8s.io/kustomize/kustomize/v5 v5.5.0/go.mod h1:AeFCmgCrXzmvjWWaeZCyBp6XzG1Y0w1svYus8GhJEOE=
+sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E=
+sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

internal/builder/instance.go

@@ -5,17 +5,74 @@ import (
 	"encoding/json"
 	"fmt"
 	"log/slog"
+	"os"
+	"path/filepath"
 	"strings"
+	"sync"
 
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/cuecontext"
+	"cuelang.org/go/cue/interpreter/embed"
 	"cuelang.org/go/cue/load"
+	"cuelang.org/go/encoding/yaml"
 	"github.com/holos-run/holos/internal/errors"
 	"github.com/holos-run/holos/internal/holos"
 	"github.com/holos-run/holos/internal/util"
 )
 
-func LoadInstance(path string, tags []string) (*Instance, error) {
+// cue context and loading is not safe for concurrent use.
+var cueMutex sync.Mutex
+
+// ExtractYAML extracts yaml encoded data from file paths.  The data is unified
+// into one [cue.Value].  If a path element is a directory, all files in the
+// directory are loaded non-recursively.
+//
+// Attribution: https://github.com/cue-lang/cue/issues/3504
+func ExtractYAML(ctxt *cue.Context, filepaths []string) (cue.Value, error) {
+	value := ctxt.CompileString("")
+	files := make([]string, 0, 10*len(filepaths))
+
+	for _, path := range filepaths {
+		info, err := os.Stat(path)
+		if err != nil {
+			return value, errors.Wrap(err)
+		}
+
+		if !info.IsDir() {
+			files = append(files, path)
+			continue
+		}
+
+		entries, err := os.ReadDir(path)
+		if err != nil {
+			return value, errors.Wrap(err)
+		}
+		for _, entry := range entries {
+			if entry.IsDir() {
+				continue
+			}
+			files = append(files, filepath.Join(path, entry.Name()))
+		}
+	}
+
+	for _, file := range files {
+		f, err := yaml.Extract(file, nil)
+		if err != nil {
+			return value, errors.Wrap(err)
+		}
+		value = value.Unify(ctxt.BuildFile(f))
+	}
+
+	return value, nil
+}
+
+// LoadInstance loads the cue configuration instance at path.  External data
+// file paths are loaded by calling [ExtractYAML] providing filepaths.  The
+// extracted data values are unified with the platform configuration [cue.Value]
+// in the returned [Instance].
+func LoadInstance(path string, filepaths []string, tags []string) (*Instance, error) {
+	cueMutex.Lock()
+	defer cueMutex.Unlock()
 	root, leaf, err := util.FindRootLeaf(path)
 	if err != nil {
 		return nil, errors.Wrap(err)
@@ -26,20 +83,26 @@ func LoadInstance(path string, tags []string) (*Instance, error) {
 		ModuleRoot: root,
 		Tags:       tags,
 	}
+	ctxt := cuecontext.New(cuecontext.Interpreter(embed.New()))
 
-	ctx := cuecontext.New()
-
-	instances := load.Instances([]string{leaf}, cfg)
-	values, err := ctx.BuildInstances(instances)
+	bis := load.Instances([]string{path}, cfg)
+	values, err := ctxt.BuildInstances(bis)
 	if err != nil {
 		return nil, errors.Wrap(err)
 	}
 
+	value, err := ExtractYAML(ctxt, filepaths)
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+	// TODO: https://cuelang.org/docs/howto/place-data-go-api/
+	value = value.Unify(values[0])
+
 	inst := &Instance{
 		path:  leaf,
-		ctx:   ctx,
+		ctx:   ctxt,
 		cfg:   cfg,
-		value: values[0],
+		value: value,
 	}
 
 	return inst, nil

internal/builder/v1alpha5/builder.go

@@ -107,6 +107,20 @@ func (c *Component) Path() string {
 	return util.DotSlash(c.Component.Path)
 }
 
+// ExtractYAML returns the path values for the --extract-yaml command line flag.
+func (c *Component) ExtractYAML() ([]string, error) {
+	if c == nil {
+		return nil, nil
+	}
+	instances := make([]string, 0, len(c.Component.Instances))
+	for _, instance := range c.Component.Instances {
+		if instance.Kind == "ExtractYAML" {
+			instances = append(instances, instance.ExtractYAML.Path)
+		}
+	}
+	return instances, nil
+}
+
 var _ holos.BuildPlan = &BuildPlan{}
 var _ task = generatorTask{}
 var _ task = transformersTask{}

internal/cli/main.go

@@ -1,110 +1 @@
 package cli
-
-import (
-	"context"
-	"fmt"
-	"log/slog"
-	"os"
-	"runtime/pprof"
-	"runtime/trace"
-
-	"connectrpc.com/connect"
-	cue "cuelang.org/go/cue/errors"
-	"github.com/holos-run/holos/internal/errors"
-	"github.com/holos-run/holos/internal/holos"
-	"google.golang.org/genproto/googleapis/rpc/errdetails"
-)
-
-func memProfile(ctx context.Context, cfg *holos.Config) {
-	if format := os.Getenv("HOLOS_MEM_PROFILE"); format != "" {
-		f, _ := os.Create(fmt.Sprintf(format, os.Getppid(), os.Getpid()))
-		defer f.Close()
-		if err := pprof.WriteHeapProfile(f); err != nil {
-			_ = HandleError(ctx, err, cfg)
-		}
-	}
-}
-
-// MakeMain makes a main function for the cli or tests.
-func MakeMain(options ...holos.Option) func() int {
-	return func() (exitCode int) {
-		cfg := holos.New(options...)
-		slog.SetDefault(cfg.Logger())
-		ctx := context.Background()
-
-		if format := os.Getenv("HOLOS_CPU_PROFILE"); format != "" {
-			f, _ := os.Create(fmt.Sprintf(format, os.Getppid(), os.Getpid()))
-			err := pprof.StartCPUProfile(f)
-			defer func() {
-				pprof.StopCPUProfile()
-				f.Close()
-			}()
-			if err != nil {
-				return HandleError(ctx, err, cfg)
-			}
-		}
-		defer memProfile(ctx, cfg)
-
-		if format := os.Getenv("HOLOS_TRACE"); format != "" {
-			f, _ := os.Create(fmt.Sprintf(format, os.Getppid(), os.Getpid()))
-			err := trace.Start(f)
-			defer func() {
-				trace.Stop()
-				f.Close()
-			}()
-			if err != nil {
-				return HandleError(ctx, err, cfg)
-			}
-		}
-
-		feature := &holos.EnvFlagger{}
-		if err := New(cfg, feature).ExecuteContext(ctx); err != nil {
-			return HandleError(ctx, err, cfg)
-		}
-		return 0
-	}
-}
-
-// HandleError is the top level error handler that unwraps and logs errors.
-func HandleError(ctx context.Context, err error, hc *holos.Config) (exitCode int) {
-	// Connect errors have codes, log them.
-	log := hc.NewTopLevelLogger().With("code", connect.CodeOf(err))
-	var cueErr cue.Error
-	var errAt *errors.ErrorAt
-
-	if errors.As(err, &errAt) {
-		loc := errAt.Source.Loc()
-		err2 := errAt.Unwrap()
-		log.ErrorContext(ctx, fmt.Sprintf("could not run: %s at %s", err2, loc), "err", err2, "loc", loc)
-	} else {
-		log.ErrorContext(ctx, fmt.Sprintf("could not run: %s", err), "err", err)
-	}
-
-	// cue errors are bundled up as a list and refer to multiple files / lines.
-	if errors.As(err, &cueErr) {
-		msg := cue.Details(cueErr, nil)
-		if _, err := fmt.Fprint(hc.Stderr(), msg); err != nil {
-			log.ErrorContext(ctx, "could not write CUE error details: "+err.Error(), "err", err)
-		}
-	}
-	// connect errors have details and codes.
-	// Refer to https://connectrpc.com/docs/go/errors
-	if connectErr := new(connect.Error); errors.As(err, &connectErr) {
-		for _, detail := range connectErr.Details() {
-			msg, valueErr := detail.Value()
-			if valueErr != nil {
-				log.WarnContext(ctx, "could not decode error detail", "err", err, "type", detail.Type(), "note", "this usually means we don't have the schema for the protobuf message type")
-				continue
-			}
-			if info, ok := msg.(*errdetails.ErrorInfo); ok {
-				logDetail := log.With("reason", info.GetReason(), "domain", info.GetDomain())
-				for k, v := range info.GetMetadata() {
-					logDetail = logDetail.With(k, v)
-				}
-				logDetail.ErrorContext(ctx, info.String())
-			}
-		}
-	}
-
-	return 1
-}

internal/cli/render/render.go

@@ -43,6 +43,8 @@ func newPlatform(cfg *holos.Config, feature holos.Flagger) *cobra.Command {
 	cmd.Flags().IntVar(&concurrency, "concurrency", runtime.NumCPU(), "number of components to render concurrently")
 	var platform string
 	cmd.Flags().StringVar(&platform, "platform", "./platform", "platform directory path")
+	var extractYAMLs holos.StringSlice
+	cmd.Flags().Var(&extractYAMLs, "extract-yaml", "data file paths to extract and unify with the platform config")
 	var selector holos.Selector
 	cmd.Flags().VarP(&selector, "selector", "l", "label selector (e.g. label==string,label!=string)")
 	tagMap := make(holos.TagMap)
@@ -57,7 +59,7 @@ func newPlatform(cfg *holos.Config, feature holos.Flagger) *cobra.Command {
 			log.WarnContext(ctx, fmt.Sprintf(msg, platform))
 		}
 
-		inst, err := builder.LoadInstance(platform, tagMap.Tags())
+		inst, err := builder.LoadInstance(platform, extractYAMLs, tagMap.Tags())
 		if err != nil {
 			return errors.Wrap(err)
 		}
@@ -107,12 +109,14 @@ func newComponent(cfg *holos.Config, feature holos.Flagger) *cobra.Command {
 	cmd.Flags().VarP(&tagMap, "inject", "t", tagHelp)
 	var concurrency int
 	cmd.Flags().IntVar(&concurrency, "concurrency", runtime.NumCPU(), "number of concurrent build steps")
+	var extractYAMLs holos.StringSlice
+	cmd.Flags().Var(&extractYAMLs, "extract-yaml", "data file paths to extract and unify with the platform config")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		ctx := cmd.Root().Context()
 		path := args[0]
 
-		inst, err := builder.LoadInstance(path, tagMap.Tags())
+		inst, err := builder.LoadInstance(path, extractYAMLs, tagMap.Tags())
 		if err != nil {
 			return errors.Wrap(err)
 		}
@@ -146,7 +150,11 @@ func makeComponentRenderFunc(w io.Writer, prefixArgs, cliTags []string) func(con
 			if err != nil {
 				return errors.Wrap(err)
 			}
-			args := make([]string, 0, 10+len(prefixArgs)+(len(tags)*2))
+			filepaths, err := component.ExtractYAML()
+			if err != nil {
+				return errors.Wrap(err)
+			}
+			args := make([]string, 0, 10+len(prefixArgs)+(len(tags)*2+len(filepaths)*2))
 			args = append(args, prefixArgs...)
 			args = append(args, "render", "component")
 			for _, tag := range cliTags {
@@ -155,6 +163,9 @@ func makeComponentRenderFunc(w io.Writer, prefixArgs, cliTags []string) func(con
 			for _, tag := range tags {
 				args = append(args, "--inject", tag)
 			}
+			for _, path := range filepaths {
+				args = append(args, "--extract-yaml", path)
+			}
 			args = append(args, component.Path())
 			if _, err := util.RunCmdA(ctx, w, "holos", args...); err != nil {
 				return errors.Format("could not render component: %w", err)

internal/cli/root.go

@@ -1,14 +1,18 @@
 package cli
 
 import (
+	"context"
 	_ "embed"
 	"fmt"
 	"log/slog"
 
+	"connectrpc.com/connect"
 	"github.com/spf13/cobra"
+	"google.golang.org/genproto/googleapis/rpc/errdetails"
 
 	"github.com/holos-run/holos/version"
 
+	"github.com/holos-run/holos/internal/errors"
 	"github.com/holos-run/holos/internal/holos"
 	"github.com/holos-run/holos/internal/logger"
 	"github.com/holos-run/holos/internal/server"
@@ -28,7 +32,8 @@ import (
 	"github.com/holos-run/holos/internal/cli/token"
 	"github.com/holos-run/holos/internal/cli/txtar"
 
-	cue "cuelang.org/go/cmd/cue/cmd"
+	cueCmd "cuelang.org/go/cmd/cue/cmd"
+	cue_errors "cuelang.org/go/cue/errors"
 )
 
 //go:embed help.txt
@@ -119,7 +124,7 @@ func newOrgCmd(feature holos.Flagger) (cmd *cobra.Command) {
 
 func newCueCmd() (cmd *cobra.Command) {
 	// Get a handle on the cue root command fields.
-	root, _ := cue.New([]string{})
+	root, _ := cueCmd.New([]string{})
 	// Copy the fields to our embedded command.
 	cmd = command.New("cue")
 	cmd.Short = root.Short
@@ -130,8 +135,52 @@ func newCueCmd() (cmd *cobra.Command) {
 
 	// We do it this way so we handle errors correctly.
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		cueRootCommand, _ := cue.New(args)
+		cueRootCommand, _ := cueCmd.New(args)
 		return cueRootCommand.Run(cmd.Root().Context())
 	}
 	return cmd
 }
+
+// HandleError is the top level error handler that unwraps and logs errors.
+func HandleError(ctx context.Context, err error, hc *holos.Config) (exitCode int) {
+	// Connect errors have codes, log them.
+	log := hc.NewTopLevelLogger().With("code", connect.CodeOf(err))
+	var cueErr cue_errors.Error
+	var errAt *errors.ErrorAt
+
+	if errors.As(err, &errAt) {
+		loc := errAt.Source.Loc()
+		err2 := errAt.Unwrap()
+		log.ErrorContext(ctx, fmt.Sprintf("could not run: %s at %s", err2, loc), "err", err2, "loc", loc)
+	} else {
+		log.ErrorContext(ctx, fmt.Sprintf("could not run: %s", err), "err", err)
+	}
+
+	// cue errors are bundled up as a list and refer to multiple files / lines.
+	if errors.As(err, &cueErr) {
+		msg := cue_errors.Details(cueErr, nil)
+		if _, err := fmt.Fprint(hc.Stderr(), msg); err != nil {
+			log.ErrorContext(ctx, "could not write CUE error details: "+err.Error(), "err", err)
+		}
+	}
+	// connect errors have details and codes.
+	// Refer to https://connectrpc.com/docs/go/errors
+	if connectErr := new(connect.Error); errors.As(err, &connectErr) {
+		for _, detail := range connectErr.Details() {
+			msg, valueErr := detail.Value()
+			if valueErr != nil {
+				log.WarnContext(ctx, "could not decode error detail", "err", err, "type", detail.Type(), "note", "this usually means we don't have the schema for the protobuf message type")
+				continue
+			}
+			if info, ok := msg.(*errdetails.ErrorInfo); ok {
+				logDetail := log.With("reason", info.GetReason(), "domain", info.GetDomain())
+				for k, v := range info.GetMetadata() {
+					logDetail = logDetail.With(k, v)
+				}
+				logDetail.ErrorContext(ctx, info.String())
+			}
+		}
+	}
+
+	return 1
+}

internal/cli/show.go

@@ -30,13 +30,15 @@ func newShowPlatformCmd() (cmd *cobra.Command) {
 
 	var platform string
 	cmd.Flags().StringVar(&platform, "platform", "./platform", "platform directory path")
+	var extractYAMLs holos.StringSlice
+	cmd.Flags().Var(&extractYAMLs, "extract-yaml", "data file paths to extract and unify with the platform config")
 	var format string
 	cmd.Flags().StringVar(&format, "format", "yaml", "yaml or json format")
 	tagMap := make(holos.TagMap)
 	cmd.Flags().VarP(&tagMap, "inject", "t", "set the value of a cue @tag field from a key=value pair")
 
 	cmd.RunE = func(c *cobra.Command, args []string) (err error) {
-		inst, err := builder.LoadInstance(platform, tagMap.Tags())
+		inst, err := builder.LoadInstance(platform, extractYAMLs, tagMap.Tags())
 		if err != nil {
 			return errors.Wrap(err)
 		}
@@ -64,6 +66,8 @@ func newShowBuildPlanCmd() (cmd *cobra.Command) {
 
 	var platform string
 	cmd.Flags().StringVar(&platform, "platform", "./platform", "platform directory path")
+	var extractYAMLs holos.StringSlice
+	cmd.Flags().Var(&extractYAMLs, "extract-yaml", "data file paths to extract and unify with the platform config")
 	var format string
 	cmd.Flags().StringVar(&format, "format", "yaml", "yaml or json format")
 	var selector holos.Selector
@@ -75,7 +79,7 @@ func newShowBuildPlanCmd() (cmd *cobra.Command) {
 
 	cmd.RunE = func(c *cobra.Command, args []string) (err error) {
 		path := platform
-		inst, err := builder.LoadInstance(path, tagMap.Tags())
+		inst, err := builder.LoadInstance(path, extractYAMLs, tagMap.Tags())
 		if err != nil {
 			return errors.Wrap(err)
 		}
@@ -122,7 +126,11 @@ func makeBuildFunc(encoder holos.OrderedEncoder, opts holos.BuildOpts) func(cont
 				return errors.Wrap(err)
 			}
 			tags = append(tags, opts.Tags...)
-			inst, err := builder.LoadInstance(component.Path(), tags)
+			filepaths, err := component.ExtractYAML()
+			if err != nil {
+				return errors.Wrap(err)
+			}
+			inst, err := builder.LoadInstance(component.Path(), filepaths, tags)
 			if err != nil {
 				return errors.Wrap(err)
 			}

internal/generate/platforms/cue.mod/gen/github.com/holos-run/holos/api/core/v1alpha5/types_go_gen.cue

@@ -290,7 +290,7 @@ package core
 	labels?: {[string]: string} @go(Labels,map[string]string)
 
 	// Annotations represents arbitrary non-identifying metadata.  For example
-	// holos uses the `cli.holos.run/description` annotation to log resources in a
+	// holos uses the `app.holos.run/description` annotation to log resources in a
 	// user customized way.
 	annotations?: {[string]: string} @go(Annotations,map[string]string)
 }
@@ -334,6 +334,11 @@ package core
 	// Injected as the tag variable "holos_component_path".
 	path: string @go(Path)
 
+	// Instances represents additional cue instance paths to unify with Path.
+	// Useful to unify data files into a component BuildPlan.  Added in holos
+	// 0.101.7.
+	instances?: [...#Instance] @go(Instances,[]Instance)
+
 	// WriteTo represents the holos render component --write-to flag.  If empty,
 	// the default value for the --write-to flag is used.
 	writeTo?: string @go(WriteTo)
@@ -350,6 +355,31 @@ package core
 	labels?: {[string]: string} @go(Labels,map[string]string)
 
 	// Annotations represents arbitrary non-identifying metadata.  Use the
-	// `cli.holos.run/description` to customize the log message of each BuildPlan.
+	// `app.holos.run/description` to customize the log message of each BuildPlan.
 	annotations?: {[string]: string} @go(Annotations,map[string]string)
 }
+
+// Instance represents a data instance to unify with the configuration.
+//
+// Useful to unify json and yaml files with cue configuration files for
+// integration with other tools.  For example, executing holos render platform
+// from a pull request workflow after [Kargo] executes the [yaml update] and
+// [git wait for pr] promotion steps.
+//
+// [Kargo]: https://docs.kargo.io/
+// [yaml update]: https://docs.kargo.io/references/promotion-steps#yaml-update
+// [git wait for pr]: https://docs.kargo.io/references/promotion-steps#git-wait-for-pr
+#Instance: {
+	// Kind is a discriminator.
+	kind: string & "ExtractYAML" @go(Kind)
+
+	// Ignored unless kind is ExtractYAML.
+	extractYAML?: #ExtractYAML @go(ExtractYAML)
+}
+
+// ExtractYAML represents a cue data instance encoded as yaml or json. If Path
+// refers to a directory all files in the directory are extracted
+// non-recursively.  Otherwise, path must refer to a file.
+#ExtractYAML: {
+	path: string @go(Path)
+}

internal/generate/platforms/cue.mod/gen/kargo.akuity.io/freight/v1alpha1/types_gen.cue

@@ -1,6 +1,6 @@
 // Code generated by timoni. DO NOT EDIT.
 
-//timoni:generate timoni vendor crd -f projects/argocd/components/kargo/vendor/1.0.3/kargo/resources/crds/kargo.akuity.io_freights.yaml
+//timoni:generate timoni vendor crd -f /Users/jeff/Holos/kargo-demo/deploy/components/kargo/kargo.gen.yaml
 
 package v1alpha1
 

internal/generate/platforms/cue.mod/gen/kargo.akuity.io/project/v1alpha1/types_gen.cue

@@ -1,6 +1,6 @@
 // Code generated by timoni. DO NOT EDIT.
 
-//timoni:generate timoni vendor crd -f projects/argocd/components/kargo/vendor/1.0.3/kargo/resources/crds/kargo.akuity.io_projects.yaml
+//timoni:generate timoni vendor crd -f /Users/jeff/Holos/kargo-demo/deploy/components/kargo/kargo.gen.yaml
 
 package v1alpha1
 

internal/generate/platforms/cue.mod/gen/kargo.akuity.io/promotion/v1alpha1/types_gen.cue

@@ -1,6 +1,6 @@
 // Code generated by timoni. DO NOT EDIT.
 
-//timoni:generate timoni vendor crd -f projects/argocd/components/kargo/vendor/1.0.3/kargo/resources/crds/kargo.akuity.io_promotions.yaml
+//timoni:generate timoni vendor crd -f /Users/jeff/Holos/kargo-demo/deploy/components/kargo/kargo.gen.yaml
 
 package v1alpha1
 
@@ -74,10 +74,90 @@ import "strings"
 		// As is the alias this step can be referred to as.
 		as?: string
 
-		// Config is the configuration for the directive.
+		// Config is opaque configuration for the PromotionStep that is
+		// understood
+		// only by each PromotionStep's implementation. It is legal to
+		// utilize
+		// expressions in defining values at any level of this block.
+		// See https://docs.kargo.io/references/expression-language for
+		// details.
 		config?: _
 
+		// Retry is the retry policy for this step.
+		retry?: {
+			// ErrorThreshold is the number of consecutive times the step must
+			// fail (for
+			// any reason) before retries are abandoned and the entire
+			// Promotion is marked
+			// as failed.
+			//
+			// If this field is set to 0, the effective default will be a
+			// step-specific
+			// one. If no step-specific default exists (i.e. is also 0), the
+			// effective
+			// default will be the system-wide default of 1.
+			//
+			// A value of 1 will cause the Promotion to be marked as failed
+			// after just
+			// a single failure; i.e. no retries will be attempted.
+			//
+			// There is no option to specify an infinite number of retries
+			// using a value
+			// such as -1.
+			//
+			// In a future release, Kargo is likely to become capable of
+			// distinguishing
+			// between recoverable and non-recoverable step failures. At that
+			// time, it is
+			// planned that unrecoverable failures will not be subject to this
+			// threshold
+			// and will immediately cause the Promotion to be marked as failed
+			// without
+			// further condition.
+			errorThreshold?: int
+
+			// Timeout is the soft maximum interval in which a step that
+			// returns a Running
+			// status (which typically indicates it's waiting for something to
+			// happen)
+			// may be retried.
+			//
+			// The maximum is a soft one because the check for whether the
+			// interval has
+			// elapsed occurs AFTER the step has run. This effectively means a
+			// step may
+			// run ONCE beyond the close of the interval.
+			//
+			// If this field is set to nil, the effective default will be a
+			// step-specific
+			// one. If no step-specific default exists (i.e. is also nil), the
+			// effective
+			// default will be the system-wide default of 0.
+			//
+			// A value of 0 will cause the step to be retried indefinitely
+			// unless the
+			// ErrorThreshold is reached.
+			timeout?: string
+		}
+
 		// Uses identifies a runner that can execute this step.
 		uses: strings.MinRunes(1)
 	}]
+
+	// Vars is a list of variables that can be referenced by
+	// expressions in
+	// promotion steps.
+	vars?: [...{
+		// Name is the name of the variable.
+		name: strings.MinRunes(1) & {
+			=~"^[a-zA-Z_]\\w*$"
+		}
+
+		// Value is the value of the variable. It is allowed to utilize
+		// expressions
+		// in the value.
+		// See https://docs.kargo.io/references/expression-language for
+		// details.
+		value: string
+	}]
 }

internal/generate/platforms/cue.mod/gen/kargo.akuity.io/stage/v1alpha1/types_gen.cue

@@ -1,6 +1,6 @@
 // Code generated by timoni. DO NOT EDIT.
 
-//timoni:generate timoni vendor crd -f projects/argocd/components/kargo/vendor/1.0.3/kargo/resources/crds/kargo.akuity.io_stages.yaml
+//timoni:generate timoni vendor crd -f /Users/jeff/Holos/kargo-demo/deploy/components/kargo/kargo.gen.yaml
 
 package v1alpha1
 
@@ -52,6 +52,11 @@ import "strings"
 // Freight into the Stage.
 #StageSpec: {
 	promotionTemplate?: {
+		// PromotionTemplateSpec describes the (partial) specification of
+		// a Promotion
+		// for a Stage. This is a template that can be used to create a
+		// Promotion for a
+		// Stage.
 		spec: {
 			// Steps specifies the directives to be executed as part of a
 			// Promotion.
@@ -62,12 +67,92 @@ import "strings"
 				// As is the alias this step can be referred to as.
 				as?: string
 
-				// Config is the configuration for the directive.
+				// Config is opaque configuration for the PromotionStep that is
+				// understood
+				// only by each PromotionStep's implementation. It is legal to
+				// utilize
+				// expressions in defining values at any level of this block.
+				// See https://docs.kargo.io/references/expression-language for
+				// details.
 				config?: _
 
+				// Retry is the retry policy for this step.
+				retry?: {
+					// ErrorThreshold is the number of consecutive times the step must
+					// fail (for
+					// any reason) before retries are abandoned and the entire
+					// Promotion is marked
+					// as failed.
+					//
+					// If this field is set to 0, the effective default will be a
+					// step-specific
+					// one. If no step-specific default exists (i.e. is also 0), the
+					// effective
+					// default will be the system-wide default of 1.
+					//
+					// A value of 1 will cause the Promotion to be marked as failed
+					// after just
+					// a single failure; i.e. no retries will be attempted.
+					//
+					// There is no option to specify an infinite number of retries
+					// using a value
+					// such as -1.
+					//
+					// In a future release, Kargo is likely to become capable of
+					// distinguishing
+					// between recoverable and non-recoverable step failures. At that
+					// time, it is
+					// planned that unrecoverable failures will not be subject to this
+					// threshold
+					// and will immediately cause the Promotion to be marked as failed
+					// without
+					// further condition.
+					errorThreshold?: int
+
+					// Timeout is the soft maximum interval in which a step that
+					// returns a Running
+					// status (which typically indicates it's waiting for something to
+					// happen)
+					// may be retried.
+					//
+					// The maximum is a soft one because the check for whether the
+					// interval has
+					// elapsed occurs AFTER the step has run. This effectively means a
+					// step may
+					// run ONCE beyond the close of the interval.
+					//
+					// If this field is set to nil, the effective default will be a
+					// step-specific
+					// one. If no step-specific default exists (i.e. is also nil), the
+					// effective
+					// default will be the system-wide default of 0.
+					//
+					// A value of 0 will cause the step to be retried indefinitely
+					// unless the
+					// ErrorThreshold is reached.
+					timeout?: string
+				}
+
 				// Uses identifies a runner that can execute this step.
 				uses: strings.MinRunes(1)
 			}] & [_, ...]
+
+			// Vars is a list of variables that can be referenced by
+			// expressions in
+			// promotion steps.
+			vars?: [...{
+				// Name is the name of the variable.
+				name: strings.MinRunes(1) & {
+					=~"^[a-zA-Z_]\\w*$"
+				}
+
+				// Value is the value of the variable. It is allowed to utilize
+				// expressions
+				// in the value.
+				// See https://docs.kargo.io/references/expression-language for
+				// details.
+				value: string
+			}]
 		}
 	}
 

internal/generate/platforms/cue.mod/gen/kargo.akuity.io/warehouse/v1alpha1/types_gen.cue

@@ -1,6 +1,6 @@
 // Code generated by timoni. DO NOT EDIT.
 
-//timoni:generate timoni vendor crd -f projects/argocd/components/kargo/vendor/1.0.3/kargo/resources/crds/kargo.akuity.io_warehouses.yaml
+//timoni:generate timoni vendor crd -f /Users/jeff/Holos/kargo-demo/deploy/components/kargo/kargo.gen.yaml
 
 package v1alpha1
 
@@ -52,6 +52,7 @@ import "strings"
 	// This field is optional. When left unspecified, the field is
 	// implicitly
 	// treated as if its value were "Automatic".
+	// Accepted values: Automatic, Manual
 	freightCreationPolicy?: "Automatic" | "Manual" | *"Automatic"
 
 	// Interval is the reconciliation interval for this Warehouse. On
@@ -170,6 +171,7 @@ import "strings"
 			// field is optional. When left unspecified, the field is
 			// implicitly treated
 			// as if its value were "NewestFromBranch".
+			// Accepted values: Lexical, NewestFromBranch, NewestTag, SemVer
 			commitSelectionStrategy?: "Lexical" | "NewestFromBranch" | "NewestTag" | "SemVer" | *"NewestFromBranch"
 
 			// DiscoveryLimit is an optional limit on the number of commits
@@ -324,6 +326,7 @@ import "strings"
 			// left unspecified, the field is implicitly treated as if its
 			// value were
 			// "SemVer".
+			// Accepted values: Digest, Lexical, NewestBuild, SemVer
 			imageSelectionStrategy?: "Digest" | "Lexical" | "NewestBuild" | "SemVer" | *"SemVer"
 
 			// InsecureSkipTLSVerify specifies whether certificate

internal/holos/interface.go

@@ -21,6 +21,8 @@ type Platform interface {
 type Component interface {
 	Describe() string
 	Path() string
+	// ExtractYAML represents the values of the --extract-yaml flag
+	ExtractYAML() ([]string, error)
 	Tags() ([]string, error)
 	WriteTo() string
 	Labels() Labels

internal/util/util.go

@@ -42,6 +42,9 @@ func FindCueMod(path string) (root string, err error) {
 	return root, nil
 }
 
+// FindRootLeaf returns the root path containing the cue.mod and the leaf path
+// relative to the root for the given target path.  FindRootLeaf calls
+// [filepath.Clean] on the returned paths.
 func FindRootLeaf(target string) (root string, leaf string, err error) {
 	if root, err = FindCueMod(target); err != nil {
 		return "", "", err

service/buf.lock

@@ -4,5 +4,5 @@ deps:
   - remote: buf.build
     owner: bufbuild
     repository: protovalidate
-    commit: 5a7b106cbb87462d9a8c9ffecdbd2e38
-    digest: shake256:2f7efa5a904668219f039d4f6eeb51e871f8f7f5966055a10663cba335bd65f76cac84da3fa758ab7b5dcb489ec599521390ce3951d119fb56df1fc2def16bb0
+    commit: a3320276596649bcad929ac829d451f4
+    digest: shake256:a6e5f64fd3fd47e3e8568e9753f59a1566f56c11ec055baf65463d3bca3499f6f16c2d6f5628fa41cfd0f4fa7e72abe65be4efd77d269749492472ed4cc4070d

tools/tools.go

@@ -6,6 +6,7 @@ package tools
 // https://go.dev/wiki/Modules
 
 import (
+  _ "sigs.k8s.io/kustomize/kustomize/v5"
 	_ "connectrpc.com/connect/cmd/protoc-gen-connect-go"
 	_ "cuelang.org/go/cmd/cue"
 	_ "github.com/bufbuild/buf/cmd/buf"

version/embedded/minor

@@ -1 +1 @@
-101
+102

version/embedded/patch

@@ -1 +1 @@
-6
+3