(#27) Add cert-manager to the mesh collection

Straight-forward helm install with no customization.

This patch also adds a "Skip" output kind which allows intermediate cue
files in the tree to signal holos to skip over the instance.  This
enables constraints to be added at intermediate layers without build
errors.

.run/render platform.run.xml

@@ -1,16 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="render platform" type="GoApplicationRunConfiguration" factoryName="Go Application">
-    <module name="holos" />
-    <working_directory value="$PROJECT_DIR$/../holos-infra" />
-    <parameters value="render --cluster-name=k2 $PROJECT_DIR$/docs/examples/platforms/reference/clusters/workload/..." />
-    <envs>
-      <env name="DEBUG" value="1" />
-    </envs>
-    <kind value="DIRECTORY" />
-    <package value="github.com/holos-run/holos" />
-    <directory value="$PROJECT_DIR$/cmd/holos" />
-    <filePath value="$PROJECT_DIR$" />
-    <output_directory value="$PROJECT_DIR$/bin" />
-    <method v="2" />
-  </configuration>
-</component>

cmd/holos/testdata/constraints.txt

@@ -0,0 +1,42 @@
+# Want support for intermediary constraints
+exec holos build ./foo/... --log-level debug
+stdout '^bf2bc7f9-9ba0-4f9e-9bd2-9a205627eb0b$'
+stderr 'processing holos component kind Skip'
+
+-- cue.mod --
+package holos
+-- foo/constraints.cue --
+package holos
+
+metadata: name: "jeff"
+-- foo/bar/bar.cue --
+package holos
+
+#KubernetesObjects & {
+  apiObjectMap: foo: bar: "bf2bc7f9-9ba0-4f9e-9bd2-9a205627eb0b"
+}
+-- schema.cue --
+package holos
+
+cluster: string @tag(cluster, string)
+
+// #OutputTypeMeta is shared among all output types
+#OutputTypeMeta: {
+	apiVersion: "holos.run/v1alpha1"
+	kind: #KubernetesObjects.kind | #NoOutput.kind
+	metadata: name: string
+}
+
+#KubernetesObjects: {
+	#OutputTypeMeta
+	kind: "KubernetesObjects"
+	apiObjectMap: {...}
+}
+
+#NoOutput: {
+	#OutputTypeMeta
+	kind: string | *"Skip"
+	metadata: name: string | *"skipped"
+}
+
+#NoOutput & {}

cmd/holos/testdata/issue15_cue_errors.txt

@@ -1,7 +1,7 @@
 # Want cue errors to show files and lines
 ! exec holos build .
-stderr 'could not decode: content: cannot convert non-concrete value string'
-stderr '/component.cue:6:1$'
+stderr '^apiObjectMap.foo.bar: cannot convert non-concrete value string'
+stderr '/component.cue:7:20$'
 
 -- cue.mod --
 package holos
@@ -11,5 +11,6 @@ package holos
 apiVersion: "holos.run/v1alpha1"
 kind: "KubernetesObjects"
 cluster: string @tag(cluster, string)
-content: foo
-foo: string
+
+apiObjectMap: foo: bar: baz
+baz: string

cmd/holos/testdata/issue25_apiobjects_cue.txt

@@ -0,0 +1,57 @@
+# Want kube api objects in the apiObjects output.
+exec holos build .
+stdout '^kind: SecretStore$'
+stdout '# Source: CUE apiObjects.SecretStore.default'
+
+-- cue.mod --
+package holos
+-- component.cue --
+package holos
+
+apiVersion: "holos.run/v1alpha1"
+kind: "KubernetesObjects"
+cluster: string @tag(cluster, string)
+
+#SecretStore: {
+    kind: string
+    metadata: name: string
+}
+
+#APIObjects & {
+  apiObjects: {
+    SecretStore: {
+      default: #SecretStore & { metadata: name: "default" }
+    }
+  }
+}
+
+
+-- schema.cue --
+package holos
+
+// #APIObjects is the output type for api objects produced by cue.  A map is used to aid debugging and clarity.
+import "encoding/yaml"
+
+#APIObjects: {
+	// apiObjects holds each the api objects produced by cue.
+	apiObjects: {
+		[Kind=_]: {
+			[Name=_]: {
+				kind: Kind
+				metadata: name: Name
+			}
+		}
+	}
+
+	// apiObjectsContent holds the marshalled representation of apiObjects
+	apiObjectMap: {
+		for kind, v in apiObjects {
+			"\(kind)": {
+				for name, obj in v {
+				  "\(name)": yaml.Marshal(obj)
+			  }
+			}
+		}
+	}
+}
+

cmd/holos/testdata/issue25_apiobjects_helm.txt

@@ -0,0 +1,58 @@
+# Want kube api objects in the apiObjects output.
+exec holos build .
+stdout '^kind: SecretStore$'
+stdout '# Source: CUE apiObjects.SecretStore.default'
+stderr 'skipping helm: no chart name specified'
+
+-- cue.mod --
+package holos
+-- component.cue --
+package holos
+
+apiVersion: "holos.run/v1alpha1"
+kind: "HelmChart"
+cluster: string @tag(cluster, string)
+
+#SecretStore: {
+    kind: string
+    metadata: name: string
+}
+
+#APIObjects & {
+  apiObjects: {
+    SecretStore: {
+      default: #SecretStore & { metadata: name: "default" }
+    }
+  }
+}
+
+
+-- schema.cue --
+package holos
+
+// #APIObjects is the output type for api objects produced by cue.  A map is used to aid debugging and clarity.
+import "encoding/yaml"
+
+#APIObjects: {
+	// apiObjects holds each the api objects produced by cue.
+	apiObjects: {
+		[Kind=_]: {
+			[Name=_]: {
+				kind: Kind
+				metadata: name: Name
+			}
+		}
+	}
+
+	// apiObjectsContent holds the marshalled representation of apiObjects
+	apiObjectMap: {
+		for kind, v in apiObjects {
+			"\(kind)": {
+				for name, obj in v {
+				  "\(name)": yaml.Marshal(obj)
+			  }
+			}
+		}
+	}
+}
+

cmd/holos/testdata/issue25_show_object_names.txt

@@ -0,0 +1,22 @@
+# Want api object kind and name in errors
+! exec holos build .
+stderr 'apiObjects.secretstore.default.foo: field not allowed'
+
+-- cue.mod --
+package holos
+-- component.cue --
+package holos
+
+apiVersion: "holos.run/v1alpha1"
+kind: "KubernetesObjects"
+cluster: string @tag(cluster, string)
+
+#SecretStore: {
+    metadata: name: string
+}
+
+apiObjects: {
+  secretstore: {
+    default: #SecretStore & { foo: "not allowed" }
+  }
+}

docs/examples/namespaces.cue

@@ -3,6 +3,7 @@ package holos
 // PlatformNamespace is a namespace to manage for Secret provisioning, SecretStore, etc...
 #PlatformNamespace: {
 	name: string
+	labels?: {[string]: string}
 }
 
 // #PlatformNamespaces is a list of namespaces to manage across the platform.

docs/examples/platforms/reference/clusters/provisioner/components/eso-creds-refresher/component.cue

@@ -8,21 +8,24 @@ package holos
 // - Namespace
 // - ServiceAccount eso-reader, eso-writer
 
-import "list"
-
-// objects are kubernetes api objects to apply.
-objects: list.FlattenN(_objects, 1)
-
-_objects: [
-	#CredsRefresherIAM.role,
-	#CredsRefresherIAM.binding,
-	for ns in #PlatformNamespaces {(#PlatformNamespaceObjects & {_ns: ns}).objects},
-]
-
 // No flux kustomization
 ksObjects: []
 
-{} & #KubernetesObjects
+#KubernetesObjects & {
+	apiObjects: {
+		let role = #CredsRefresherIAM.role
+		let binding = #CredsRefresherIAM.binding
+		ClusterRole: "\(role.metadata.name)":           role
+		ClusterRoleBinding: "\(binding.metadata.name)": binding
+		for ns in #PlatformNamespaces {
+			for obj in (#PlatformNamespaceObjects & {_ns: ns}).objects {
+				let Kind = obj.kind
+				let Name = obj.metadata.name
+				"\(Kind)": "\(ns.name)/\(Name)": obj
+			}
+		}
+	}
+}
 
 #InputKeys: {
 	cluster:   "provisioner"

docs/examples/platforms/reference/clusters/provisioner/components/namespaces/component.cue

@@ -1,7 +1,5 @@
 package holos
 
-import "list"
-
 #TargetNamespace: "default"
 
 #InputKeys: {
@@ -20,12 +18,14 @@ import "list"
 	]
 }
 
-objects: list.FlattenN(_objects, 1)
-
-_objects: [
-	for ns in #PlatformNamespaces {
-		(#PlatformNamespaceObjects & {_ns: ns}).objects
-	},
-]
-
-{} & #KubernetesObjects
+#KubernetesObjects & {
+	apiObjects: {
+		for ns in #PlatformNamespaces {
+			for obj in (#PlatformNamespaceObjects & {_ns: ns}).objects {
+				let Kind = obj.kind
+				let Name = obj.metadata.name
+				"\(Kind)": "\(Name)": obj
+			}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/workload/cloud/mesh/certmanager/certmanager.cue

@@ -0,0 +1,24 @@
+package holos
+
+// https://cert-manager.io/docs/
+
+#TargetNamespace: "cert-manager"
+
+#InputKeys: {
+	project: "mesh"
+	component: "certmanager"
+	service: "cert-manager"
+}
+
+#HelmChart & {
+	values: installCrds: true
+	namespace: #TargetNamespace
+	chart: {
+		name:    "cert-manager"
+		version: "1.14.3"
+		repository: {
+			name: "jetstack"
+			url:  "https://charts.jetstack.io"
+		}
+	}
+}

docs/examples/platforms/reference/clusters/workload/cloud/mesh/mesh.cue

@@ -0,0 +1,10 @@
+package holos
+
+// All components are share this collection
+#InputKeys: project: "mesh"
+
+// Shared dependencies for all components in this collection.
+#Kustomization: spec: {
+	dependsOn: [{name: "\(#StageName)-secrets-namespaces"}, ...]
+	targetNamespace: #TargetNamespace
+}

docs/examples/platforms/reference/clusters/workload/cloud/secrets/components/eso-creds-refresher/component.cue

@@ -2,11 +2,16 @@ package holos
 
 import "encoding/json"
 
-// objects are kubernetes api objects to apply
-objects: #CredsRefresherService.objects
-
 // output kubernetes api objects for holos
-{} & #KubernetesObjects
+#KubernetesObjects & {
+	apiObjects: {
+		for obj in #CredsRefresherService.objects {
+			let Kind = obj.kind
+			let Name = obj.metadata.name
+			"\(Kind)": "\(Name)": obj
+		}
+	}
+}
 
 #InputKeys: {
 	project:   "secrets"

docs/examples/platforms/reference/clusters/workload/cloud/secrets/components/namespaces/component.cue

@@ -1,7 +1,5 @@
 package holos
 
-import "list"
-
 #TargetNamespace: "default"
 
 #InputKeys: {
@@ -14,18 +12,18 @@ import "list"
 	_ns: #PlatformNamespace
 
 	objects: [
-		#Namespace & {
-			metadata: name: _ns.name
-		},
+		#Namespace & {metadata: _ns},
 	]
 }
 
-objects: list.FlattenN(_objects, 1)
-
-_objects: [
-	for ns in #PlatformNamespaces {
-		(#PlatformNamespaceObjects & {_ns: ns}).objects
-	},
-]
-
-{} & #KubernetesObjects
+#KubernetesObjects & {
+	apiObjects: {
+		for ns in #PlatformNamespaces {
+			for obj in (#PlatformNamespaceObjects & {_ns: ns}).objects {
+				let Kind = obj.kind
+				let Name = obj.metadata.name
+				"\(Kind)": "\(Name)": obj
+			}
+		}
+	}
+}

docs/examples/platforms/reference/clusters/workload/cloud/secrets/components/validate/component.cue

@@ -11,12 +11,13 @@ package holos
 
 #Kustomization: spec: dependsOn: [{name: #InstancePrefix + "-eso"}]
 
-objects: [
-	#SecretStore,
-	#ExternalSecret & {
-		_name: "validate"
-		spec: data: [{remoteRef: key: _name}]
-	},
-]
 
-{} & #KubernetesObjects
+#KubernetesObjects & {
+	apiObjects: {
+		SecretStore: default: #SecretStore
+
+		ExternalSecret: validate: #ExternalSecret & {
+			_name: "validate"
+		}
+	}
+}

docs/examples/platforms/reference/clusters/workload/metal/ceph/ceph.cue

@@ -0,0 +1,37 @@
+package holos
+
+// Manage Ceph CSI to provide PersistentVolumeClaims to a cluster.
+
+#TargetNamespace: "ceph-system"
+
+#SecretName: "\(#ClusterName)-ceph-csi-rbd"
+
+#InputKeys: {
+	project:   "metal"
+	service:   "ceph"
+	component: "ceph"
+}
+
+#Kustomization: spec: {
+	dependsOn: [{name: "prod-secrets-namespaces"}]
+	targetNamespace: #TargetNamespace
+}
+
+#HelmChart & {
+	namespace: #TargetNamespace
+	chart: {
+		name:    "ceph-csi-rbd"
+		version: "3.10.2"
+		repository: {
+			name: "ceph-csi"
+			url:  "https://ceph.github.io/csi-charts"
+		}
+	}
+
+	apiObjects: {
+		SecretStore: default: #SecretStore
+		ExternalSecret: "\(#SecretName)": #ExternalSecret & {
+			_name: #SecretName
+		}
+	}
+}

docs/examples/platforms/reference/clusters/workload/metal/ceph/values.holos.cue

@@ -0,0 +1,177 @@
+package holos
+
+#Input: {
+	config: {
+		// (required) String representing a Ceph cluster to provision storage from.
+		// Should be unique across all Ceph clusters in use for provisioning,
+		// cannot be greater than 36 bytes in length, and should remain immutable for
+		// the lifetime of the StorageClass in use.
+		clusterID: string
+		// (required) []String list of ceph monitor "address:port" values.
+		monitors: [...string]
+	}
+}
+
+// Imported from https://github.com/holos-run/holos-infra/blob/0ae58858f5583d25fa7543e47b5f5e9f0b2f3c83/components/core/metal/ceph-csi-rbd/values.holos.yaml
+
+#ChartValues: {
+	// Necessary for Talos see https://github.com/siderolabs/talos/discussions/8163
+	selinuxMount: false
+
+	csiConfig: [#Input.config]
+
+	storageClass: {
+		annotations: "storageclass.kubernetes.io/is-default-class": "true"
+
+		// Specifies whether the storageclass should be created
+		create: true
+		name:   "ceph-ssd"
+
+		// (optional) Prefix to use for naming RBD images.
+		// If omitted, defaults to "csi-vol-".
+		// NOTE: Set this to a cluster specific value, e.g. vol-k1-
+		volumeNamePrefix: "vol-\(#ClusterName)-"
+
+		// (required) String representing a Ceph cluster to provision storage from.
+		// Should be unique across all Ceph clusters in use for provisioning,
+		// cannot be greater than 36 bytes in length, and should remain immutable for
+		// the lifetime of the StorageClass in use.
+		clusterID: #Input.config.clusterID
+
+		// (optional) If you want to use erasure coded pool with RBD, you need to
+		// create two pools. one erasure coded and one replicated.
+		// You need to specify the replicated pool here in the `pool` parameter, it is
+		// used for the metadata of the images.
+		// The erasure coded pool must be set as the `dataPool` parameter below.
+		// dataPool: <ec-data-pool>
+		dataPool: ""
+
+		// (required) Ceph pool into which the RBD image shall be created
+		// eg: pool: replicapool
+		pool: "k8s-dev"
+
+		// (optional) RBD image features, CSI creates image with image-format 2 CSI
+		// RBD currently supports `layering`, `journaling`, `exclusive-lock`,
+		// `object-map`, `fast-diff`, `deep-flatten` features.
+		// Refer https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features
+		// for image feature dependencies.
+		// imageFeatures: layering,journaling,exclusive-lock,object-map,fast-diff
+		imageFeatures: "layering"
+
+		// (optional) Specifies whether to try other mounters in case if the current
+		// mounter fails to mount the rbd image for any reason. True means fallback
+		// to next mounter, default is set to false.
+		// Note: tryOtherMounters is currently useful to fallback from krbd to rbd-nbd
+		// in case if any of the specified imageFeatures is not supported by krbd
+		// driver on node scheduled for application pod launch, but in the future this
+		// should work with any mounter type.
+		// tryOtherMounters: false
+		// (optional) uncomment the following to use rbd-nbd as mounter
+		// on supported nodes
+		// mounter: rbd-nbd
+		mounter: ""
+
+		// (optional) ceph client log location, eg: rbd-nbd
+		// By default host-path /var/log/ceph of node is bind-mounted into
+		// csi-rbdplugin pod at /var/log/ceph mount path. This is to configure
+		// target bindmount path used inside container for ceph clients logging.
+		// See docs/rbd-nbd.md for available configuration options.
+		// cephLogDir: /var/log/ceph
+		cephLogDir: ""
+
+		// (optional) ceph client log strategy
+		// By default, log file belonging to a particular volume will be deleted
+		// on unmap, but you can choose to just compress instead of deleting it
+		// or even preserve the log file in text format as it is.
+		// Available options `remove` or `compress` or `preserve`
+		// cephLogStrategy: remove
+		cephLogStrategy: ""
+
+		// (optional) Instruct the plugin it has to encrypt the volume
+		// By default it is disabled. Valid values are "true" or "false".
+		// A string is expected here, i.e. "true", not true.
+		// encrypted: "true"
+		encrypted: ""
+
+		// (optional) Use external key management system for encryption passphrases by
+		// specifying a unique ID matching KMS ConfigMap. The ID is only used for
+		// correlation to configmap entry.
+		encryptionKMSID: ""
+
+		// Add topology constrained pools configuration, if topology based pools
+		// are setup, and topology constrained provisioning is required.
+		// For further information read TODO<doc>
+		// topologyConstrainedPools: |
+		//   [{"poolName":"pool0",
+		//     "dataPool":"ec-pool0" # optional, erasure-coded pool for data
+		//     "domainSegments":[
+		//       {"domainLabel":"region","value":"east"},
+		//       {"domainLabel":"zone","value":"zone1"}]},
+		//    {"poolName":"pool1",
+		//     "dataPool":"ec-pool1" # optional, erasure-coded pool for data
+		//     "domainSegments":[
+		//       {"domainLabel":"region","value":"east"},
+		//       {"domainLabel":"zone","value":"zone2"}]},
+		//    {"poolName":"pool2",
+		//     "dataPool":"ec-pool2" # optional, erasure-coded pool for data
+		//     "domainSegments":[
+		//       {"domainLabel":"region","value":"west"},
+		//       {"domainLabel":"zone","value":"zone1"}]}
+		//   ]
+		topologyConstrainedPools: []
+
+		// (optional) mapOptions is a comma-separated list of map options.
+		// For krbd options refer
+		// https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
+		// For nbd options refer
+		// https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
+		// Format:
+		// mapOptions: "<mounter>:op1,op2;<mounter>:op1,op2"
+		// An empty mounter field is treated as krbd type for compatibility.
+		// eg:
+		// mapOptions: "krbd:lock_on_read,queue_depth=1024;nbd:try-netlink"
+		mapOptions: ""
+
+		// (optional) unmapOptions is a comma-separated list of unmap options.
+		// For krbd options refer
+		// https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options
+		// For nbd options refer
+		// https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options
+		// Format:
+		// unmapOptions: "<mounter>:op1,op2;<mounter>:op1,op2"
+		// An empty mounter field is treated as krbd type for compatibility.
+		// eg:
+		// unmapOptions: "krbd:force;nbd:force"
+		unmapOptions: ""
+
+		// The secrets have to contain Ceph credentials with required access
+		// to the 'pool'.
+		provisionerSecret: #SecretName
+		// If Namespaces are left empty, the secrets are assumed to be in the
+		// Release namespace.
+		provisionerSecretNamespace:      ""
+		controllerExpandSecret:          #SecretName
+		controllerExpandSecretNamespace: ""
+		nodeStageSecret:                 #SecretName
+		nodeStageSecretNamespace:        ""
+		// Specify the filesystem type of the volume. If not specified,
+		// csi-provisioner will set default as `ext4`.
+		fstype:               "ext4"
+		reclaimPolicy:        "Delete"
+		allowVolumeExpansion: true
+		mountOptions: []
+	}
+
+	secret: {
+		// Specifies whether the secret should be created
+		create: false
+		name:   #SecretName
+		// Key values correspond to a user name and its key, as defined in the
+		// ceph cluster. User ID should have required access to the 'pool'
+		// specified in the storage class
+		userID:  "admin"
+		userKey: "$(ceph auth get-key client.admin)"
+		// Encryption passphrase
+		encryptionPassphrase: "$(python -c 'import secrets; print(secrets.token_hex(32));')"
+	}
+}

docs/examples/platforms/reference/clusters/workload/metal/ceph/values.input.cue

@@ -0,0 +1,8 @@
+package holos
+
+#Input: {
+	config: {
+		clusterID: "a6de32ab-c84f-49a6-b97e-e31dc2a70931"
+		monitors: ["10.64.1.21:6789", "10.64.1.31:6789", "10.64.1.41:6789"]
+	}
+}

docs/examples/platforms/reference/clusters/workload/metal/readme.md

@@ -0,0 +1,23 @@
+# Metal Clusters
+
+This cluster type is overlaid onto other cluster types to add services necessary outside of a cloud like GKE or EKS.  Ceph for PersistenVolumeClaim support on a Talos Proxmox cluster is the primary use case.
+
+## Test Script
+
+Test ceph is working with:
+
+```bash
+apply -n default -f-<<EOF
+heredoc> apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: test
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: 1Gi
+EOF
+```

docs/examples/platforms/reference/namespaces.cue

@@ -1,10 +1,15 @@
 package holos
 
+// #PlatformNamespaces is the union of all namespaces across all cluster types.  Namespaces are created in all clusters regardless of if they're
+// used within the cluster or not.  The is important for security and consistency with IAM, RBAC, and Secrets sync between clusters.
 #PlatformNamespaces: [
 	{name: "external-secrets"},
 	{name: "holos-system"},
 	{name: "flux-system"},
-	{name: "ceph-system"},
+	{
+		name: "ceph-system"
+		labels: "pod-security.kubernetes.io/enforce": "privileged"
+	},
 	{name: "istio-system"},
 	{name: "istio-ingress"},
 	{name: "cert-manager"},

docs/examples/schema.cue

@@ -11,28 +11,34 @@ import (
 	"encoding/yaml"
 )
 
+// _apiVersion is the version of this schema.  Defines the interface between CUE output and the holos cli.
 _apiVersion: "holos.run/v1alpha1"
 
-// #Name defines the name: string key value pair used all over the place.
-#Name: name: string
+// #ClusterName is the cluster name for cluster scoped resources.
+#ClusterName: #InputKeys.cluster
+// #StageName is prod, dev, stage, etc...  Usually prod for platform components.
+#StageName: #InputKeys.stage
+// #CollectionName is the preferred handle to the collection element of the instance name.  A collection name mapes to an "application name" as described in the kubernetes recommended labels documentation.  Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
+#CollectionName: #InputKeys.project
+// #ComponentName is the name of the holos component.
+#ComponentName: #InputKeys.component
+// #InstanceName is the name of the holos component instance being managed varying by stage, project, and component names.
+#InstanceName: "\(#StageName)-\(#CollectionName)-\(#ComponentName)"
+// #InstancePrefix is the stage and project without the component name.  Useful for dependency management among multiple components for a project stage.
+#InstancePrefix: "\(#StageName)-\(#CollectionName)"
 
 // #TargetNamespace is the target namespace for a holos component.
 #TargetNamespace: string
 
-// #InstanceName is the name of the holos component instance being managed varying by stage, project, and component names.
-#InstanceName: "\(#InputKeys.stage)-\(#InputKeys.project)-\(#InputKeys.component)"
-
-// #InstancePrefix is the stage and project without the component name.  Useful for dependency management among multiple components for a project stage.
-#InstancePrefix: "\(#InputKeys.stage)-\(#InputKeys.project)"
-
-// TypeMeta indicates a kubernetes api object
-#TypeMeta: metav1.#TypeMeta
-
 // #CommonLabels are mixed into every kubernetes api object.
 #CommonLabels: {
-	"holos.run/stage.name":     #InputKeys.stage
-	"holos.run/project.name":   #InputKeys.project
-	"holos.run/component.name": #InputKeys.component
+	"holos.run/stage.name":        #StageName
+	"holos.run/project.name":      #CollectionName
+	"holos.run/component.name":    #ComponentName
+	"app.kubernetes.io/part-of":   #StageName
+	"app.kubernetes.io/name":      #CollectionName
+	"app.kubernetes.io/component": #ComponentName
+	"app.kubernetes.io/instance":  #InstanceName
 	...
 }
 
@@ -79,10 +85,10 @@ _apiVersion: "holos.run/v1alpha1"
 			kind: string | *"GitRepository"
 			name: string | *"flux-system"
 		}
-		suspend?: bool
+		suspend?:         bool
 		targetNamespace?: string
-		timeout: string | *"3m0s"
-		wait:    bool | *true
+		timeout:          string | *"3m0s"
+		wait:             bool | *true
 	}
 }
 
@@ -90,8 +96,8 @@ _apiVersion: "holos.run/v1alpha1"
 #ExternalSecret: #NamespaceObject & es.#ExternalSecret & {
 	_name: string
 	metadata: {
-		namespace: #TargetNamespace
 		name:      _name
+		namespace: #TargetNamespace
 	}
 	spec: {
 		refreshInterval: string | *"1h"
@@ -100,8 +106,12 @@ _apiVersion: "holos.run/v1alpha1"
 			name: string | *"default"
 		}
 		target: {
+			name:           _name
 			creationPolicy: string | *"Owner"
+			deletionPolicy: string | *"Retain"
 		}
+		// Copy fields 1:1 from external Secret to target Secret.
+		dataFrom: [{extract: key: _name}]
 	}
 }
 
@@ -115,11 +125,11 @@ _apiVersion: "holos.run/v1alpha1"
 			remoteNamespace: #TargetNamespace
 			auth: token: bearerToken: {
 				name: string | *"eso-reader"
-				key: string | *"token"
+				key:  string | *"token"
 			}
 			server: {
 				caBundle: #InputKeys.provisionerCABundle
-				url: #InputKeys.provisionerURL
+				url:      #InputKeys.provisionerURL
 			}
 		}
 	}
@@ -131,8 +141,6 @@ _apiVersion: "holos.run/v1alpha1"
 	cluster: string @tag(cluster, type=string)
 	// stage is usually set by the platform or project.
 	stage: *"prod" | string @tag(stage, type=string)
-	// project is usually set by the platform or project.
-	project: string @tag(project, type=string)
 	// service is usually set by the component.
 	service: string @tag(service, type=string)
 	// component is the name of the component
@@ -142,9 +150,9 @@ _apiVersion: "holos.run/v1alpha1"
 	gcpProjectID:     string @tag(gcpProjectID, type=string)
 	gcpProjectNumber: int    @tag(gcpProjectNumber, type=int)
 
-  // Same as cluster certificate-authority-data field in ~/.holos/kubeconfig.provisioner
+	// Same as cluster certificate-authority-data field in ~/.holos/kubeconfig.provisioner
 	provisionerCABundle: string @tag(provisionerCABundle, type=string)
-  // Same as the cluster server field in ~/.holos/kubeconfig.provisioner
+	// Same as the cluster server field in ~/.holos/kubeconfig.provisioner
 	provisionerURL: string @tag(provisionerURL, type=string)
 }
 
@@ -161,7 +169,7 @@ _apiVersion: "holos.run/v1alpha1"
 	}
 	stages: [ID=_]: {
 		name: string & ID
-		environments: [...#Name]
+		environments: [...{name: string}]
 	}
 	projects: [ID=_]: {
 		name: string & ID
@@ -171,32 +179,53 @@ _apiVersion: "holos.run/v1alpha1"
 	}
 }
 
+// #APIObjects is the output type for api objects produced by cue.  A map is used to aid debugging and clarity.
+#APIObjects: {
+	// apiObjects holds each the api objects produced by cue.
+	apiObjects: {
+		[Kind=_]: {
+			[Name=_]: metav1.#TypeMeta & {
+				kind: Kind
+			}
+		}
+	}
+
+	// apiObjectsContent holds the marshalled representation of apiObjects
+	apiObjectMap: {
+		for kind, v in apiObjects {
+			"\(kind)": {
+				for name, obj in v {
+					"\(name)": yaml.Marshal(obj)
+				}
+			}
+		}
+	}
+}
+
 // #OutputTypeMeta is shared among all output types
 #OutputTypeMeta: {
 	// apiVersion is the output api version
 	apiVersion: _apiVersion
 	// kind is a discriminator of the type of output
-	kind: #PlatformSpec.kind | #KubernetesObjects.kind | #HelmChart.kind
+	kind: #PlatformSpec.kind | #KubernetesObjects.kind | #HelmChart.kind | #NoOutput.kind
 	// name holds a unique name suitable for a filename
 	metadata: name: string
-	// contentType is the standard MIME type indicating the content type of the content field
-	contentType: *"application/yaml" | "application/json"
-	// content holds the content text output
-	content: string | *""
 	// debug returns arbitrary debug output.
 	debug?: _
 }
 
+#NoOutput: {
+	#OutputTypeMeta
+	kind: string | *"Skip"
+	metadata: name: string | *"skipped"
+}
+
 // #KubernetesObjectOutput is the output schema of a single component.
 #KubernetesObjects: {
 	#OutputTypeMeta
-
-	// kind KubernetesObjects provides a yaml text stream of kubernetes api objects in the out field.
+	#APIObjects
 	kind: "KubernetesObjects"
-	// objects holds a list of the kubernetes api objects to configure.
-	objects: [...metav1.#TypeMeta] | *[]
-	// out holds the rendered yaml text stream of kubernetes api objects.
-	content: yaml.MarshalStream(objects)
+	metadata: name: #InstanceName
 	// ksObjects holds the flux Kustomization objects for gitops
 	ksObjects: [...#Kustomization] | *[#Kustomization]
 	// ksContent is the yaml representation of kustomization
@@ -215,10 +244,15 @@ _apiVersion: "holos.run/v1alpha1"
 	}
 }
 
+// #ChartValues represent the values provided to a helm chart.  Existing values may be imorted using cue import values.yaml -p holos then wrapping the values.cue content in #Values: {}
+#ChartValues: {...}
+
 // #HelmChart is a holos component which produces kubernetes api objects from cue values provided to the helm template command.
 #HelmChart: {
 	#OutputTypeMeta
+	#APIObjects
 	kind: "HelmChart"
+	metadata: name: #InstanceName
 	// ksObjects holds the flux Kustomization objects for gitops.
 	ksObjects: [...#Kustomization] | *[#Kustomization]
 	// ksContent is the yaml representation of kustomization.
@@ -228,7 +262,7 @@ _apiVersion: "holos.run/v1alpha1"
 	// chart defines the upstream helm chart to process.
 	chart: #Chart
 	// values represents the helm values to provide to the chart.
-	values: {...}
+	values: #ChartValues
 	// valuesContent holds the values yaml
 	valuesContent: yaml.Marshal(values)
 	// platform returns the platform data structure for visibility / troubleshooting.
@@ -243,7 +277,10 @@ _apiVersion: "holos.run/v1alpha1"
 	kind: "PlatformSpec"
 }
 
-#Output: #PlatformSpec | #KubernetesObjects | #HelmChart
+// #SecretName is the name of a Secret, ususally coupling a Deployment to an ExternalSecret
+#SecretName: string
 
-// Holos component name
-metadata: name: #InstanceName
+// By default, render kind: Skipped so holos knows to skip over intermediate cue files.
+// This enables the use of holos render ./foo/bar/baz/... when bar contains intermediary constraints which are not complete components.
+// Holos skips over these intermediary cue instances.
+{} & #NoOutput

pkg/cli/build/build.go

@@ -20,7 +20,10 @@ func makeBuildRunFunc(cfg *holos.Config) command.RunFunc {
 		}
 		outs := make([]string, 0, len(results))
 		for _, result := range results {
-			outs = append(outs, result.Content)
+			if result.Skip {
+				continue
+			}
+			outs = append(outs, result.FinalOutput())
 		}
 		out := strings.Join(outs, "---\n")
 		if _, err := fmt.Fprintln(cmd.OutOrStdout(), out); err != nil {

pkg/cli/command/cmd.go

@@ -27,11 +27,3 @@ func New(name string) *cobra.Command {
 	}
 	return cmd
 }
-
-// EnsureNewline adds a trailing newline if not already there.
-func EnsureNewline(b []byte) []byte {
-	if len(b) > 0 && b[len(b)-1] != '\n' {
-		b = append(b, '\n')
-	}
-	return b
-}

pkg/cli/kv/get.go

@@ -7,6 +7,7 @@ import (
 	"github.com/holos-run/holos/pkg/cli/secret"
 	"github.com/holos-run/holos/pkg/holos"
 	"github.com/holos-run/holos/pkg/logger"
+	"github.com/holos-run/holos/pkg/util"
 	"github.com/holos-run/holos/pkg/wrapper"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -77,7 +78,7 @@ func makeGetRunFunc(cfg *holos.Config, cf getConfig) command.RunFunc {
 			//  Print one file to stdout
 			if key := *cf.file; key != "" {
 				if data, found := secret.Data[key]; found {
-					cfg.Write(command.EnsureNewline(data))
+					cfg.Write(util.EnsureNewline(data))
 					return nil
 				}
 				return wrapper.Wrap(fmt.Errorf("not found: %s have %#v", key, keys))
@@ -89,7 +90,7 @@ func makeGetRunFunc(cfg *holos.Config, cf getConfig) command.RunFunc {
 
 			for k, v := range secret.Data {
 				cfg.Printf("-- %s --\n", k)
-				cfg.Write(command.EnsureNewline(v))
+				cfg.Write(util.EnsureNewline(v))
 			}
 		}
 		return nil

pkg/cli/main.go

@@ -16,14 +16,14 @@ func MakeMain(options ...holos.Option) func() int {
 		slog.SetDefault(cfg.Logger())
 		ctx := context.Background()
 		if err := New(cfg).ExecuteContext(ctx); err != nil {
-			return handleError(ctx, err, cfg)
+			return HandleError(ctx, err, cfg)
 		}
 		return 0
 	}
 }
 
-// handleError is the top level error handler that unwraps and logs errors.
-func handleError(ctx context.Context, err error, hc *holos.Config) (exitCode int) {
+// HandleError is the top level error handler that unwraps and logs errors.
+func HandleError(ctx context.Context, err error, hc *holos.Config) (exitCode int) {
 	log := hc.NewTopLevelLogger()
 	var cueErr errors.Error
 	var errAt *wrapper.ErrorAt

pkg/cli/render/render.go

@@ -27,9 +27,12 @@ func makeRenderRunFunc(cfg *holos.Config) command.RunFunc {
 		// the same file path. Write files into a blank temporary directory, error if a
 		// file exists, then move the directory into place.
 		for _, result := range results {
+			if result.Skip {
+				continue
+			}
 			// API Objects
 			path := result.Filename(cfg.WriteTo(), cfg.ClusterName())
-			if err := result.Save(ctx, path, result.Content); err != nil {
+			if err := result.Save(ctx, path, result.FinalOutput()); err != nil {
 				return wrapper.Wrap(err)
 			}
 			// Kustomization

pkg/cli/secret/create.go

@@ -7,6 +7,7 @@ import (
 	"github.com/holos-run/holos/pkg/logger"
 	"github.com/holos-run/holos/pkg/wrapper"
 	"github.com/spf13/cobra"
+	"io"
 	"io/fs"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,6 +28,7 @@ func NewCreateCmd(hc *holos.Config) *cobra.Command {
 	flagSet.Var(&cfg.files, "from-file", "store files as keys in the secret")
 	cfg.dryRun = flagSet.Bool("dry-run", false, "dry run")
 	cfg.appendHash = flagSet.Bool("append-hash", true, "append hash to kubernetes secret name")
+	cfg.dataStdin = flagSet.Bool("data-stdin", false, "read data field as json from stdin if")
 
 	cmd.Flags().SortFlags = false
 	cmd.Flags().AddGoFlagSet(flagSet)
@@ -46,8 +48,9 @@ func makeCreateRunFunc(hc *holos.Config, cfg *config) command.RunFunc {
 				APIVersion: "v1",
 			},
 			ObjectMeta: metav1.ObjectMeta{
-				Name:   secretName,
-				Labels: map[string]string{NameLabel: secretName},
+				Name:      secretName,
+				Namespace: *cfg.namespace,
+				Labels:    map[string]string{NameLabel: secretName},
 			},
 			Data: make(secretData),
 		}
@@ -60,6 +63,22 @@ func makeCreateRunFunc(hc *holos.Config, cfg *config) command.RunFunc {
 			}
 		}
 
+		if *cfg.dataStdin {
+			log.InfoContext(ctx, "reading data keys from stdin...")
+			var obj map[string]string
+			data, err := io.ReadAll(hc.Stdin())
+			if err != nil {
+				return wrapper.Wrap(err)
+			}
+			err = yaml.Unmarshal(data, &obj)
+			if err != nil {
+				return wrapper.Wrap(err)
+			}
+			for k, v := range obj {
+				secret.Data[k] = []byte(v)
+			}
+		}
+
 		for _, file := range cfg.files {
 			if err := filepath.WalkDir(file, makeWalkFunc(secret.Data, file)); err != nil {
 				return wrapper.Wrap(err)

pkg/cli/secret/get.go

@@ -63,7 +63,7 @@ func makeGetRunFunc(hc *holos.Config, cfg *config) command.RunFunc {
 
 			log.DebugContext(ctx, "results", "len", len(list.Items))
 			if len(list.Items) < 1 {
-				continue
+				return wrapper.Wrap(fmt.Errorf("not found: %v", secretName))
 			}
 
 			// Sort oldest first.

pkg/cli/secret/secret.go

@@ -17,6 +17,7 @@ type config struct {
 	extract    *bool
 	dryRun     *bool
 	appendHash *bool
+	dataStdin  *bool
 	cluster    *string
 	namespace  *string
 	extractTo  *string

pkg/cli/secret/secret_test.go

@@ -55,11 +55,12 @@ func cmdHolos(ts *testscript.TestScript, neg bool, args []string) {
 	cmd := cli.New(cfg)
 	cmd.SetArgs(args)
 	err := cmd.Execute()
+
 	if neg {
 		if err == nil {
-			ts.Fatalf("want: error\nhave: %v", err)
+			ts.Fatalf("\nwant: error\nhave: %v", err)
 		} else {
-			ts.Logf("want: error\nhave: %v", err)
+			cli.HandleError(cmd.Context(), err, cfg)
 		}
 	} else {
 		ts.Check(err)

pkg/cli/secret/testdata/issue20_secret_not_found.txt

@@ -0,0 +1,3 @@
+# Want missing secrets to exit non-zero https://github.com/holos-run/holos/issues/20
+! holos get secret does-not-exist
+stderr 'not found: does-not-exist'

pkg/cli/txtar/txtar.go

@@ -67,9 +67,9 @@ func printFile(w io.Writer, idx int, a *txtar.Archive) (err error) {
 		return wrapper.Wrap(fmt.Errorf("idx cannot be 0"))
 	}
 	if idx > 0 {
-		_, err = w.Write(command.EnsureNewline(a.Files[idx-1].Data))
+		_, err = w.Write(util.EnsureNewline(a.Files[idx-1].Data))
 	} else {
-		_, err = w.Write(command.EnsureNewline(a.Files[len(a.Files)+idx].Data))
+		_, err = w.Write(util.EnsureNewline(a.Files[len(a.Files)+idx].Data))
 	}
 	return
 }

pkg/internal/builder/builder.go

@@ -10,10 +10,13 @@ import (
 	"fmt"
 	"github.com/holos-run/holos"
 	"github.com/holos-run/holos/pkg/logger"
+	"github.com/holos-run/holos/pkg/util"
 	"github.com/holos-run/holos/pkg/wrapper"
+	"log/slog"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 
 	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/load"
@@ -26,6 +29,8 @@ const (
 	// Helm is the value of the kind field of holos build output indicating helm
 	// values and helm command information.
 	Helm = "HelmChart"
+	// Skip is the value when the instance should be skipped
+	Skip = "Skip"
 	// ChartDir is the chart cache directory name.
 	ChartDir = "vendor"
 )
@@ -72,11 +77,17 @@ type Metadata struct {
 	Name string `json:"name,omitempty"`
 }
 
+// apiObjectMap is the shape of marshalled api objects returned from cue to the
+// holos cli. A map is used to improve the clarity of error messages from cue.
+type apiObjectMap map[string]map[string]string
+
 // Result is the build result for display or writing.
 type Result struct {
-	Metadata  Metadata `json:"metadata,omitempty"`
-	Content   string   `json:"content,omitempty"`
-	KsContent string   `json:"ksContent,omitempty"`
+	Metadata     Metadata     `json:"metadata,omitempty"`
+	KsContent    string       `json:"ksContent,omitempty"`
+	APIObjectMap apiObjectMap `json:"apiObjectMap,omitempty"`
+	finalOutput  string
+	Skip         bool
 }
 
 type Repository struct {
@@ -92,13 +103,14 @@ type Chart struct {
 
 // A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
 type HelmChart struct {
-	APIVersion    string   `json:"apiVersion"`
-	Kind          string   `json:"kind"`
-	Metadata      Metadata `json:"metadata"`
-	KsContent     string   `json:"ksContent"`
-	Namespace     string   `json:"namespace"`
-	Chart         Chart    `json:"chart"`
-	ValuesContent string   `json:"valuesContent"`
+	APIVersion    string       `json:"apiVersion"`
+	Kind          string       `json:"kind"`
+	Metadata      Metadata     `json:"metadata"`
+	KsContent     string       `json:"ksContent"`
+	Namespace     string       `json:"namespace"`
+	Chart         Chart        `json:"chart"`
+	ValuesContent string       `json:"valuesContent"`
+	APIObjectMap  apiObjectMap `json:"APIObjectMap"`
 }
 
 // Name returns the metadata name of the result. Equivalent to the
@@ -115,6 +127,42 @@ func (r *Result) KustomizationFilename(writeTo string, cluster string) string {
 	return filepath.Join(writeTo, "clusters", cluster, "holos", "components", r.Name()+"-kustomization.gen.yaml")
 }
 
+// FinalOutput returns the final rendered output.
+func (r *Result) FinalOutput() string {
+	return r.finalOutput
+}
+
+// addAPIObjects adds the overlay api objects to finalOutput.
+func (r *Result) addOverlayObjects(log *slog.Logger) {
+	b := []byte(r.FinalOutput())
+	kinds := make([]string, 0, len(r.APIObjectMap))
+	// Sort the keys
+	for kind := range r.APIObjectMap {
+		kinds = append(kinds, kind)
+	}
+	slices.Sort(kinds)
+
+	for _, kind := range kinds {
+		v := r.APIObjectMap[kind]
+		// Sort the keys
+		names := make([]string, 0, len(v))
+		for name := range v {
+			names = append(names, name)
+		}
+		slices.Sort(names)
+
+		for _, name := range names {
+			yamlString := v[name]
+			log.Debug(fmt.Sprintf("%s/%s", kind, name), "kind", kind, "name", name)
+			util.EnsureNewline(b)
+			header := fmt.Sprintf("---\n# Source: CUE apiObjects.%s.%s\n", kind, name)
+			b = append(b, []byte(header+yamlString)...)
+			util.EnsureNewline(b)
+		}
+	}
+	r.finalOutput = string(b)
+}
+
 // Save writes the content to the filesystem for git ops.
 func (r *Result) Save(ctx context.Context, path string, content string) error {
 	log := logger.FromContext(ctx)
@@ -206,11 +254,14 @@ func (b *Builder) Run(ctx context.Context) (results []*Result, err error) {
 
 		log.DebugContext(ctx, "cue: processing holos component kind "+info.Kind)
 		switch kind := info.Kind; kind {
+		case Skip:
+			result.Skip = true
 		case Kube:
 			// CUE directly provides the kubernetes api objects in result.Content
 			if err := value.Decode(&result); err != nil {
 				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
 			}
+			result.addOverlayObjects(log)
 		case Helm:
 			var helmChart HelmChart
 			// First decode into the result.  Helm will populate the api objects later.
@@ -225,6 +276,7 @@ func (b *Builder) Run(ctx context.Context) (results []*Result, err error) {
 			if err := runHelm(ctx, &helmChart, &result, holos.PathComponent(instance.Dir)); err != nil {
 				return nil, err
 			}
+			result.addOverlayObjects(log)
 		default:
 			return nil, wrapper.Wrap(fmt.Errorf("build kind not implemented: %v", kind))
 		}
@@ -286,6 +338,10 @@ func runCmd(ctx context.Context, name string, args ...string) (result runResult,
 // the rendered kubernetes api objects in the result.
 func runHelm(ctx context.Context, hc *HelmChart, r *Result, path holos.PathComponent) error {
 	log := logger.FromContext(ctx).With("chart", hc.Chart.Name)
+	if hc.Chart.Name == "" {
+		log.WarnContext(ctx, "skipping helm: no chart name specified, use a different component type")
+		return nil
+	}
 
 	cachedChartPath := filepath.Join(string(path), ChartDir, hc.Chart.Name)
 	if isNotExist(cachedChartPath) {
@@ -328,7 +384,7 @@ func runHelm(ctx context.Context, hc *HelmChart, r *Result, path holos.PathCompo
 		return wrapper.Wrap(fmt.Errorf("could not run helm template: %w", err))
 	}
 
-	r.Content = helmOut.stdout.String()
+	r.finalOutput = helmOut.stdout.String()
 
 	return nil
 }
@@ -348,7 +404,7 @@ func isNotExist(path string) bool {
 	return os.IsNotExist(err)
 }
 
-// cacheChart stores a cached copy of Chart in the chart sub-directory of path.
+// cacheChart stores a cached copy of Chart in the chart subdirectory of path.
 func cacheChart(ctx context.Context, path holos.PathComponent, chartDir string, chart Chart) error {
 	log := logger.FromContext(ctx)
 

pkg/util/util.go

@@ -0,0 +1,9 @@
+package util
+
+// EnsureNewline adds a trailing newline if not already there.
+func EnsureNewline(b []byte) []byte {
+	if len(b) > 0 && b[len(b)-1] != '\n' {
+		b = append(b, '\n')
+	}
+	return b
+}

pkg/version/embedded/minor

@@ -1 +1 @@
-46
+48

pkg/version/embedded/patch

@@ -1 +1 @@
-1
+2