Compare commits

..

4 Commits

Author SHA1 Message Date
Jeff McCune
f60db8fa1f (#25) Show name of api object in errors
This patch changes the interface between CUE and Holos to remove the
content field and replace it with an api object map.  The map is a
`map[string]map[string]string` with the rendered yaml as the value of a
kind/name nesting.

This structure enables better error messages, cue disjunction errors
indicate the type and the name of the resource instead of just the list
index number.
2024-02-29 11:23:49 -08:00
Jeff McCune
eefc092ea9 (#22) Copy external secret data files one for one
Without this patch the secret data was nested under a key with the same
name as the secret name.  This caused the ceph controller to not find
the values.

This patch changes the golden path for #ExternalSecret to copy all data
keys 1:1 from the external to the target in the cluster.
2024-02-28 16:51:26 -08:00
Jeff McCune
0860ac3409 (#22) Rename ceph secret to include ClusterName
Without this patch all clusters would use the same ceph secret from the
provisioner cluster.  This is a problem because ceph credentials are
unique per cluster.

This patch renames the ceph secret to have a cluster name prefix.

The secret is created with:

```bash
vault kv get -format=json -field data kv/k2/kube-namespace/ceph-csi-rbd/csi-rbd-secret \
  | holos create secret --namespace ceph-system k2-ceph-csi-rbd --cluster-name=k2 --data-stdin --append-hash=false
```
2024-02-28 16:14:22 -08:00
Jeff McCune
6b156e9883 (#22) Label ns ceph-system with pod-security enforce: privileged
This patch adds the `pod-security.kubernetes.io/enforce: privileged`
label to the ceph-system namespace.

The Namespace resources are managed all over the map, it would be a good
idea to consolidate the PlatformNamespaces data into one well known
place for the entire platform.  Eschewing for now.
2024-02-28 15:57:01 -08:00
26 changed files with 343 additions and 118 deletions

View File

@@ -1,7 +1,7 @@
# Want cue errors to show files and lines
! exec holos build .
stderr 'could not decode: content: cannot convert non-concrete value string'
stderr '/component.cue:6:1$'
stderr '^apiObjectMap.foo.bar: cannot convert non-concrete value string'
stderr '/component.cue:7:20$'
-- cue.mod --
package holos
@@ -11,5 +11,6 @@ package holos
apiVersion: "holos.run/v1alpha1"
kind: "KubernetesObjects"
cluster: string @tag(cluster, string)
content: foo
foo: string
apiObjectMap: foo: bar: baz
baz: string

View File

@@ -0,0 +1,57 @@
# Want kube api objects in the apiObjects output.
exec holos build .
stdout '^kind: SecretStore$'
stdout '# Source: holos component cue api objects SecretStore/default'
-- cue.mod --
package holos
-- component.cue --
package holos
apiVersion: "holos.run/v1alpha1"
kind: "KubernetesObjects"
cluster: string @tag(cluster, string)
#SecretStore: {
kind: string
metadata: name: string
}
#APIObjects & {
apiObjects: {
SecretStore: {
default: #SecretStore & { metadata: name: "default" }
}
}
}
-- schema.cue --
package holos
// #APIObjects is the output type for api objects produced by cue. A map is used to aid debugging and clarity.
import "encoding/yaml"
#APIObjects: {
// apiObjects holds each the api objects produced by cue.
apiObjects: {
[Kind=_]: {
[Name=_]: {
kind: Kind
metadata: name: Name
}
}
}
// apiObjectsContent holds the marshalled representation of apiObjects
apiObjectMap: {
for kind, v in apiObjects {
"\(kind)": {
for name, obj in v {
"\(name)": yaml.Marshal(obj)
}
}
}
}
}

View File

@@ -0,0 +1,58 @@
# Want kube api objects in the apiObjects output.
exec holos build .
stdout '^kind: SecretStore$'
stdout '# Source: holos component cue api objects SecretStore/default'
stderr 'skipping helm: no chart name specified'
-- cue.mod --
package holos
-- component.cue --
package holos
apiVersion: "holos.run/v1alpha1"
kind: "HelmChart"
cluster: string @tag(cluster, string)
#SecretStore: {
kind: string
metadata: name: string
}
#APIObjects & {
apiObjects: {
SecretStore: {
default: #SecretStore & { metadata: name: "default" }
}
}
}
-- schema.cue --
package holos
// #APIObjects is the output type for api objects produced by cue. A map is used to aid debugging and clarity.
import "encoding/yaml"
#APIObjects: {
// apiObjects holds each the api objects produced by cue.
apiObjects: {
[Kind=_]: {
[Name=_]: {
kind: Kind
metadata: name: Name
}
}
}
// apiObjectsContent holds the marshalled representation of apiObjects
apiObjectMap: {
for kind, v in apiObjects {
"\(kind)": {
for name, obj in v {
"\(name)": yaml.Marshal(obj)
}
}
}
}
}

View File

@@ -0,0 +1,22 @@
# Want api object kind and name in errors
! exec holos build .
stderr 'apiObjects.secretstore.default.foo: field not allowed'
-- cue.mod --
package holos
-- component.cue --
package holos
apiVersion: "holos.run/v1alpha1"
kind: "KubernetesObjects"
cluster: string @tag(cluster, string)
#SecretStore: {
metadata: name: string
}
apiObjects: {
secretstore: {
default: #SecretStore & { foo: "not allowed" }
}
}

View File

@@ -3,6 +3,7 @@ package holos
// PlatformNamespace is a namespace to manage for Secret provisioning, SecretStore, etc...
#PlatformNamespace: {
name: string
labels?: {[string]: string}
}
// #PlatformNamespaces is a list of namespaces to manage across the platform.

View File

@@ -1,8 +0,0 @@
package holos
#Input: {
config: {
clusterID: "a6de32ab-c84f-49a6-b97e-e31dc2a70931"
monitors: ["10.64.1.21:6789", "10.64.1.31:6789", "10.64.1.41:6789"]
}
}

View File

@@ -1,3 +0,0 @@
# Metal Clusters
This cluster type is overlaid onto other cluster types to add services necessary outside of a cloud like GKE or EKS. Ceph for PersistenVolumeClaim support on a Talos Proxmox cluster is the primary use case.

View File

@@ -8,21 +8,24 @@ package holos
// - Namespace
// - ServiceAccount eso-reader, eso-writer
import "list"
// objects are kubernetes api objects to apply.
objects: list.FlattenN(_objects, 1)
_objects: [
#CredsRefresherIAM.role,
#CredsRefresherIAM.binding,
for ns in #PlatformNamespaces {(#PlatformNamespaceObjects & {_ns: ns}).objects},
]
// No flux kustomization
ksObjects: []
{} & #KubernetesObjects
#KubernetesObjects & {
apiObjects: {
let role = #CredsRefresherIAM.role
let binding = #CredsRefresherIAM.binding
ClusterRole: "\(role.metadata.name)": role
ClusterRoleBinding: "\(binding.metadata.name)": binding
for ns in #PlatformNamespaces {
for obj in (#PlatformNamespaceObjects & {_ns: ns}).objects {
let Kind = obj.kind
let Name = obj.metadata.name
"\(Kind)": "\(ns.name)/\(Name)": obj
}
}
}
}
#InputKeys: {
cluster: "provisioner"

View File

@@ -1,7 +1,5 @@
package holos
import "list"
#TargetNamespace: "default"
#InputKeys: {
@@ -20,12 +18,14 @@ import "list"
]
}
objects: list.FlattenN(_objects, 1)
_objects: [
for ns in #PlatformNamespaces {
(#PlatformNamespaceObjects & {_ns: ns}).objects
},
]
{} & #KubernetesObjects
#KubernetesObjects & {
apiObjects: {
for ns in #PlatformNamespaces {
for obj in (#PlatformNamespaceObjects & {_ns: ns}).objects {
let Kind = obj.kind
let Name = obj.metadata.name
"\(Kind)": "\(Name)": obj
}
}
}
}

View File

@@ -2,11 +2,16 @@ package holos
import "encoding/json"
// objects are kubernetes api objects to apply
objects: #CredsRefresherService.objects
// output kubernetes api objects for holos
{} & #KubernetesObjects
#KubernetesObjects & {
apiObjects: {
for obj in #CredsRefresherService.objects {
let Kind = obj.kind
let Name = obj.metadata.name
"\(Kind)": "\(Name)": obj
}
}
}
#InputKeys: {
project: "secrets"

View File

@@ -1,7 +1,5 @@
package holos
import "list"
#TargetNamespace: "default"
#InputKeys: {
@@ -14,18 +12,18 @@ import "list"
_ns: #PlatformNamespace
objects: [
#Namespace & {
metadata: name: _ns.name
},
#Namespace & {metadata: _ns},
]
}
objects: list.FlattenN(_objects, 1)
_objects: [
for ns in #PlatformNamespaces {
(#PlatformNamespaceObjects & {_ns: ns}).objects
},
]
{} & #KubernetesObjects
#KubernetesObjects & {
apiObjects: {
for ns in #PlatformNamespaces {
for obj in (#PlatformNamespaceObjects & {_ns: ns}).objects {
let Kind = obj.kind
let Name = obj.metadata.name
"\(Kind)": "\(Name)": obj
}
}
}
}

View File

@@ -11,9 +11,13 @@ package holos
#Kustomization: spec: dependsOn: [{name: #InstancePrefix + "-eso"}]
objects: [
#SecretStore,
#ExternalSecret & { _name: "validate" },
]
{} & #KubernetesObjects
#KubernetesObjects & {
apiObjects: {
SecretStore: default: #SecretStore
ExternalSecret: validate: #ExternalSecret & {
_name: "validate"
}
}
}

View File

@@ -4,6 +4,8 @@ package holos
#TargetNamespace: "ceph-system"
#SecretName: "\(#ClusterName)-ceph-csi-rbd"
#InputKeys: {
project: "metal"
service: "ceph"
@@ -25,5 +27,11 @@ package holos
url: "https://ceph.github.io/csi-charts"
}
}
objects: [#ExternalSecret & { _name: "ceph-csi-rbd" }]
apiObjects: {
SecretStore: default: #SecretStore
ExternalSecret: "\(#SecretName)": #ExternalSecret & {
_name: #SecretName
}
}
}

View File

@@ -1,14 +1,14 @@
package holos
#Input: {
config : {
// (required) String representing a Ceph cluster to provision storage from.
// Should be unique across all Ceph clusters in use for provisioning,
// cannot be greater than 36 bytes in length, and should remain immutable for
// the lifetime of the StorageClass in use.
clusterID: string
// (required) []String list of ceph monitor "address:port" values.
monitors: [...string]
config: {
// (required) String representing a Ceph cluster to provision storage from.
// Should be unique across all Ceph clusters in use for provisioning,
// cannot be greater than 36 bytes in length, and should remain immutable for
// the lifetime of the StorageClass in use.
clusterID: string
// (required) []String list of ceph monitor "address:port" values.
monitors: [...string]
}
}
@@ -30,7 +30,7 @@ package holos
// (optional) Prefix to use for naming RBD images.
// If omitted, defaults to "csi-vol-".
// NOTE: Set this to a cluster specific value, e.g. vol-k1-
volumeNamePrefix: "vol-\(#InputKeys.cluster)-"
volumeNamePrefix: "vol-\(#ClusterName)-"
// (required) String representing a Ceph cluster to provision storage from.
// Should be unique across all Ceph clusters in use for provisioning,
@@ -146,13 +146,13 @@ package holos
// The secrets have to contain Ceph credentials with required access
// to the 'pool'.
provisionerSecret: "csi-rbd-secret"
provisionerSecret: #SecretName
// If Namespaces are left empty, the secrets are assumed to be in the
// Release namespace.
provisionerSecretNamespace: ""
controllerExpandSecret: "csi-rbd-secret"
controllerExpandSecret: #SecretName
controllerExpandSecretNamespace: ""
nodeStageSecret: "csi-rbd-secret"
nodeStageSecret: #SecretName
nodeStageSecretNamespace: ""
// Specify the filesystem type of the volume. If not specified,
// csi-provisioner will set default as `ext4`.
@@ -165,7 +165,7 @@ package holos
secret: {
// Specifies whether the secret should be created
create: false
name: "csi-rbd-secret"
name: #SecretName
// Key values correspond to a user name and its key, as defined in the
// ceph cluster. User ID should have required access to the 'pool'
// specified in the storage class

View File

@@ -0,0 +1,8 @@
package holos
#Input: {
config: {
clusterID: "a6de32ab-c84f-49a6-b97e-e31dc2a70931"
monitors: ["10.64.1.21:6789", "10.64.1.31:6789", "10.64.1.41:6789"]
}
}

View File

@@ -0,0 +1,23 @@
# Metal Clusters
This cluster type is overlaid onto other cluster types to add services necessary outside of a cloud like GKE or EKS. Ceph for PersistenVolumeClaim support on a Talos Proxmox cluster is the primary use case.
## Test Script
Test ceph is working with:
```bash
apply -n default -f-<<EOF
heredoc> apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: test
spec:
accessModes:
- ReadWriteOnce
volumeMode: Filesystem
resources:
requests:
storage: 1Gi
EOF
```

View File

@@ -6,7 +6,10 @@ package holos
{name: "external-secrets"},
{name: "holos-system"},
{name: "flux-system"},
{name: "ceph-system"},
{
name: "ceph-system"
labels: "pod-security.kubernetes.io/enforce": "privileged"
},
{name: "istio-system"},
{name: "istio-ingress"},
{name: "cert-manager"},

View File

@@ -11,6 +11,9 @@ import (
"encoding/yaml"
)
// #ClusterName is the cluster name for cluster scoped resources.
#ClusterName: #InputKeys.cluster
_apiVersion: "holos.run/v1alpha1"
// #Name defines the name: string key value pair used all over the place.
@@ -90,8 +93,8 @@ _apiVersion: "holos.run/v1alpha1"
#ExternalSecret: #NamespaceObject & es.#ExternalSecret & {
_name: string
metadata: {
namespace: #TargetNamespace
name: _name
namespace: #TargetNamespace
}
spec: {
refreshInterval: string | *"1h"
@@ -100,12 +103,12 @@ _apiVersion: "holos.run/v1alpha1"
name: string | *"default"
}
target: {
name: _name
creationPolicy: string | *"Owner"
deletionPolicy: string | *"Retain"
}
data: [{
remoteRef: key: _name
secretKey: _name
}]
// Copy fields 1:1 from external Secret to target Secret.
dataFrom: [{extract: key: _name}]
}
}
@@ -175,6 +178,29 @@ _apiVersion: "holos.run/v1alpha1"
}
}
// #APIObjects is the output type for api objects produced by cue. A map is used to aid debugging and clarity.
#APIObjects: {
// apiObjects holds each the api objects produced by cue.
apiObjects: {
[Kind=_]: {
[Name=_]: metav1.#TypeMeta & {
kind: Kind
}
}
}
// apiObjectsContent holds the marshalled representation of apiObjects
apiObjectMap: {
for kind, v in apiObjects {
"\(kind)": {
for name, obj in v {
"\(name)": yaml.Marshal(obj)
}
}
}
}
}
// #OutputTypeMeta is shared among all output types
#OutputTypeMeta: {
// apiVersion is the output api version
@@ -194,14 +220,10 @@ _apiVersion: "holos.run/v1alpha1"
// #KubernetesObjectOutput is the output schema of a single component.
#KubernetesObjects: {
#OutputTypeMeta
#APIObjects
// kind KubernetesObjects provides a yaml text stream of kubernetes api objects in the out field.
kind: "KubernetesObjects"
// objects holds a list of the kubernetes api objects to configure.
objects: [...metav1.#TypeMeta] | *[]
// content holds the rendered yaml text stream of kubernetes api objects.
content: yaml.MarshalStream(objects)
contentType: "application/yaml"
// ksObjects holds the flux Kustomization objects for gitops
ksObjects: [...#Kustomization] | *[#Kustomization]
// ksContent is the yaml representation of kustomization
@@ -210,6 +232,8 @@ _apiVersion: "holos.run/v1alpha1"
platform: #Platform
}
objects: "not allowed"
// #Chart defines an upstream helm chart
#Chart: {
name: string
@@ -226,6 +250,8 @@ _apiVersion: "holos.run/v1alpha1"
// #HelmChart is a holos component which produces kubernetes api objects from cue values provided to the helm template command.
#HelmChart: {
#OutputTypeMeta
#APIObjects
kind: "HelmChart"
// ksObjects holds the flux Kustomization objects for gitops.
ksObjects: [...#Kustomization] | *[#Kustomization]
@@ -243,11 +269,6 @@ _apiVersion: "holos.run/v1alpha1"
platform: #Platform
// instance returns the key values of the holos component instance.
instance: #InputKeys
// objects holds a list of the kubernetes api objects to configure.
objects: [...metav1.#TypeMeta] | *[]
// content holds the rendered yaml text stream of kubernetes api objects.
content: yaml.MarshalStream(objects)
contentType: "application/yaml"
}
// #PlatformSpec is the output schema of a platform specification.
@@ -260,3 +281,6 @@ _apiVersion: "holos.run/v1alpha1"
// Holos component name
metadata: name: #InstanceName
// #SecretName is the name of a Secret, ususally coupling a Deployment to an ExternalSecret
#SecretName: string

View File

@@ -20,7 +20,7 @@ func makeBuildRunFunc(cfg *holos.Config) command.RunFunc {
}
outs := make([]string, 0, len(results))
for _, result := range results {
outs = append(outs, result.Content)
outs = append(outs, result.FinalOutput())
}
out := strings.Join(outs, "---\n")
if _, err := fmt.Fprintln(cmd.OutOrStdout(), out); err != nil {

View File

@@ -29,7 +29,7 @@ func makeRenderRunFunc(cfg *holos.Config) command.RunFunc {
for _, result := range results {
// API Objects
path := result.Filename(cfg.WriteTo(), cfg.ClusterName())
if err := result.Save(ctx, path, result.Content); err != nil {
if err := result.Save(ctx, path, result.FinalOutput()); err != nil {
return wrapper.Wrap(err)
}
// Kustomization

View File

@@ -12,6 +12,7 @@ import (
"github.com/holos-run/holos/pkg/logger"
"github.com/holos-run/holos/pkg/util"
"github.com/holos-run/holos/pkg/wrapper"
"log/slog"
"os"
"os/exec"
"path/filepath"
@@ -73,12 +74,16 @@ type Metadata struct {
Name string `json:"name,omitempty"`
}
// apiObjectMap is the shape of marshalled api objects returned from cue to the
// holos cli. A map is used to improve the clarity of error messages from cue.
type apiObjectMap map[string]map[string]string
// Result is the build result for display or writing.
type Result struct {
Metadata Metadata `json:"metadata,omitempty"`
Content string `json:"content,omitempty"`
ContentType string `json:"contentType"`
KsContent string `json:"ksContent,omitempty"`
Metadata Metadata `json:"metadata,omitempty"`
KsContent string `json:"ksContent,omitempty"`
APIObjectMap apiObjectMap `json:"apiObjectMap,omitempty"`
finalOutput string
}
type Repository struct {
@@ -94,15 +99,14 @@ type Chart struct {
// A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
type HelmChart struct {
APIVersion string `json:"apiVersion"`
Kind string `json:"kind"`
Metadata Metadata `json:"metadata"`
KsContent string `json:"ksContent"`
Namespace string `json:"namespace"`
Chart Chart `json:"chart"`
ValuesContent string `json:"valuesContent"`
ContentType string `json:"contentType"`
Content string `json:"content"`
APIVersion string `json:"apiVersion"`
Kind string `json:"kind"`
Metadata Metadata `json:"metadata"`
KsContent string `json:"ksContent"`
Namespace string `json:"namespace"`
Chart Chart `json:"chart"`
ValuesContent string `json:"valuesContent"`
APIObjectMap apiObjectMap `json:"APIObjectMap"`
}
// Name returns the metadata name of the result. Equivalent to the
@@ -119,6 +123,26 @@ func (r *Result) KustomizationFilename(writeTo string, cluster string) string {
return filepath.Join(writeTo, "clusters", cluster, "holos", "components", r.Name()+"-kustomization.gen.yaml")
}
// FinalOutput returns the final rendered output.
func (r *Result) FinalOutput() string {
return r.finalOutput
}
// addAPIObjects adds the overlay api objects to finalOutput.
func (r *Result) addOverlayObjects(log *slog.Logger) {
b := []byte(r.FinalOutput())
for kind, v := range r.APIObjectMap {
for name, yamlString := range v {
log.Debug(fmt.Sprintf("%s/%s", kind, name), "kind", kind, "name", name)
util.EnsureNewline(b)
header := fmt.Sprintf("---\n# Source: holos component cue api objects %s/%s\n", kind, name)
b = append(b, []byte(header+yamlString)...)
util.EnsureNewline(b)
}
}
r.finalOutput = string(b)
}
// Save writes the content to the filesystem for git ops.
func (r *Result) Save(ctx context.Context, path string, content string) error {
log := logger.FromContext(ctx)
@@ -215,6 +239,7 @@ func (b *Builder) Run(ctx context.Context) (results []*Result, err error) {
if err := value.Decode(&result); err != nil {
return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
}
result.addOverlayObjects(log)
case Helm:
var helmChart HelmChart
// First decode into the result. Helm will populate the api objects later.
@@ -229,15 +254,7 @@ func (b *Builder) Run(ctx context.Context) (results []*Result, err error) {
if err := runHelm(ctx, &helmChart, &result, holos.PathComponent(instance.Dir)); err != nil {
return nil, err
}
// Append any cue api objects defined alongside the helm holos component.
if helmChart.Content != "" && helmChart.ContentType == "application/yaml" {
buf := []byte(result.Content)
util.EnsureNewline(buf)
buf = append(buf, []byte("---\n# Source: holos component overlay objects\n")...)
buf = append(buf, []byte(helmChart.Content)...)
log.DebugContext(ctx, "added additional api objects", "bytes", len(buf))
result.Content = string(buf)
}
result.addOverlayObjects(log)
default:
return nil, wrapper.Wrap(fmt.Errorf("build kind not implemented: %v", kind))
@@ -300,6 +317,10 @@ func runCmd(ctx context.Context, name string, args ...string) (result runResult,
// the rendered kubernetes api objects in the result.
func runHelm(ctx context.Context, hc *HelmChart, r *Result, path holos.PathComponent) error {
log := logger.FromContext(ctx).With("chart", hc.Chart.Name)
if hc.Chart.Name == "" {
log.WarnContext(ctx, "skipping helm: no chart name specified, use a different component type")
return nil
}
cachedChartPath := filepath.Join(string(path), ChartDir, hc.Chart.Name)
if isNotExist(cachedChartPath) {
@@ -342,7 +363,7 @@ func runHelm(ctx context.Context, hc *HelmChart, r *Result, path holos.PathCompo
return wrapper.Wrap(fmt.Errorf("could not run helm template: %w", err))
}
r.Content = helmOut.stdout.String()
r.finalOutput = helmOut.stdout.String()
return nil
}

View File

@@ -1 +1 @@
47
48

View File

@@ -1 +1 @@
1
0