Cache helm charts

This patch speeds up rendering by storing a copy of helm charts in the
holos component directory.

docs/examples/platforms/reference/projects/secrets/components/eso/eso.cue

@@ -0,0 +1,3 @@
+package holos
+
+#InputKeys: component: "eso"

docs/examples/platforms/reference/projects/secrets/components/eso/output.cue

@@ -0,0 +1,16 @@
+package holos
+
+#Kustomization: spec: dependsOn: [{name: #InstancePrefix+"-namespaces"}]
+
+#HelmChart & {
+	values: installCrds: true
+	namespace: #TargetNamespace
+	chart: {
+		name: "external-secrets"
+		version: "0.9.12"
+		repository: {
+			name: "external-secrets"
+		  url: "https://charts.external-secrets.io"
+		}
+	}
+}

docs/examples/platforms/reference/projects/secrets/components/namespaces/namespaces.cue

@@ -5,6 +5,6 @@ package holos
 metadata: name: #InstanceName
 objects: [
 	#Namespace & {
-		metadata: name: "external-secrets"
+		metadata: name: #TargetNamespace
 	}
 ]

docs/examples/platforms/reference/projects/secrets/components/namespaces/output.cue

@@ -4,8 +4,3 @@ package holos
 {} & #KubernetesObjects & {
 	ksObjects: [#Kustomization]
 }
-
-#InputKeys: {
-	project: "secrets"
-	service: "eso"
-}

docs/examples/platforms/reference/projects/secrets/inputkeys.cue

@@ -0,0 +1,11 @@
+package holos
+
+#TargetNamespace: "external-secrets"
+
+#InputKeys: {
+	project: "secrets"
+	service: "eso"
+}
+
+// Holos component name
+metadata: name: #InstanceName

docs/examples/schema.cue

@@ -14,6 +14,8 @@ _apiVersion: "holos.run/v1alpha1"
 
 // #InstanceName is the name of the holos component instance being managed varying by stage, project, and component names.
 #InstanceName: "\(#InputKeys.stage)-\(#InputKeys.project)-\(#InputKeys.component)"
+// #InstancePrefix is the stage and project without the component name.  Useful for dependency management among multiple components for a project stage.
+#InstancePrefix: "\(#InputKeys.stage)-\(#InputKeys.project)"
 
 // #NamespaceMeta defines standard metadata for namespaces.
 // Refer to https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-metadata-name
@@ -25,6 +27,9 @@ _apiVersion: "holos.run/v1alpha1"
 	...
 }
 
+// #TargetNamespace is the target namespace for a holos component.
+#TargetNamespace: string
+
 // Kubernetes API Objects
 #Namespace: corev1.#Namespace & #NamespaceMeta
 #ConfigMap: corev1.#ConfigMap
@@ -92,7 +97,7 @@ _Platform: #Platform
 	// apiVersion is the output api version
 	apiVersion: _apiVersion
 	// kind is a discriminator of the type of output
-	kind: #PlatformSpec.kind | #KubernetesObjects.kind | #ChartValues.kind
+	kind: #PlatformSpec.kind | #KubernetesObjects.kind | #HelmChart.kind
 	// name holds a unique name suitable for a filename
 	metadata: name: string
 	// contentType is the standard MIME type indicating the content type of the content field
@@ -113,17 +118,43 @@ _Platform: #Platform
 	// out holds the rendered yaml text stream of kubernetes api objects.
 	content: yaml.MarshalStream(objects)
 	// ksObjects holds the flux Kustomization objects for gitops
-	ksObjects: [...#Kustomization] | *[]
+	ksObjects: [...#Kustomization] | *[#Kustomization]
 	// ksContent is the yaml representation of kustomization
 	ksContent: yaml.MarshalStream(ksObjects)
 	// platform returns the platform data structure for visibility / troubleshooting.
 	platform: _Platform
 }
 
-// #ChartValues is the output schema of a holos component which produces values for a helm chart.
-#ChartValues: {
+// #Chart defines an upstream helm chart
+#Chart: {
+	name: string
+	version: string
+	repository: {
+		name: string
+		url: string
+	}
+}
+
+// #HelmChart is a holos component which produces kubernetes api objects from cue values provided to the helm template command.
+#HelmChart: {
 	#OutputTypeMeta
-	kind: "ChartValues"
+	kind: "HelmChart"
+	// ksObjects holds the flux Kustomization objects for gitops.
+	ksObjects: [...#Kustomization] | *[#Kustomization]
+	// ksContent is the yaml representation of kustomization.
+	ksContent: yaml.MarshalStream(ksObjects)
+	// namespace defines the value passed to the helm --namespace flag
+	namespace: #TargetNamespace
+	// chart defines the upstream helm chart to process.
+	chart: #Chart
+	// values represents the helm values to provide to the chart.
+	values: {...}
+	// valuesContent holds the values yaml
+	valuesContent: yaml.Marshal(values)
+	// platform returns the platform data structure for visibility / troubleshooting.
+	platform: _Platform
+	// instance returns the key values of the holos component instance.
+	instance: #InputKeys
 }
 
 // #PlatformSpec is the output schema of a platform specification.
@@ -132,4 +163,4 @@ _Platform: #Platform
 	kind: "PlatformSpec"
 }
 
-#Output: #PlatformSpec | #KubernetesObjects | #ChartValues
+#Output: #PlatformSpec | #KubernetesObjects | #HelmChart

holos.go

@@ -0,0 +1,10 @@
+// Package holos defines types for the rest of the system.
+package holos
+
+// A PathCueMod is a string representing the filesystem path of a cue module.
+// It is given a unique type so the API is clear.
+type PathCueMod string
+
+// A PathComponent is a string representing the filesystem path of a holos component.
+// It is given a unique type so the API is clear.
+type PathComponent string

pkg/cli/render.go

@@ -36,7 +36,7 @@ func makeRenderRunFunc(cfg *config.Config) runFunc {
 			if err := result.Save(ctx, path, result.KsContent); err != nil {
 				return wrapper.Wrap(err)
 			}
-			log.InfoContext(ctx, "rendered "+result.Name(), "status", "ok", "action", "save", "path", path, "name", result.Name())
+			log.InfoContext(ctx, "rendered "+result.Name(), "status", "ok", "action", "rendered", "name", result.Name())
 		}
 		return nil
 	}

pkg/config/config.go

@@ -9,7 +9,8 @@ import (
 	"os"
 )
 
-// An Option configures a Config
+// An Option configures a Config using [functional
+// options](https://commandcenter.blogspot.com/2014/01/self-referential-functions-and-design.html).
 type Option func(o *options)
 
 type options struct {
@@ -17,12 +18,12 @@ type options struct {
 	stderr io.Writer
 }
 
-// Stdout redirects standard output to w
+// Stdout redirects standard output to w, useful for test capture.
 func Stdout(w io.Writer) Option {
 	return func(o *options) { o.stdout = w }
 }
 
-// Stderr redirects standard error to w
+// Stderr redirects standard error to w, useful for test capture.
 func Stderr(w io.Writer) Option {
 	return func(o *options) { o.stderr = w }
 }
@@ -51,7 +52,9 @@ func New(opts ...Option) *Config {
 	return cfg
 }
 
-// Config holds configuration for the whole program, used by main()
+// Config holds configuration for the whole program, used by main(). The config
+// should be initialized early at a well known location in the program lifecycle
+// then remain immutable.
 type Config struct {
 	logConfig      *logger.Config
 	writeTo        string
@@ -63,7 +66,7 @@ type Config struct {
 	clusterFlagSet *flag.FlagSet
 }
 
-// LogFlagSet returns the logging *flag.FlagSet
+// LogFlagSet returns the logging *flag.FlagSet for use by the command handler.
 func (c *Config) LogFlagSet() *flag.FlagSet {
 	return c.logConfig.FlagSet()
 }
@@ -93,12 +96,12 @@ func (c *Config) Finalize() error {
 	return nil
 }
 
-// Vet validates the config
+// Vet validates the config.
 func (c *Config) Vet() error {
 	return c.logConfig.Vet()
 }
 
-// Logger returns a *slog.Logger configured by the user
+// Logger returns a *slog.Logger configured by the user.
 func (c *Config) Logger() *slog.Logger {
 	if c.logger != nil {
 		return c.logger
@@ -127,12 +130,12 @@ func (c *Config) WriteTo() string {
 	return c.writeTo
 }
 
-// ClusterName returns the configured cluster name
+// ClusterName returns the cluster name configured by flags.
 func (c *Config) ClusterName() string {
 	return c.clusterName
 }
 
-// getenv is equivalent to os.Getenv() with a default value
+// getenv is equivalent to os.LookupEnv with a default value.
 func getenv(key, defaultValue string) string {
 	if value, exists := os.LookupEnv(key); exists {
 		return value

pkg/internal/builder/builder.go

@@ -4,17 +4,32 @@
 package builder
 
 import (
+	"bytes"
 	"context"
+	"cuelang.org/go/cue/build"
 	"fmt"
+	"github.com/holos-run/holos"
 	"github.com/holos-run/holos/pkg/logger"
 	"github.com/holos-run/holos/pkg/wrapper"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/load"
 )
 
+const (
+	// Kube is the value of the kind field of holos build output indicating
+	// kubernetes api objects.
+	Kube = "KubernetesObjects"
+	// Helm is the value of the kind field of holos build output indicating helm
+	// values and helm command information.
+	Helm = "HelmChart"
+	// ChartDir is the chart cache directory name.
+	ChartDir = "vendor"
+)
+
 // An Option configures a Builder
 type Option func(*config)
 
@@ -64,6 +79,28 @@ type Result struct {
 	KsContent string   `json:"ksContent,omitempty"`
 }
 
+type Repository struct {
+	Name string `json:"name"`
+	URL  string `json:"url"`
+}
+
+type Chart struct {
+	Name       string     `json:"name"`
+	Version    string     `json:"version"`
+	Repository Repository `json:"repository"`
+}
+
+// A HelmChart represents a helm command to provide chart values in order to render kubernetes api objects.
+type HelmChart struct {
+	APIVersion    string   `json:"apiVersion"`
+	Kind          string   `json:"kind"`
+	Metadata      Metadata `json:"metadata"`
+	KsContent     string   `json:"ksContent"`
+	Namespace     string   `json:"namespace"`
+	Chart         Chart    `json:"chart"`
+	ValuesContent string   `json:"valuesContent"`
+}
+
 // Name returns the metadata name of the result. Equivalent to the
 // OrderedComponent name specified in platform.yaml in the holos prototype.
 func (r *Result) Name() string {
@@ -91,7 +128,7 @@ func (r *Result) Save(ctx context.Context, path string, content string) error {
 		log.WarnContext(ctx, "could not write", "path", path, "err", err)
 		return wrapper.Wrap(err)
 	}
-	log.DebugContext(ctx, "wrote "+path, "action", "write", "path", path, "status", "ok")
+	log.DebugContext(ctx, "out: wrote "+path, "action", "write", "path", path, "status", "ok")
 	return nil
 }
 
@@ -100,15 +137,15 @@ func (b *Builder) Cluster() string {
 	return b.cfg.cluster
 }
 
-func (b *Builder) Run(ctx context.Context) ([]*Result, error) {
+// Instances returns the cue build instances being built.
+func (b *Builder) Instances(ctx context.Context) ([]*build.Instance, error) {
 	log := logger.FromContext(ctx)
-	cueCtx := cuecontext.New()
-	results := make([]*Result, 0, len(b.cfg.args))
 
-	dir, err := b.findCueMod()
+	mod, err := b.findCueMod()
 	if err != nil {
 		return nil, wrapper.Wrap(err)
 	}
+	dir := string(mod)
 
 	cfg := load.Config{Dir: dir}
 
@@ -126,39 +163,68 @@ func (b *Builder) Run(ctx context.Context) ([]*Result, error) {
 		relPath = "./" + relPath
 		args[idx] = relPath
 		equiv := fmt.Sprintf("cue export --out yaml -t cluster=%v %v", b.Cluster(), relPath)
-		log.Debug(equiv)
+		log.Debug("cue: equivalent command: " + equiv)
 	}
 
 	// Refer to https://github.com/cue-lang/cue/blob/v0.7.0/cmd/cue/cmd/common.go#L429
 	cfg.Tags = append(cfg.Tags, "cluster="+b.Cluster())
-	log.DebugContext(ctx, fmt.Sprintf("configured cue tags: %v", cfg.Tags))
+	log.DebugContext(ctx, fmt.Sprintf("cue: tags %v", cfg.Tags))
 
-	instances := load.Instances(args, &cfg)
+	return load.Instances(args, &cfg), nil
+}
+
+func (b *Builder) Run(ctx context.Context) (results []*Result, err error) {
+	results = make([]*Result, 0, len(b.cfg.args))
+	cueCtx := cuecontext.New()
+	logger.FromContext(ctx).DebugContext(ctx, "cue: building instances")
+	instances, err := b.Instances(ctx)
+	if err != nil {
+		return results, err
+	}
 
 	for _, instance := range instances {
 		var info buildInfo
 		var result Result
+		log := logger.FromContext(ctx).With("dir", instance.Dir)
 		results = append(results, &result)
 		if err := instance.Err; err != nil {
 			return nil, wrapper.Wrap(fmt.Errorf("could not load: %w", err))
 		}
+		log.DebugContext(ctx, "cue: building instance")
 		value := cueCtx.BuildInstance(instance)
 		if err := value.Err(); err != nil {
 			return nil, wrapper.Wrap(fmt.Errorf("could not build: %w", err))
 		}
+		log.DebugContext(ctx, "cue: validating instance")
 		if err := value.Validate(); err != nil {
 			return nil, wrapper.Wrap(fmt.Errorf("could not validate: %w", err))
 		}
-
+		log.DebugContext(ctx, "cue: decoding holos component build info")
 		if err := value.Decode(&info); err != nil {
 			return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
 		}
 
+		log.DebugContext(ctx, "cue: processing holos component kind "+info.Kind)
 		switch kind := info.Kind; kind {
-		case "KubernetesObjects":
+		case Kube:
+			// CUE directly provides the kubernetes api objects in result.Content
 			if err := value.Decode(&result); err != nil {
 				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
 			}
+		case Helm:
+			var helmChart HelmChart
+			// First decode into the result.  Helm will populate the api objects later.
+			if err := value.Decode(&result); err != nil {
+				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
+			}
+			// Decode again into the helm chart struct to get the values content to provide to helm.
+			if err := value.Decode(&helmChart); err != nil {
+				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
+			}
+			// runHelm populates result.Content from helm template output.
+			if err := runHelm(ctx, &helmChart, &result, holos.PathComponent(instance.Dir)); err != nil {
+				return nil, err
+			}
 		default:
 			return nil, wrapper.Wrap(fmt.Errorf("build kind not implemented: %v", kind))
 		}
@@ -170,14 +236,15 @@ func (b *Builder) Run(ctx context.Context) ([]*Result, error) {
 // findCueMod returns the root module location containing the cue.mod file or
 // directory or an error if the builder arguments do not share a common root
 // module.
-func (b *Builder) findCueMod() (dir string, err error) {
+func (b *Builder) findCueMod() (dir holos.PathCueMod, err error) {
 	for _, origPath := range b.cfg.args {
-		var path string
-		if path, err = filepath.Abs(origPath); err != nil {
-			return
+		absPath, err := filepath.Abs(origPath)
+		if err != nil {
+			return "", err
 		}
+		path := holos.PathCueMod(absPath)
 		for {
-			if _, err := os.Stat(filepath.Join(path, "cue.mod")); err == nil {
+			if _, err := os.Stat(filepath.Join(string(path), "cue.mod")); err == nil {
 				if dir != "" && dir != path {
 					return "", fmt.Errorf("multiple modules not supported: %v is not %v", dir, path)
 				}
@@ -186,7 +253,7 @@ func (b *Builder) findCueMod() (dir string, err error) {
 			} else if !os.IsNotExist(err) {
 				return "", err
 			}
-			parentPath := filepath.Dir(path)
+			parentPath := holos.PathCueMod(filepath.Dir(string(path)))
 			if parentPath == path {
 				return "", fmt.Errorf("no cue.mod from root to leaf: %v", origPath)
 			}
@@ -195,3 +262,114 @@ func (b *Builder) findCueMod() (dir string, err error) {
 	}
 	return dir, nil
 }
+
+type runResult struct {
+	stdout *bytes.Buffer
+	stderr *bytes.Buffer
+}
+
+func runCmd(ctx context.Context, name string, args ...string) (result runResult, err error) {
+	result = runResult{
+		stdout: new(bytes.Buffer),
+		stderr: new(bytes.Buffer),
+	}
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Stdout = result.stdout
+	cmd.Stderr = result.stderr
+	log := logger.FromContext(ctx)
+	log.DebugContext(ctx, "run: "+name, "name", name, "args", args)
+	err = cmd.Run()
+	return
+}
+
+// runHelm provides the values produced by CUE to helm template and returns
+// the rendered kubernetes api objects in the result.
+func runHelm(ctx context.Context, hc *HelmChart, r *Result, path holos.PathComponent) error {
+	log := logger.FromContext(ctx).With("chart", hc.Chart.Name)
+
+	cachedChartPath := filepath.Join(string(path), ChartDir, hc.Chart.Name)
+	if isNotExist(cachedChartPath) {
+		// Add repositories
+		repo := hc.Chart.Repository
+		out, err := runCmd(ctx, "helm", "repo", "add", repo.Name, repo.URL)
+		if err != nil {
+			log.ErrorContext(ctx, "could not run helm", "stderr", out.stderr.String(), "stdout", out.stdout.String())
+			return wrapper.Wrap(fmt.Errorf("could not run helm repo add: %w", err))
+		}
+		// Update repository
+		out, err = runCmd(ctx, "helm", "repo", "update", repo.Name)
+		if err != nil {
+			log.ErrorContext(ctx, "could not run helm", "stderr", out.stderr.String(), "stdout", out.stdout.String())
+			return wrapper.Wrap(fmt.Errorf("could not run helm repo update: %w", err))
+		}
+		// Cache the chart
+		if err := cacheChart(ctx, path, ChartDir, hc.Chart); err != nil {
+			return fmt.Errorf("could not cache chart: %w", err)
+		}
+	}
+
+	// Write values file
+	tempDir, err := os.MkdirTemp("", "holos")
+	if err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+	}
+	defer remove(ctx, tempDir)
+
+	valuesPath := filepath.Join(tempDir, "values.yaml")
+	if err := os.WriteFile(valuesPath, []byte(hc.ValuesContent), 0644); err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not write values: %w", err))
+	}
+	log.DebugContext(ctx, "helm: wrote values", "path", valuesPath, "bytes", len(hc.ValuesContent))
+
+	// Run charts
+	chart := hc.Chart
+	helmOut, err := runCmd(ctx, "helm", "template", "--values", valuesPath, "--namespace", hc.Namespace, "--kubeconfig", "/dev/null", "--version", chart.Version, chart.Name, cachedChartPath)
+	if err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not run helm template: %w", err))
+	}
+
+	r.Content = helmOut.stdout.String()
+
+	return nil
+}
+
+// remove cleans up path, useful for temporary directories.
+func remove(ctx context.Context, path string) {
+	log := logger.FromContext(ctx)
+	if err := os.RemoveAll(path); err != nil {
+		log.WarnContext(ctx, "tmp: could not remove", "err", err, "path", path)
+	} else {
+		log.DebugContext(ctx, "tmp: removed", "path", path)
+	}
+}
+
+func isNotExist(path string) bool {
+	_, err := os.Stat(path)
+	return os.IsNotExist(err)
+}
+
+// cacheChart stores a cached copy of Chart in the chart sub-directory of path.
+func cacheChart(ctx context.Context, path holos.PathComponent, chartDir string, chart Chart) error {
+	log := logger.FromContext(ctx)
+
+	cacheTemp, err := os.MkdirTemp(string(path), chartDir)
+	if err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not make temp dir: %w", err))
+	}
+	defer remove(ctx, cacheTemp)
+
+	chartName := fmt.Sprintf("%s/%s", chart.Repository.Name, chart.Name)
+	helmOut, err := runCmd(ctx, "helm", "pull", "--destination", cacheTemp, "--untar=true", "--version", chart.Version, chartName)
+	if err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not run helm pull: %w", err))
+	}
+	log.Debug("helm pull", "stdout", helmOut.stdout, "stderr", helmOut.stderr)
+
+	cachePath := filepath.Join(string(path), chartDir)
+	if err := os.Rename(cacheTemp, cachePath); err != nil {
+		return wrapper.Wrap(fmt.Errorf("could not rename: %w", err))
+	}
+	log.InfoContext(ctx, "cached", "chart", chart.Name, "version", chart.Version, "path", cachePath)
+
+	return nil
+}

pkg/version/embedded/minor

@@ -1 +1 @@
-40
+42

pkg/version/embedded/patch

@@ -1 +1 @@
-4
+0