Normalize log messages

Make the log messages clear and readable, for example:

    holos render --log-format=json --log-level=debug \
      --cluster-name core2 ./docs/examples/platforms/reference/projects/secrets/components/namespaces/ \
      2> >(jq -r '"\(.source.file):\(.source.line)\t" + .msg')

The msg field is intended to have an imperative verb, ideally in the
past tense, followed by an actionable noun.  Past tense indicates
success where as the "could not foo: "+err error form indicates an
attempt to do something that failed.

    config.go:91    finalized config from flags
    builder.go:115  cue export --out yaml ./platforms/reference/projects/secrets/components/namespaces
    builder.go:85   wrote deploy/clusters/core2/components/prod-secrets-namespaces/prod-secrets-namespaces.gen.yaml
    render.go:30    rendered prod-secrets-namespaces

.gitignore

@@ -1,6 +1,7 @@
-bin
-/vendor
-/.idea
+bin/
+vendor/
+.idea/
 coverage.out
-
 dist/
+*.hold/
+/deploy/

docs/examples/platforms/reference/projects/secrets/components/namespaces/namespaces.cue

@@ -1,5 +1,7 @@
 package holos
 
+// e.g. prod-secrets-namespaces
+metadata: name: "\(#InputKeys.stage)-\(#InputKeys.project)-namespaces"
 objects: [
 	#Namespace & {
 		metadata: name: "external-secrets"

docs/examples/platforms/reference/projects/secrets/output.cue

@@ -2,3 +2,8 @@ package holos
 
 // Output schema
 {} & #KubernetesObjects
+
+#InputKeys: {
+	project: "secrets"
+	service: "eso"
+}

docs/examples/schema.cue

@@ -30,7 +30,7 @@ _apiVersion: "holos.run/v1alpha1"
 	// cluster is usually the only key necessary when working with a component on the command line.
 	cluster: string @tag(cluster, type=string)
 	// stage is usually set by the platform or project.
-	stage: string @tag(stage, type=string)
+	stage: *"prod" | string @tag(stage, type=string)
 	// project is usually set by the platform or project.
 	project: string @tag(project, type=string)
 	// service is usually set by the component.
@@ -68,8 +68,12 @@ _Platform: #Platform
 	apiVersion: _apiVersion
 	// kind is a discriminator of the type of output
 	kind: #PlatformSpec.kind | #KubernetesObjects.kind | #ChartValues.kind
-	// out holds the text output
-	out: string | *""
+	// name holds a unique name suitable for a filename
+	metadata: name: string
+	// contentType is the standard MIME type indicating the content type of the content field
+	contentType: *"application/yaml" | "application/json"
+	// content holds the content text output
+	content: string | *""
 	// debug returns arbitrary debug output.
 	debug?: _
 }
@@ -82,7 +86,7 @@ _Platform: #Platform
 	// objects holds a list of the kubernetes api objects to configure.
 	objects: [...metav1.#TypeMeta] | *[]
 	// out holds the rendered yaml text stream of kubernetes api objects.
-	out: yaml.MarshalStream(objects)
+	content: yaml.MarshalStream(objects)
 	// platform returns the platform data structure for visibility / troubleshooting.
 	platform: _Platform
 }

pkg/cli/build.go

@@ -4,33 +4,27 @@ import (
 	"fmt"
 	"github.com/holos-run/holos/pkg/config"
 	"github.com/holos-run/holos/pkg/internal/builder"
-	"github.com/holos-run/holos/pkg/version"
 	"github.com/holos-run/holos/pkg/wrapper"
 	"github.com/spf13/cobra"
+	"strings"
 )
 
-// newCmd returns a new subcommand
-func newCmd(name string) *cobra.Command {
-	cmd := &cobra.Command{
-		Use:     name,
-		Version: version.Version,
-		Args:    cobra.NoArgs,
-		CompletionOptions: cobra.CompletionOptions{
-			HiddenDefaultCmd: true,
-		},
-		RunE: func(c *cobra.Command, args []string) error {
-			return wrapper.Wrap(fmt.Errorf("could not run %v: not implemented", c.Name()))
-		},
-		SilenceUsage:  true,
-		SilenceErrors: true,
-	}
-	return cmd
-}
-
 // build is the internal implementation of the build cli command
 func build(cmd *cobra.Command, args []string) error {
 	build := builder.New(builder.Entrypoints(args))
-	return build.Run(cmd.Context())
+	results, err := build.Run(cmd.Context())
+	if err != nil {
+		return err
+	}
+	outs := make([]string, 0, len(results))
+	for _, result := range results {
+		outs = append(outs, result.Content)
+	}
+	out := strings.Join(outs, "---\n")
+	if _, err := fmt.Fprintln(cmd.OutOrStdout(), out); err != nil {
+		return wrapper.Wrap(err)
+	}
+	return nil
 }
 
 // newBuildCmd returns the build subcommand for the root command

pkg/cli/render.go

@@ -0,0 +1,46 @@
+package cli
+
+import (
+	"fmt"
+	"github.com/holos-run/holos/pkg/config"
+	"github.com/holos-run/holos/pkg/internal/builder"
+	"github.com/holos-run/holos/pkg/logger"
+	"github.com/holos-run/holos/pkg/wrapper"
+	"github.com/spf13/cobra"
+)
+
+func makeRenderRunFunc(cfg *config.Config) runFunc {
+	return func(cmd *cobra.Command, args []string) error {
+		if cfg.ClusterName() == "" {
+			return wrapper.Wrap(fmt.Errorf("missing cluster name"))
+		}
+
+		ctx := cmd.Context()
+		log := logger.FromContext(ctx)
+		build := builder.New(builder.Entrypoints(args))
+		results, err := build.Run(cmd.Context())
+		if err != nil {
+			return wrapper.Wrap(err)
+		}
+		for _, result := range results {
+			path := result.Filename(cfg.WriteTo(), cfg.ClusterName())
+			if err := result.Save(ctx, path); err != nil {
+				return wrapper.Wrap(err)
+			}
+			log.InfoContext(ctx, "rendered "+result.Name(), "status", "ok", "action", "save", "path", path, "name", result.Name())
+		}
+		return nil
+	}
+}
+
+// newRenderCmd returns the render subcommand for the root command
+func newRenderCmd(cfg *config.Config) *cobra.Command {
+	cmd := newCmd("render [directory...]")
+	cmd.Args = cobra.MinimumNArgs(1)
+	cmd.Short = "write kubernetes api objects to the filesystem"
+	cmd.Flags().SortFlags = false
+	cmd.Flags().AddGoFlagSet(cfg.WriteFlagSet())
+	cmd.Flags().AddGoFlagSet(cfg.ClusterFlagSet())
+	cmd.RunE = makeRenderRunFunc(cfg)
+	return cmd
+}

pkg/cli/root.go

@@ -1,13 +1,17 @@
 package cli
 
 import (
+	"fmt"
 	"github.com/holos-run/holos/pkg/config"
 	"github.com/holos-run/holos/pkg/logger"
 	"github.com/holos-run/holos/pkg/version"
+	"github.com/holos-run/holos/pkg/wrapper"
 	"github.com/spf13/cobra"
 	"log/slog"
 )
 
+type runFunc func(c *cobra.Command, args []string) error
+
 // New returns a new root *cobra.Command for command line execution.
 func New(cfg *config.Config) *cobra.Command {
 	rootCmd := &cobra.Command{
@@ -32,16 +36,35 @@ func New(cfg *config.Config) *cobra.Command {
 			return nil
 		},
 		RunE: func(c *cobra.Command, args []string) error {
-			cfg.Logger().InfoContext(c.Context(), "hello")
-			return nil
+			return c.Usage()
 		},
 	}
 	rootCmd.SetVersionTemplate("{{.Version}}\n")
-	rootCmd.Flags().SortFlags = false
-	rootCmd.Flags().AddGoFlagSet(cfg.LogFlagSet())
+	rootCmd.SetOut(cfg.Stdout())
+	rootCmd.PersistentFlags().SortFlags = false
+	rootCmd.PersistentFlags().AddGoFlagSet(cfg.LogFlagSet())
 
-	// build subcommand
+	// subcommands
 	rootCmd.AddCommand(newBuildCmd(cfg))
+	rootCmd.AddCommand(newRenderCmd(cfg))
 
 	return rootCmd
 }
+
+// newCmd returns a new subcommand
+func newCmd(name string) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     name,
+		Version: version.Version,
+		Args:    cobra.NoArgs,
+		CompletionOptions: cobra.CompletionOptions{
+			HiddenDefaultCmd: true,
+		},
+		RunE: func(c *cobra.Command, args []string) error {
+			return wrapper.Wrap(fmt.Errorf("could not run %v: not implemented", c.Name()))
+		},
+		SilenceUsage:  true,
+		SilenceErrors: true,
+	}
+	return cmd
+}

pkg/cli/root_test.go

@@ -11,9 +11,10 @@ import (
 )
 
 func newCommand() (*cobra.Command, *bytes.Buffer) {
-	var b bytes.Buffer
-	cmd := New(config.New(config.Stderr(&b)))
-	return cmd, &b
+	var b1, b2 bytes.Buffer
+	// discard stdout for now, it's a bunch of usage messages.
+	cmd := New(config.New(config.Stdout(&b1), config.Stderr(&b2)))
+	return cmd, &b2
 }
 
 func TestNewRoot(t *testing.T) {
@@ -61,9 +62,10 @@ func TestLogOutput(t *testing.T) {
 	if err := cmd.Execute(); err != nil {
 		t.Fatalf("could not execute: %v", err)
 	}
-	stderr := b.String()
-	if !strings.Contains(stderr, "config lifecycle") {
-		t.Fatalf("lifecycle message missing: stderr: %v", stderr)
+	have := strings.TrimSpace(b.String())
+	want := "finalized config from flags"
+	if !strings.Contains(have, want) {
+		t.Fatalf("have does not contain want\n\thave: %#v\n\twant: %#v", have, want)
 	}
 }
 

pkg/config/config.go

@@ -29,25 +29,38 @@ func Stderr(w io.Writer) Option {
 
 // New returns a new top level cli Config.
 func New(opts ...Option) *Config {
-	o := &options{
+	cfgOptions := &options{
 		stdout: os.Stdout,
 		stderr: os.Stderr,
 	}
-	for _, f := range opts {
-		f(o)
+	for _, option := range opts {
+		option(cfgOptions)
 	}
-	return &Config{
-		logConfig: logger.NewConfig(),
-		options:   o,
+	writeFlagSet := flag.NewFlagSet("", flag.ContinueOnError)
+	clusterFlagSet := flag.NewFlagSet("", flag.ContinueOnError)
+	cfg := &Config{
+		logConfig:      logger.NewConfig(),
+		writeTo:        getenv("HOLOS_WRITE_TO", "deploy"),
+		clusterName:    getenv("HOLOS_CLUSTER_NAME", ""),
+		writeFlagSet:   writeFlagSet,
+		clusterFlagSet: clusterFlagSet,
+		options:        cfgOptions,
 	}
+	writeFlagSet.StringVar(&cfg.writeTo, "write-to", cfg.writeTo, "write to directory")
+	clusterFlagSet.StringVar(&cfg.clusterName, "cluster-name", cfg.clusterName, "cluster name")
+	return cfg
 }
 
 // Config holds configuration for the whole program, used by main()
 type Config struct {
-	logConfig *logger.Config
-	logger    *slog.Logger
-	options   *options
-	finalized bool
+	logConfig      *logger.Config
+	writeTo        string
+	clusterName    string
+	logger         *slog.Logger
+	options        *options
+	finalized      bool
+	writeFlagSet   *flag.FlagSet
+	clusterFlagSet *flag.FlagSet
 }
 
 // LogFlagSet returns the logging *flag.FlagSet
@@ -55,6 +68,16 @@ func (c *Config) LogFlagSet() *flag.FlagSet {
 	return c.logConfig.FlagSet()
 }
 
+// WriteFlagSet returns a *flag.FlagSet wired to c *Config.  Useful for commands that write files.
+func (c *Config) WriteFlagSet() *flag.FlagSet {
+	return c.writeFlagSet
+}
+
+// ClusterFlagSet returns a *flag.FlagSet wired to c *Config.  Useful for commands scoped to one cluster.
+func (c *Config) ClusterFlagSet() *flag.FlagSet {
+	return c.clusterFlagSet
+}
+
 // Finalize validates the config and finalizes the startup lifecycle based on user configuration.
 func (c *Config) Finalize() error {
 	if c.finalized {
@@ -65,7 +88,7 @@ func (c *Config) Finalize() error {
 	}
 	l := c.Logger()
 	c.logger = l
-	l.Debug("config lifecycle", "state", "finalized")
+	l.Debug("finalized config from flags", "state", "finalized")
 	c.finalized = true
 	return nil
 }
@@ -89,6 +112,30 @@ func (c *Config) NewTopLevelLogger() *slog.Logger {
 	return c.logConfig.NewTopLevelLogger(c.options.stderr)
 }
 
+// Stderr should be used instead of os.Stderr to capture output for tests.
 func (c *Config) Stderr() io.Writer {
 	return c.options.stderr
 }
+
+// Stdout should be used instead of os.Stdout to capture output for tests.
+func (c *Config) Stdout() io.Writer {
+	return c.options.stdout
+}
+
+// WriteTo returns the write to path configured by flags.
+func (c *Config) WriteTo() string {
+	return c.writeTo
+}
+
+// ClusterName returns the configured cluster name
+func (c *Config) ClusterName() string {
+	return c.clusterName
+}
+
+// getenv is equivalent to os.Getenv() with a default value
+func getenv(key, defaultValue string) string {
+	if value, exists := os.LookupEnv(key); exists {
+		return value
+	}
+	return defaultValue
+}

pkg/config/config_test.go

@@ -5,10 +5,10 @@ import (
 	"testing"
 )
 
-func newConfig() (*Config, *bytes.Buffer) {
+// newConfig returns a new *Config with stderr wired to a bytes.Buffer.
+func newConfig() (cfg *Config, stderr *bytes.Buffer) {
 	var b bytes.Buffer
-	c := New(Stdout(&b))
-	return c, &b
+	return New(Stderr(&b)), &b
 }
 
 func TestConfigFinalize(t *testing.T) {
@@ -22,11 +22,22 @@ func TestConfigFinalize(t *testing.T) {
 }
 
 func TestConfigFinalizeTwice(t *testing.T) {
-	cfg, _ := newConfig()
+	cfg, stderr := newConfig()
 	if err := cfg.Finalize(); err != nil {
-		t.Fatalf("unexpected error: %v", err)
+		t.Fatalf("want: %#v have: %#v", nil, err)
 	}
 	if err := cfg.Finalize(); err == nil {
-		t.Fatalf("want error got nil")
+		t.Fatalf("want: error have: %#v", err)
+	} else {
+		want := "could not finalize: already finalized"
+		have := err.Error()
+		if want != have {
+			t.Fatalf("want: %#v have: %#v", want, have)
+		}
+	}
+	want := ""
+	have := stderr.String()
+	if want != have {
+		t.Fatalf("want: %#v have: %#v", want, have)
 	}
 }

pkg/internal/builder/builder.go

@@ -6,6 +6,7 @@ package builder
 import (
 	"context"
 	"fmt"
+	"github.com/holos-run/holos/pkg/logger"
 	"github.com/holos-run/holos/pkg/wrapper"
 	"os"
 	"path/filepath"
@@ -45,16 +46,54 @@ type buildInfo struct {
 	Kind       string `json:"kind,omitempty"`
 }
 
-type out struct {
-	Out string `json:"out,omitempty"`
+// Metadata represents the standard metadata fields of the cue output
+type Metadata struct {
+	Name string `json:"name,omitempty"`
 }
 
-func (b *Builder) Run(ctx context.Context) error {
+// Result is the build result for display or writing.
+type Result struct {
+	Metadata Metadata `json:"metadata,omitempty"`
+	Content  string   `json:"content,omitempty"`
+}
+
+// Name returns the metadata name of the result. Equivalent to the
+// OrderedComponent name specified in platform.yaml in the holos prototype.
+func (r *Result) Name() string {
+	return r.Metadata.Name
+}
+
+func (r *Result) Filename(writeTo string, cluster string) string {
+	return filepath.Join(writeTo, "clusters", cluster, "components", r.Name(), r.Name()+".gen.yaml")
+}
+
+// Save writes the content to the filesystem for git ops.
+func (r *Result) Save(ctx context.Context, path string) error {
+	if r.Name() == "" {
+		return wrapper.Wrap(fmt.Errorf("missing name from cue result"))
+	}
+	log := logger.FromContext(ctx)
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, os.FileMode(0775)); err != nil {
+		log.WarnContext(ctx, "could not mkdir", "path", dir, "err", err)
+		return wrapper.Wrap(err)
+	}
+	if err := os.WriteFile(path, []byte(r.Content), os.FileMode(0644)); err != nil {
+		log.WarnContext(ctx, "could not write", "path", path, "err", err)
+		return wrapper.Wrap(err)
+	}
+	log.DebugContext(ctx, "wrote "+path, "action", "mkdir", "path", path, "status", "ok")
+	return nil
+}
+
+func (b *Builder) Run(ctx context.Context) ([]*Result, error) {
+	log := logger.FromContext(ctx)
 	cueCtx := cuecontext.New()
+	results := make([]*Result, 0, len(b.cfg.args))
 
 	dir, err := b.findCueMod()
 	if err != nil {
-		return wrapper.Wrap(err)
+		return nil, wrapper.Wrap(err)
 	}
 
 	cfg := load.Config{Dir: dir}
@@ -64,47 +103,50 @@ func (b *Builder) Run(ctx context.Context) error {
 	for idx, path := range b.cfg.args {
 		target, err := filepath.Abs(path)
 		if err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not find absolute path: %w", err))
+			return nil, wrapper.Wrap(fmt.Errorf("could not find absolute path: %w", err))
 		}
 		relPath, err := filepath.Rel(dir, target)
 		if err != nil {
-			return wrapper.Wrap(fmt.Errorf("invalid argument, must be relative to cue.mod: %w", err))
+			return nil, wrapper.Wrap(fmt.Errorf("invalid argument, must be relative to cue.mod: %w", err))
 		}
-		args[idx] = "./" + relPath
+		relPath = "./" + relPath
+		args[idx] = relPath
+		equiv := fmt.Sprintf("cue export --out yaml %v", relPath)
+		log.Debug(equiv)
 	}
 
 	instances := load.Instances(args, &cfg)
 
 	for _, instance := range instances {
 		var info buildInfo
+		var result Result
+		results = append(results, &result)
 		if err := instance.Err; err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not load: %w", err))
+			return nil, wrapper.Wrap(fmt.Errorf("could not load: %w", err))
 		}
 		value := cueCtx.BuildInstance(instance)
 		if err := value.Err(); err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not build: %w", err))
+			return nil, wrapper.Wrap(fmt.Errorf("could not build: %w", err))
 		}
 		if err := value.Validate(); err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not validate: %w", err))
+			return nil, wrapper.Wrap(fmt.Errorf("could not validate: %w", err))
 		}
 
 		if err := value.Decode(&info); err != nil {
-			return wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
+			return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
 		}
 
 		switch kind := info.Kind; kind {
 		case "KubernetesObjects":
-			var out out
-			if err := value.Decode(&out); err != nil {
-				return wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
+			if err := value.Decode(&result); err != nil {
+				return nil, wrapper.Wrap(fmt.Errorf("could not decode: %w", err))
 			}
-			fmt.Printf(out.Out)
 		default:
-			return wrapper.Wrap(fmt.Errorf("build kind not implemented: %v", kind))
+			return nil, wrapper.Wrap(fmt.Errorf("build kind not implemented: %v", kind))
 		}
 	}
 
-	return nil
+	return results, nil
 }
 
 // findCueMod returns the root module location containing the cue.mod file or

pkg/logger/logger.go

@@ -166,9 +166,9 @@ func (c *Config) NewLogger(w io.Writer) *slog.Logger {
 func NewConfig() *Config {
 	f := flag.NewFlagSet("", flag.ContinueOnError)
 	c := &Config{flagSet: f}
-	f.StringVar(&c.level, "log-level", "info", fmt.Sprintf("Log Level (%s)", strings.Join(validLogLevels, "|")))
-	f.StringVar(&c.format, "log-format", "text", fmt.Sprintf("Log format (%s)", strings.Join(validLogFormats, "|")))
-	f.Var(&c.dropAttrs, "log-drop", "Log attributes to drop, e.g. \"user-agent,version\"")
+	f.StringVar(&c.level, "log-level", getenv("HOLOS_LOG_LEVEL", "info"), fmt.Sprintf("log level (%s)", strings.Join(validLogLevels, "|")))
+	f.StringVar(&c.format, "log-format", getenv("HOLOS_LOG_FORMAT", "text"), fmt.Sprintf("log format (%s)", strings.Join(validLogFormats, "|")))
+	f.Var(&c.dropAttrs, "log-drop", "log attributes to drop (example \"user-agent,version\")")
 	return c
 }
 
@@ -207,3 +207,11 @@ func (c *Config) vetFormat() error {
 	err := fmt.Errorf("invalid log format: %s is not one of %s", c.format, strings.Join(validLogFormats, ", "))
 	return wrapper.Wrap(err)
 }
+
+// getenv is equivalent to os.Getenv() with a default value
+func getenv(key, defaultValue string) string {
+	if value, exists := os.LookupEnv(key); exists {
+		return value
+	}
+	return defaultValue
+}

pkg/version/embedded/patch

@@ -1 +1 @@
-1
+2

pkg/wrapper/wrap.go

@@ -39,6 +39,8 @@ func (e *ErrorAt) Error() string {
 
 // Wrap wraps err in a ErrorAt or returns err if err is nil, already a
 // ErrorAt, or caller info is not available.
+//
+// XXX: Refactor to wrap.Err(error, ...slog.Attr).  Often want to add attributes for the top level logger.
 func Wrap(err error) error {
 	// Nothing to do
 	if err == nil {

scripts/msgs

@@ -0,0 +1,2 @@
+#! /bin/bash
+exec jq -r '"\(.source.file):\(.source.line)\t" + .msg'