(#6) Holos kv put command to create secrets

A "holos secret" is a Secret in the secrets namespace of the provisioner
cluster.  The put command creates a unique secret from files and
directories listed as arguments, or from a txtar archive provided on
standard input.

Secret data may come from any or all of the following sources:

1. Create a secret from raw data on standard input.  --name and --file
   must be specified.
2. Create a secret from txtar data on standard input.  The secret name
   is taken from the --name flag if provided, otherwise is taken from
   the first line of the txtar comment.
3. Create a secret from files and directories specified as arguments.
   The secret name is the base name of the first argument unless it is
   overridden by the --name flag.

This is likely doing too much, really all we care about is this use
case:

holos kv put talosconfig

holos kv get talosconfig | holos txtar

Additionally, I want to get get one command without writing a file:

DATA="$(holos kv get talosconfig --file talosconfig)

go.mod

@@ -54,6 +54,7 @@ require (
 	k8s.io/api v0.29.2 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/kubectl v0.29.2 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

go.sum

@@ -183,6 +183,8 @@ k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/kubectl v0.29.2 h1:uaDYaBhumvkwz0S2XHt36fK0v5IdNgL7HyUniwb2IUo=
+k8s.io/kubectl v0.29.2/go.mod h1:BhizuYBGcKaHWyq+G7txGw2fXg576QbPrrnQdQDZgqI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=

pkg/cli/kv/get.go

@@ -1,6 +1,8 @@
 package kv
 
 import (
+	"flag"
+	"fmt"
 	"github.com/holos-run/holos/pkg/cli/command"
 	"github.com/holos-run/holos/pkg/config"
 	"github.com/holos-run/holos/pkg/logger"
@@ -10,17 +12,28 @@ import (
 	"sort"
 )
 
+type getConfig struct {
+	file *string
+}
+
 func newGetCmd(cfg *config.Config) *cobra.Command {
 	cmd := command.New("get")
 	cmd.Args = cobra.MinimumNArgs(1)
 	cmd.Short = "print secret data in txtar format"
+
+	cf := getConfig{}
+	flagSet := flag.NewFlagSet("", flag.ContinueOnError)
+	cf.file = flagSet.String("file", "", "file to print to stdout")
+
 	cmd.Flags().SortFlags = false
-	cmd.RunE = makeGetRunFunc(cfg)
+	cmd.Flags().AddGoFlagSet(cfg.ClusterFlagSet())
+	cmd.Flags().AddGoFlagSet(flagSet)
+	cmd.RunE = makeGetRunFunc(cfg, cf)
 
 	return cmd
 }
 
-func makeGetRunFunc(cfg *config.Config) command.RunFunc {
+func makeGetRunFunc(cfg *config.Config, cf getConfig) command.RunFunc {
 	return func(cmd *cobra.Command, args []string) error {
 		ctx := cmd.Context()
 		log := logger.FromContext(ctx)
@@ -35,6 +48,9 @@ func makeGetRunFunc(cfg *config.Config) command.RunFunc {
 			opts := metav1.ListOptions{
 				LabelSelector: NameLabel + "=" + name,
 			}
+			if name := cfg.ClusterName(); name != "" {
+				opts.LabelSelector += fmt.Sprintf(",%s=%s", ClusterLabel, name)
+			}
 			list, err := cs.CoreV1().Secrets(cfg.KVNamespace()).List(ctx, opts)
 			if err != nil {
 				return wrapper.Wrap(err)
@@ -51,13 +67,25 @@ func makeGetRunFunc(cfg *config.Config) command.RunFunc {
 			// most recent secret is the one we want.
 			secret := list.Items[len(list.Items)-1]
 
+			keys := make([]string, 0, len(secret.Data))
 			for k, v := range secret.Data {
+				keys = append(keys, k)
 				nlog.DebugContext(ctx, "data", "name", secret.Name, "key", k, "len", len(v))
 			}
 
+			//  Print one file to stdout
+			if key := *cf.file; key != "" {
+				if data, found := secret.Data[key]; found {
+					cfg.Write(command.EnsureNewline(data))
+					return nil
+				}
+				return wrapper.Wrap(fmt.Errorf("not found: %s have %#v", key, keys))
+			}
+
 			if len(secret.Data) > 0 {
 				cfg.Println(secret.Name)
 			}
+
 			for k, v := range secret.Data {
 				cfg.Printf("-- %s --\n", k)
 				cfg.Write(command.EnsureNewline(v))

pkg/cli/kv/kv.go

@@ -10,6 +10,8 @@ import (
 )
 
 const NameLabel = "holos.run/secret.name"
+const OwnerLabel = "holos.run/secret.owner"
+const ClusterLabel = "holos.run/cluster.name"
 
 // New returns the kv root command for the cli
 func New(cfg *config.Config) *cobra.Command {
@@ -25,6 +27,7 @@ func New(cfg *config.Config) *cobra.Command {
 	// subcommands
 	cmd.AddCommand(newGetCmd(cfg))
 	cmd.AddCommand(newListCmd(cfg))
+	cmd.AddCommand(newPutCmd(cfg))
 	return cmd
 }
 

pkg/cli/kv/list.go

@@ -13,6 +13,7 @@ func newListCmd(cfg *config.Config) *cobra.Command {
 	cmd.Args = cobra.NoArgs
 	cmd.Short = "list secrets"
 	cmd.Flags().SortFlags = false
+	cmd.Flags().AddGoFlagSet(cfg.ClusterFlagSet())
 	cmd.RunE = makeListRunFunc(cfg)
 
 	return cmd

pkg/cli/kv/put.go

@@ -0,0 +1,199 @@
+package kv
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"github.com/holos-run/holos/pkg/cli/command"
+	"github.com/holos-run/holos/pkg/config"
+	"github.com/holos-run/holos/pkg/logger"
+	"github.com/holos-run/holos/pkg/wrapper"
+	"github.com/spf13/cobra"
+	"golang.org/x/tools/txtar"
+	"io"
+	"io/fs"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubectl/pkg/util/hash"
+	"os"
+	"path/filepath"
+	"sigs.k8s.io/yaml"
+	"strings"
+)
+
+type putConfig struct {
+	secretName *string
+	file       *string
+	dryRun     *bool
+}
+
+func newPutCmd(cfg *config.Config) *cobra.Command {
+	cmd := command.New("put")
+	cmd.Args = cobra.MinimumNArgs(0)
+	cmd.Short = "put a secret from stdin or file args"
+	cmd.Flags().SortFlags = false
+
+	pcfg := putConfig{}
+	flagSet := flag.NewFlagSet("", flag.ContinueOnError)
+	pcfg.secretName = flagSet.String("name", "", "secret name to use instead of txtar comment")
+	pcfg.file = flagSet.String("file", "", "file name to use instead of txtar path")
+	pcfg.dryRun = flagSet.Bool("dry-run", false, "print to standard output instead of creating")
+
+	cmd.Flags().AddGoFlagSet(flagSet)
+	cmd.Flags().AddGoFlagSet(cfg.ClusterFlagSet())
+	cmd.RunE = makePutRunFunc(cfg, pcfg)
+
+	return cmd
+}
+
+func makePutRunFunc(cfg *config.Config, pcfg putConfig) command.RunFunc {
+	return func(cmd *cobra.Command, args []string) error {
+		a := &txtar.Archive{}
+
+		// Add stdin to the archive.
+		if len(args) == 0 {
+			data, err := io.ReadAll(cfg.Stdin())
+			if err != nil {
+				return wrapper.Wrap(err)
+			}
+
+			if *pcfg.file != "" {
+				file := txtar.File{
+					Name: *pcfg.file,
+					Data: data,
+				}
+				a.Files = append(a.Files, file)
+			} else {
+				a = txtar.Parse(data)
+			}
+		}
+
+		// Do we have a secret name?
+		if *pcfg.secretName != "" {
+			a.Comment = []byte(*pcfg.secretName)
+		}
+		if len(a.Comment) == 0 {
+			// Use the first argument if not
+			if len(args) > 0 {
+				a.Comment = []byte(filepath.Base(args[0]))
+			} else {
+				err := fmt.Errorf("missing secret name from name, args, or txtar comment")
+				return wrapper.Wrap(err)
+			}
+		}
+
+		head, _, _ := bytes.Cut(a.Comment, []byte("\n"))
+		secretName := string(head)
+
+		// Add files from the filesystem to the archive
+		for _, name := range args {
+			if err := filepath.WalkDir(name, makeWalkFunc(a, name)); err != nil {
+				return wrapper.Wrap(err)
+			}
+		}
+
+		log := logger.FromContext(cmd.Context())
+		ctx := cmd.Context()
+
+		// Nothing to do?
+		if len(a.Files) == 0 {
+			log.WarnContext(ctx, "nothing to do")
+			return nil
+		}
+
+		// Create the secret.
+		secret, err := createSecret(ctx, cfg, pcfg, a, secretName)
+		if err != nil {
+			return wrapper.Wrap(err)
+		}
+
+		if *pcfg.dryRun {
+			data, err := yaml.Marshal(secret)
+			if err != nil {
+				return wrapper.Wrap(err)
+			}
+			cfg.Println(string(data))
+			return nil
+		}
+
+		// Make the API call
+		cs, err := newClientSet(cfg)
+		if err != nil {
+			return wrapper.Wrap(err)
+		}
+
+		secret, err = cs.CoreV1().Secrets(cfg.KVNamespace()).Create(ctx, secret, metav1.CreateOptions{})
+		if err != nil {
+			return wrapper.Wrap(err)
+		}
+
+		log.InfoContext(ctx, "created: "+secret.Name, "secret", secret.Name, "name", secretName, "namespace", secret.Namespace)
+		return nil
+	}
+}
+
+func createSecret(ctx context.Context, cfg *config.Config, pcfg putConfig, a *txtar.Archive, secretName string) (*v1.Secret, error) {
+	secretData := make(map[string][]byte)
+	for _, file := range a.Files {
+		secretData[file.Name] = file.Data
+	}
+
+	labels := map[string]string{NameLabel: secretName}
+	if owner := os.Getenv("USER"); owner != "" {
+		labels[OwnerLabel] = owner
+	}
+	if cluster := cfg.ClusterName(); cluster != "" {
+		labels[ClusterLabel] = cluster
+	}
+
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   secretName,
+			Labels: labels,
+		},
+		Data: secretData,
+	}
+
+	secretHash, err := hash.SecretHash(secret)
+	if err != nil {
+		return nil, wrapper.Wrap(err)
+	}
+	secret.Name = fmt.Sprintf("%s-%s", secret.Name, secretHash)
+
+	return secret, nil
+}
+
+func makeWalkFunc(a *txtar.Archive, rootDir string) fs.WalkDirFunc {
+	return func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Depth is the count of path separators from the root
+		depth := strings.Count(path[len(rootDir):], string(filepath.Separator))
+
+		if depth > 1 {
+			if d.IsDir() {
+				return filepath.SkipDir
+			}
+		}
+
+		if !d.IsDir() {
+			if file, err := file(path); err != nil {
+				return wrapper.Wrap(err)
+			} else {
+				file.Name = filepath.Base(path)
+				a.Files = append(a.Files, file)
+			}
+		}
+
+		return nil
+	}
+}
+
+func file(path string) (file txtar.File, err error) {
+	file.Name = path
+	file.Data, err = os.ReadFile(path)
+	return
+}

pkg/cli/txtar/txtar.go

@@ -5,11 +5,11 @@ import (
 	"fmt"
 	"github.com/holos-run/holos/pkg/cli/command"
 	"github.com/holos-run/holos/pkg/config"
+	"github.com/holos-run/holos/pkg/util"
 	"github.com/holos-run/holos/pkg/wrapper"
 	"github.com/spf13/cobra"
 	"golang.org/x/tools/txtar"
 	"io"
-	"io/fs"
 	"log/slog"
 	"os"
 	"path/filepath"
@@ -36,7 +36,7 @@ func makeRunFunc(cfg *config.Config) command.RunFunc {
 		// create an archive
 		a := &txtar.Archive{}
 		for _, name := range args {
-			if err := filepath.WalkDir(name, makeWalkFunc(a)); err != nil {
+			if err := filepath.WalkDir(name, util.MakeWalkFunc(a)); err != nil {
 				return wrapper.Wrap(err)
 			}
 		}
@@ -47,30 +47,6 @@ func makeRunFunc(cfg *config.Config) command.RunFunc {
 	}
 }
 
-func makeWalkFunc(a *txtar.Archive) fs.WalkDirFunc {
-	return func(path string, d os.DirEntry, err error) error {
-		if err != nil {
-			return wrapper.Wrap(err)
-		}
-
-		if !d.IsDir() {
-			if file, err := file(path); err != nil {
-				return wrapper.Wrap(err)
-			} else {
-				a.Files = append(a.Files, file)
-			}
-		}
-
-		return nil
-	}
-}
-
-func file(path string) (file txtar.File, err error) {
-	file.Name = path
-	file.Data, err = os.ReadFile(path)
-	return
-}
-
 // extract files from the configured Stdin to Stdout or the filesystem.
 func extract(cfg *config.Config) error {
 	input, err := io.ReadAll(cfg.Stdin())

pkg/util/txtar.go

@@ -0,0 +1,32 @@
+package util
+
+import (
+	"github.com/holos-run/holos/pkg/wrapper"
+	"golang.org/x/tools/txtar"
+	"io/fs"
+	"os"
+)
+
+func MakeWalkFunc(a *txtar.Archive) fs.WalkDirFunc {
+	return func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return wrapper.Wrap(err)
+		}
+
+		if !d.IsDir() {
+			if file, err := file(path); err != nil {
+				return wrapper.Wrap(err)
+			} else {
+				a.Files = append(a.Files, file)
+			}
+		}
+
+		return nil
+	}
+}
+
+func file(path string) (file txtar.File, err error) {
+	file.Name = path
+	file.Data, err = os.ReadFile(path)
+	return
+}

pkg/version/embedded/minor

@@ -1 +1 @@
-43
+44

pkg/version/embedded/patch

@@ -1 +1 @@
-2
+0