docs: make generate for docs preview

To properly support the kubectl-slice use case the Artifact map needs to
write multiple files out to the filesystem.  This needs to be dynamic in
the sense holos and the end user don't know what files the kubectl-slice
transformer is producing.

As a hack, which may actually turn out to be "good enough" this patch
makes the Slice transformer behave like so:

1. Execute kubectl-slice outputting to an empty temp directory.
2. Holos saves all files in this directory into the artifact map.
3. At the end of the Artifact pipeline, if the final artifact produced
   ends in a /, then all keys in the artifact map having the prefix are
   written to the output directory.

This should be sufficient for the use case, but we'll need to consider
how this transformer and apporach works when subsequent transformers are
used in the pipeline.  I haven't thought deeply about it, but it should
ideally work pretty well if the tools involved truly only care about
directories and not the files within the directory.

Makefile

@@ -78,6 +78,12 @@ build: ## Build holos executable.
 	@echo "GOPATH=${GOPATH}"
 	go build -trimpath -o bin/$(BIN_NAME) -ldflags $(LD_FLAGS) $(REPO_PATH)/cmd/$(BIN_NAME)
 
+.PHONY: debug
+debug: ## Build debug executable.
+	@echo "building ${BIN_NAME}-debug ${VERSION}"
+	@echo "GOPATH=${GOPATH}"
+	go build -o bin/$(BIN_NAME)-debug $(REPO_PATH)/cmd/$(BIN_NAME)
+
 linux: ## Build holos executable for tilt.
 	@echo "building ${BIN_NAME}.linux ${VERSION}"
 	@echo "GOPATH=${GOPATH}"

api/core/v1alpha5/types.go

@@ -190,11 +190,13 @@ type AuthSource struct {
 //  1. [Kustomize] - Patch and transform the output from prior generators or
 //     transformers.  See [Introduction to Kustomize].
 //  2. [Join] - Concatenate multiple prior outputs into one output.
+//  3. [Slice] - Slice an artifact into multiple artifacts using [kubectl-slice].
 //
 // [Introduction to Kustomize]: https://kubectl.docs.kubernetes.io/guides/config_management/introduction/
+// [kubectl-slice]: https://github.com/patrickdappollonio/kubectl-slice
 type Transformer struct {
 	// Kind represents the kind of transformer. Must be Kustomize, or Join.
-	Kind string `json:"kind" yaml:"kind" cue:"\"Kustomize\" | \"Join\""`
+	Kind string `json:"kind" yaml:"kind" cue:"\"Kustomize\" | \"Join\" | \"Slice\""`
 	// Inputs represents the files to transform. The Output of prior Generators
 	// and Transformers.
 	Inputs []FilePath `json:"inputs" yaml:"inputs"`

doc/md/api/core.md

@@ -464,11 +464,12 @@ Transformer combines multiple inputs from prior [Generator](<#Generator>) or [Tr
 
 1. [Kustomize](<#Kustomize>) \- Patch and transform the output from prior generators or transformers. See [Introduction to Kustomize](<https://kubectl.docs.kubernetes.io/guides/config_management/introduction/>).
 2. [Join](<#Join>) \- Concatenate multiple prior outputs into one output.
+3. \[Slice\] \- Slice an artifact into multiple artifacts using [kubectl\\\-slice](<https://github.com/patrickdappollonio/kubectl-slice>).
 
 ```go
 type Transformer struct {
     // Kind represents the kind of transformer. Must be Kustomize, or Join.
-    Kind string `json:"kind" yaml:"kind" cue:"\"Kustomize\" | \"Join\""`
+    Kind string `json:"kind" yaml:"kind" cue:"\"Kustomize\" | \"Join\" | \"Slice\""`
     // Inputs represents the files to transform. The Output of prior Generators
     // and Transformers.
     Inputs []FilePath `json:"inputs" yaml:"inputs"`

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.15
 	github.com/mennanov/fieldmask-utils v1.1.2
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/patrickdappollonio/kubectl-slice v1.4.1
 	github.com/princjef/gomarkdoc v1.1.0
 	github.com/prometheus/client_golang v1.19.1
 	github.com/rogpeppe/go-internal v1.13.2-0.20241226121412-a5dc8ff20d0a
@@ -248,6 +249,7 @@ require (
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mb0/glob v0.0.0-20160210091149-1eb79d2de6c4 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/miekg/dns v1.1.58 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
@@ -324,7 +326,7 @@ require (
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.7.0 // indirect
-	github.com/spf13/viper v1.18.2 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
@@ -368,7 +370,7 @@ require (
 	golang.org/x/term v0.28.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
 	google.golang.org/grpc v1.65.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect

go.sum

@@ -737,6 +737,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mb0/glob v0.0.0-20160210091149-1eb79d2de6c4 h1:NK3O7S5FRD/wj7ORQ5C3Mx1STpyEMuFe+/F0Lakd1Nk=
+github.com/mb0/glob v0.0.0-20160210091149-1eb79d2de6c4/go.mod h1:FqD3ES5hx6zpzDainDaHgkTIqrPaI9uX4CVWqYZoQjY=
 github.com/mennanov/fieldmask-utils v1.1.2 h1:f5hd3hYeWdl+q2thiKYyZZmqTqn90uayWG03bca9U+E=
 github.com/mennanov/fieldmask-utils v1.1.2/go.mod h1:xRqd9Fjz/gFEDYCQw7pxGouxqLhSPrkOdx2yhEAXEls=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -831,6 +833,8 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
+github.com/patrickdappollonio/kubectl-slice v1.4.1 h1:JtH3IhvbWpNmIoIaZrHNSA0KpffRsdxh5z5UxeUIMmo=
+github.com/patrickdappollonio/kubectl-slice v1.4.1/go.mod h1:XPVl2GUsmGuSInYe+fk58BySbrZpZEwbF9rSoYSMgpc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
@@ -956,8 +960,8 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
-github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
+github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
+github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
 github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
 github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -1414,8 +1418,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20220531173845-685668d2de03/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
-google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ=
-google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro=
+google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y=
+google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s=
 google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc=
 google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA=

internal/artifact/artifact.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/holos-run/holos/internal/errors"
@@ -60,6 +61,17 @@ func (a *MapStore) Get(path string) (data []byte, ok bool) {
 func (a *MapStore) Save(dir, path string) error {
 	fullPath := filepath.Join(dir, path)
 	msg := fmt.Sprintf("could not save %s", fullPath)
+	// TODO(jjm): replace slash hack with artifact directory support maybe
+	if strings.HasSuffix(path, "/") {
+		for _, key := range a.Keys() {
+			if strings.HasPrefix(key, path) {
+				if err := a.Save(dir, key); err != nil {
+					return errors.Wrap(err)
+				}
+			}
+		}
+		return nil
+	}
 	data, ok := a.Get(path)
 	if !ok {
 		return errors.Format("%s: missing %s", msg, path)

internal/builder/v1alpha5/builder.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"log/slog"
 	"os"
 	"path/filepath"
@@ -366,6 +367,10 @@ func (t transformersTask) run(ctx context.Context) error {
 			if err := t.opts.Store.Set(string(transformer.Output), data); err != nil {
 				return errors.Format("%s: %w", msg, err)
 			}
+		case "Slice":
+			if err := slice(ctx, transformer, t.taskParams); err != nil {
+				return errors.Wrap(err)
+			}
 		default:
 			return errors.Format("%s: unsupported kind %s", msg, transformer.Kind)
 		}
@@ -651,6 +656,80 @@ func kustomize(ctx context.Context, t core.Transformer, p taskParams) error {
 	return nil
 }
 
+func slice(ctx context.Context, t core.Transformer, p taskParams) error {
+	log := logger.FromContext(ctx)
+	msg := fmt.Sprintf(
+		"could not transform %s for %s path %s",
+		t.Output,
+		p.buildPlanName,
+		p.opts.Path,
+	)
+	// TODO(jjm): replace slash hack with artifact directory support maybe
+	//
+	// NOTE: We do not actually store an artifact with a trailing slash into the
+	// artifact map, this could trigger an infinite recursive loop with the way
+	// this hack is implemented in the artifact Save() method.
+	if !strings.HasSuffix(string(t.Output), "/") {
+		return errors.Format("%s: slice output must end in /", msg)
+	}
+
+	tempDir, err := os.MkdirTemp("", "holos.slice")
+	if err != nil {
+		return errors.Wrap(err)
+	}
+	defer util.Remove(ctx, tempDir)
+
+	// Write the inputs
+	for _, input := range t.Inputs {
+		path := string(input)
+		if err := p.opts.Store.Save(tempDir, path); err != nil {
+			return errors.Format("%s: %w", msg, err)
+		}
+	}
+
+	// TODO(jjm): useless?
+	var tempDirSlice = filepath.Join(tempDir, "slice") // TODO(jjm): hack for expedience
+
+	// Execute kubectl-slice for each input file, writing to one output directory.
+	for _, input := range t.Inputs {
+		in := filepath.Join(tempDir, string(input))
+		_, err := util.RunCmdW(
+			ctx, p.opts.Stderr,
+			"kubectl-slice", "-f", in, "-o", tempDirSlice)
+		if err != nil {
+			return errors.Format("%s: could not run kubectl-slice: %w", msg, err)
+		}
+	}
+
+	// Store each file located in the output directory as an artifact.
+	// TODO(jjm): model a directory entry in the artifact map.
+	err = filepath.WalkDir(tempDirSlice, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() {
+			// artifact key, relative to the output field from the transformer
+			artifact := filepath.Join(string(t.Output), filepath.Base(path))
+			log.DebugContext(ctx, fmt.Sprintf("storing: %s to %s", path, artifact), "label", "slice", "artifact", artifact)
+			// Read the file
+			data, err := os.ReadFile(path)
+			if err != nil {
+				return errors.Format("%s: %w", msg, err)
+			}
+			// Store into the artifact map
+			if err := p.opts.Store.Set(artifact, data); err != nil {
+				return errors.Format("%s: %w", msg, err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return errors.Format("%s: could not walk slice output dir: %w", msg, err)
+	}
+
+	return nil
+}
+
 func validate(ctx context.Context, validator core.Validator, p taskParams) error {
 	store := p.opts.Store
 	tempDir, err := os.MkdirTemp("", "holos.validate")

internal/generate/platforms/cue.mod/gen/github.com/holos-run/holos/api/core/v1alpha5/types_go_gen.cue

@@ -211,11 +211,13 @@ package core
 //  1. [Kustomize] - Patch and transform the output from prior generators or
 //     transformers.  See [Introduction to Kustomize].
 //  2. [Join] - Concatenate multiple prior outputs into one output.
+//  3. [Slice] - Slice an artifact into multiple artifacts using [kubectl-slice].
 //
 // [Introduction to Kustomize]: https://kubectl.docs.kubernetes.io/guides/config_management/introduction/
+// [kubectl-slice]: https://github.com/patrickdappollonio/kubectl-slice
 #Transformer: {
 	// Kind represents the kind of transformer. Must be Kustomize, or Join.
-	kind: string & ("Kustomize" | "Join") @go(Kind)
+	kind: string & ("Kustomize" | "Join" | "Slice") @go(Kind)
 
 	// Inputs represents the files to transform. The Output of prior Generators
 	// and Transformers.

service/buf.lock

@@ -4,5 +4,5 @@ deps:
   - remote: buf.build
     owner: bufbuild
     repository: protovalidate
-    commit: a3320276596649bcad929ac829d451f4
-    digest: shake256:a6e5f64fd3fd47e3e8568e9753f59a1566f56c11ec055baf65463d3bca3499f6f16c2d6f5628fa41cfd0f4fa7e72abe65be4efd77d269749492472ed4cc4070d
+    commit: d39267d9df8f4053bbac6b956a23169f
+    digest: shake256:bfb2a6f67179429b8fec30ddd03fc42538cc4d2d207e3c89e325de75fd3716335be06114666d2dd0a099eaf37cbc685c5d1b1a347537af74017e780280e64a9c

tools/tools.go

@@ -6,14 +6,15 @@ package tools
 // https://go.dev/wiki/Modules
 
 import (
-  _ "sigs.k8s.io/kustomize/kustomize/v5"
 	_ "connectrpc.com/connect/cmd/protoc-gen-connect-go"
 	_ "cuelang.org/go/cmd/cue"
 	_ "github.com/bufbuild/buf/cmd/buf"
 	_ "github.com/fullstorydev/grpcurl/cmd/grpcurl"
 	_ "github.com/google/ko"
+	_ "github.com/patrickdappollonio/kubectl-slice"
 	_ "github.com/princjef/gomarkdoc/cmd/gomarkdoc"
 	_ "golang.org/x/tools/cmd/godoc"
 	_ "google.golang.org/protobuf/cmd/protoc-gen-go"
 	_ "honnef.co/go/tools/cmd/staticcheck"
+	_ "sigs.k8s.io/kustomize/kustomize/v5"
 )