component: refactor to consistently use absolute paths

Testing is problematic because the current working directory is not the
platform root.  This patch refactors the codebase to consistently store
and use the platform root directory to construct absolute paths for
reading and writing files during a BuldPlan execution.
This commit is contained in:
Jeff McCune
2025-04-04 13:29:52 -07:00
parent 33bc8f8e23
commit fafae89403
11 changed files with 240 additions and 103 deletions

View File

@@ -8,8 +8,6 @@ import (
"path/filepath"
"strings"
"sync"
"github.com/holos-run/holos/internal/errors"
)
// NewStore should provide a concrete Store.
@@ -54,7 +52,7 @@ func (a *MapStore) Set(path string, data []byte) error {
a.mu.Lock()
defer a.mu.Unlock()
if _, ok := a.m[path]; ok {
return errors.Format("%s already set", path)
return fmt.Errorf("%s already set", path)
}
a.m[path] = data
slog.Debug(fmt.Sprintf("store: set path %s", path), "component", "store", "op", "set", "path", path, "bytes", len(data))
@@ -70,10 +68,15 @@ func (a *MapStore) Get(path string) (data []byte, ok bool) {
return
}
// Save writes a file or directory tree to the filesystem.
// Save writes a file or directory tree to the filesystem. dir must be an
// absolute path.
func (a *MapStore) Save(dir, path string) error {
if !filepath.IsAbs(dir) {
return fmt.Errorf("path not absolute: %s", dir)
}
if strings.HasSuffix(path, "/") {
return errors.Format("path must not end in a /")
return fmt.Errorf("path must not end in a /: %s", path)
}
fullPath := filepath.Join(dir, path)
@@ -82,10 +85,10 @@ func (a *MapStore) Save(dir, path string) error {
// Save a single file and return.
if data, ok := a.Get(path); ok {
if err := os.MkdirAll(filepath.Dir(fullPath), 0777); err != nil {
return errors.Format("%s: %w", msg, err)
return fmt.Errorf("%s: %w", msg, err)
}
if err := os.WriteFile(fullPath, data, 0666); err != nil {
return errors.Format("%s: %w", msg, err)
return fmt.Errorf("%s: %w", msg, err)
}
return nil
}
@@ -98,10 +101,10 @@ func (a *MapStore) Save(dir, path string) error {
data, _ := a.Get(key)
fullPath = filepath.Join(dir, key)
if err := os.MkdirAll(filepath.Dir(fullPath), 0777); err != nil {
return errors.Format("%s: %w", msg, err)
return fmt.Errorf("%s: %w", msg, err)
}
if err := os.WriteFile(fullPath, data, 0666); err != nil {
return errors.Format("%s: %w", msg, err)
return fmt.Errorf("%s: %w", msg, err)
}
}
}
@@ -111,28 +114,31 @@ func (a *MapStore) Save(dir, path string) error {
// Load saves a file or directory tree to the store.
func (a *MapStore) Load(dir, path string) error {
fileSystem := os.DirFS(dir)
err := fs.WalkDir(fileSystem, path, func(path string, d fs.DirEntry, err error) error {
if !filepath.IsAbs(dir) {
return fmt.Errorf("path not absolute: %s", dir)
}
fsys := os.DirFS(dir)
err := fs.WalkDir(fsys, path, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return errors.Wrap(err)
return err
}
// Skip over directories.
if d.IsDir() {
return nil
}
// Load files into the store.
data, err := fs.ReadFile(fileSystem, path)
data, err := fs.ReadFile(fsys, path)
if err != nil {
return errors.Wrap(err)
return err
}
if err := a.Set(path, data); err != nil {
return errors.Wrap(err)
return err
}
return nil
})
if err != nil {
return errors.Wrap(err)
return err
}
return nil

View File

@@ -31,8 +31,8 @@ func NewShowCmd(cfg *platform.Config) (cmd *cobra.Command) {
cmd.AddCommand(spCmd)
sbp := &showBuildPlans{
Format: "yaml",
Out: cfg.Stdout,
format: "yaml",
cfg: cfg,
}
sbCmd := platform.NewCommand(cfg, sbp.Run)
sbCmd.Use = "buildplans"
@@ -66,18 +66,18 @@ func (s *showPlatform) Run(ctx context.Context, p *platform.Platform) error {
}
type showBuildPlans struct {
Format string
Out io.Writer
format string
cfg *platform.Config
}
func (s *showBuildPlans) flagSet() *pflag.FlagSet {
fs := pflag.NewFlagSet("", pflag.ContinueOnError)
fs.StringVar(&s.Format, "format", "yaml", "yaml or json format")
fs.StringVar(&s.format, "format", "yaml", "yaml or json format")
return fs
}
func (s *showBuildPlans) Run(ctx context.Context, p *platform.Platform) error {
encoder, err := holos.NewSequentialEncoder(s.Format, s.Out)
encoder, err := holos.NewSequentialEncoder(s.format, s.cfg.Stdout)
if err != nil {
return errors.Wrap(err)
}
@@ -90,8 +90,7 @@ func (s *showBuildPlans) Run(ctx context.Context, p *platform.Platform) error {
if err != nil {
return errors.Wrap(err)
}
opts := holos.NewBuildOpts(pc.Path())
opts.BuildContext.TempDir = "${TMPDIR_PLACEHOLDER}"
opts := holos.NewBuildOpts(p.Root(), pc.Path(), s.cfg.WriteTo, "${TMPDIR_PLACEHOLDER}")
// TODO(jjm): refactor into [holos.NewBuildOpts] as functional options.
// Component name, label, annotations passed via tags to cue.

View File

@@ -95,7 +95,7 @@ func (c *Component) BuildPlan(tm holos.TypeMeta, opts holos.BuildOpts) (BuildPla
switch tm.APIVersion {
case "v1alpha6":
// Prepare runtime build context for injection as a cue tag.
bc := v1alpha6.NewBuildContext(opts.BuildContext)
bc := v1alpha6.NewBuildContext(opts.TempDir())
buildContextTags, err := bc.Tags()
if err != nil {
return bp, errors.Format("could not get build context tag: %w", err)
@@ -150,11 +150,9 @@ func (c *Component) render(ctx context.Context, tm holos.TypeMeta) error {
defer util.Remove(ctx, tempDir)
// Runtime configuration of the build.
opts := holos.NewBuildOpts(c.Path)
opts := holos.NewBuildOpts(c.Root, c.Path, c.WriteTo, tempDir)
opts.Stderr = c.Stderr
opts.Concurrency = c.Concurrency
opts.WriteTo = filepath.Join(c.Root, c.WriteTo)
opts.BuildContext.TempDir = tempDir
log := logger.FromContext(ctx)
log.DebugContext(ctx, fmt.Sprintf("rendering %s kind %s version %s", c.Path, tm.Kind, tm.APIVersion), "kind", tm.Kind, "apiVersion", tm.APIVersion, "path", c.Path)
@@ -187,10 +185,9 @@ func (c *Component) renderAlpha5(ctx context.Context) error {
defer util.Remove(ctx, tempDir)
// Runtime configuration of the build.
opts := holos.NewBuildOpts(c.Path)
opts := holos.NewBuildOpts(c.Root, c.Path, c.WriteTo, "")
opts.Stderr = c.Stderr
opts.Concurrency = c.Concurrency
opts.WriteTo = filepath.Join(c.Root, c.WriteTo)
tm := holos.TypeMeta{
Kind: "BuildPlan",

View File

@@ -117,7 +117,12 @@ type taskParams struct {
}
func (t taskParams) id() string {
return fmt.Sprintf("%s:%s/%s", t.opts.Path, t.buildPlanName, t.taskName)
return fmt.Sprintf(
"%s:%s/%s",
filepath.Clean(t.opts.Leaf()),
t.buildPlanName,
t.taskName,
)
}
type generatorTask struct {
@@ -149,7 +154,7 @@ func (t generatorTask) run(ctx context.Context) error {
}
func (t generatorTask) file() error {
data, err := os.ReadFile(filepath.Join(string(t.opts.Path), string(t.generator.File.Source)))
data, err := os.ReadFile(filepath.Join(t.opts.AbsPath(), string(t.generator.File.Source)))
if err != nil {
return errors.Wrap(err)
}
@@ -162,7 +167,7 @@ func (t generatorTask) file() error {
func (t generatorTask) helm(ctx context.Context) error {
h := t.generator.Helm
// Cache the chart by version to pull new versions. (#273)
cacheDir := filepath.Join(string(t.opts.Path), "vendor", t.generator.Helm.Chart.Version)
cacheDir := filepath.Join(t.opts.AbsPath(), "vendor", t.generator.Helm.Chart.Version)
cachePath := filepath.Join(cacheDir, filepath.Base(h.Chart.Name))
log := logger.FromContext(ctx)
@@ -455,11 +460,11 @@ func buildArtifact(ctx context.Context, idx int, artifact core.Artifact, tasks c
// Write the final artifact
out := string(artifact.Artifact)
if err := opts.Store.Save(opts.WriteTo, out); err != nil {
if err := opts.Store.Save(opts.AbsWriteTo(), out); err != nil {
return errors.Format("%s: %w", msg, err)
}
log := logger.FromContext(ctx)
log.DebugContext(ctx, fmt.Sprintf("wrote %s", filepath.Join(opts.WriteTo, out)))
log.DebugContext(ctx, fmt.Sprintf("wrote %s", filepath.Join(opts.AbsWriteTo(), out)))
return nil
}
@@ -472,8 +477,7 @@ type BuildPlan struct {
func (b *BuildPlan) Build(ctx context.Context) error {
name := b.BuildPlan.Metadata.Name
path := b.Opts.Path
log := logger.FromContext(ctx).With("name", name, "path", path)
log := logger.FromContext(ctx).With("name", name, "path", filepath.Clean(b.Opts.Leaf()))
msg := fmt.Sprintf("could not build %s", name)
if b.BuildPlan.Spec.Disabled {
@@ -595,7 +599,7 @@ func kustomize(ctx context.Context, t core.Transformer, p taskParams) error {
"could not transform %s for %s path %s",
t.Output,
p.buildPlanName,
p.opts.Path,
filepath.Clean(p.opts.Leaf()),
)
// Write the kustomization

View File

@@ -24,32 +24,8 @@ import (
"helm.sh/helm/v3/pkg/cli"
)
// Platform represents a platform builder.
type Platform struct {
Platform core.Platform
}
// Load loads from a cue value.
func (p *Platform) Load(v cue.Value) error {
return errors.Wrap(v.Decode(&p.Platform))
}
func (p *Platform) Export(encoder holos.Encoder) error {
if err := encoder.Encode(&p.Platform); err != nil {
return errors.Wrap(err)
}
return nil
}
func (p *Platform) Select(selectors ...holos.Selector) []holos.Component {
components := make([]holos.Component, 0, len(p.Platform.Spec.Components))
for _, component := range p.Platform.Spec.Components {
if holos.IsSelected(component.Labels, selectors...) {
components = append(components, &Component{component})
}
}
return components
}
// TODO(jjm) accept an interface to run commands to inject a mock runner from
// the tests.
type Component struct {
Component core.Component
@@ -122,7 +98,7 @@ func (c *Component) ExtractYAML() ([]string, error) {
}
var _ holos.BuildPlan = &BuildPlan{}
var _ task = generatorTask{}
var _ task = &generatorTask{}
var _ task = transformersTask{}
var _ task = validatorTask{}
@@ -138,7 +114,16 @@ type taskParams struct {
}
func (t taskParams) id() string {
return fmt.Sprintf("%s:%s/%s", t.opts.Path, t.buildPlanName, t.taskName)
path := filepath.Clean(t.opts.Leaf())
return fmt.Sprintf("%s:%s/%s", path, t.buildPlanName, t.taskName)
}
func (t taskParams) tempDir() (string, error) {
if tempDir := t.opts.TempDir(); tempDir == "" {
return "", errors.Format("missing build context temp directory")
} else {
return tempDir, nil
}
}
type generatorTask struct {
@@ -147,7 +132,7 @@ type generatorTask struct {
wg *sync.WaitGroup
}
func (t generatorTask) run(ctx context.Context) error {
func (t *generatorTask) run(ctx context.Context) error {
defer t.wg.Done()
msg := fmt.Sprintf("could not build %s", t.id())
switch t.generator.Kind {
@@ -163,14 +148,18 @@ func (t generatorTask) run(ctx context.Context) error {
if err := t.file(); err != nil {
return errors.Format("%s: could not generate file: %w", msg, err)
}
case "Command":
if err := t.command(ctx); err != nil {
return errors.Format("%s: could not generate from command: %w", msg, err)
}
default:
return errors.Format("%s: unsupported kind %s", msg, t.generator.Kind)
}
return nil
}
func (t generatorTask) file() error {
data, err := os.ReadFile(filepath.Join(string(t.opts.Path), string(t.generator.File.Source)))
func (t *generatorTask) file() error {
data, err := os.ReadFile(filepath.Join(t.opts.AbsPath(), string(t.generator.File.Source)))
if err != nil {
return errors.Wrap(err)
}
@@ -180,10 +169,10 @@ func (t generatorTask) file() error {
return nil
}
func (t generatorTask) helm(ctx context.Context) error {
func (t *generatorTask) helm(ctx context.Context) error {
h := t.generator.Helm
// Cache the chart by version to pull new versions. (#273)
cacheDir := filepath.Join(string(t.opts.Path), "vendor", t.generator.Helm.Chart.Version)
cacheDir := filepath.Join(t.opts.AbsPath(), "vendor", t.generator.Helm.Chart.Version)
cachePath := filepath.Join(cacheDir, filepath.Base(h.Chart.Name))
log := logger.FromContext(ctx)
@@ -307,7 +296,7 @@ func (t generatorTask) helm(ctx context.Context) error {
return nil
}
func (t generatorTask) resources() error {
func (t *generatorTask) resources() error {
var size int
for _, m := range t.generator.Resources {
size += len(m)
@@ -338,6 +327,39 @@ func (t generatorTask) resources() error {
return nil
}
func (t *generatorTask) command(ctx context.Context) error {
store := t.opts.Store
msg := fmt.Sprintf("could not generate from command %s", t.id())
args := t.generator.Command.Args
if len(args) < 1 {
return errors.Format("%s: command args length must be at least 1", msg)
}
r, err := util.RunCmdA(ctx, t.opts.Stderr, args[0], args[1:]...)
if err != nil {
return errors.Format("%s: %w", msg, err)
}
if t.generator.Command.Stdout {
// Store the command stdout as the artifact.
if err := store.Set(string(t.generator.Output), r.Stdout.Bytes()); err != nil {
return errors.Format("%s: %w", msg, err)
}
} else {
// Store the file tree as the artifact.
tempDir, err := t.taskParams.tempDir()
if err != nil {
return errors.Format("%s: %w", msg, err)
}
if err := store.Load(tempDir, string(t.generator.Output)); err != nil {
return errors.Format("%s: %w", msg, err)
}
}
return nil
}
type transformersTask struct {
taskParams
transformers []core.Transformer
@@ -422,7 +444,7 @@ func buildArtifact(ctx context.Context, idx int, artifact core.Artifact, tasks c
msg := fmt.Sprintf("could not build %s artifact %s", buildPlanName, artifact.Artifact)
// Process Generators concurrently
for gid, gen := range artifact.Generators {
task := generatorTask{
task := &generatorTask{
taskParams: taskParams{
taskName: fmt.Sprintf("artifact/%d/generator/%d", idx, gid),
buildPlanName: buildPlanName,
@@ -480,11 +502,11 @@ func buildArtifact(ctx context.Context, idx int, artifact core.Artifact, tasks c
// Write the final artifact
out := string(artifact.Artifact)
if err := opts.Store.Save(opts.WriteTo, out); err != nil {
if err := opts.Store.Save(opts.AbsWriteTo(), out); err != nil {
return errors.Format("%s: %w", msg, err)
}
log := logger.FromContext(ctx)
log.DebugContext(ctx, fmt.Sprintf("wrote %s", filepath.Join(opts.WriteTo, out)))
log.DebugContext(ctx, fmt.Sprintf("wrote %s", filepath.Join(opts.AbsWriteTo(), out)))
return nil
}
@@ -497,8 +519,10 @@ type BuildPlan struct {
func (b *BuildPlan) Build(ctx context.Context) error {
name := b.BuildPlan.Metadata.Name
path := b.Opts.Path
log := logger.FromContext(ctx).With("name", name, "path", path)
log := logger.FromContext(ctx).With(
"name", name,
"path", filepath.Clean(b.Opts.Leaf()),
)
msg := fmt.Sprintf("could not build %s", name)
if b.BuildPlan.Spec.Disabled {
@@ -617,11 +641,11 @@ func commandTransformer(ctx context.Context, t core.Transformer, p taskParams) e
"could not transform %s for %s path %s",
t.Output,
p.buildPlanName,
p.opts.Path,
p.opts.Leaf(),
)
// Sanity checks.
tempDir := p.opts.BuildContext.TempDir
tempDir := p.opts.TempDir()
if tempDir == "" {
return errors.Format("%s: holos maintainer error: BuildContext.TempDir not provided by holos", msg)
}
@@ -666,7 +690,7 @@ func kustomize(ctx context.Context, t core.Transformer, p taskParams) error {
"could not transform %s for %s path %s",
t.Output,
p.buildPlanName,
p.opts.Path,
filepath.Clean(p.opts.Leaf()),
)
// Write the kustomization
@@ -701,6 +725,7 @@ func kustomize(ctx context.Context, t core.Transformer, p taskParams) error {
return nil
}
// TODO(jjm) move to transformerTask command
func validate(ctx context.Context, validator core.Validator, p taskParams) error {
store := p.opts.Store
tempDir, err := os.MkdirTemp("", "holos.validate")
@@ -753,10 +778,10 @@ func (bc BuildContext) Tags() ([]string, error) {
}
// NewBuildContext returns a new BuildContext
func NewBuildContext(bc holos.BuildContext) BuildContext {
func NewBuildContext(tempDir string) BuildContext {
return BuildContext{
BuildContext: core.BuildContext{
TempDir: bc.TempDir,
TempDir: tempDir,
},
}
}

View File

@@ -1,6 +1,7 @@
package v1alpha6_test
import (
"fmt"
"testing"
"github.com/holos-run/holos/internal/holos"
@@ -13,30 +14,51 @@ const apiVersion string = "v1alpha6"
func TestComponents(t *testing.T) {
tempDir := testutil.SetupPlatform(t, apiVersion)
h := testutil.NewComponentHarness(t, tempDir, apiVersion)
root := h.Root()
assert.NotEmpty(t, root)
t.Run("Minimal", func(t *testing.T) {
msg := "Expected a minimal component to work, but do nothing"
h := testutil.NewComponentHarness(t, tempDir, apiVersion)
root := h.Root()
assert.NotEmpty(t, root)
t.Run("WithNoArtifacts", func(t *testing.T) {
leaf := "components/minimal"
c := h.Component(leaf)
msg := fmt.Sprintf("Expected %s with no artifacts to work, but do nothing", leaf)
componentPath := "components/minimal"
c := h.Component(componentPath)
tm, err := c.TypeMeta()
require.NoError(t, err, msg)
t.Run("TypeMeta", func(t *testing.T) {
tm, err := c.TypeMeta()
require.NoError(t, err, msg)
assert.Equal(t, apiVersion, tm.APIVersion)
assert.Equal(t, "BuildPlan", tm.Kind)
assert.Equal(t, apiVersion, tm.APIVersion, msg)
assert.Equal(t, "BuildPlan", tm.Kind, msg)
})
t.Run("BuildPlan", func(t *testing.T) {
tm, err := c.TypeMeta()
require.NoError(t, err, msg)
bp, err := c.BuildPlan(tm, holos.NewBuildOpts(componentPath))
bp, err := c.BuildPlan(tm, holos.NewBuildOpts(root, leaf, "deploy", t.TempDir()))
require.NoError(t, err, msg)
err = bp.Build(h.Ctx())
require.NoError(t, err, msg)
})
})
t.Run("BuildPlan", func(t *testing.T) {
t.Run("Command", func(t *testing.T) {
t.Run("Generator", func(t *testing.T) {
leaf := "components/commands/generator/simple"
c := h.Component(leaf)
msg := fmt.Sprintf("Expected %s with command generator to render config manifests", leaf)
tm, err := c.TypeMeta()
require.NoError(t, err, msg)
assert.Equal(t, tm.APIVersion, apiVersion)
t.Run("Build", func(t *testing.T) {
bp, err := c.BuildPlan(tm, holos.NewBuildOpts(root, leaf, "deploy", t.TempDir()))
require.NoError(t, err, msg)
err = bp.Build(h.Ctx())
require.NoError(t, err, msg)
// TODO: Check the rendered manifests.
})
})
t.Run("Transformer", func(t *testing.T) {})
t.Run("Validator", func(t *testing.T) {})
})
})
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
@@ -327,22 +328,54 @@ type BuildOpts struct {
// Tags represents user managed tags including a component name, labels, and
// annotations.
Tags []string
// BuildContext represents holos managed tags.
BuildContext BuildContext
root string
leaf string
writeTo string
tempDir string
}
// NewBuildOpts returns a [BuildOpts] configured to build the component at path.
func NewBuildOpts(path string) BuildOpts {
// NewBuildOpts returns a [BuildOpts] configured to build the component at leaf
// from the platform module at root writing rendered manifests into the deploy
// directory.
func NewBuildOpts(root, leaf, deploy, tempDir string) BuildOpts {
return BuildOpts{
Store: artifact.NewStore(),
Concurrency: min(runtime.NumCPU(), 8),
Stderr: os.Stderr,
WriteTo: "deploy",
Path: path,
Tags: make([]string, 0, 10),
root: root,
leaf: leaf,
writeTo: deploy,
tempDir: tempDir,
}
}
// Leaf returns the cleaned component path relative to the platform root. For
// example "components/podinfo"
func (b *BuildOpts) Leaf() string {
return filepath.Clean(b.leaf)
}
// TODO(jjm) rename to AbsLeaf() and document.
func (b *BuildOpts) AbsPath() string {
return filepath.Join(b.root, b.leaf)
}
// AbsDeploy returns the absolute path to the write to directory, usually the
// deploy sub directory of the platform module root.
func (b *BuildOpts) AbsWriteTo() string {
return filepath.Join(b.root, b.writeTo)
}
// TempDir returns the temporary directory managed by holos and injected into
// cue using a [BuildContext] so artifacts can refer to the same path in the
// configuration.
func (b *BuildOpts) TempDir() string {
return b.tempDir
}
// BuildContext represents build context values provided by the holos render
// component command. These values are expected to be randomly generated and
// late binding, meaning they cannot be known ahead of time in a static

View File

@@ -0,0 +1,33 @@
package holos
import (
"encoding/json"
"github.com/holos-run/holos/api/core/v1alpha6:core"
)
// Example of a simple v1alpha6 command generator.
holos: core.#BuildPlan & {
metadata: {
name: "simple"
labels: "holos.run/component.name": name
annotations: "app.holos.run/description": "\(name) command generator"
}
spec: artifacts: [{
artifact: "components/\(metadata.name)/\(metadata.name).gen.yaml"
generators: [{
kind: "Command"
output: artifact
command: {
args: ["/bin/echo", json.Marshal(_ConfigMap)]
stdout: true
}
}]
}]
}
_ConfigMap: {
apiVersion: "v1"
kind: "ConfigMap"
metadata: name: "simple"
}

View File

@@ -0,0 +1,16 @@
@extern(embed)
package holos
import (
"encoding/json"
"github.com/holos-run/holos/api/core/v1alpha6:core"
)
_BuildContext: string | *"{}" @tag(holos_build_context, type=string)
BuildContext: core.#BuildContext & json.Unmarshal(_BuildContext)
holos: core.#BuildPlan & {
buildContext: BuildContext
}
holos: _ @embed(file=typemeta.yaml)

View File

@@ -0,0 +1,2 @@
kind: BuildPlan
apiVersion: v1alpha6

View File

@@ -12,7 +12,7 @@ import (
"github.com/holos-run/holos/internal/component"
"github.com/holos-run/holos/internal/errors"
"github.com/holos-run/holos/internal/generate"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
)
//go:embed all:fixtures