Compare commits

..

4 Commits

Author SHA1 Message Date
Jeff McCune
9e60ddbe85 (#175) Add holos generate component cue command
This patch adds a command to generate CUE based holos components from
examples embedded in the executable.  The examples are passed through
the go template rendering engine with values pulled from flags.

Each directory in the embedded filesystem becomes a unique command for
nice tab completion.  The `--name` flag defaults to "example" and is the
resulting component name.

A follow up patch with more flags will set the stage for a Helm
component schematic.

```
holos generate component cue minimal
```

```txt
3:07PM INF component.go:91 generated component version=0.80.2 name=example path=/home/jeff/holos/dev/bare/components/example
```
2024-05-20 15:10:54 -07:00
Jeff McCune
44334fca52 (#175) Fix lint 2024-05-20 12:39:43 -07:00
Jeff McCune
2e2ed398c6 (#175) Fix tests 2024-05-20 11:32:29 -07:00
Jeff McCune
34f2a52cb7 (#175) Add holos render platform command
Split holos render into component and platform.

This patch splits the previous `holos render` command into subcommands.
`holos render component ./path/to/component/` behaves as the previous
`holos render` command and renders an individual component.

The new `holos render platform ./path/to/platform/` subcommand makes
space to render the entire platform using the platform model pulled from
the PlatformService.

Starting with an empty directory:

```sh
holos register user
holos generate platform bare
holos pull platform config .
holos render platform ./platform/
```

```txt
10:01AM INF platform.go:29 ok render component version=0.80.2 path=components/configmap cluster=k1 num=1 total=1 duration=448.133038ms
```

The bare platform has a single component which refers to the platform
model pulled from the PlatformService:

```sh
cat deploy/clusters/mycluster/components/platform-configmap/platform-configmap.gen.yaml
```

```yaml
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: platform
  namespace: default
data:
  platform: |
    spec:
      model:
        cloud:
          providers:
            - cloudflare
        cloudflare:
          email: platform@openinfrastructure.co
        org:
          displayName: Open Infrastructure Services
          name: ois
```
2024-05-20 10:41:24 -07:00
30 changed files with 538 additions and 166 deletions

View File

@@ -9,8 +9,8 @@ import "google.golang.org/protobuf/types/known/structpb"
// model, and holos components to build into one resource.
type Platform struct {
TypeMeta `json:",inline" yaml:",inline"`
Metadata ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
Spec PlatformSpec `json:"spec,omitempty" yaml:"spec,omitempty"`
Metadata ObjectMeta `json:"metadata" yaml:"metadata"`
Spec PlatformSpec `json:"spec" yaml:"spec"`
}
// PlatformSpec represents the platform build plan specification.
@@ -18,5 +18,15 @@ type PlatformSpec struct {
// Model represents the platform model holos gets from from the
// holos.platform.v1alpha1.PlatformService.GetPlatform method and provides to
// CUE using a tag.
Model structpb.Struct `json:"model,omitempty" yaml:"model,omitempty"`
Model structpb.Struct `json:"model" yaml:"model"`
Components []PlatformSpecComponent `json:"components" yaml:"components"`
}
// PlatformSpecComponent represents a component to build or render with flags to
// pass, for example the cluster name.
type PlatformSpecComponent struct {
// Path is the path of the component relative to the platform root.
Path string `json:"path" yaml:"path"`
// Cluster is the cluster name to use when building the component.
Cluster string `json:"cluster" yaml:"cluster"`
}

View File

@@ -2,6 +2,8 @@
exec holos build ./foo/... --log-level debug
stdout '^bf2bc7f9-9ba0-4f9e-9bd2-9a205627eb0b$'
-- platform.config.json --
{}
-- cue.mod --
package holos
-- foo/constraints.cue --
@@ -20,6 +22,7 @@ spec: components: KubernetesObjectsList: [
package holos
_cluster: string @tag(cluster, string)
_platform_config: string @tag(platform_config, string)
#KubernetesObjects: {
apiVersion: "holos.run/v1alpha1"

View File

@@ -3,12 +3,15 @@
stderr 'apiObjectMap.foo.bar: cannot convert incomplete value'
stderr '/component.cue:\d+:\d+$'
-- platform.config.json --
{}
-- cue.mod --
package holos
-- component.cue --
package holos
_cluster: string @tag(cluster, string)
_platform_config: string @tag(platform_config, string)
apiVersion: "holos.run/v1alpha1"
kind: "BuildPlan"

View File

@@ -3,6 +3,8 @@ exec holos build .
stdout '^kind: SecretStore$'
stdout '# Source: CUE apiObjects.SecretStore.default'
-- platform.config.json --
{}
-- cue.mod --
package holos
-- component.cue --
@@ -13,6 +15,7 @@ kind: "BuildPlan"
spec: components: KubernetesObjectsList: [{apiObjectMap: #APIObjects.apiObjectMap}]
_cluster: string @tag(cluster, string)
_platform_config: string @tag(platform_config, string)
#SecretStore: {
kind: string

View File

@@ -4,6 +4,8 @@ stdout '^kind: SecretStore$'
stdout '# Source: CUE apiObjects.SecretStore.default'
stderr 'skipping helm: no chart name specified'
-- platform.config.json --
{}
-- cue.mod --
package holos
-- component.cue --
@@ -14,6 +16,7 @@ kind: "BuildPlan"
spec: components: HelmChartList: [{apiObjectMap: #APIObjects.apiObjectMap}]
_cluster: string @tag(cluster, string)
_platform_config: string @tag(platform_config, string)
#SecretStore: {
kind: string

View File

@@ -2,6 +2,8 @@
! exec holos build .
stderr 'apiObjects.secretstore.default.foo: field not allowed'
-- platform.config.json --
{}
-- cue.mod --
package holos
-- component.cue --
@@ -10,6 +12,7 @@ package holos
apiVersion: "holos.run/v1alpha1"
kind: "KubernetesObjects"
cluster: string @tag(cluster, string)
_platform_config: string @tag(platform_config, string)
#SecretStore: {
metadata: name: string

View File

@@ -2,6 +2,8 @@
! exec holos build .
stderr 'Error: execution error at \(zitadel/templates/secret_zitadel-masterkey.yaml:2:4\): Either set .Values.zitadel.masterkey xor .Values.zitadel.masterkeySecretName'
-- platform.config.json --
{}
-- cue.mod --
package holos
-- zitadel.cue --
@@ -12,6 +14,7 @@ kind: "BuildPlan"
spec: components: HelmChartList: [_HelmChart]
_cluster: string @tag(cluster, string)
_platform_config: string @tag(platform_config, string)
_HelmChart: {
apiVersion: "holos.run/v1alpha1"

View File

@@ -1,15 +1,18 @@
# Kustomize is a supported holos component kind
exec holos render --cluster-name=mycluster . --log-level=debug
exec holos render component --cluster-name=mycluster . --log-level=debug
# Want generated output
cmp want.yaml deploy/clusters/mycluster/components/kstest/kstest.gen.yaml
-- platform.config.json --
{}
-- cue.mod --
package holos
-- component.cue --
package holos
_cluster: string @tag(cluster, string)
_platform_config: string @tag(platform_config, string)
apiVersion: "holos.run/v1alpha1"
kind: "BuildPlan"

View File

@@ -3,11 +3,14 @@
! exec holos build .
stderr 'unknown field \\"TypoKubernetesObjectsList\\"'
-- platform.config.json --
{}
-- cue.mod --
package holos
-- component.cue --
package holos
_cluster: string @tag(cluster, string)
_platform_config: string @tag(platform_config, string)
apiVersion: "holos.run/v1alpha1"
kind: "BuildPlan"

View File

@@ -186,14 +186,6 @@ func (b Builder) runInstance(ctx context.Context, instance *build.Instance) (res
return
}
results, err = b.buildPlan(ctx, &bp, path)
// TODO(jeff) Platform should not return a Result like an individual holos component does.
case "Platform":
var pf v1alpha1.Platform
if err = decoder.Decode(&pf); err != nil {
err = errors.Wrap(fmt.Errorf("could not decode Platform %s: %w", instance.Dir, err))
return
}
results, err = b.buildPlatform(ctx, &pf)
default:
err = errors.Wrap(fmt.Errorf("unknown kind: %v", tm.Kind))
}
@@ -201,33 +193,6 @@ func (b Builder) runInstance(ctx context.Context, instance *build.Instance) (res
return
}
// buildPlatform builds all of the holos components specified in a Platform resource returned from CUE.
//
// TODO(jeff): There is technical debt here in the way results are handled.
// Results were intended as a way to define how holos should build various kinds
// of individual components, not a whole platform though. After launch,
// refactor the platform building to return something else. The result itself
// also needs to be refactored to an interface instead of a struct. Each kind
// of component should implement the interface.
func (b *Builder) buildPlatform(ctx context.Context, pf *v1alpha1.Platform) (results []*v1alpha1.Result, err error) {
log := logger.FromContext(ctx)
// TODO: Iterate over each platform component in Platform.spec.components.
// each component should note the cluster name and path to the holos
// component.
data, err := pf.Spec.Model.MarshalJSON()
if err != nil {
return nil, errors.Wrap(err)
}
if len(data) > 0 {
data = append(data, '\n')
}
buf := bytes.NewBuffer(data)
if _, err := buf.WriteTo(os.Stdout); err != nil {
log.ErrorContext(ctx, "could not write", "err", err)
}
return nil, errors.Wrap(fmt.Errorf("not implemeneted"))
}
func (b *Builder) buildPlan(ctx context.Context, buildPlan *v1alpha1.BuildPlan, path holos.InstancePath) (results []*v1alpha1.Result, err error) {
log := logger.FromContext(ctx)

View File

@@ -0,0 +1,90 @@
package builder
import (
"bytes"
"context"
"encoding/json"
"fmt"
"cuelang.org/go/cue/build"
"cuelang.org/go/cue/cuecontext"
"github.com/holos-run/holos"
"github.com/holos-run/holos/api/v1alpha1"
"github.com/holos-run/holos/internal/client"
"github.com/holos-run/holos/internal/errors"
"github.com/holos-run/holos/internal/logger"
)
// Platform builds a platform
func (b *Builder) Platform(ctx context.Context, cfg *client.Config) (*v1alpha1.Platform, error) {
log := logger.FromContext(ctx)
log.DebugContext(ctx, "cue: building platform instance")
instances, err := b.Instances(ctx, cfg)
if err != nil {
return nil, errors.Wrap(err)
}
if len(instances) != 1 {
return nil, errors.Wrap(errors.New(fmt.Sprintf("instances length %d must be exactly 1", len(instances))))
}
// We only process the first instance, assume the render platform subcommand enforces this.
instance := instances[0]
log.DebugContext(ctx, "cue: building instance", "dir", instance.Dir)
p, err := b.runPlatform(ctx, instance)
if err != nil {
return nil, errors.Wrap(fmt.Errorf("could not build platform: %w", err))
}
return p, nil
}
func (b Builder) runPlatform(ctx context.Context, instance *build.Instance) (*v1alpha1.Platform, error) {
path := holos.InstancePath(instance.Dir)
log := logger.FromContext(ctx).With("dir", path)
if err := instance.Err; err != nil {
return nil, errors.Wrap(fmt.Errorf("could not load: %w", err))
}
cueCtx := cuecontext.New()
value := cueCtx.BuildInstance(instance)
if err := value.Err(); err != nil {
return nil, errors.Wrap(fmt.Errorf("could not build %s: %w", instance.Dir, err))
}
log.DebugContext(ctx, "cue: validating instance")
if err := value.Validate(); err != nil {
return nil, errors.Wrap(fmt.Errorf("could not validate: %w", err))
}
log.DebugContext(ctx, "cue: decoding holos platform")
jsonBytes, err := value.MarshalJSON()
if err != nil {
return nil, errors.Wrap(fmt.Errorf("could not marshal cue instance %s: %w", instance.Dir, err))
}
decoder := json.NewDecoder(bytes.NewReader(jsonBytes))
// Discriminate the type of build plan.
tm := &v1alpha1.TypeMeta{}
err = decoder.Decode(tm)
if err != nil {
return nil, errors.Wrap(fmt.Errorf("invalid platform: %s: %w", instance.Dir, err))
}
log.DebugContext(ctx, "cue: discriminated build kind: "+tm.Kind, "kind", tm.Kind, "apiVersion", tm.APIVersion)
// New decoder for the full object
decoder = json.NewDecoder(bytes.NewReader(jsonBytes))
decoder.DisallowUnknownFields()
var pf v1alpha1.Platform
switch tm.Kind {
case "Platform":
if err = decoder.Decode(&pf); err != nil {
err = errors.Wrap(fmt.Errorf("could not decode platform %s: %w", instance.Dir, err))
return nil, err
}
return &pf, nil
default:
err = errors.Wrap(fmt.Errorf("unknown kind: %v", tm.Kind))
}
return nil, err
}

View File

@@ -1,9 +1,6 @@
package command
import (
"fmt"
"github.com/holos-run/holos/internal/errors"
"github.com/holos-run/holos/version"
"github.com/spf13/cobra"
)
@@ -20,9 +17,6 @@ func New(name string) *cobra.Command {
CompletionOptions: cobra.CompletionOptions{
HiddenDefaultCmd: true,
},
RunE: func(c *cobra.Command, args []string) error {
return errors.Wrap(fmt.Errorf("could not run %v: not implemented", c.Name()))
},
SilenceUsage: true,
SilenceErrors: true,
}

View File

@@ -20,6 +20,7 @@ func New(cfg *holos.Config) *cobra.Command {
cmd.Args = cobra.NoArgs
cmd.AddCommand(NewPlatform(cfg))
cmd.AddCommand(NewComponent())
return cmd
}
@@ -45,3 +46,44 @@ func NewPlatform(cfg *holos.Config) *cobra.Command {
return cmd
}
// NewComponent returns a command to generate a holos component
func NewComponent() *cobra.Command {
cmd := command.New("component")
cmd.Short = "generate a component from an embedded schematic"
cmd.AddCommand(NewCueComponent())
return cmd
}
func NewCueComponent() *cobra.Command {
cmd := command.New("cue")
cmd.Short = "generate a cue component from an embedded schematic"
components := generate.CueComponents()
cmd.Long = fmt.Sprintf("Embedded cue components available to generate:\n\n %s", strings.Join(components, "\n "))
for _, name := range components {
cmd.AddCommand(makeCueCommand(name))
}
return cmd
}
func makeCueCommand(name string) *cobra.Command {
cmd := command.New(name)
cmd.Short = fmt.Sprintf("generate a %s cue component from an embedded schematic", name)
cmd.Args = cobra.NoArgs
cfg := &generate.CueConfig{}
cmd.Flags().AddGoFlagSet(cfg.FlagSet())
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Root().Context()
if err := generate.GenerateCueComponent(ctx, name, cfg); err != nil {
return errors.Wrap(err)
}
return nil
}
return cmd
}

View File

@@ -11,15 +11,24 @@ import (
"github.com/holos-run/holos/internal/errors"
"github.com/holos-run/holos/internal/holos"
"github.com/holos-run/holos/internal/logger"
"github.com/holos-run/holos/internal/render"
"github.com/spf13/cobra"
)
// New returns the render subcommand for the root command
func New(cfg *holos.Config) *cobra.Command {
cmd := command.New("render [directory...]")
cmd := command.New("render")
cmd.Args = cobra.NoArgs
cmd.Short = "render platform configuration"
cmd.AddCommand(NewComponent(cfg))
cmd.AddCommand(NewPlatform(cfg))
return cmd
}
// New returns the component subcommand for the render command
func NewComponent(cfg *holos.Config) *cobra.Command {
cmd := command.New("component [directory...]")
cmd.Args = cobra.MinimumNArgs(1)
cmd.Short = "write kubernetes api objects to the filesystem"
cmd.Flags().SortFlags = false
cmd.Flags().AddGoFlagSet(cfg.WriteFlagSet())
cmd.Flags().AddGoFlagSet(cfg.ClusterFlagSet())
@@ -78,6 +87,30 @@ func New(cfg *holos.Config) *cobra.Command {
return cmd
}
func NewPlatform(cfg *holos.Config) *cobra.Command {
cmd := command.New("platform [directory]")
cmd.Args = cobra.ExactArgs(1)
cmd.Short = "render all platform components"
config := client.NewConfig(cfg)
cmd.PersistentFlags().AddGoFlagSet(config.ClientFlagSet())
cmd.PersistentFlags().AddGoFlagSet(config.TokenFlagSet())
cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Root().Context()
build := builder.New(builder.Entrypoints(args))
platform, err := build.Platform(ctx, config)
if err != nil {
return errors.Wrap(err)
}
return render.Platform(ctx, platform, cmd.ErrOrStderr())
}
return cmd
}
type Result interface {
Continue() bool
Name() string

View File

@@ -1,7 +0,0 @@
# Want no hash appended
holos create secret test --namespace holos-system --from-file $WORK/test --append-hash=false
stderr ' created: test '
stderr ' secret=test '
-- test --
sekret

View File

@@ -384,7 +384,7 @@ export class PlatformConfig extends Message<PlatformConfig> {
/**
* Platform Model.
*
* @generated from field: optional google.protobuf.Struct platform_model = 2;
* @generated from field: google.protobuf.Struct platform_model = 2;
*/
platformModel?: Struct;
@@ -397,7 +397,7 @@ export class PlatformConfig extends Message<PlatformConfig> {
static readonly typeName = "holos.object.v1alpha1.PlatformConfig";
static readonly fields: FieldList = proto3.util.newFieldList(() => [
{ no: 1, name: "platform_id", kind: "scalar", T: 9 /* ScalarType.STRING */ },
{ no: 2, name: "platform_model", kind: "message", T: Struct, opt: true },
{ no: 2, name: "platform_model", kind: "message", T: Struct },
]);
static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): PlatformConfig {

View File

@@ -0,0 +1,93 @@
package generate
import (
"bytes"
"context"
"embed"
"flag"
"io/fs"
"log/slog"
"os"
"path/filepath"
"text/template"
"github.com/holos-run/holos/internal/errors"
"github.com/holos-run/holos/internal/server/middleware/logger"
)
//go:embed all:components
var components embed.FS
// componentsRoot is the root path to copy component cue code from.
const componentsRoot = "components"
// CueConfig represents the config values passed to cue go templates.
type CueConfig struct {
ComponentName string
flagSet *flag.FlagSet
}
func (c *CueConfig) FlagSet() *flag.FlagSet {
if c == nil {
return nil
}
if c.flagSet != nil {
return c.flagSet
}
fs := flag.NewFlagSet("", flag.ContinueOnError)
fs.StringVar(&c.ComponentName, "name", "example", "component name")
c.flagSet = fs
return fs
}
// CueComponents returns a slice of embedded component schematics or nil if there are none.
func CueComponents() []string {
entries, err := fs.ReadDir(components, filepath.Join(componentsRoot, "cue"))
if err != nil {
return nil
}
dirs := make([]string, 0, len(entries))
for _, entry := range entries {
dirs = append(dirs, entry.Name())
}
return dirs
}
// makeRenderFunc makes a template rendering function for embedded files.
func makeRenderFunc(log *slog.Logger, path string, cfg *CueConfig) func([]byte) *bytes.Buffer {
return func(content []byte) *bytes.Buffer {
tmpl, err := template.New(filepath.Base(path)).Parse(string(content))
if err != nil {
log.Error("could not load template", "err", err)
return bytes.NewBuffer(content)
}
var rendered bytes.Buffer
if err := tmpl.Execute(&rendered, cfg); err != nil {
log.Error("could not execute template", "err", err)
return bytes.NewBuffer(content)
}
return &rendered
}
}
// GenerateCueComponent writes the cue code for a component to the local working
// directory.
func GenerateCueComponent(ctx context.Context, name string, cfg *CueConfig) error {
path := filepath.Join(componentsRoot, "cue", name)
dstPath := filepath.Join(getCwd(ctx), cfg.ComponentName)
log := logger.FromContext(ctx).With("name", cfg.ComponentName, "path", dstPath)
log.DebugContext(ctx, "mkdir")
if err := os.MkdirAll(dstPath, os.ModePerm); err != nil {
return errors.Wrap(err)
}
mapper := makeRenderFunc(log, path, cfg)
if err := copyEmbedFS(ctx, components, path, dstPath, mapper); err != nil {
return errors.Wrap(err)
}
log.InfoContext(ctx, "generated component")
return nil
}

View File

@@ -0,0 +1,32 @@
package holos
import v1 "github.com/holos-run/holos/api/v1alpha1"
import "encoding/yaml"
let ComponentName = "{{ .ComponentName }}"
// The BuildPlan represents the kubernetes api objects to manage. CUE returns
// the build plan to the holos CLI for rendering to plain yaml files.
v1.#BuildPlan & {
spec: components: resources: "\(ComponentName)": {
metadata: name: ComponentName
apiObjectMap: OBJECTS.apiObjectMap
}
}
// OBJECTS represents the kubernetes api objects to manage.
let OBJECTS = v1.#APIObjects & {
// Add Kubernetes API Objects to manage here.
apiObjects: ConfigMap: "\(ComponentName)": {
metadata: {
name: ComponentName
namespace: "default"
}
data: platform: yaml.Marshal(PLATFORM)
}
}
// This is an example of how to refer to the Platform model.
let PLATFORM = {
spec: model: _Platform.spec.model
}

View File

@@ -1,97 +1,17 @@
package generate
import (
"bytes"
"context"
"embed"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"github.com/holos-run/holos/internal/client"
"github.com/holos-run/holos/internal/errors"
"github.com/holos-run/holos/internal/server/middleware/logger"
platform "github.com/holos-run/holos/service/gen/holos/platform/v1alpha1"
)
//go:embed all:platforms
var platforms embed.FS
// root is the root path to copy platform cue code from.
const root = "platforms"
// Platforms returns a slice of embedded platforms or nil if there are none.
func Platforms() []string {
entries, err := fs.ReadDir(platforms, root)
if err != nil {
return nil
}
dirs := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() && entry.Name() != "cue.mod" {
dirs = append(dirs, entry.Name())
}
}
return dirs
}
// GeneratePlatform writes the cue code for a platform to the local working
// directory.
func GeneratePlatform(ctx context.Context, rpc *client.Client, orgID string, name string) error {
log := logger.FromContext(ctx)
// Check for a valid platform
platformPath := filepath.Join(root, name)
if !dirExists(platforms, platformPath) {
return errors.Wrap(fmt.Errorf("cannot generate: have: [%s] want: %+v", name, Platforms()))
}
// Link the local platform the SaaS platform ID.
rpcPlatforms, err := rpc.Platforms(ctx, orgID)
if err != nil {
return errors.Wrap(err)
}
var rpcPlatform *platform.Platform
for _, p := range rpcPlatforms {
if p.GetName() == name {
rpcPlatform = p
break
}
}
if rpcPlatform == nil {
return errors.Wrap(errors.New("cannot generate: platform not found in the holos server"))
}
// Write the platform data.
data, err := json.MarshalIndent(rpcPlatform, "", " ")
if err != nil {
return errors.Wrap(err)
}
if len(data) > 0 {
data = append(data, '\n')
}
log = log.With("platform_id", rpcPlatform.GetId())
if err := os.WriteFile(client.PlatformMetadataFile, data, 0644); err != nil {
return errors.Wrap(fmt.Errorf("could not write platform metadata: %w", err))
}
log.InfoContext(ctx, "wrote "+client.PlatformMetadataFile, "path", filepath.Join(getCwd(ctx), client.PlatformMetadataFile))
// Copy the cue.mod directory
if err := copyEmbedFS(ctx, platforms, filepath.Join(root, "cue.mod"), "cue.mod"); err != nil {
return errors.Wrap(err)
}
// Copy the named platform
if err := copyEmbedFS(ctx, platforms, platformPath, "."); err != nil {
return errors.Wrap(err)
}
log.InfoContext(ctx, "generated platform "+name, "path", getCwd(ctx))
return nil
}
func dirExists(srcFS embed.FS, path string) bool {
entries, err := fs.ReadDir(srcFS, path)
if err != nil {
@@ -100,7 +20,9 @@ func dirExists(srcFS embed.FS, path string) bool {
return len(entries) > 0
}
func copyEmbedFS(ctx context.Context, srcFS embed.FS, srcPath, dstPath string) error {
// copyEmbedFS copies embedded files from srcPath to dstPath passing the
// contents through mapFunc.
func copyEmbedFS(ctx context.Context, srcFS embed.FS, srcPath, dstPath string, mapFunc func([]byte) *bytes.Buffer) error {
log := logger.FromContext(ctx)
return fs.WalkDir(srcFS, srcPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
@@ -124,7 +46,8 @@ func copyEmbedFS(ctx context.Context, srcFS embed.FS, srcPath, dstPath string) e
if err != nil {
return errors.Wrap(err)
}
if err := os.WriteFile(dstFullPath, data, os.ModePerm); err != nil {
buf := mapFunc(data)
if err := os.WriteFile(dstFullPath, buf.Bytes(), os.ModePerm); err != nil {
return errors.Wrap(err)
}
log.DebugContext(ctx, "wrote", "file", dstFullPath)

View File

@@ -0,0 +1,94 @@
package generate
import (
"bytes"
"context"
"embed"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"github.com/holos-run/holos/internal/client"
"github.com/holos-run/holos/internal/errors"
"github.com/holos-run/holos/internal/logger"
platform "github.com/holos-run/holos/service/gen/holos/platform/v1alpha1"
)
//go:embed all:platforms
var platforms embed.FS
// platformsRoot is the root path to copy platform cue code from.
const platformsRoot = "platforms"
// Platforms returns a slice of embedded platforms or nil if there are none.
func Platforms() []string {
entries, err := fs.ReadDir(platforms, platformsRoot)
if err != nil {
return nil
}
dirs := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() && entry.Name() != "cue.mod" {
dirs = append(dirs, entry.Name())
}
}
return dirs
}
// GeneratePlatform writes the cue code for a platform to the local working
// directory.
func GeneratePlatform(ctx context.Context, rpc *client.Client, orgID string, name string) error {
log := logger.FromContext(ctx)
// Check for a valid platform
platformPath := filepath.Join(platformsRoot, name)
if !dirExists(platforms, platformPath) {
return errors.Wrap(fmt.Errorf("cannot generate: have: [%s] want: %+v", name, Platforms()))
}
// Link the local platform the SaaS platform ID.
rpcPlatforms, err := rpc.Platforms(ctx, orgID)
if err != nil {
return errors.Wrap(err)
}
var rpcPlatform *platform.Platform
for _, p := range rpcPlatforms {
if p.GetName() == name {
rpcPlatform = p
break
}
}
if rpcPlatform == nil {
return errors.Wrap(errors.New("cannot generate: platform not found in the holos server"))
}
// Write the platform data.
data, err := json.MarshalIndent(rpcPlatform, "", " ")
if err != nil {
return errors.Wrap(err)
}
if len(data) > 0 {
data = append(data, '\n')
}
log = log.With("platform_id", rpcPlatform.GetId())
if err := os.WriteFile(client.PlatformMetadataFile, data, 0644); err != nil {
return errors.Wrap(fmt.Errorf("could not write platform metadata: %w", err))
}
log.InfoContext(ctx, "wrote "+client.PlatformMetadataFile, "path", filepath.Join(getCwd(ctx), client.PlatformMetadataFile))
// Copy the cue.mod directory
if err := copyEmbedFS(ctx, platforms, filepath.Join(platformsRoot, "cue.mod"), "cue.mod", bytes.NewBuffer); err != nil {
return errors.Wrap(err)
}
// Copy the named platform
if err := copyEmbedFS(ctx, platforms, platformPath, ".", bytes.NewBuffer); err != nil {
return errors.Wrap(err)
}
log.InfoContext(ctx, "generated platform "+name, "path", getCwd(ctx))
return nil
}

View File

@@ -3,8 +3,6 @@ package holos
import "encoding/yaml"
import v1 "github.com/holos-run/holos/api/v1alpha1"
let PLATFORM = {message: "TODO: Load the platform from the API."}
// Provide a BuildPlan to the holos cli to render k8s api objects.
v1.#BuildPlan & {
spec: components: resources: platformConfigmap: {
@@ -20,6 +18,12 @@ let OBJECTS = v1.#APIObjects & {
name: "platform"
namespace: "default"
}
// Output the platform model which is derived from the web app form the
// platform engineer provides and the form values the end user provides.
data: platform: yaml.Marshal(PLATFORM)
}
}
let PLATFORM = {
spec: model: _Platform.spec.model
}

View File

@@ -22,5 +22,23 @@ _Platform: v1.#Platform & {
spec: {
// model represents the web form values provided by the user.
model: _PlatformConfig.platform_model
components: [for c in _components {c}]
_components: [string]: v1.#PlatformSpecComponent
_components: {
for WorkloadCluster in _Clusters.Workload {
"\(WorkloadCluster)-configmap": {
path: "components/configmap"
cluster: WorkloadCluster
}
}
}
}
}
// _Clusters represents the clusters in the platform. The default values are
// intended to be provided by the user in a file which is not written over by
// `holos generate`.
_Clusters: {
Workload: [...string] | *["mycluster"]
}

View File

@@ -13,8 +13,8 @@ import "google.golang.org/protobuf/types/known/structpb"
// model, and holos components to build into one resource.
#Platform: {
#TypeMeta
metadata?: #ObjectMeta @go(Metadata)
spec?: #PlatformSpec @go(Spec)
metadata: #ObjectMeta @go(Metadata)
spec: #PlatformSpec @go(Spec)
}
// PlatformSpec represents the platform build plan specification.
@@ -22,5 +22,16 @@ import "google.golang.org/protobuf/types/known/structpb"
// Model represents the platform model holos gets from from the
// holos.platform.v1alpha1.PlatformService.GetPlatform method and provides to
// CUE using a tag.
model?: structpb.#Struct @go(Model)
model: structpb.#Struct @go(Model)
components: [...#PlatformSpecComponent] @go(Components,[]PlatformSpecComponent)
}
// PlatformSpecComponent represents a component to build or render with flags to
// pass, for example the cluster name.
#PlatformSpecComponent: {
// Path is the path of the component relative to the platform root.
path: string @go(Path)
// Cluster is the cluster name to use when building the component.
cluster: string @go(Cluster)
}

View File

@@ -122,5 +122,5 @@ _#isResourceOwner_ResourceOwner: _
platform_id?: string @go(PlatformId) @protobuf(1,bytes,opt,json=platformId,proto3)
// Platform Model.
platform_model?: null | structpb.#Struct @go(PlatformModel,*structpb.Struct) @protobuf(2,bytes,opt,json=platformModel,proto3,oneof)
platform_model?: null | structpb.#Struct @go(PlatformModel,*structpb.Struct) @protobuf(2,bytes,opt,json=platformModel,proto3)
}

View File

@@ -0,0 +1,16 @@
package object
// Override the optional fields which result from the omitempty struct tags.
// Make them required so they work with yaml.Marshal
import "google.golang.org/protobuf/types/known/structpb"
// PlatformConfig represents the data passed from the holos cli to CUE when
// rendering configuration.
#PlatformConfig: {
// Platform UUID.
platform_id: string
// Platform Model.
platform_model: structpb.#Struct
}

View File

@@ -0,0 +1,32 @@
package render
import (
"context"
"fmt"
"io"
"time"
"github.com/holos-run/holos/api/v1alpha1"
"github.com/holos-run/holos/internal/errors"
"github.com/holos-run/holos/internal/server/middleware/logger"
"github.com/holos-run/holos/internal/util"
)
func Platform(ctx context.Context, pf *v1alpha1.Platform, stderr io.Writer) error {
total := len(pf.Spec.Components)
for idx, component := range pf.Spec.Components {
start := time.Now()
log := logger.FromContext(ctx).With("path", component.Path, "cluster", component.Cluster, "num", idx+1, "total", total)
log.DebugContext(ctx, "render component")
// Execute a sub-process to limit CUE memory usage.
args := []string{"render", "component", "--cluster-name", component.Cluster, component.Path}
result, err := util.RunCmd(ctx, "holos", args...)
if err != nil {
_, _ = io.Copy(stderr, result.Stderr)
return errors.Wrap(fmt.Errorf("could not render component: %w", err))
}
duration := time.Since(start)
log.InfoContext(ctx, "ok render component", "duration", duration)
}
return nil
}

View File

@@ -544,7 +544,7 @@ type PlatformConfig struct {
// Platform UUID.
PlatformId string `protobuf:"bytes,1,opt,name=platform_id,json=platformId,proto3" json:"platform_id,omitempty"`
// Platform Model.
PlatformModel *structpb.Struct `protobuf:"bytes,2,opt,name=platform_model,json=platformModel,proto3,oneof" json:"platform_model,omitempty"`
PlatformModel *structpb.Struct `protobuf:"bytes,2,opt,name=platform_model,json=platformModel,proto3" json:"platform_model,omitempty"`
}
func (x *PlatformConfig) Reset() {
@@ -678,21 +678,20 @@ var file_holos_object_v1alpha1_object_proto_rawDesc = []byte{
0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x43, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x73, 0x22, 0x93, 0x01, 0x0a, 0x0e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f,
0x72, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x29, 0x0a, 0x0b, 0x70, 0x6c, 0x61, 0x74,
0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xba,
0x48, 0x05, 0x72, 0x03, 0xb0, 0x01, 0x01, 0x52, 0x0a, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72,
0x6d, 0x49, 0x64, 0x12, 0x43, 0x0a, 0x0e, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x5f,
0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74,
0x72, 0x75, 0x63, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d,
0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x88, 0x01, 0x01, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x70, 0x6c, 0x61,
0x74, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x42, 0x45, 0x5a, 0x43, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x6f, 0x6c, 0x6f, 0x73, 0x2d,
0x72, 0x75, 0x6e, 0x2f, 0x68, 0x6f, 0x6c, 0x6f, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x68, 0x6f, 0x6c, 0x6f, 0x73, 0x2f, 0x6f, 0x62, 0x6a, 0x65,
0x63, 0x74, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b, 0x6f, 0x62, 0x6a, 0x65,
0x63, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6e, 0x66, 0x69, 0x67, 0x73, 0x22, 0x7b, 0x0a, 0x0e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72,
0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x29, 0x0a, 0x0b, 0x70, 0x6c, 0x61, 0x74, 0x66,
0x6f, 0x72, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xba, 0x48,
0x05, 0x72, 0x03, 0xb0, 0x01, 0x01, 0x52, 0x0a, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d,
0x49, 0x64, 0x12, 0x3e, 0x0a, 0x0e, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x6d,
0x6f, 0x64, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72,
0x75, 0x63, 0x74, 0x52, 0x0d, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x4d, 0x6f, 0x64,
0x65, 0x6c, 0x42, 0x45, 0x5a, 0x43, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x68, 0x6f, 0x6c, 0x6f, 0x73, 0x2d, 0x72, 0x75, 0x6e, 0x2f, 0x68, 0x6f, 0x6c, 0x6f, 0x73,
0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x68, 0x6f, 0x6c,
0x6f, 0x73, 0x2f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68,
0x61, 0x31, 0x3b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
}
var (
@@ -855,7 +854,6 @@ func file_holos_object_v1alpha1_object_proto_init() {
(*ResourceOwner_OrgId)(nil),
(*ResourceOwner_UserId)(nil),
}
file_holos_object_v1alpha1_object_proto_msgTypes[7].OneofWrappers = []interface{}{}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{

View File

@@ -94,5 +94,5 @@ message PlatformConfig {
// Platform UUID.
string platform_id = 1 [(buf.validate.field).string.uuid = true];
// Platform Model.
optional google.protobuf.Struct platform_model = 2;
google.protobuf.Struct platform_model = 2;
}

View File

@@ -1 +1 @@
80
81

View File

@@ -1 +1 @@
1
0