Compare commits

...

3 Commits

Author SHA1 Message Date
Jeff McCune
4127804092 quickstart: v0.93.2 with schema.#Platform
Make sure go install works from the quickstart documentation by doing a
release.  Otherwise, v0.93.1 is installed which doesn't include the
platform schema.
2024-09-09 17:04:32 -07:00
Jeff McCune
8f424cfabe quickstart: sync docs to this commit
Sync the documentation to the current output of the code at this commit.
2024-09-09 17:02:53 -07:00
Jeff McCune
699148abdd quickstart: define a convenince schema for the Platform
Previously, the quickstart step of generating the pod info component and
generating the platform as a whole left the task of integrating the
Component into the Platform as an exercise for the reader.  This is a
problem because it creates unnecessary friction.

This patch addresses the problem by lifting up the Platform concept
into the user-facing Schema API.  The generated platform includes a top
level #Platform definition which exposes the core Platform specification
on the Output field.

The Platform CUE instance then reduces to a simple `#Platform.Output`
which provides the Platform spec to holos for rendering each component
for each cluster.

The CUE code for the schema.#Platform iterates over each
Component to derive the list of components to manage for the Platform.

The CUE code for the generated quickstart platform links the definition
of StandardFleets, which is a Workload fleet and a Management cluster
fleet to the Platform conveninece wrapper.

Finally, the generated podinfo component drops a CUE file at the
repository root to automatically add the component to every workload
cluster.

The result is the only task left for the end user is to define at least
one workload cluster.  Once defined, the component is automatically
managed because it is managed on all workload clusters.

This approach futher opens the door to allow generated components to
define their namespaces and generated secrets on the management cluster
separate from their workloads on the workload clusters.

This patch includes a behavior change, from now on all generated
components should assume they are writing to the root of the user's Git
repository so that they can generate files through the whole tree.

In the future, we should template output paths for generated components.
A simple approach might be to embed a file with a .target suffix, with
the contents being a simple Go template of the file path to write to.
The holos generate subcommand can then check if any given embedded file
foo has a foo.target companion, then write the target to the rendered
template value.
2024-09-09 16:05:00 -07:00
13 changed files with 356 additions and 78 deletions

View File

@@ -4,7 +4,10 @@
// plate code and generating component build plans in a consistent manner.
package v1alpha3
import core "github.com/holos-run/holos/api/core/v1alpha3"
import (
core "github.com/holos-run/holos/api/core/v1alpha3"
"google.golang.org/protobuf/types/known/structpb"
)
//go:generate ../../../hack/gendoc
@@ -97,3 +100,51 @@ type ArgoConfig struct {
// main.
TargetRevision string `cue:"string | *\"main\""`
}
// Cluster represents a cluster managed by the Platform.
type Cluster struct {
// Name represents the cluster name, for example "east1", "west1", or
// "management".
Name string `json:"name"`
// Primary represents if the cluster is marked as the primary among a set of
// candidate clusters. Useful for promotion of database leaders.
Primary bool `json:"primary" cue:"true | *false"`
}
// Fleet represents a named collection of similarly configured Clusters. Useful
// to segregate workload clusters from their management cluster.
type Fleet struct {
Name string `json:"name"`
// Clusters represents a mapping of Clusters by their name.
Clusters map[string]Cluster `json:"clusters" cue:"{[Name=_]: name: Name}"`
}
// StandardFleets represents the standard set of Clusters in a Platform
// segmented into Fleets by their purpose. The management Fleet contains a
// single Cluster, for example a GKE autopilot cluster with no workloads
// deployed for reliability and cost efficiency. The workload Fleet contains
// all other Clusters which contain workloads and sync Secrets from the
// management cluster.
type StandardFleets struct {
// Workload represents a Fleet of zero or more workload Clusters.
Workload Fleet `json:"workload" cue:"{name: \"workload\"}"`
// Management represents a Fleet with one Cluster named management.
Management Fleet `json:"management" cue:"{name: \"management\", clusters: management: _}"`
}
// Platform is a convenience structure to produce a core Platform specification
// value in the Output field. Useful to collect components at the root of the
// Platform configuration tree as a struct, which are automatically converted
// into a list for the core Platform spec output.
type Platform struct {
// Name represents the Platform name.
Name string `cue:"string | *\"holos\""`
// Components is a structured map of components to manage by their name.
Components map[string]core.PlatformSpecComponent
// Model represents the Platform model holos gets from from the
// PlatformService.GetPlatform rpc method and provides to CUE using a tag.
Model structpb.Struct `cue:"{...}"`
// Output represents the core Platform spec for the holos cli to iterate over
// and render each listed Component, injecting the Model.
Output core.Platform
}

View File

@@ -11,7 +11,11 @@ Package v1alpha3 contains CUE definitions intended as convenience wrappers aroun
## Index
- [type ArgoConfig](<#ArgoConfig>)
- [type Cluster](<#Cluster>)
- [type Fleet](<#Fleet>)
- [type Helm](<#Helm>)
- [type Platform](<#Platform>)
- [type StandardFleets](<#StandardFleets>)
<a name="ArgoConfig"></a>
@@ -40,6 +44,35 @@ type ArgoConfig struct {
}
```
<a name="Cluster"></a>
## type Cluster {#Cluster}
Cluster represents a cluster managed by the Platform.
```go
type Cluster struct {
// Name represents the cluster name, for example "east1", "west1", or
// "management".
Name string `json:"name"`
// Primary represents if the cluster is marked as the primary among a set of
// candidate clusters. Useful for promotion of database leaders.
Primary bool `json:"primary" cue:"true | *false"`
}
```
<a name="Fleet"></a>
## type Fleet {#Fleet}
Fleet represents a named collection of similarly configured Clusters. Useful to segregate workload clusters from their management cluster.
```go
type Fleet struct {
Name string `json:"name"`
// Clusters represents a mapping of Clusters by their name.
Clusters map[string]Cluster `json:"clusters" cue:"{[Name=_]: name: Name}"`
}
```
<a name="Helm"></a>
## type Helm {#Helm}
@@ -98,4 +131,38 @@ type Helm struct {
}
```
<a name="Platform"></a>
## type Platform {#Platform}
Platform is a convenience structure to produce a core Platform specification value in the Output field. Useful to collect components at the root of the Platform configuration tree as a struct, which are automatically converted into a list for the core Platform spec output.
```go
type Platform struct {
// Name represents the Platform name.
Name string `cue:"string | *\"holos\""`
// Components is a structured map of components to manage by their name.
Components map[string]core.PlatformSpecComponent
// Model represents the Platform model holos gets from from the
// PlatformService.GetPlatform rpc method and provides to CUE using a tag.
Model structpb.Struct `cue:"{...}"`
// Output represents the core Platform spec for the holos cli to iterate over
// and render each listed Component, injecting the Model.
Output core.Platform
}
```
<a name="StandardFleets"></a>
## type StandardFleets {#StandardFleets}
StandardFleets represents the standard set of Clusters in a Platform segmented into Fleets by their purpose. The management Fleet contains a single Cluster, for example a GKE autopilot cluster with no workloads deployed for reliability and cost efficiency. The workload Fleet contains all other Clusters which contain workloads and sync Secrets from the management cluster.
```go
type StandardFleets struct {
// Workload represents a Fleet of zero or more workload Clusters.
Workload Fleet `json:"workload" cue:"{name: \"workload\"}"`
// Management represents a Fleet with one Cluster named management.
Management Fleet `json:"management" cue:"{name: \"management\", clusters: management: _}"`
}
```
Generated by [gomarkdoc](<https://github.com/princjef/gomarkdoc>)

View File

@@ -13,10 +13,17 @@ and more consistent.
You'll need the following tools installed to complete this guide.
1. [holos](/docs/install) - to build the platform.
2. [helm](https://helm.sh/docs/intro/install/) - to render Holos components that wrap upstream Helm charts.
3. [k3d](https://k3d.io/#installation) - to provide a Kubernetes API server.
4. [OrbStack](https://docs.orbstack.dev/install) or [Docker](https://docs.docker.com/get-docker/) - to use k3d.
5. [kubectl](https://kubernetes.io/docs/tasks/tools/) - to interact with the k8s api server.
2. [helm](https://helm.sh/docs/intro/install/) - to render Holos components that
wrap upstream Helm charts.
Optionally, if you'd like to apply the rendered manifests to a real cluster
you'll need:
1. [k3d](https://k3d.io/#installation) - to provide a Kubernetes API server.
2. [OrbStack](https://docs.orbstack.dev/install) or
[Docker](https://docs.docker.com/get-docker/) - to use k3d.
3. [kubectl](https://kubernetes.io/docs/tasks/tools/) - to interact with
kubernetes.
## Install Holos
@@ -53,8 +60,8 @@ repository unless otherwise stated.
## Generate the Platform {#Generate-Platform}
Generate the Platform code in the repository root. A Platform refers to all of
the software and services integrated together to provide your organization's
software development platform. In this guide the platform will contain a single
the software holistically integrated to provide a software development platform
for your organization. In this guide the platform will contain a single
Component to demonstrate how the concepts fit together.
```bash
@@ -63,47 +70,31 @@ holos generate platform quickstart
Commit the generated platform config to the repository.
```bash
git add .
git commit -m "holos generate platform quickstart - $(holos --version)"
```
## Create a Component
The platform you generated is empty. Rendering the platform succeeds, but does
nothing because there are no Components listed in the Platform spec:
```cue
package holos
import core "github.com/holos-run/holos/api/core/v1alpha3"
core.#Platform & {
metadata: name: "quickstart"
}
```
<Tabs groupId="render">
<Tabs groupId="commit-platform">
<TabItem value="command" label="Command">
```bash
holos render platform ./platform
git add .
git commit -m "holos generate platform quickstart - $(holos --version)"
```
</TabItem>
<TabItem value="output" label="Output">
```bash
# No output is produced, the Platform contains no Components.
```txt
[main (root-commit) 0b17b7f] holos generate platform quickstart
213 files changed, 72349 insertions(+)
...
```
</TabItem>
</Tabs>
Generate the CUE code definition for a Component that wraps the podinfo Helm
chart.
## Generate a Component {#generate-component}
The platform you generated is empty. Generate the CUE code definition for a
Component that wraps the podinfo Helm chart.
<Tabs groupId="gen-podinfo">
<TabItem value="command" label="Command">
```bash
mkdir -p components
(cd components && holos generate component helm podinfo)
holos generate component helm podinfo
```
</TabItem>
<TabItem value="output" label="Output">
@@ -113,35 +104,82 @@ chart.
</TabItem>
</Tabs>
The HelmChart Component is defined in the `components/podinfo/podinfo.cue` file, for example:
This command produces two files. A leaf `components/podinfo/podinfo.gen.cue`
file, and a root `podinfo.gen.cue` file. Holos takes advantage of the fact that
[order is irrelevant](https://cuelang.org/docs/tour/basics/order-irrelevance/)
in CUE to register the component with the Platform specification by adding a
file to the root of the Git repository in addition to defining the component
itself in the leaf component directory.
```cue
package holos
The Helm chart Component is defined in the `components/podinfo/podinfo.cue`
file, for example:
// Produce a helm chart build plan.
(#Helm & Chart).Output
<Tabs groupId="podinfo-files">
<TabItem value="components/podinfo/podinfo.gen.cue" label="Leaf">
`components/podinfo/podinfo.gen.cue`
```cue
package holos
let Chart = {
Name: "podinfo"
Version: "6.6.2"
Namespace: "default"
// Produce a helm chart build plan.
(#Helm & Chart).Output
Repo: name: "podinfo"
Repo: url: "https://stefanprodan.github.io/podinfo"
let Chart = {
Name: "podinfo"
Version: "6.6.2"
Namespace: "default"
Values: {}
}
```
Repo: name: "podinfo"
Repo: url: "https://stefanprodan.github.io/podinfo"
Values: {}
}
```
</TabItem>
<TabItem value="podinfo.gen.cue" label="Root">
`podinfo.gen.cue`
```cue
package holos
// Manage podinfo on workload clusters only
for Cluster in #Fleets.workload.clusters {
#Platform: Components: "\(Cluster.name)/podinfo": {
path: "components/podinfo"
cluster: Cluster.name
}
}
```
</TabItem>
</Tabs>
In this example we're providing the minimal information about the Helm chart we
want to manage in this Component. The name, version, Kubernetes namespace to
deploy into, and the chart repository location.
want to manage. The name, version, Kubernetes namespace to deploy into, and the
chart repository location.
This chart deploys cleanly with no values provided, but we include an empty
struct to illustrate how Holos improves the consistency and safety of Helm
values by taking advantage the strong type checking in CUE. Shared values, such
as the organization domain name, can safely be passed to all Components across
all clusters in the Platform.
Values struct to illustrate how Holos improves the consistency and safety of
Helm by taking advantage the strong type checking in CUE. Shared values,
such as the organization domain name, can safely be passed to all Components
across all clusters in the Platform by defining them at the root of the
configuration.
Commit the generated component config to the repository.
<Tabs groupId="commit-component">
<TabItem value="command" label="Command">
```bash
git add .
git commit -m "holos generate component helm podinfo - $(holos --version)"
```
</TabItem>
<TabItem value="output" label="Output">
```txt
[main cc0e90c] holos generate component helm podinfo
2 files changed, 24 insertions(+)
create mode 100644 components/podinfo/podinfo.gen.cue
create mode 100644 podinfo.gen.cue
```
</TabItem>
</Tabs>
## Render the Component
@@ -180,6 +218,8 @@ written into the deploy directory.
└── components
└── podinfo
└── podinfo.gen.yaml
5 directories, 1 file
```
</TabItem>
</Tabs>
@@ -187,7 +227,13 @@ written into the deploy directory.
The component is deployed to one cluster named default. The same component is
often deployed to multiple clusters, for example east and west for reliability.
This example is equivalent to executing `helm template` on the chart.
:::tip
This example is equivalent to executing `helm template` on the chart and saving
the output to a file. Holos simplifies this task by making it safer and more
consistent across multiple charts.
:::
## Mix in an ArgoCD Application
@@ -238,9 +284,22 @@ With this file in place, render the component again.
</TabItem>
</Tabs>
The render command uses the cached chart this time around. Then, the helm
template output is rendered along with an additional ArgoCD Application resource
for GitOps in the `podinfo.application.gen.yaml` file.
Holos uses the locally cached copy of the chart to render the output to improve
performance and reliability. Then, the Helm template output is rendered along
with an additional ArgoCD Application resource for GitOps in the
`podinfo.application.gen.yaml` file.
:::tip
By defining the ArgoCD configuration at the root, we again take advantage of the
fact that [order is
irrelevant](https://cuelang.org/docs/tour/basics/order-irrelevance/) in CUE.
:::
Defining the configuration at the root causes all of the leaf Components to take
on the ArgoCD configuration and render an Application resource for the
Component.
<Tabs groupId="tree-podinfo-argocd">
<TabItem value="command" label="Command">
@@ -258,11 +317,15 @@ for GitOps in the `podinfo.application.gen.yaml` file.
│   └── podinfo.gen.yaml
└── gitops
└── podinfo.application.gen.yaml
6 directories, 2 files
```
</TabItem>
</Tabs>
The Application resource looks like:
Note the new `podinfo.application.gen.yaml` created by enabling the ArgoCD in
the Helm component. The Application resource in the file looks like the
following.
<Tabs groupId="podinfo-application">
<TabItem value="file" label="podinfo.application.gen.yaml">
@@ -291,14 +354,31 @@ added to your Platform.
:::
Finally, add and commit the results to your platform Git repository.
<Tabs groupId="commit-argo">
<TabItem value="command" label="Command">
```bash
git add .
git commit -m "holos render component ./components/podinfo --cluster-name=default"
```
</TabItem>
<TabItem value="output" label="Output">
```txt
[main f95cef1] holos render component ./components/podinfo --cluster-name=default
3 files changed, 134 insertions(+)
create mode 100644 argocd.cue
create mode 100644 deploy/clusters/default/components/podinfo/podinfo.gen.yaml
create mode 100644 deploy/clusters/default/gitops/podinfo.application.gen.yaml
```
</TabItem>
</Tabs>
In this section we learned how Holos provides a simple way to add an ArgoCD
Application resource for the podinfo Component. Holos further provides
consistency by creating a similar Application resource for every subsequent
Component added to the platform in the future, all by defining the configuration
of ArgoCD in a single place.
[application]: https://argo-cd.readthedocs.io/en/stable/user-guide/application-specification/
Application resource for the podinfo Component which wraps a Helm chart. Holos
provides consistency by managing an Application resource for every Component
added to the platform, all by defining the configuration of ArgoCD in the
`argocd.cue` file in the root of the Git repository.
## Quickstart Recap {#quickstart-recap}
@@ -309,3 +389,5 @@ In this guide we learned how to:
3. Create a Component that wraps the upstream podinfo Helm Chart without modifications.
4. Render individual components.
5. Mix in an ArgoCD Application resource to every Component in the Platform.
[application]: https://argo-cd.readthedocs.io/en/stable/user-guide/application-specification/

View File

@@ -131,8 +131,8 @@ func makeRenderFunc[T any](log *slog.Logger, path string, cfg T) func([]byte) *b
func GenerateComponent(ctx context.Context, kind string, name string, cfg *Schematic) error {
// use name from args to build the source path
path := filepath.Join(componentsRoot, kind, name)
// use cfg.Name from flags to build the destination path
dstPath := filepath.Join(getCwd(ctx), cfg.Name)
// write to the current directory.
dstPath := filepath.Join(getCwd(ctx))
log := logger.FromContext(ctx).With("name", cfg.Name, "path", dstPath)
log.DebugContext(ctx, "mkdir")
if err := os.MkdirAll(dstPath, os.ModePerm); err != nil {

View File

@@ -0,0 +1,9 @@
package holos
// Manage podinfo on workload clusters only
for Cluster in #Fleets.workload.clusters {
#Platform: Components: "\(Cluster.name)/podinfo": {
path: "components/podinfo"
cluster: Cluster.name
}
}

View File

@@ -8,7 +8,10 @@
// plate code and generating component build plans in a consistent manner.
package v1alpha3
import core "github.com/holos-run/holos/api/core/v1alpha3"
import (
core "github.com/holos-run/holos/api/core/v1alpha3"
"google.golang.org/protobuf/types/known/structpb"
)
// Helm provides a BuildPlan via the Output field which contains one HelmChart
// from package core. Useful as a convenience wrapper to render a HelmChart
@@ -92,3 +95,57 @@ import core "github.com/holos-run/holos/api/core/v1alpha3"
// main.
TargetRevision: string & (string | *"main")
}
// Cluster represents a cluster managed by the Platform.
#Cluster: {
// Name represents the cluster name, for example "east1", "west1", or
// "management".
name: string @go(Name)
// Primary represents if the cluster is marked as the primary among a set of
// candidate clusters. Useful for promotion of database leaders.
primary: bool & (true | *false) @go(Primary)
}
// Fleet represents a named collection of similarly configured Clusters. Useful
// to segregate workload clusters from their management cluster.
#Fleet: {
name: string @go(Name)
// Clusters represents a mapping of Clusters by their name.
clusters: {[string]: #Cluster} & {[Name=_]: name: Name} @go(Clusters,map[string]Cluster)
}
// StandardFleets represents the standard set of Clusters in a Platform
// segmented into Fleets by their purpose. The management Fleet contains a
// single Cluster, for example a GKE autopilot cluster with no workloads
// deployed for reliability and cost efficiency. The workload Fleet contains
// all other Clusters which contain workloads and sync Secrets from the
// management cluster.
#StandardFleets: {
// Workload represents a Fleet of zero or more workload Clusters.
workload: #Fleet & {name: "workload"} @go(Workload)
// Management represents a Fleet with one Cluster named management.
management: #Fleet & {name: "management", clusters: management: _} @go(Management)
}
// Platform is a convenience structure to produce a core Platform specification
// value in the Output field. Useful to collect components at the root of the
// Platform configuration tree as a struct, which are automatically converted
// into a list for the core Platform spec output.
#Platform: {
// Name represents the Platform name.
Name: string & (string | *"holos")
// Components is a structured map of components to manage by their name.
Components: {[string]: core.#PlatformSpecComponent} @go(,map[string]core.PlatformSpecComponent)
// Model represents the Platform model holos gets from from the
// PlatformService.GetPlatform rpc method and provides to CUE using a tag.
Model: structpb.#Struct & {...}
// Output represents the core Platform spec for the holos cli to iterate over
// and render each listed Component, injecting the Model.
Output: core.#Platform
}

View File

@@ -199,3 +199,12 @@ import (
}
}
}
#Platform: {
Name: _
Model: _
Components: [string]: _
Output: metadata: name: Name
Output: spec: model: Model
Output: spec: components: [for c in Components {c}]
}

View File

@@ -1,7 +0,0 @@
package holos
import core "github.com/holos-run/holos/api/core/v1alpha3"
core.#Platform & {
metadata: name: "quickstart"
}

View File

@@ -0,0 +1,3 @@
package holos
#Platform.Output

View File

@@ -0,0 +1,3 @@
package holos
#Platform: Name: "quickstart"

View File

@@ -9,3 +9,7 @@ import schema "github.com/holos-run/holos/api/schema/v1alpha3"
#ArgoConfig: schema.#ArgoConfig & {
ClusterName: _ClusterName
}
#Fleets: schema.#StandardFleets
#Platform: schema.#Platform

View File

@@ -1 +1 @@
1
2