Compare commits

...

15 Commits

Author SHA1 Message Date
Jeff McCune
c6a13059f3 v0.97.1 2024-10-28 11:12:55 -07:00
Jeff McCune
ff3eb896f3 webite: put ois logo back
Until we get a better logo.
2024-10-28 10:46:59 -07:00
Jeff McCune
70f70ae6b9 website: fix launch announcement 2024-10-28 10:45:43 -07:00
Jeff McCune
2580ec1c5f website: fix order of api references
The api references are in reverse order and don't have good descriptions
in the index listings.  This patch adds front matter to each generated
document to order them correctly and add a nice description.
2024-10-27 20:43:54 -07:00
Jeff McCune
4fa99e0faa website: add helm prometheus blackbox httpbin guide
The purpose of this guide is to demonstrate how holos offers value
above and beyond helm and kustomize alone.
2024-10-27 19:48:42 -07:00
Jeff McCune
7341d25483 website: add at proto did for bsky 2024-10-25 14:26:42 -07:00
Jeff McCune
3074b3a241 website: add discord invite link 2024-10-24 10:27:37 -07:00
Jeff McCune
9a5e7869c6 v1alpha4: omit the platform model if empty
The platform model distracts from getting started:

  cue export --out yaml ./platform

  kind: Platform
  apiVersion: v1alpha4
  metadata:
    name: default
  spec:
    components:
      - name: prometheus
        component: projects/platform/components/prometheus
        cluster: local
        model: {}

With this patch it's absent by default.
2024-10-23 13:46:58 -07:00
Jeff McCune
1064ceba31 v1alpha4: manage a single workload cluster named local by default
Manage a single cluster by default after generating the platform.
Assume it's a local cluster for use with the guides.
2024-10-23 13:42:46 -07:00
Jeff McCune
4bccaa3710 v1alpha4: _Platform not #Platform for component registration
`_Platform` is a hidden field representing the platform components,
`#Platform` defines the schema of the hidden field.
2024-10-23 13:33:28 -07:00
Jeff McCune
95efae1343 docs: update rendered manifest figure in technical overview
It's too wide with KubeAPI, take it out.
2024-10-20 10:11:57 -07:00
Jeff McCune
ba88125877 v1alpha4: enable config map generator for Kustomize
Without this patch it's difficult to mix in a plain file as a config
map.  This is necessary for the use case of using a Job to generate a
secret in-cluster.  We want a plain shell script to be carried through
and transformed into the job.

We already have the KustomizeConfig fields to support this, they just
weren't wired up to the #Kustomization component kind.

I didn't check if it's wired up to Helm and Kustomize for expedience.
They may be missing there as well.
2024-10-19 10:57:37 -07:00
Jeff McCune
d12c1a0c11 Merge pull request #284 from holos-run/gl/deploy-a-service-v4alpha-update
Update deploy-a-service guide for Author API v1alpha4
2024-10-18 20:35:51 -07:00
Jeff McCune
d56d3400a7 docs: replace tabs in deploy-a-service guide 2024-10-18 20:33:54 -07:00
Gary Larizza
4f0f9dced5 Update deploy-a-service guide for Author API v1alpha4
PROBLEM:

Version v1alpha4 of the Author API has been updated with backwards
incompatible changes, and the deploy-a-service guide uses code from
version v1alpha3.

SOLUTION:

Update any code, links, and data that is out of date, and then run
through the guide to make sure it works locally.

OUTCOME:

The instructions in the deploy-a-service guide will work successfully
with version v1alpha4 of the Author API.
2024-10-18 15:21:56 -07:00
43 changed files with 5209 additions and 221 deletions

View File

@@ -7,34 +7,52 @@
"words": [
"acraccesstokens",
"admissionregistration",
"alertmanager",
"alertmanagers",
"anthos",
"apiextensions",
"apimachinery",
"apiobjects",
"apiservers",
"applicationset",
"applicationsets",
"appproject",
"appprojects",
"argoproj",
"argumentless",
"authcode",
"authorizationpolicies",
"authorizationpolicy",
"authpolicy",
"authproxy",
"authroutes",
"automount",
"automounting",
"autoscaler",
"balancereader",
"blackbox",
"buildplan",
"builtinpluginloadingoptions",
"cadvisor",
"cainjector",
"CAROOT",
"certificaterequest",
"certificaterequests",
"certificatesigningrequests",
"clsx",
"clusterexternalsecret",
"clusterexternalsecrets",
"clusterissuer",
"clusterissuers",
"clusterrole",
"clusterrolebinding",
"clustersecretstore",
"clustersecretstores",
"clusterwide",
"CNCF",
"CODEOWNERS",
"configmap",
"configmapargs",
"cookiesecret",
"coredns",
"corev",
@@ -42,16 +60,22 @@
"crds",
"creds",
"crossplane",
"crunchydata",
"cuecontext",
"cuelang",
"customresourcedefinition",
"daemonset",
"deploymentruntimeconfig",
"destinationrule",
"destinationrules",
"devicecode",
"dnsmasq",
"dscacheutil",
"ecrauthorizationtokens",
"edns",
"endpointslices",
"entgo",
"envoyfilter",
"envoyfilters",
"errgroup",
"etcdsnapshotfiles",
@@ -59,51 +83,78 @@
"externalsecrets",
"fctr",
"fieldmaskpb",
"fieldspec",
"flushcache",
"fullname",
"gatewayclass",
"gatewayclasses",
"gcraccesstokens",
"gendoc",
"generationbehavior",
"generatorargs",
"generatoroptions",
"ggnpl",
"ghaction",
"githubaccesstokens",
"gitops",
"godoc",
"golangci",
"gomarkdoc",
"goreleaser",
"gotypesalias",
"grpcreflect",
"grpcroute",
"grpcroutes",
"grpcurl",
"healthchecks",
"healthz",
"helmchartargs",
"helmchartconfigs",
"helmcharts",
"Hiera",
"holos",
"holoslogger",
"horizontalpodautoscaler",
"horizontalpodautoscalers",
"Hostaliases",
"Hostnames",
"htpasswd",
"httpbin",
"httproute",
"httproutes",
"iampolicygenerator",
"Infima",
"intstr",
"isatty",
"istiod",
"jbrx",
"jeffmccune",
"jetstack",
"jiralert",
"Jsonnet",
"kfbh",
"killall",
"kubeadm",
"kubeconfig",
"kubelet",
"kubelogin",
"kubernetesobjects",
"Kustomization",
"Kustomizations",
"kustomize",
"kustomizebuild",
"kvpairsources",
"labeldrop",
"labelmap",
"ldflags",
"leaderelection",
"ledgerwriter",
"libnss",
"limitranges",
"livez",
"loadbalancer",
"loadrestrictions",
"logfmt",
"mattn",
"mccutchen",
"mindmap",
@@ -112,29 +163,58 @@
"mtls",
"Multicluster",
"mutatingwebhookconfiguration",
"mutatingwebhookconfigurations",
"mxcl",
"myhostname",
"myRegistrKeySecretName",
"mysecret",
"nameofclusterrole",
"nameserver",
"namespacedname",
"ndots",
"networkpolicies",
"nodename",
"nolint",
"oauthproxy",
"objectmap",
"objectmeta",
"organizationconnect",
"orgid",
"otelconnect",
"overriden",
"Parentspanid",
"patchstrategicmerge",
"pcjc",
"peerauthentication",
"peerauthentications",
"persistentvolumeclaim",
"persistentvolumeclaims",
"persistentvolumes",
"pflag",
"pgadmin",
"pgupgrade",
"pipefail",
"PKCE",
"platformconnect",
"pluginconfig",
"pluginrestrictions",
"podcli",
"poddisruptionbudget",
"poddisruptionbudgets",
"podinfo",
"portmapping",
"postgrescluster",
"privs",
"prometheuses",
"promhttp",
"protobuf",
"protojson",
"providerconfig",
"proxyconfig",
"proxyconfigs",
"Pulumi",
"pushgateway",
"pushsecret",
"pushsecrets",
"putenv",
"qjbp",
@@ -143,11 +223,19 @@
"readyz",
"referencegrant",
"referencegrants",
"Registr",
"replacementfield",
"replicasets",
"replicationcontrollers",
"requestauthentication",
"requestauthentications",
"resourcequotas",
"retryable",
"rolebinding",
"rootfs",
"ropc",
"seccomp",
"secretargs",
"SECRETKEY",
"secretstore",
"secretstores",
@@ -156,29 +244,47 @@
"serviceaccount",
"servicebindings",
"serviceentries",
"serviceentry",
"servicemonitor",
"somevalue",
"SOMEVAR",
"sortoptions",
"spanid",
"spiffe",
"stackdriver",
"startupapicheck",
"statefulset",
"statefulsets",
"stefanprodan",
"storageclasses",
"streamwatcher",
"struct",
"structpb",
"subcharts",
"subjectaccessreviews",
"svclb",
"sysfs",
"systemconnect",
"tablewriter",
"templatable",
"thanos",
"Tiltfile",
"timestamppb",
"Timoni",
"tlsclientconfig",
"tokencache",
"Tokener",
"tolerations",
"Traceid",
"traefik",
"transactionhistory",
"tsdb",
"typemeta",
"udev",
"uibutton",
"unstage",
"untar",
"upbound",
"Upsert",
"urandom",
"usecases",
@@ -186,11 +292,18 @@
"userdata",
"userservice",
"validatingwebhookconfiguration",
"validatingwebhookconfigurations",
"vaultdynamicsecrets",
"virtualservice",
"virtualservices",
"volumeattachments",
"wasmplugin",
"wasmplugins",
"workloadentries",
"workloadentry",
"workloadgroup",
"workloadgroups",
"yournamespace",
"zerolog",
"zitadel",
"ztunnel"

View File

@@ -0,0 +1,4 @@
---
description: Simplified abstraction to generate core v1alpha3 components.
sidebar_position: 997
---

View File

@@ -0,0 +1,4 @@
---
description: Simplified abstraction to generate core v1alpha4 build plans.
sidebar_position: 996
---

View File

@@ -0,0 +1,4 @@
---
description: Core v1alpha4 schema for advanced use cases.
sidebar_position: 998
---

View File

@@ -0,0 +1,4 @@
---
description: Core v1alpha3 schema for advanced use cases.
sidebar_position: 997
---

View File

@@ -0,0 +1,4 @@
---
description: Core v1alpha2 schema for advanced use cases.
sidebar_position: 996
---

View File

@@ -365,7 +365,7 @@ type Component struct {
// Model represents the platform model holos gets from from the
// PlatformService.GetPlatform rpc method and provides to CUE using a tag.
// Injected as the tag "holos_model".
Model map[string]any `json:"model"`
Model map[string]any `json:"model,omitempty"`
// Tags represents cue @tag variables injected into the holos render component
// command from the holos render platform command. Tags with a "holos_"
// prefix are reserved for use by the Holos Authors.

View File

@@ -1,3 +1,7 @@
---
description: Simplified abstraction to generate core v1alpha3 components.
sidebar_position: 997
---
<!-- Code generated by gomarkdoc. DO NOT EDIT -->
# v1alpha3

View File

@@ -1,3 +1,7 @@
---
description: Simplified abstraction to generate core v1alpha4 build plans.
sidebar_position: 996
---
<!-- Code generated by gomarkdoc. DO NOT EDIT -->
# v1alpha4

View File

@@ -1,3 +1,7 @@
---
description: Core v1alpha4 schema for advanced use cases.
sidebar_position: 998
---
<!-- Code generated by gomarkdoc. DO NOT EDIT -->
# v1alpha2

View File

@@ -1,3 +1,7 @@
---
description: Core v1alpha3 schema for advanced use cases.
sidebar_position: 997
---
<!-- Code generated by gomarkdoc. DO NOT EDIT -->
# v1alpha3

View File

@@ -1,3 +1,7 @@
---
description: Core v1alpha2 schema for advanced use cases.
sidebar_position: 996
---
<!-- Code generated by gomarkdoc. DO NOT EDIT -->
# v1alpha4
@@ -234,7 +238,7 @@ type Component struct {
// Model represents the platform model holos gets from from the
// PlatformService.GetPlatform rpc method and provides to CUE using a tag.
// Injected as the tag "holos_model".
Model map[string]any `json:"model"`
Model map[string]any `json:"model,omitempty"`
// Tags represents cue @tag variables injected into the holos render component
// command from the holos render platform command. Tags with a "holos_"
// prefix are reserved for use by the Holos Authors.

View File

@@ -94,8 +94,8 @@ an ArgoCD Application or Flux Kustomization.
consistently add common labels.
:::tip
[ComponentFields] in the Author API describes the fields common to all kinds of
component.
[ComponentConfig] in the Author API describes the fields common to all kinds of
components.
:::
We'll start with a [Helm] component to deploy the service, then compare it to a
@@ -356,10 +356,17 @@ import ks "sigs.k8s.io/kustomize/api/types"
let Chart = {
// highlight-next-line
Name: "podinfo"
Version: "6.6.2"
// highlight-next-line
Namespace: #Migration.Namespace
Chart: {
version: "6.6.2"
repository: {
name: "podinfo"
url: "https://stefanprodan.github.io/podinfo"
}
}
// Necessary to ensure the resources go to the correct namespace.
// highlight-next-line
EnableKustomizePostProcessor: true
@@ -368,9 +375,6 @@ let Chart = {
namespace: Namespace
}
Repo: name: "podinfo"
Repo: url: "https://stefanprodan.github.io/podinfo"
// Allow the platform team to route traffic into our namespace.
// highlight-next-line
Resources: ReferenceGrant: grant: #ReferenceGrant & {
@@ -396,19 +400,19 @@ Name as the sub-directory name when it writes the rendered manifest into
`deploy/`. Normally this name also matches the directory and file name of the
component, `podinfo/podinfo.cue`, but `holos` doesn't enforce this convention.
**Line 11**: We use the same namespace we registered with the `namespaces`
**Line 10**: We use the same namespace we registered with the `namespaces`
component as the value we pass to Helm. This is a good example of Holos
offering safety and consistency with CUE, if we change the value of
`#Migration.Namespace`, multiple components stay consistent.
**Lines 14-15**: Unfortunately, the Helm chart doesn't set the
**Lines 20-21**: Unfortunately, the Helm chart doesn't set the
`metadata.namespace` field for the resources it generates, which creates a
security problem. The resources will be created in the wrong namespace. We
don't want to modify the upstream chart because it creates a maintenance burden.
We solve the problem by having Holos post-process the Helm output with
Kustomize. This ensures all resources go into the correct namespace.
**Lines 23**: The migration team grants the platform team permission to route
**Lines 27**: The migration team grants the platform team permission to route
traffic into the `migration` Namespace using a [ReferenceGrant].
:::note
@@ -780,7 +784,7 @@ the bank to register with. The `#HTTPRoutes` struct is similar to the
As a member of the migration team, we'll add the file
`projects/migration-routes.cue` to expose the service we're migrating.
Go ahead and create this file with the following content.
Go ahead and create this file (if it hasn't been created previously) with the following content.
<Tabs groupId="6F9044EC-1737-4926-BD07-455536BA6573">
<TabItem value="projects/migration-routes.cue" label="projects/migration-routes.cue">
@@ -813,7 +817,7 @@ import v1 "gateway.networking.k8s.io/httproute/v1"
// For the guides, we simplify this down to a flat namespace.
// highlight-next-line
[Name=string]: v1.#HTTPRoute & {
let HOST = Name + "." + #Platform.Domain
let HOST = Name + "." + #Organization.Domain
// highlight-next-line
_backendRefs: [NAME=string]: {
@@ -925,37 +929,38 @@ git diff
<TabItem value="output" label="Output">
```diff
diff --git a/deploy/clusters/workload/components/httproutes/httproutes.gen.yaml b/deploy/clusters/workload/components/httproutes/httproutes.gen.yaml
index 4b476da..a150015 100644
index 06f7c91..349e070 100644
--- a/deploy/clusters/workload/components/httproutes/httproutes.gen.yaml
+++ b/deploy/clusters/workload/components/httproutes/httproutes.gen.yaml
@@ -46,3 +46,27 @@ spec:
- name: frontend
namespace: bank-frontend
port: 80
@@ -47,3 +47,28 @@ spec:
- path:
type: PathPrefix
value: /
+---
+# Source: CUE apiObjects.HTTPRoute.migration-podinfo
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+ name: migration-podinfo
+ namespace: istio-ingress
+ labels:
+ app: migration-podinfo
+ argocd.argoproj.io/instance: httproutes
+ holos.run/component.name: httproutes
+ name: migration-podinfo
+ namespace: istio-ingress
+spec:
+ hostnames:
+ - migration-podinfo.holos.localhost
+ - migration-podinfo.holos.localhost
+ parentRefs:
+ - name: default
+ namespace: istio-ingress
+ - name: default
+ namespace: istio-ingress
+ rules:
+ - matches:
+ - path:
+ type: PathPrefix
+ value: /
+ backendRefs:
+ - name: podinfo
+ port: 9898
+ namespace: migration
+ - backendRefs:
+ - name: podinfo
+ namespace: migration
+ port: 9898
+ matches:
+ - path:
+ type: PathPrefix
+ value: /
```
</TabItem>
@@ -1428,10 +1433,10 @@ for some time.
[Quickstart]: /docs/quickstart/
[Change a Service]: /docs/guides/change-a-service/
[Helm]: /docs/api/author/v1alpha3/#Helm
[Kubernetes]: /docs/api/author/v1alpha3/#Kubernetes
[Kustomize]: /docs/api/author/v1alpha3/#Kustomize
[ComponentFields]: /docs/api/author/v1alpha3/#ComponentFields
[Helm]: /docs/api/author/v1alpha4/#Helm
[Kubernetes]: /docs/api/author/v1alpha4/#Kubernetes
[Kustomize]: /docs/api/author/v1alpha4/#Kustomize
[ComponentConfig]: /docs/api/author/v1alpha4/#ComponentConfig
[platform-files]: /docs/quickstart/#how-platform-rendering-works
[AppProject]: https://argo-cd.readthedocs.io/en/stable/user-guide/projects/
[unification operator]: https://cuelang.org/docs/reference/spec/#unification

View File

@@ -1,15 +0,0 @@
---
description: Helm Component
slug: /guides/helm-component
sidebar_position: 400
---
# Helm Component
The [Deploy a Service](/docs/guides/deploy-a-service/) guide is the best guide
we have on wrapping a Helm chart in a Holos Component. The [Helm] section of
the Author API may also be useful.
[Helm]: /docs/api/author/v1alpha3/#Helm
[Kubernetes]: /docs/api/author/v1alpha3/#Kubernetes
[Kustomize]: /docs/api/author/v1alpha3/#Kustomize

4793
doc/md/guides/helm.mdx Normal file

File diff suppressed because it is too large Load Diff

Binary file not shown.

After

Width:  |  Height:  |  Size: 248 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 206 KiB

View File

@@ -1,20 +0,0 @@
---
description: Kubernetes Component
slug: /guides/kubernetes-component
sidebar_position: 500
---
# Kubernetes Component
:::warning
TODO
:::
This is a placeholder for a guide for managing Kubernetes resources directly
from a Holos Component with strong type checking.
In the meantime, please refer to the [Kubernetes] section of the Author API.
[Helm]: /docs/api/author/v1alpha3/#Helm
[Kubernetes]: /docs/api/author/v1alpha3/#Kubernetes
[Kustomize]: /docs/api/author/v1alpha3/#Kustomize

View File

@@ -1,20 +0,0 @@
---
description: Wrap a Kustomize Kustomization in a Holos Component.
slug: /guides/kustomize-component
sidebar_position: 600
---
# Kustomize Component
:::warning
TODO
:::
This is a placeholder for a guide on wrapping a Kustomize Kustomization base
with a Holos component.
In the meantime, please refer to the [Kustomize] section of the Author API.
[Helm]: /docs/api/author/v1alpha3/#Helm
[Kubernetes]: /docs/api/author/v1alpha3/#Kubernetes
[Kustomize]: /docs/api/author/v1alpha3/#Kustomize

View File

@@ -42,9 +42,9 @@ graph TB
Kustomize[<a href="#component">Kustomize</a>]
CUE[<a href="#component">CUE</a>]
Cluster --> Platform
Fleet --> Cluster
Component --> Fleet
Fleet --> Platform
Cluster --> Fleet
Component --> Cluster
Helm --> Component
Kustomize --> Component
CUE --> Component

View File

@@ -8,7 +8,7 @@ sidebar_position: 900
## Community Support
You can ask questions in our community forums in [GitHub Discussions](https://github.com/holos-run/holos/discussions) or [Google Groups](https://groups.google.com/g/holos-discuss).
You can ask questions in our community forums in [GitHub Discussions](https://github.com/holos-run/holos/discussions), [Discord](https://discord.gg/JgDVbNpye7), or [Google Groups](https://groups.google.com/g/holos-discuss).
## Commercial Support and Services

View File

@@ -45,7 +45,7 @@ Go command line tool leveraging [CUE] to fill this gap.
```mermaid
---
title: Figure 1 - v1alpha4 Render Pipeline
title: Figure 1 - v1alpha4 Rendered Manifest Pipeline
---
graph LR
Platform[<a href="/docs/api/author/v1alpha4/#Platform">Platform</a>]
@@ -60,18 +60,19 @@ graph LR
ResourcesArtifact[<a href="/docs/api/core/v1alpha4/#artifact">Resources<br/>Artifact</a>]
GitOpsArtifact[<a href="/docs/api/core/v1alpha4/#artifact">GitOps<br/>Artifact</a>]
Generator[<a href="/docs/api/core/v1alpha4/#generators">Generator</a>]
Transformer[<a href="/docs/api/core/v1alpha4/#transformer">Transformer</a>]
Generators[<a href="/docs/api/core/v1alpha4/#generators">Generators</a>]
Transformers[<a href="/docs/api/core/v1alpha4/#transformer">Transformers</a>]
Files[Manifest<br/>Files]
Platform --> Component
Component --> Helm --> BuildPlan
Component --> Kubernetes --> BuildPlan
Component --> Kustomize --> BuildPlan
BuildPlan --> ResourcesArtifact --> Generator
BuildPlan --> GitOpsArtifact --> Generator
BuildPlan --> ResourcesArtifact --> Generators
BuildPlan --> GitOpsArtifact --> Generators
Generator --> Transformer --> Files --> KubeAPI
Generators --> Transformers --> Files
```
## Use Case

View File

@@ -1,94 +0,0 @@
---
slug: holos-platform-manager
title: Holos Platform Manager
authors: [jeff]
tags: [holos]
---
## Introducing Holos
Im excited to announce Holos, a tool designed to help engineering teams
manage their software development platforms built on the Kubernetes resource
model.
:::tip
For a hands-on introduction, check out our [Quickstart] Guide.
:::
<!-- truncate -->
### The Backstory
In our roles at [Open Infrastructure Services], and earlier at Puppet, we helped
many companies automate infrastructure management. In 2017, we had the
opportunity to work with Twitter to improve their configuration management
system. This opportunity gave us insight into the challenges of managing a
large-scale platform with multiple engineering teams. Our work involved
everything from observability systems to application deployment workflows and of
course, managing the core infrastructure.
This experience demonstrated the value of platform engineering. As the pandemic
hit, I began thinking about what a fully cloud-native platform might look like
using the Kubernetes resource model. Around the same time, I came across the
Hacker News post, “[Why Are We Templating YAML]?”, which sparked a good
discussion. It was clear I wasnt alone in my frustration with managing YAML
files and ensuring clear, predictable changes before merging them into
production.
A common pain point and theme is the complexity of working with nested YAML
configurations, especially with tools like ArgoCD and Helm. The lack of a
standard for rendering YAML templates makes it difficult to see what changes are
actually being applied to the Kubernetes API. This often results in trial and
error, costly blue-green deployments, and hours of debugging.
During the pandemic, I began experimenting with a tool to address this issue,
drawing on lessons from our work at Twitter. The key problems we aimed to solve
are:
- **Lack of visibility**: Engineers struggled to foresee the impact of small changes.
- **Large blast radius**: Small changes affected global systems, with no way to limit the impact.
- **Incomplete tooling**: While processes were in place, the right information wasnt surfaced at the right time.
We built several iterations of a reference platform based on Kubernetes,
initially focusing on fully rendering manifests into plain files—a pattern now
called the [rendered manifests pattern]. Over time, we realized we were spending
most of our time maintaining bash scripts and YAML templates. This led back to
the question: Why are we templating YAML? What _should_ replace templates?
We'd previously seen a colleague use CUE effectively to generate large scale
configurations for Envoy, and ran into CUE again when we worked on a project
involving Dagger, but I still hadn't taken a deep look at CUE.
At the end of 2023, I decided to dive deep with [CUE]. I quickly came to
appreciate CUEs unified approach where **order is irrelevant**. Before CUE, we
handled configuration data in a hierarchy with a precedence ordering, similar to
how we handled data in Puppet with Hiera. CUE's promise of no longer needing to
think about ordering and precedence rules held, alleviating a large cognitive
burden when dealing with complex configurations. CUE quickly allowed me to
replace the unmaintainable bash scripts and complex Helm templates, simplifying
our workflow.
### Enter Holos
Holos adds CUE as a well-specified integration layer over tools like Helm,
Kustomize, ArgoCD, and Crossplane. With Holos, we can now efficiently integrate
upstream Helm charts and Kustomize bases into our platform without the
complexity of templates and scripts. This has also made it easy for one team to
define "golden paths" that other teams can follow—like automatically configuring
namespaces and security policies when dev teams start new projects.
We've found Holos incredibly useful and hope you do too. Let us know your
thoughts!
[Guides]: /docs/guides/
[API Reference]: /docs/api/
[Quickstart]: /docs/quickstart/
[CUE]: https://cuelang.org/
[Author API]: /docs/api/author/
[Core API]: /docs/api/core/
[Open Infrastructure Services]: https://openinfrastructure.co/
[Why are we templating YAML]: https://hn.algolia.com/?dateRange=all&page=0&prefix=false&query=https%3A%2F%2Fleebriggs.co.uk%2Fblog%2F2019%2F02%2F07%2Fwhy-are-we-templating-yaml&sort=byDate&type=story
[Holos]: https://holos.run/
[Quickstart]: /docs/quickstart/
[rendered manifests pattern]: https://akuity.io/blog/the-rendered-manifests-pattern/

View File

@@ -0,0 +1,118 @@
---
slug: announcing-holos
title: Announcing Holos
authors: [jeff]
tags: [holos, launch]
---
Im excited to share Holos, a Go command line tool we developed to make it
easier to manage a platform built on Kubernetes. Holos implements the rendered
manifests pattern as a data pipeline to fully render manifests generated from
[Helm], [Kustomize], or [CUE] in a holistic way.
[Helm]: https://helm.sh/
[Kustomize]: https://kustomize.io/
[CUE]: https://cuelang.org/
```mermaid
---
title: Rendered Manifest Pipeline
---
graph LR
Platform[<a href="/docs/api/author/v1alpha4/#Platform">Platform</a>]
Component[<a href="/docs/api/author/v1alpha4/#ComponentConfig">Components</a>]
Helm[<a href="/docs/api/author/v1alpha4/#Helm">Helm</a>]
Kustomize[<a href="/docs/api/author/v1alpha4/#Kustomize">Kustomize</a>]
Kubernetes[<a href="/docs/api/author/v1alpha4/#Kubernetes">Kubernetes</a>]
BuildPlan[<a href="/docs/api/core/v1alpha4/#buildplan">BuildPlan</a>]
ResourcesArtifact[<a href="/docs/api/core/v1alpha4/#artifact">Resources<br/>Artifact</a>]
GitOpsArtifact[<a href="/docs/api/core/v1alpha4/#artifact">GitOps<br/>Artifact</a>]
Generators[<a href="/docs/api/core/v1alpha4/#generators">Generators</a>]
Transformers[<a href="/docs/api/core/v1alpha4/#transformer">Transformers</a>]
Files[Manifest<br/>Files]
Platform --> Component
Component --> Helm --> BuildPlan
Component --> Kubernetes --> BuildPlan
Component --> Kustomize --> BuildPlan
BuildPlan --> ResourcesArtifact --> Generators
BuildPlan --> GitOpsArtifact --> Generators
Generators --> Transformers --> Files
```
<!-- truncate -->
At the start of the pandemic I was migrating our platform from VMs managed by
Puppet to Kubernetes. My primary goal was to build an observability system
similar to what we had when we managed Puppet at Twitter prior to the
acquisition. I started building the observability system with the official
[prometheus community charts], but quickly ran into issues where the
individual charts didnt work with each other. I was frustrated with how
complicated and difficult to configure these charts were. They werent well
integrated, so I switched to the [kube-prometheus-stack] umbrella chart which
attempts to solve this integration problem.
The umbrella chart got us further, as long as we didnt stray too far from the
default values, but we quickly ran into operational challenges. Upgrading the
chart introduced breaking changes we couldnt see until they were applied,
causing incidents. We needed to manage secrets securely so we mixed in
ExternalSecrets with many of the charts. We decided to handle these
customizations by implementing the [rendered manifests pattern] using scripts in
our CI pipeline.
These scripts got us further, but we found them costly to maintain.
Teammates needed to be careful to execute them with the same context they were
executed in CI. We realized we were reinventing Hiera to manage a hierarchy of
helm values.yaml files to inject into multiple charts.
At this point I started looking for a more holistic solution to this problem of
integrating multiple charts together. We saw the value in the rendered
manifests pattern, but we couldnt find an agreed upon implementation. We built
a Go command line tool to implement the pattern as a data pipeline. Id been
thinking about the comments from the [Why are we templating YAML] posts and
wondering what an answer to this question would look like.
The Go command line tool was an incremental improvement over the CI scripts, but
we still didnt have a good way to handle the data values. We were still
templating YAML which didnt catch errors early enough. It was too easy to
render invalid resources Kubernetes rejected, causing deployment problems. I
searched for a solution to manage helm values, something like Hiera which we
knew well from Puppet, but not hierarchical because we knew it was important to
trace where config values came from in an outage. A few HN comments mentioned
CUE, and an engineer we worked with at Twitter used CUE to configure Envoy at
scale, so I gave it a try. I quickly appreciated how CUE provides both strong
type checking and validation of constraints, unifies all configuration data, and
provides clarity into where values originate from.
Take a look at Holos if youre looking to implement the rendered manifests
pattern or cant shake that feeling it should be easier to integrate third party
software into Kubernetes like we felt.
1. [Helm Guide] Walks through how we solved the challenges we faced with the prometheus Helm charts.
2. [Quickstart] Works through how a platform team can define golden paths for other teams using CUE.
3. [Author API] provides an ergonomic way to work with Helm, Kustomize, and CUE resources.
[Helm Guide]: /docs/guides/helm/
[Guides]: /docs/guides/
[API Reference]: /docs/api/
[Quickstart]: /docs/quickstart/
[Author API]: /docs/api/author/
[Core API]: /docs/api/core/
[Open Infrastructure Services]: https://openinfrastructure.co/
[Why are we templating YAML]: https://hn.algolia.com/?dateRange=all&page=0&prefix=false&query=https%3A%2F%2Fleebriggs.co.uk%2Fblog%2F2019%2F02%2F07%2Fwhy-are-we-templating-yaml&sort=byDate&type=story
[Holos]: https://holos.run/
[Quickstart]: /docs/quickstart/
[Helm]: https://helm.sh/
[Kustomize]: https://kustomize.io/
[CUE]: https://cuelang.org/
[rendered manifests pattern]: https://akuity.io/blog/the-rendered-manifests-pattern/
[prometheus community charts]: https://github.com/prometheus-community/helm-charts
[kube-prometheus-stack]: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack

View File

@@ -116,6 +116,11 @@ const config: Config = {
label: 'GitHub',
position: 'right',
},
{
href: 'https://discord.gg/JgDVbNpye7',
label: 'Discord',
position: 'right',
},
],
},
footer: {
@@ -150,8 +155,8 @@ const config: Config = {
href: '/docs/support',
},
{
label: 'Announcements List',
href: 'https://groups.google.com/g/holos-announce',
label: 'Discord',
href: 'https://discord.gg/JgDVbNpye7',
},
{
label: 'Discussion List',

View File

@@ -0,0 +1 @@
did:plc:7jly72mfd42u4fj4mfyovxqz

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.7 KiB

View File

@@ -0,0 +1 @@
<svg width="98" height="96" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M48.854 0C21.839 0 0 22 0 49.217c0 21.756 13.993 40.172 33.405 46.69 2.427.49 3.316-1.059 3.316-2.362 0-1.141-.08-5.052-.08-9.127-13.59 2.934-16.42-5.867-16.42-5.867-2.184-5.704-5.42-7.17-5.42-7.17-4.448-3.015.324-3.015.324-3.015 4.934.326 7.523 5.052 7.523 5.052 4.367 7.496 11.404 5.378 14.235 4.074.404-3.178 1.699-5.378 3.074-6.6-10.839-1.141-22.243-5.378-22.243-24.283 0-5.378 1.94-9.778 5.014-13.2-.485-1.222-2.184-6.275.486-13.038 0 0 4.125-1.304 13.426 5.052a46.97 46.97 0 0 1 12.214-1.63c4.125 0 8.33.571 12.213 1.63 9.302-6.356 13.427-5.052 13.427-5.052 2.67 6.763.97 11.816.485 13.038 3.155 3.422 5.015 7.822 5.015 13.2 0 18.905-11.404 23.06-22.324 24.283 1.78 1.548 3.316 4.481 3.316 9.126 0 6.6-.08 11.897-.08 13.526 0 1.304.89 2.853 3.316 2.364 19.412-6.52 33.405-24.935 33.405-46.691C97.707 22 75.788 0 48.854 0z" fill="#fff"/></svg>

After

Width:  |  Height:  |  Size: 960 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

View File

@@ -0,0 +1 @@
<svg width="98" height="96" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M48.854 0C21.839 0 0 22 0 49.217c0 21.756 13.993 40.172 33.405 46.69 2.427.49 3.316-1.059 3.316-2.362 0-1.141-.08-5.052-.08-9.127-13.59 2.934-16.42-5.867-16.42-5.867-2.184-5.704-5.42-7.17-5.42-7.17-4.448-3.015.324-3.015.324-3.015 4.934.326 7.523 5.052 7.523 5.052 4.367 7.496 11.404 5.378 14.235 4.074.404-3.178 1.699-5.378 3.074-6.6-10.839-1.141-22.243-5.378-22.243-24.283 0-5.378 1.94-9.778 5.014-13.2-.485-1.222-2.184-6.275.486-13.038 0 0 4.125-1.304 13.426 5.052a46.97 46.97 0 0 1 12.214-1.63c4.125 0 8.33.571 12.213 1.63 9.302-6.356 13.427-5.052 13.427-5.052 2.67 6.763.97 11.816.485 13.038 3.155 3.422 5.015 7.822 5.015 13.2 0 18.905-11.404 23.06-22.324 24.283 1.78 1.548 3.316 4.481 3.316 9.126 0 6.6-.08 11.897-.08 13.526 0 1.304.89 2.853 3.316 2.364 19.412-6.52 33.405-24.935 33.405-46.691C97.707 22 75.788 0 48.854 0z" fill="#24292f"/></svg>

After

Width:  |  Height:  |  Size: 963 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 645 KiB

View File

@@ -0,0 +1,4 @@
<svg width="200" height="200" xmlns="http://www.w3.org/2000/svg">
<circle cx="100" cy="100" r="90" fill="#FDF6E3" stroke="#073642" stroke-width="4" />
<text x="50%" y="50%" text-anchor="middle" fill="#073642" font-size="96" dy=".35em" font-family="Arial, sans-serif" font-weight="bold">H</text>
</svg>

After

Width:  |  Height:  |  Size: 306 B

View File

@@ -0,0 +1,4 @@
<svg width="200" height="200" xmlns="http://www.w3.org/2000/svg">
<circle cx="100" cy="100" r="90" stroke="#FDF6E3" fill="#073642" stroke-width="4" />
<text x="50%" y="50%" text-anchor="middle" fill="#FDF6E3" font-size="96" dy=".35em" font-family="Arial, sans-serif" font-weight="bold">H</text>
</svg>

After

Width:  |  Height:  |  Size: 306 B

View File

@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Created with Vectornator (http://vectornator.io/) -->
<svg style="fill-rule:nonzero;clip-rule:evenodd;stroke-linecap:round;stroke-linejoin:round;" version="1.1" viewBox="0 0 605.044 336.948" xmlns="http://www.w3.org/2000/svg" xmlns:vectornator="http://vectornator.io" xmlns:xlink="http://www.w3.org/1999/xlink">
<defs/>
<g id="Logo" vectornator:layerName="Logo">
<g opacity="1">
<path d="M591.109 167.89C580.229 111.687 546.936 77.6261 491.282 65.5687C433.469 53.0447 384.362 69.3554 342.802 110.398C310.616 142.185 278.709 174.259 246.482 205.995C225.922 226.226 205.922 247.343 179.109 259.626C149.202 273.334 119.469 272.37 93.4157 251.847C66.8557 230.91 61.2157 201.791 69.349 169.809C81.5757 121.787 138.509 102.695 182.402 131.721C187.656 135.193 192.829 138.787 198.149 142.406C200.682 140.009 202.496 138.35 204.242 136.622C224.509 116.598 244.109 95.8634 265.162 76.7034C293.082 51.2821 325.909 36.9234 364.469 39.9821C377.656 41.0274 390.656 44.4154 406.242 47.1781C374.122 16.6794 338.802 2.73806 296.776 9.35939C257.362 15.5701 226.149 36.7381 199.789 65.6101C193.602 72.3927 188.029 74.0394 179.256 70.7394C168.722 66.7727 158.376 64.2394 148.229 63.0127L148.229 63.0087C148.216 63.0074 148.189 63.0074 148.176 63.0061C148.056 62.9914 147.936 62.9714 147.816 62.9581C147.802 62.9687 147.802 62.9767 147.789 62.9874C21.4957 55.3727 9.17567 165.405 9.17567 165.405C-2.58433 286.525 80.029 311.302 80.029 311.302L80.069 311.302C90.0823 315.263 100.456 318.062 111.029 320.229L111.042 320.25C111.042 320.25 112.962 320.745 116.509 321.295C117.456 321.47 118.402 321.653 119.349 321.819C119.376 321.781 119.402 321.745 119.416 321.706C138.362 324.111 187.416 325.538 246.122 289.946C246.242 289.951 246.376 289.939 246.496 289.95C284.176 252.225 321.882 214.525 359.722 176.97C377.829 159.006 394.296 138.981 418.056 127.409C455.242 109.302 501.269 117.934 522.642 147.982C539.616 171.834 541.616 197.893 529.402 223.943C516.682 251.057 493.776 265.733 464.016 266.622C423.082 267.846 382.082 267.358 341.122 267.277C332.309 267.259 325.469 269.946 319.336 276.357C306.549 289.741 293.216 302.61 280.202 315.782C278.402 317.595 275.269 320.245 272.429 323.437C276.522 323.878 281.362 324.021 283.762 324.023C330.429 324.065 377.096 324.385 423.762 323.905C446.856 323.666 470.549 324.903 492.896 320.317C561.376 306.267 604.602 237.579 591.109 167.89" fill="#fdf6e3" fill-rule="nonzero" opacity="1" stroke="none"/>
<path d="M202.896 194.097C202.896 228.629 174.909 256.622 140.376 256.622C105.842 256.622 77.8423 228.629 77.8423 194.097C77.8423 159.565 105.842 131.57 140.376 131.57C174.909 131.57 202.896 159.565 202.896 194.097" fill="#fdf6e3" fill-rule="nonzero" opacity="1" stroke="none"/>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.8 KiB

View File

@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Created with Vectornator (http://vectornator.io/) -->
<svg style="fill-rule:nonzero;clip-rule:evenodd;stroke-linecap:round;stroke-linejoin:round;" version="1.1" viewBox="0 0 605.044 336.948" xmlns="http://www.w3.org/2000/svg" xmlns:vectornator="http://vectornator.io" xmlns:xlink="http://www.w3.org/1999/xlink">
<defs/>
<g id="Logo" vectornator:layerName="Logo">
<g opacity="1">
<path d="M591.109 167.89C580.229 111.687 546.936 77.6261 491.282 65.5687C433.469 53.0447 384.362 69.3554 342.802 110.398C310.616 142.185 278.709 174.259 246.482 205.995C225.922 226.226 205.922 247.343 179.109 259.626C149.202 273.334 119.469 272.37 93.4157 251.847C66.8557 230.91 61.2157 201.791 69.349 169.809C81.5757 121.787 138.509 102.695 182.402 131.721C187.656 135.193 192.829 138.787 198.149 142.406C200.682 140.009 202.496 138.35 204.242 136.622C224.509 116.598 244.109 95.8634 265.162 76.7034C293.082 51.2821 325.909 36.9234 364.469 39.9821C377.656 41.0274 390.656 44.4154 406.242 47.1781C374.122 16.6794 338.802 2.73806 296.776 9.35939C257.362 15.5701 226.149 36.7381 199.789 65.6101C193.602 72.3927 188.029 74.0394 179.256 70.7394C168.722 66.7727 158.376 64.2394 148.229 63.0127L148.229 63.0087C148.216 63.0074 148.189 63.0074 148.176 63.0061C148.056 62.9914 147.936 62.9714 147.816 62.9581C147.802 62.9687 147.802 62.9767 147.789 62.9874C21.4957 55.3727 9.17567 165.405 9.17567 165.405C-2.58433 286.525 80.029 311.302 80.029 311.302L80.069 311.302C90.0823 315.263 100.456 318.062 111.029 320.229L111.042 320.25C111.042 320.25 112.962 320.745 116.509 321.295C117.456 321.47 118.402 321.653 119.349 321.819C119.376 321.781 119.402 321.745 119.416 321.706C138.362 324.111 187.416 325.538 246.122 289.946C246.242 289.951 246.376 289.939 246.496 289.95C284.176 252.225 321.882 214.525 359.722 176.97C377.829 159.006 394.296 138.981 418.056 127.409C455.242 109.302 501.269 117.934 522.642 147.982C539.616 171.834 541.616 197.893 529.402 223.943C516.682 251.057 493.776 265.733 464.016 266.622C423.082 267.846 382.082 267.358 341.122 267.277C332.309 267.259 325.469 269.946 319.336 276.357C306.549 289.741 293.216 302.61 280.202 315.782C278.402 317.595 275.269 320.245 272.429 323.437C276.522 323.878 281.362 324.021 283.762 324.023C330.429 324.065 377.096 324.385 423.762 323.905C446.856 323.666 470.549 324.903 492.896 320.317C561.376 306.267 604.602 237.579 591.109 167.89" fill="#142831" fill-rule="nonzero" opacity="1" stroke="none"/>
<path d="M202.896 194.097C202.896 228.629 174.909 256.622 140.376 256.622C105.842 256.622 77.8423 228.629 77.8423 194.097C77.8423 159.565 105.842 131.57 140.376 131.57C174.909 131.57 202.896 159.565 202.896 194.097" fill="#142831" fill-rule="nonzero" opacity="1" stroke="none"/>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.8 KiB

View File

@@ -1,5 +1,12 @@
#! /bin/bash
#
tmpdir="$(mktemp -d)"
finish() {
rm -rf "$tmpdir"
}
trap finish EXIT
set -euo pipefail
# Generate the documentation for the package the calls go:generate
@@ -10,8 +17,5 @@ gomarkdoc --output "doc/md/${package%/}.md" "./${package}"
# Fix heading anchors by making them explicit
# Refer to https://docusaurus.io/docs/markdown-features/toc#heading-ids
stamp=$RANDOM
# sed 's/^## type /## /' "doc/md/${package%/}.md" > "doc/md/${package%/}.md.${stamp}"
sed -E 's/## type ([A-Za-z0-9_]+)/## type \1 {#\1}/' "doc/md/${package%/}.md" > "doc/md/${package%/}.md.${stamp}"
mv "doc/md/${package%/}.md.${stamp}" "doc/md/${package%/}.md"
sed -E 's/## type ([A-Za-z0-9_]+)/## type \1 {#\1}/' "doc/md/${package%/}.md" > "${tmpdir}/doc.md"
cat "./${package%%/}/header.yaml" "${tmpdir}/doc.md" > "doc/md/${package%/}.md"

View File

@@ -392,7 +392,7 @@ package v1alpha4
// Model represents the platform model holos gets from from the
// PlatformService.GetPlatform rpc method and provides to CUE using a tag.
// Injected as the tag "holos_model".
model: {...} @go(Model,map[string]any)
model?: {...} @go(Model,map[string]any)
// Tags represents cue @tag variables injected into the holos render component
// command from the holos render platform command. Tags with a "holos_"

View File

@@ -59,18 +59,33 @@ import (
artifact: "\(_path)/\(Name).gen.yaml"
let ResourcesOutput = "resources.gen.yaml"
let IntermediateOutput = "combined.gen.yaml"
generators: [{
kind: "Resources"
output: ResourcesOutput
resources: Resources
}]
generators: [
{
kind: "Resources"
output: ResourcesOutput
resources: Resources
},
for x in KustomizeConfig.Files {
kind: "File"
output: x.Source
file: source: x.Source
},
for x in KustomizeConfig.Resources {
kind: "File"
output: x.Source
file: source: x.Source
},
]
transformers: [
core.#Transformer & {
kind: "Kustomize"
inputs: [ResourcesOutput]
inputs: [for x in generators {x.output}]
output: IntermediateOutput
kustomize: kustomization: KustomizeConfig.Kustomization & {
resources: inputs
resources: [
ResourcesOutput,
for x in KustomizeConfig.Resources {x.Source},
]
}
},
_Transformer & {

View File

@@ -1,16 +1,19 @@
package platforms
// TODO: Remove env GODEBUG=gotypesalias=0 when cue 0.11 is released and used.
// See: https://github.com/cue-lang/cue/issues/3539
//go:generate rm -rf cue.mod/gen/github.com/holos-run/holos/api/v1alpha1
//go:generate cue get go github.com/holos-run/holos/api/v1alpha1/...
//go:generate env GODEBUG=gotypesalias=0 cue get go github.com/holos-run/holos/api/v1alpha1/...
//go generate rm -rf cue.mod/gen/github.com/holos-run/holos/api/core
//go:generate cue get go github.com/holos-run/holos/api/core/...
//go:generate env GODEBUG=gotypesalias=0 cue get go github.com/holos-run/holos/api/core/...
//go generate rm -rf cue.mod/gen/github.com/holos-run/holos/api/meta
//go:generate cue get go github.com/holos-run/holos/api/meta/...
//go:generate env GODEBUG=gotypesalias=0 cue get go github.com/holos-run/holos/api/meta/...
//go generate rm -rf cue.mod/gen/github.com/holos-run/holos/api/author
//go:generate cue get go github.com/holos-run/holos/api/author/...
//go:generate env GODEBUG=gotypesalias=0 cue get go github.com/holos-run/holos/api/author/...
//go generate rm -rf cue.mod/gen/github.com/holos-run/holos/service/gen/holos/object
//go:generate cue import ../../../service/holos/object/v1alpha1/object.proto -o cue.mod/gen/github.com/holos-run/holos/service/gen/holos/object/v1alpha1/object.proto_gen.cue -I ../../../proto -f

View File

@@ -0,0 +1,8 @@
package holos
import api "github.com/holos-run/holos/api/author/v1alpha4"
// Manage a workload cluster named local for use with the guides.
_Fleets: api.#StandardFleets & {
workload: clusters: local: _
}

View File

@@ -2,9 +2,9 @@ package holos
import api "github.com/holos-run/holos/api/author/v1alpha4"
#Platform: api.#Platform & {
Name: "guide"
_Platform: api.#Platform & {
Name: "default"
}
// Render a Platform resource for holos to process
#Platform.Resource
_Platform.Resource

View File

@@ -2,9 +2,6 @@ package holos
import api "github.com/holos-run/holos/api/author/v1alpha4"
// Manage a workload cluster named workload for use with the guides.
#Fleets: api.#StandardFleets
// Define the default organization name.
#Organization: api.#OrganizationStrict & {
DisplayName: string | *"Bank of Holos"

View File

@@ -1 +1 @@
0
1