(#66) Project managed namespaces

This patch uses the existing #ManagedNamespaces definition to create and
manage namespaces on the provisioner and workload clusters so that
SecretStore and eso-creds-refresher resources are managed in the project
environment namespaces and the project stage system namespace.

.gitignore

@@ -5,3 +5,4 @@ coverage.out
 dist/
 *.hold/
 /deploy/
+.vscode/

api/v1alpha1/buildplan.go

@@ -19,9 +19,10 @@ type BuildPlanSpec struct {
 }
 
 type BuildPlanComponents struct {
-	HelmChartList         []HelmChart         `json:"helmChartList,omitempty" yaml:"helmChartList,omitempty"`
-	KubernetesObjectsList []KubernetesObjects `json:"kubernetesObjectsList,omitempty" yaml:"kubernetesObjectsList,omitempty"`
-	KustomizeBuildList    []KustomizeBuild    `json:"kustomizeBuildList,omitempty" yaml:"kustomizeBuildList,omitempty"`
+	HelmChartList         []HelmChart                  `json:"helmChartList,omitempty" yaml:"helmChartList,omitempty"`
+	KubernetesObjectsList []KubernetesObjects          `json:"kubernetesObjectsList,omitempty" yaml:"kubernetesObjectsList,omitempty"`
+	KustomizeBuildList    []KustomizeBuild             `json:"kustomizeBuildList,omitempty" yaml:"kustomizeBuildList,omitempty"`
+	Resources             map[string]KubernetesObjects `json:"resources,omitempty" yaml:"resources,omitempty"`
 }
 
 func (bp *BuildPlan) Validate() error {

api/v1alpha1/helm.go

@@ -35,7 +35,7 @@ type Repository struct {
 	URL  string `json:"url"`
 }
 
-func (hc *HelmChart) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+func (hc *HelmChart) Render(ctx context.Context, path holos.InstancePath) (*Result, error) {
 	result := Result{HolosComponent: hc.HolosComponent}
 	if err := hc.helm(ctx, &result, path); err != nil {
 		return nil, err
@@ -49,7 +49,7 @@ func (hc *HelmChart) Render(ctx context.Context, path holos.PathComponent) (*Res
 
 // runHelm provides the values produced by CUE to helm template and returns
 // the rendered kubernetes api objects in the result.
-func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.PathComponent) error {
+func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.InstancePath) error {
 	log := logger.FromContext(ctx).With("chart", hc.Chart.Name)
 	if hc.Chart.Name == "" {
 		log.WarnContext(ctx, "skipping helm: no chart name specified, use a different component type")
@@ -121,7 +121,7 @@ func (hc *HelmChart) helm(ctx context.Context, r *Result, path holos.PathCompone
 }
 
 // cacheChart stores a cached copy of Chart in the chart subdirectory of path.
-func cacheChart(ctx context.Context, path holos.PathComponent, chartDir string, chart Chart) error {
+func cacheChart(ctx context.Context, path holos.InstancePath, chartDir string, chart Chart) error {
 	log := logger.FromContext(ctx)
 
 	cacheTemp, err := os.MkdirTemp(string(path), chartDir)

api/v1alpha1/kubernetesobjects.go

@@ -2,6 +2,7 @@ package v1alpha1
 
 import (
 	"context"
+
 	"github.com/holos-run/holos"
 )
 
@@ -13,7 +14,7 @@ type KubernetesObjects struct {
 }
 
 // Render produces kubernetes api objects from the APIObjectMap
-func (o *KubernetesObjects) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+func (o *KubernetesObjects) Render(ctx context.Context, path holos.InstancePath) (*Result, error) {
 	result := Result{HolosComponent: o.HolosComponent}
 	result.addObjectMap(ctx, o.APIObjectMap)
 	return &result, nil

api/v1alpha1/kustomize.go

@@ -2,6 +2,7 @@ package v1alpha1
 
 import (
 	"context"
+
 	"github.com/holos-run/holos"
 	"github.com/holos-run/holos/pkg/logger"
 	"github.com/holos-run/holos/pkg/util"
@@ -29,7 +30,7 @@ type KustomizeBuild struct {
 
 // Render produces a Result by executing kubectl kustomize on the holos
 // component path. Useful for processing raw yaml files.
-func (kb *KustomizeBuild) Render(ctx context.Context, path holos.PathComponent) (*Result, error) {
+func (kb *KustomizeBuild) Render(ctx context.Context, path holos.InstancePath) (*Result, error) {
 	log := logger.FromContext(ctx)
 	result := Result{HolosComponent: kb.HolosComponent}
 	// Run kustomize.

api/v1alpha1/objectmap.go

@@ -1,8 +1,14 @@
 package v1alpha1
 
+// Label is an arbitrary unique identifier.  Defined as a type for clarity and type checking.
+type Label string
+
+// Kind is a kubernetes api object kind. Defined as a type for clarity and type checking.
+type Kind string
+
 // APIObjectMap is the shape of marshalled api objects returned from cue to the
 // holos cli. A map is used to improve the clarity of error messages from cue.
-type APIObjectMap map[string]map[string]string
+type APIObjectMap map[Kind]map[Label]string
 
 // FileContentMap is a map of file names to file contents.
 type FileContentMap map[string]string

api/v1alpha1/render.go

@@ -2,12 +2,13 @@ package v1alpha1
 
 import (
 	"context"
+
 	"github.com/holos-run/holos"
 )
 
 type Renderer interface {
 	GetKind() string
-	Render(ctx context.Context, path holos.PathComponent) (*Result, error)
+	Render(ctx context.Context, path holos.InstancePath) (*Result, error)
 }
 
 // Render produces a Result representing the kubernetes api objects to
@@ -16,6 +17,6 @@ type Renderer interface {
 // conceptualized as a data pipeline, for example a component may render a
 // result by first calling helm template, then passing the result through
 // kustomize, then mixing in overlay api objects.
-func Render(ctx context.Context, r Renderer, path holos.PathComponent) (*Result, error) {
+func Render(ctx context.Context, r Renderer, path holos.InstancePath) (*Result, error) {
 	return r.Render(ctx, path)
 }

api/v1alpha1/result.go

@@ -3,12 +3,13 @@ package v1alpha1
 import (
 	"context"
 	"fmt"
-	"github.com/holos-run/holos/pkg/logger"
-	"github.com/holos-run/holos/pkg/util"
-	"github.com/holos-run/holos/pkg/wrapper"
 	"os"
 	"path/filepath"
 	"slices"
+
+	"github.com/holos-run/holos/pkg/logger"
+	"github.com/holos-run/holos/pkg/util"
+	"github.com/holos-run/holos/pkg/wrapper"
 )
 
 // Result is the build result for display or writing.  Holos components Render the Result as a data pipeline.
@@ -40,7 +41,7 @@ func (r *Result) AccumulatedOutput() string {
 func (r *Result) addObjectMap(ctx context.Context, objectMap APIObjectMap) {
 	log := logger.FromContext(ctx)
 	b := []byte(r.AccumulatedOutput())
-	kinds := make([]string, 0, len(objectMap))
+	kinds := make([]Kind, 0, len(objectMap))
 	// Sort the keys
 	for kind := range objectMap {
 		kinds = append(kinds, kind)
@@ -50,7 +51,7 @@ func (r *Result) addObjectMap(ctx context.Context, objectMap APIObjectMap) {
 	for _, kind := range kinds {
 		v := objectMap[kind]
 		// Sort the keys
-		names := make([]string, 0, len(v))
+		names := make([]Label, 0, len(v))
 		for name := range v {
 			names = append(names, name)
 		}

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/buildplan_go_gen.cue

@@ -19,7 +19,8 @@ package v1alpha1
 }
 
 #BuildPlanComponents: {
-	helmCharts?: [...#HelmChart] @go(HelmCharts,[]HelmChart)
-	kubernetesObjects?: [...#KubernetesObjects] @go(KubernetesObjects,[]KubernetesObjects)
-	kustomizeBuilds?: [...#KustomizeBuild] @go(KustomizeBuilds,[]KustomizeBuild)
+	helmChartList?: [...#HelmChart] @go(HelmChartList,[]HelmChart)
+	kubernetesObjectsList?: [...#KubernetesObjects] @go(KubernetesObjectsList,[]KubernetesObjects)
+	kustomizeBuildList?: [...#KustomizeBuild] @go(KustomizeBuildList,[]KustomizeBuild)
+	resources?: {[string]: #KubernetesObjects} @go(Resources,map[string]KubernetesObjects)
 }

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/constants_go_gen.cue

@@ -11,5 +11,5 @@ package v1alpha1
 // ChartDir is the directory name created in the holos component directory to cache a chart.
 #ChartDir: "vendor"
 
-// ResourcesFile is the file name used to store component output when post processing with kustomize.
+// ResourcesFile is the file name used to store component output when post-processing with kustomize.
 #ResourcesFile: "resources.yaml"

docs/examples/cue.mod/gen/github.com/holos-run/holos/api/v1alpha1/objectmap_go_gen.cue

@@ -4,6 +4,12 @@
 
 package v1alpha1
 
+// Label is an arbitrary unique identifier.  Defined as a type for clarity and type checking.
+#Label: string
+
+// Kind is a kubernetes api object kind. Defined as a type for clarity and type checking.
+#Kind: string
+
 // APIObjectMap is the shape of marshalled api objects returned from cue to the
 // holos cli. A map is used to improve the clarity of error messages from cue.
 #APIObjectMap: {[string]: [string]: string}

docs/examples/helpers.cue

@@ -7,14 +7,16 @@ import "encoding/yaml"
 	// apiObjects holds each the api objects produced by cue.
 	apiObjects: {
 		[Kind=_]: {
-			[Name=_]: {
+			[string]: {
 				kind: Kind
 				...
 			}
 		}
+		Namespace?: [Name=_]: #Namespace & {metadata: name: Name}
 		ExternalSecret?: [Name=_]: #ExternalSecret & {_name: Name}
 		VirtualService?: [Name=_]: #VirtualService & {metadata: name: Name}
 		Issuer?: [Name=_]: #Issuer & {metadata: name: Name}
+		Gateway?: [Name=_]: #Gateway & {metadata: name: Name}
 	}
 
 	// apiObjectMap holds the marshalled representation of apiObjects

docs/examples/platforms/reference/clusters/projects.cue

@@ -0,0 +1,31 @@
+package holos
+
+_Projects: #Projects & {
+	holos: {
+		clusters: {
+			k1: _
+			k2: _
+		}
+		environments: {
+			prod: stage: "prod"
+			dev: stage:  "dev"
+			jeff: stage: dev.stage
+			gary: stage: dev.stage
+			nate: stage: dev.stage
+		}
+	}
+
+	iam: {
+		clusters: {
+			core1: _
+			core2: _
+		}
+	}
+}
+
+// Manage namespaces for platform project environments.
+for project in _Projects {
+	for ns in project.managedNamespaces {
+		#ManagedNamespaces: (ns.namespace.metadata.name): ns
+	}
+}

docs/examples/platforms/reference/clusters/projects/projects.cue

@@ -1,3 +0,0 @@
-package holos
-
-#InputKeys: project: "projects"

docs/examples/platforms/reference/clusters/provisioner/projects/projects.cue

@@ -0,0 +1,5 @@
+package holos
+
+for Project in _Projects {
+	spec: components: resources: (#ProjectTemplate & {project: Project}).provisioner.resources
+}

docs/examples/platforms/reference/clusters/workload/projects/projects.cue

@@ -0,0 +1,5 @@
+package holos
+
+for Project in _Projects {
+	spec: components: resources: (#ProjectTemplate & {project: Project}).workload.resources
+}

docs/examples/platforms/reference/platform_projects.cue

@@ -0,0 +1,180 @@
+package holos
+
+import "strings"
+
+// Platform level definition of a project.
+#Project: {
+	name: string
+	// All projects have at least a prod environment and stage.
+	environments: prod: stage: "prod"
+	environments: prod: dnsSegments: []
+	stages: prod: _
+	stages: dev:  _
+	// Short hostnames to construct fqdns.
+	hosts: (name): _
+}
+
+#ProjectTemplate: {
+	project: #Project
+
+	// ExtAuthzHosts maps host names to the backend environment namespace for ExtAuthz.
+	let ExtAuthzHosts = {
+		// Initialize all stages, even if they have no environments.
+		for stage in project.stages {
+			(stage.name): {}
+		}
+
+		for env in project.environments {
+			(env.stage): {
+				for host in project.hosts {
+					let NAME = "https-\(project.name)-\(env.name)-\(host.name)"
+					let SEGMENTS = [host.name] + env.dnsSegments + [#Platform.org.domain]
+					let HOST = strings.Join(SEGMENTS, ".")
+					(NAME): #GatewayServer & {
+						hosts: ["\(env.namespace)/\(HOST)"]
+						// name must be unique across all servers in all gateways
+						port: name:     NAME
+						port: number:   443
+						port: protocol: "HTTPS"
+						// TODO: Manage a certificate with each host in the dns alt names.
+						tls: credentialName: HOST
+						tls: mode:           "SIMPLE"
+					}
+
+					for cluster in project.clusters {
+						let NAME = "https-\(cluster.name)-\(project.name)-\(env.name)-\(host.name)"
+						let SEGMENTS = [host.name] + env.dnsSegments + [cluster.name, #Platform.org.domain]
+						let HOST = strings.Join(SEGMENTS, ".")
+						(NAME): #GatewayServer & {
+							hosts: ["\(env.namespace)/\(HOST)"]
+							// name must be unique across all servers in all gateways
+							port: name:     NAME
+							port: number:   443
+							port: protocol: "HTTPS"
+							// TODO: Manage a certificate with each host in the dns alt names.
+							tls: credentialName: HOST
+							tls: mode:           "SIMPLE"
+						}
+					}
+				}
+			}
+		}
+	}
+
+	workload: resources: {
+		for stage in project.stages {
+			// Istio Gateway
+			"\(stage.slug)-gateway": #KubernetesObjects & {
+				apiObjectMap: (#APIObjects & {
+					apiObjects: Gateway: (stage.slug): #Gateway & {
+						spec: servers: [for host in ExtAuthzHosts[stage.name] {host}]
+					}
+
+					for host in ExtAuthzHosts[stage.name] {
+						apiObjects: ExternalSecret: (host.tls.credentialName): metadata: namespace: "istio-ingress"
+					}
+				}).apiObjectMap
+			}
+		}
+	}
+
+	provisioner: resources: {
+		for stage in project.stages {
+			"\(stage.slug)-certs": #KubernetesObjects & {
+				apiObjectMap: (#APIObjects & {
+					for host in ExtAuthzHosts[stage.name] {
+						let CN = host.tls.credentialName
+						apiObjects: Certificate: (CN): #Certificate & {
+							metadata: name:      CN
+							metadata: namespace: "istio-ingress"
+							spec: {
+								commonName: CN
+								dnsNames: [CN]
+								secretName: CN
+								issuerRef: {
+									kind: "ClusterIssuer"
+									name: "letsencrypt"
+								}
+							}
+						}
+					}
+
+				}).apiObjectMap
+			}
+		}
+	}
+}
+
+// #GatewayServer defines the value of the istio Gateway.spec.servers field.
+#GatewayServer: {
+	// The ip or the Unix domain socket to which the listener should
+	// be bound to.
+	bind?:            string
+	defaultEndpoint?: string
+
+	// One or more hosts exposed by this gateway.
+	hosts: [...string]
+
+	// An optional name of the server, when set must be unique across
+	// all servers.
+	name?: string
+
+	// The Port on which the proxy should listen for incoming
+	// connections.
+	port: {
+		// Label assigned to the port.
+		name: string
+
+		// A valid non-negative integer port number.
+		number: int
+
+		// The protocol exposed on the port.
+		protocol:    string
+		targetPort?: int
+	}
+
+	// Set of TLS related options that govern the server's behavior.
+	tls?: {
+		// REQUIRED if mode is `MUTUAL` or `OPTIONAL_MUTUAL`.
+		caCertificates?: string
+
+		// Optional: If specified, only support the specified cipher list.
+		cipherSuites?: [...string]
+
+		// For gateways running on Kubernetes, the name of the secret that
+		// holds the TLS certs including the CA certificates.
+		credentialName?: string
+
+		// If set to true, the load balancer will send a 301 redirect for
+		// all http connections, asking the clients to use HTTPS.
+		httpsRedirect?: bool
+
+		// Optional: Maximum TLS protocol version.
+		maxProtocolVersion?: "TLS_AUTO" | "TLSV1_0" | "TLSV1_1" | "TLSV1_2" | "TLSV1_3"
+
+		// Optional: Minimum TLS protocol version.
+		minProtocolVersion?: "TLS_AUTO" | "TLSV1_0" | "TLSV1_1" | "TLSV1_2" | "TLSV1_3"
+
+		// Optional: Indicates whether connections to this port should be
+		// secured using TLS.
+		mode?: "PASSTHROUGH" | "SIMPLE" | "MUTUAL" | "AUTO_PASSTHROUGH" | "ISTIO_MUTUAL" | "OPTIONAL_MUTUAL"
+
+		// REQUIRED if mode is `SIMPLE` or `MUTUAL`.
+		privateKey?: string
+
+		// REQUIRED if mode is `SIMPLE` or `MUTUAL`.
+		serverCertificate?: string
+
+		// A list of alternate names to verify the subject identity in the
+		// certificate presented by the client.
+		subjectAltNames?: [...string]
+
+		// An optional list of hex-encoded SHA-256 hashes of the
+		// authorized client certificates.
+		verifyCertificateHash?: [...string]
+
+		// An optional list of base64-encoded SHA-256 hashes of the SPKIs
+		// of authorized client certificates.
+		verifyCertificateSpki?: [...string]
+	}
+}

docs/examples/platforms/reference/project_constraints.cue

@@ -0,0 +1 @@
+package holos

docs/examples/project_types.cue

@@ -1,11 +1,94 @@
 package holos
 
+import h "github.com/holos-run/holos/api/v1alpha1"
+
+// #Projects is a map of all the projects in the platform.
+#Projects: [Name=_]: #Project & {name: Name}
+
 #Project: {
 	name: string
+	let ProjectName = name
+	description: string
+	environments: [Name=string]: #Environment & {
+		name:    Name
+		project: ProjectName
+	}
+	stages: [Name=string]: #Stage & {
+		name:    Name
+		project: ProjectName
+	}
+
+	// hosts are short hostnames to configure for the project.
+	hosts: [Name=string]: #Host & {name: Name}
+	// clusters are the cluster names the project is configured on.
+	clusters: [Name=string]: #Cluster & {name: Name}
+
+	// managedNamespaces ensures project namespaces have SecretStores that can sync ExternalSecrets from the provisioner cluster.
+	managedNamespaces: {
+		// Define the shape of a managed namespace.
+		[Name=_]: #ManagedNamespace & {
+			namespace: metadata: name: Name
+			clusterNames: ["provisioner", for c in clusters {c.name}]
+		}
+
+		// Manage a system namespace for each stage in the project.
+		for stage in stages {
+			for ns in stage.namespaces {
+				(ns.name): _
+			}
+		}
+
+		// Manage a namespace for each environment in the project.
+		for env in environments {
+			(env.namespace): _
+		}
+	}
+
+	// features is YAGNI maybe? 
+	features: [Name=string]: #Feature & {name: Name}
 }
 
-#Projects: {
-	[Name=_]: #Project & {
-		name: Name
+// #Cluster defines a cluster
+#Cluster: name: string
+
+// #Host defines a short hostname
+#Host: name: string
+
+#Environment: {
+	// name uniquely identifies the environment within the scope of the project.
+	name:      string
+	project:   string
+	stage:     string | "dev" | "prod"
+	slug:      "\(name)-\(project)"
+	namespace: "\(name)-\(project)"
+	dnsSegments: [...string] | *[name]
+}
+
+#Stage: {
+	name:    string
+	project: string
+	slug:    "\(name)-\(project)"
+	// Manage a system namespace for each stage
+	namespaces: [Name=_]: name: Name
+	namespaces: "\(name)-\(project)-system": _
+}
+
+#Feature: {
+	name:        string
+	description: string
+	enabled:     *true | false
+}
+
+#ProjectTemplate: {
+	project: #Project
+
+	// workload cluster resources
+	workload: resources: [Name=_]: h.#KubernetesObjects & {
+		metadata: name: Name
+	}
+
+	// provisioner cluster resources
+	provisioner: resources: [Name=_]: h.#KubernetesObjects & {
+		metadata: name: Name
 	}
 }

docs/examples/schema.cue

@@ -73,11 +73,15 @@ _apiVersion: "holos.run/v1alpha1"
 #Job:             #NamespaceObject & batchv1.#Job
 #CronJob:         #NamespaceObject & batchv1.#CronJob
 #Deployment:      #NamespaceObject & appsv1.#Deployment
-#Gateway:         #NamespaceObject & gw.#Gateway
 #VirtualService:  #NamespaceObject & vs.#VirtualService
 #Certificate:     #NamespaceObject & crt.#Certificate
 #PostgresCluster: #NamespaceObject & pg.#PostgresCluster
 
+#Gateway: #NamespaceObject & gw.#Gateway & {
+	metadata: namespace: string | *"istio-ingress"
+	spec: selector: istio: string | *"ingressgateway"
+}
+
 // #HTTP01Cert defines a http01 certificate.
 #HTTP01Cert: {
 	_name:      string

holos.go

@@ -5,6 +5,6 @@ package holos
 // It is given a unique type so the API is clear.
 type PathCueMod string
 
-// A PathComponent is a string representing the filesystem path of a holos component.
+// A InstancePath is a string representing the filesystem path of a holos instance.
 // It is given a unique type so the API is clear.
-type PathComponent string
+type InstancePath string

pkg/internal/builder/builder.go

@@ -155,22 +155,29 @@ func (b *Builder) Run(ctx context.Context) (results []*v1alpha1.Result, err erro
 		}
 
 		// TODO: concurrent renders
+		for _, component := range buildPlan.Spec.Components.Resources {
+			if result, err := component.Render(ctx, holos.InstancePath(instance.Dir)); err != nil {
+				return nil, wrapper.Wrap(fmt.Errorf("could not render: %w", err))
+			} else {
+				results = append(results, result)
+			}
+		}
 		for _, component := range buildPlan.Spec.Components.KubernetesObjectsList {
-			if result, err := component.Render(ctx, holos.PathComponent(instance.Dir)); err != nil {
+			if result, err := component.Render(ctx, holos.InstancePath(instance.Dir)); err != nil {
 				return nil, wrapper.Wrap(fmt.Errorf("could not render: %w", err))
 			} else {
 				results = append(results, result)
 			}
 		}
 		for _, component := range buildPlan.Spec.Components.HelmChartList {
-			if result, err := component.Render(ctx, holos.PathComponent(instance.Dir)); err != nil {
+			if result, err := component.Render(ctx, holos.InstancePath(instance.Dir)); err != nil {
 				return nil, wrapper.Wrap(fmt.Errorf("could not render: %w", err))
 			} else {
 				results = append(results, result)
 			}
 		}
 		for _, component := range buildPlan.Spec.Components.KustomizeBuildList {
-			if result, err := component.Render(ctx, holos.PathComponent(instance.Dir)); err != nil {
+			if result, err := component.Render(ctx, holos.InstancePath(instance.Dir)); err != nil {
 				return nil, wrapper.Wrap(fmt.Errorf("could not render: %w", err))
 			} else {
 				results = append(results, result)

pkg/version/embedded/minor

@@ -1 +1 @@
-60
+61

pkg/version/embedded/patch

@@ -1 +1 @@
-3
+1