From c22a6792c2e8f997178a9fa3bd27a22ed0b7abd1 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 25 Jun 2024 12:46:11 +0200 Subject: [PATCH 1/4] add tenant nginx-ingress Signed-off-by: Andrei Kvapil --- packages/apps/kubernetes/README.md | 21 +++++++++------ .../templates/helmreleases/delete.yaml | 2 ++ .../apps/kubernetes/templates/ingress.yaml | 8 +++--- packages/apps/kubernetes/values.schema.json | 17 ++++++------ packages/apps/kubernetes/values.yaml | 27 ++++++++++++++----- 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 434ca7ee..e3134e00 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -31,11 +31,16 @@ kubectl get secret -n kubernetes--admin-kubeconfig -o g ### Common parameters -| Name | Description | Value | -| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------- | -| `host` | The hostname used to access the Kubernetes cluster externally (defaults to using the cluster name as a subdomain for the tenant host). | `""` | -| `controlPlane.replicas` | Number of replicas for Kubernetes contorl-plane components | `2` | -| `nodeGroups` | nodeGroups configuration | `{}` | -| `addons.certManager.enabled` | Enables the certificate manager which automatically creates and manages SSL/TLS certificates | `true` | -| `addons.ingressNginx.enabled` | Enables Ingress-NGINX Controller on nodes with 'ingress-nginx' role | `true` | -| `addons.ingressNginx.host` | The domain name that should be passtrough to the cluster by upper ingress. | `example.org` | +| Name | Description | Value | +| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ----- | +| `host` | The hostname used to access the Kubernetes cluster externally (defaults to using the cluster name as a subdomain for the tenant host). | `""` | +| `controlPlane.replicas` | Number of replicas for Kubernetes contorl-plane components | `2` | +| `nodeGroups` | nodeGroups configuration | `{}` | + +### Cluster Addons + +| Name | Description | Value | +| ----------------------------- | ---------------------------------------------------------------------------------- | ------- | +| `addons.certManager.enabled` | Enables the cert-manager | `false` | +| `addons.ingressNginx.enabled` | Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role) | `false` | +| `addons.ingressNginx.hosts` | List of domain names that should be passed through to the cluster by upper cluster | `[]` | diff --git a/packages/apps/kubernetes/templates/helmreleases/delete.yaml b/packages/apps/kubernetes/templates/helmreleases/delete.yaml index f89ac771..5874e878 100644 --- a/packages/apps/kubernetes/templates/helmreleases/delete.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/delete.yaml @@ -32,6 +32,7 @@ spec: {{ .Release.Name }}-cilium {{ .Release.Name }}-csi {{ .Release.Name }}-cert-manager + {{ .Release.Name }}-ingress-nginx -p '{"spec": {"suspend": true}}' --type=merge --field-manager=flux-client-side-apply || true --- @@ -64,6 +65,7 @@ rules: - {{ .Release.Name }}-cilium - {{ .Release.Name }}-csi - {{ .Release.Name }}-cert-manager + - {{ .Release.Name }}-ingress-nginx --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/packages/apps/kubernetes/templates/ingress.yaml b/packages/apps/kubernetes/templates/ingress.yaml index 3951e1de..e677afab 100644 --- a/packages/apps/kubernetes/templates/ingress.yaml +++ b/packages/apps/kubernetes/templates/ingress.yaml @@ -1,6 +1,6 @@ {{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }} {{- $ingress := index $myNS.metadata.annotations "namespace.cozystack.io/ingress" }} -{{- if .Values.addons.ingressNginx.enabled }} +{{- if .Values.addons.ingressNginx.hosts }} --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -12,16 +12,18 @@ metadata: spec: ingressClassName: "{{ $ingress }}" rules: - - host: {{ .Values.addons.ingressNginx.host | quote }} + {{- range .Values.addons.ingressNginx.hosts }} + - host: {{ . | quote }} http: paths: - path: / pathType: ImplementationSpecific backend: service: - name: {{ .Release.Name }}-ingress-nginx + name: {{ $.Release.Name }}-ingress-nginx port: number: 443 + {{- end }} --- apiVersion: v1 kind: Service diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 2455d5f8..8d3fa1c7 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -25,8 +25,8 @@ "properties": { "enabled": { "type": "boolean", - "description": "Enables the certificate manager which automatically creates and manages SSL/TLS certificates", - "default": true + "description": "Enables the cert-manager", + "default": false } } }, @@ -35,13 +35,14 @@ "properties": { "enabled": { "type": "boolean", - "description": "Enables Ingress-NGINX Controller on nodes with 'ingress-nginx' role", - "default": true + "description": "Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role)", + "default": false }, - "host": { - "type": "string", - "description": "The domain name that should be passtrough to the cluster by upper ingress.", - "default": "example.org" + "hosts": { + "type": "array", + "description": "List of domain names that should be passed through to the cluster by upper cluster", + "default": [], + "items": {} } } } diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index 8d8f536a..d591a107 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -19,13 +19,26 @@ nodeGroups: roles: - ingress-nginx +## @section Cluster Addons +## addons: - ## @param addons.certManager.enabled Enables the certificate manager which automatically creates and manages SSL/TLS certificates - certManager: - enabled: true - ## @param addons.ingressNginx.enabled Enables Ingress-NGINX Controller on nodes with 'ingress-nginx' role - ## @param addons.ingressNginx.host The domain name that should be passtrough to the cluster by upper ingress. + ## Cert-manager: automatically creates and manages SSL/TLS certificate + ## + certManager: + ## @param addons.certManager.enabled Enables the cert-manager + enabled: false + + ## Ingress-NGINX Controller + ## ingressNginx: - enabled: true - host: "example.org" + ## @param addons.ingressNginx.enabled Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role) + ## + enabled: false + ## @param addons.ingressNginx.hosts List of domain names that should be passed through to the cluster by upper cluster + ## e.g: + ## hosts: + ## - example.org + ## - foo.example.net + ## + hosts: [] From c01604fb7fce10cee5adea1446572ed11a8108f3 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 26 Jun 2024 03:07:53 +0200 Subject: [PATCH 2/4] fix typo in cert-manager addon --- .../apps/kubernetes/templates/helmreleases/cert-manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml index 175b94b2..71d395f5 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml @@ -8,7 +8,7 @@ metadata: coztstack.io/target-cluster-name: {{ .Release.Name }} spec: interval: 1m - releaseName: cert-mnager + releaseName: cert-manager chart: spec: chart: cozy-cert-manager From 7462be79be12a289f55ca24cebc484a75d5d7bfc Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 26 Jun 2024 03:08:09 +0200 Subject: [PATCH 3/4] add fluxcd addon --- packages/apps/kubernetes/README.md | 2 + .../templates/helmreleases/delete.yaml | 4 + .../templates/helmreleases/fluxcd.yaml | 74 +++++++++++++++++++ packages/apps/kubernetes/values.schema.json | 10 +++ packages/apps/kubernetes/values.yaml | 7 ++ 5 files changed, 97 insertions(+) create mode 100644 packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index e3134e00..ea801630 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -44,3 +44,5 @@ kubectl get secret -n kubernetes--admin-kubeconfig -o g | `addons.certManager.enabled` | Enables the cert-manager | `false` | | `addons.ingressNginx.enabled` | Enable Ingress-NGINX controller (expect nodes with 'ingress-nginx' role) | `false` | | `addons.ingressNginx.hosts` | List of domain names that should be passed through to the cluster by upper cluster | `[]` | +| `addons.fluxcd.enabled` | Enables Flux CD | `false` | + diff --git a/packages/apps/kubernetes/templates/helmreleases/delete.yaml b/packages/apps/kubernetes/templates/helmreleases/delete.yaml index 5874e878..21925476 100644 --- a/packages/apps/kubernetes/templates/helmreleases/delete.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/delete.yaml @@ -33,6 +33,8 @@ spec: {{ .Release.Name }}-csi {{ .Release.Name }}-cert-manager {{ .Release.Name }}-ingress-nginx + {{ .Release.Name }}-fluxcd-operator + {{ .Release.Name }}-fluxcd -p '{"spec": {"suspend": true}}' --type=merge --field-manager=flux-client-side-apply || true --- @@ -66,6 +68,8 @@ rules: - {{ .Release.Name }}-csi - {{ .Release.Name }}-cert-manager - {{ .Release.Name }}-ingress-nginx + - {{ .Release.Name }}-fluxcd-operator + - {{ .Release.Name }}-fluxcd --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml new file mode 100644 index 00000000..17c1e352 --- /dev/null +++ b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml @@ -0,0 +1,74 @@ +{{- if .Values.addons.fluxcd.enabled }} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-fluxcd-operator + labels: + cozystack.io/repository: system + coztstack.io/target-cluster-name: {{ .Release.Name }} +spec: + interval: 1m + releaseName: fluxcd-operator + chart: + spec: + chart: cozy-fluxcd-operator + reconcileStrategy: Revision + sourceRef: + kind: HelmRepository + name: cozystack-system + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-kubeconfig + targetNamespace: cozy-fluxcd + storageNamespace: cozy-fluxcd + install: + createNamespace: true + values: + flux-operator: + fullnameOverride: flux-operator + tolerations: [] + hostNetwork: false + dependsOn: + {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- end }} + - name: {{ .Release.Name }}-cilium + namespace: {{ .Release.Namespace }} +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-fluxcd + labels: + cozystack.io/repository: system + coztstack.io/target-cluster-name: {{ .Release.Name }} +spec: + interval: 1m + releaseName: fluxcd + chart: + spec: + chart: cozy-fluxcd + reconcileStrategy: Revision + sourceRef: + kind: HelmRepository + name: cozystack-system + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-kubeconfig + targetNamespace: cozy-fluxcd + storageNamespace: cozy-fluxcd + install: + createNamespace: true + dependsOn: + {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- end }} + - name: {{ .Release.Name }}-cilium + namespace: {{ .Release.Namespace }} + - name: {{ .Release.Name }}-fluxcd-operator + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 8d3fa1c7..cceff4d3 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -45,6 +45,16 @@ "items": {} } } + }, + "fluxcd": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enables Flux CD", + "default": false + } + } } } } diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index d591a107..548e344d 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -42,3 +42,10 @@ addons: ## - foo.example.net ## hosts: [] + + ## Flux CD + ## + fluxcd: + ## @param addons.fluxcd.enabled Enables Flux CD + ## + enabled: false From 07384c40f885c5135f706530bd0c6d4f1cc1bcda Mon Sep 17 00:00:00 2001 From: Kingdon Barrett Date: Fri, 28 Jun 2024 03:02:41 -0400 Subject: [PATCH 4/4] Tenant nginx ingress (fixes) (#191) I am testing install with this PR #183 and I had some issues, these should help --------- Signed-off-by: Kingdon Barrett --- .../templates/helmreleases/ingress-nginx.yaml | 2 +- scripts/installer.sh | 1 + scripts/package-system.mk | 11 +++++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml index 13749a9f..2255b07b 100644 --- a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml @@ -8,7 +8,7 @@ metadata: coztstack.io/target-cluster-name: {{ .Release.Name }} spec: interval: 1m - releaseName: cert-mnager + releaseName: ingress-nginx chart: spec: chart: cozy-ingress-nginx diff --git a/scripts/installer.sh b/scripts/installer.sh index 80fce275..63121180 100755 --- a/scripts/installer.sh +++ b/scripts/installer.sh @@ -20,6 +20,7 @@ run_migrations() { flux_is_ok() { kubectl wait --for=condition=available -n cozy-fluxcd deploy/source-controller deploy/helm-controller --timeout=1s + kubectl wait --for=condition=ready -n cozy-fluxcd helmrelease/fluxcd --timeout=1s # to call "apply resume" below } ensure_fluxcd() { diff --git a/scripts/package-system.mk b/scripts/package-system.mk index 3f9c908d..37d12fe3 100644 --- a/scripts/package-system.mk +++ b/scripts/package-system.mk @@ -5,13 +5,16 @@ help: ## Show this help. @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) show: ## Show output of rendered templates - kubectl get hr -n $(NAMESPACE) $(NAME) -o jsonpath='{.spec.values}' | helm template --dry-run=server --post-renderer ../../../scripts/fluxcd-kustomize.sh -n $(NAMESPACE) $(NAME) . -f - + kubectl get hr -n $(NAMESPACE) $(NAME) -o jsonpath='{.spec.values}' | NAMESPACE=$(NAMESPACE) NAME=$(NAME) \ + helm template --dry-run=server --post-renderer ../../../scripts/fluxcd-kustomize.sh -n $(NAMESPACE) $(NAME) . -f - -apply: suspend ## Apply Helm release to a Kubernetes cluster - kubectl get hr -n $(NAMESPACE) $(NAME) -o jsonpath='{.spec.values}' | helm upgrade -i --post-renderer ../../../scripts/fluxcd-kustomize.sh -n $(NAMESPACE) $(NAME) . -f - +apply: suspend ## Apply Helm release to a Kubernetes cluster + kubectl get hr -n $(NAMESPACE) $(NAME) -o jsonpath='{.spec.values}' | NAMESPACE=$(NAMESPACE) NAME=$(NAME) \ + helm upgrade -i --post-renderer ../../../scripts/fluxcd-kustomize.sh -n $(NAMESPACE) $(NAME) . -f - diff: ## Diff Helm release against objects in a Kubernetes cluster - kubectl get hr -n $(NAMESPACE) $(NAME) -o jsonpath='{.spec.values}' | helm diff upgrade --allow-unreleased --post-renderer ../../../scripts/fluxcd-kustomize.sh -n $(NAMESPACE) $(NAME) . -f - + kubectl get hr -n $(NAMESPACE) $(NAME) -o jsonpath='{.spec.values}' | NAMESPACE=$(NAMESPACE) NAME=$(NAME) \ + helm diff upgrade --allow-unreleased --post-renderer ../../../scripts/fluxcd-kustomize.sh -n $(NAMESPACE) $(NAME) . -f - suspend: ## Suspend reconciliation for an existing Helm release kubectl patch hr -n $(NAMESPACE) $(NAME) -p '{"spec": {"suspend": true}}' --type=merge --field-manager=flux-client-side-apply