[platform] Introduce expose-services, expose-ingress and expose-external-ips options (#929)

docs update: https://github.com/cozystack/website/pull/197

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Summary by CodeRabbit

- **New Features**
- Added automated migration script to transition configuration from
HelmRelease to ConfigMap for service exposure and external IPs.
- Introduced new ingress templates for API, CDI upload proxy, and VM
export proxy services, enabling dynamic exposure based on centralized
configuration.

- **Bug Fixes**
  - Updated NGINX Ingress Controller Helm chart version to 1.6.0.

- **Refactor**
- Centralized ingress configuration using a ConfigMap, simplifying and
unifying service exposure and ingress class management.
- Removed legacy parameters and templates for dashboard, CDI upload
proxy, and VM export proxy from values and schema files.
- Simplified ingress templates for dashboard and Keycloak to rely on
centralized ConfigMap data and exposure lists.
- Adjusted ingress controller service to conditionally use external IPs
based on centralized configuration.

- **Documentation**
- Updated documentation to reflect the removal of deprecated parameters
and clarify current configuration options.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

hack/e2e.sh

@@ -334,8 +334,8 @@ if ! kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr monitorin
   kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr monitoring
 fi
 
-kubectl patch -n tenant-root ingresses.apps.cozystack.io ingress --type=merge -p '{"spec":{
-  "dashboard": true
+kubectl patch -n cozy-system cm cozystack --type=merge -p '{"data":{
+  "expose-services": "api,dashboard,cdi-uploadproxy,vm-exportproxy,keycloak"
 }}'
 
 # Wait for nginx-ingress-controller

packages/extra/ingress/Chart.yaml

@@ -3,4 +3,4 @@ name: ingress
 description: NGINX Ingress Controller
 icon: /logos/ingress-nginx.svg
 type: application
-version: 1.5.1
+version: 1.6.0

packages/extra/ingress/README.md

@@ -4,13 +4,10 @@
 
 ### Common parameters
 
-| Name              | Description                                                       | Value   |
-| ----------------- | ----------------------------------------------------------------- | ------- |
-| `replicas`        | Number of ingress-nginx replicas                                  | `2`     |
-| `externalIPs`     | List of externalIPs for service.                                  | `[]`    |
-| `whitelist`       | List of client networks                                           | `[]`    |
-| `clouflareProxy`  | Restoring original visitor IPs when Cloudflare proxied is enabled | `false` |
-| `dashboard`       | Should ingress serve Cozystack service dashboard                  | `false` |
-| `cdiUploadProxy`  | Should ingress serve CDI upload proxy                             | `false` |
-| `virtExportProxy` | Should ingress serve KubeVirt export proxy                        | `false` |
+| Name             | Description                                                       | Value   |
+| ---------------- | ----------------------------------------------------------------- | ------- |
+| `replicas`       | Number of ingress-nginx replicas                                  | `2`     |
+| `externalIPs`    | List of externalIPs for service.                                  | `[]`    |
+| `whitelist`      | List of client networks                                           | `[]`    |
+| `clouflareProxy` | Restoring original visitor IPs when Cloudflare proxied is enabled | `false` |
 

packages/extra/ingress/templates/cdi-uploadproxy.yaml

@@ -1,37 +0,0 @@
-{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
-{{- $issuerType := (index $cozyConfig.data "clusterissuer") | default "http01" }}
-
-{{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }}
-{{- $host := index $myNS.metadata.annotations "namespace.cozystack.io/host" }}
-
-{{- if .Values.cdiUploadProxy }}
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  annotations:
-    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
-    cert-manager.io/cluster-issuer: letsencrypt-prod
-    {{- if eq $issuerType "cloudflare" }} 
-    {{- else }}
-    acme.cert-manager.io/http01-ingress-class: {{ .Release.Namespace }}
-    {{- end }}
-  name: cdi-uploadproxy-{{ .Release.Namespace }}
-  namespace: cozy-kubevirt-cdi
-spec:
-  ingressClassName: {{ .Release.Namespace }}
-  rules:
-  - host: cdi-uploadproxy.{{ $host }}
-    http:
-      paths:
-      - backend:
-          service:
-            name: cdi-uploadproxy
-            port:
-              number: 443
-        path: /
-        pathType: Prefix
-  tls:
-  - hosts:
-    - cdi-uploadproxy.{{ $host }}
-    secretName: cdi-uploadproxy-{{ .Release.Namespace }}-tls
-{{- end }}

packages/extra/ingress/templates/nginx-ingress.yaml

@@ -1,3 +1,6 @@
+{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
+{{- $exposeIngress := index $cozyConfig.data "expose-ingress" | default "tenant-root" }}
+{{- $exposeExternalIPs := splitList "," ((index $cozyConfig.data "expose-external-ips") | default "") }}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -31,9 +34,9 @@ spec:
           enabled: false
         {{- end }}
         service:
-          {{- if .Values.externalIPs }}
+          {{- if and (eq $exposeIngress .Release.Namespace) $exposeExternalIPs }}
           externalIPs:
-            {{- toYaml .Values.externalIPs | nindent 12 }}
+            {{- toYaml $exposeExternalIPs | nindent 12 }}
           type: ClusterIP
           externalTrafficPolicy: Cluster
           {{- else }}

packages/extra/ingress/values.schema.json

@@ -25,21 +25,6 @@
             "type": "boolean",
             "description": "Restoring original visitor IPs when Cloudflare proxied is enabled",
             "default": false
-        },
-        "dashboard": {
-            "type": "boolean",
-            "description": "Should ingress serve Cozystack service dashboard",
-            "default": false
-        },
-        "cdiUploadProxy": {
-            "type": "boolean",
-            "description": "Should ingress serve CDI upload proxy",
-            "default": false
-        },
-        "virtExportProxy": {
-            "type": "boolean",
-            "description": "Should ingress serve KubeVirt export proxy",
-            "default": false
         }
     }
 }

packages/extra/ingress/values.yaml

@@ -4,17 +4,6 @@
 ##
 replicas: 2
 
-## @param externalIPs [array] List of externalIPs for service.
-## Optional. If not specified will use LoadBalancer service by default.
-## 
-## e.g:
-## externalIPs:
-##   - "11.22.33.44"
-##   - "11.22.33.45"
-##   - "11.22.33.46"
-##
-externalIPs: []
-
 ## @param whitelist List of client networks
 ## Example:
 ## whitelist:
@@ -24,12 +13,3 @@ whitelist: []
 
 ## @param clouflareProxy Restoring original visitor IPs when Cloudflare proxied is enabled
 clouflareProxy: false
-
-## @param dashboard Should ingress serve Cozystack service dashboard
-dashboard: false
-
-## @param cdiUploadProxy Should ingress serve CDI upload proxy
-cdiUploadProxy: false
-
-## @param virtExportProxy Should ingress serve KubeVirt export proxy
-virtExportProxy: false

packages/extra/ingress/vm-exportproxy.yaml

@@ -1,37 +0,0 @@
-{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
-{{- $issuerType := (index $cozyConfig.data "clusterissuer") | default "http01" }}
-
-{{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }}
-{{- $host := index $myNS.metadata.annotations "namespace.cozystack.io/host" }}
-
-{{- if .Values.virtExportProxy }}
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  annotations:
-    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
-    cert-manager.io/cluster-issuer: letsencrypt-prod
-    {{- if eq $issuerType "cloudflare" }} 
-    {{- else }}
-    acme.cert-manager.io/http01-ingress-class: {{ .Release.Namespace }}
-    {{- end }}
-  name: virt-exportproxy-{{ .Release.Namespace }}
-  namespace: cozy-kubevirt
-spec:
-  ingressClassName: {{ .Release.Namespace }}
-  rules:
-  - host: virt-exportproxy.{{ $host }}
-    http:
-      paths:
-      - backend:
-          service:
-            name: virt-exportproxy
-            port:
-              number: 443
-        path: /
-        pathType: ImplementationSpecific
-  tls:
-  - hosts:
-    virt-exportproxy.{{ $host }}
-    secretName: virt-exportproxy-{{ .Release.Namespace }}-tls
-{{- end }}

packages/extra/versions_map

@@ -19,7 +19,7 @@ ingress 1.2.0 28fca4ef
 ingress 1.3.0 fde4bcfa
 ingress 1.4.0 fd240701
 ingress 1.5.0 93bdf411
-ingress 1.5.1 HEAD
+ingress 1.6.0 HEAD
 monitoring 1.0.0 d7cfa53c
 monitoring 1.1.0 25221fdc
 monitoring 1.2.0 f81be075

packages/system/cozystack-api/templates/api-ingress.yaml

@@ -0,0 +1,28 @@
+{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
+{{- $host := index $cozyConfig.data "root-host" }}
+{{- $exposeServices := splitList "," ((index $cozyConfig.data "expose-services") | default "") }}
+{{- $exposeIngress := index $cozyConfig.data "expose-ingress" | default "tenant-root" }}
+
+{{- if and (has "api" $exposeServices) }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  name: kubernetes
+  namespace: default
+spec:
+  ingressClassName: {{ $exposeIngress }}
+  rules:
+  - host: api.{{ $host }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: kubernetes
+            port:
+              number: 443
+        path: /
+        pathType: Prefix
+{{- end }}

packages/system/dashboard/templates/dashboard-ingress.yaml

@@ -1,19 +1,10 @@
 {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
 {{- $issuerType := (index $cozyConfig.data "clusterissuer") | default "http01" }}
+{{- $host := index $cozyConfig.data "root-host" }}
+{{- $exposeServices := splitList "," ((index $cozyConfig.data "expose-services") | default "") }}
+{{- $exposeIngress := index $cozyConfig.data "expose-ingress" | default "tenant-root" }}
 
-{{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }}
-{{- $host := index $myNS.metadata.annotations "namespace.cozystack.io/host" }}
-
-{{- $tenantRoot := dict }}
-{{- if .Capabilities.APIVersions.Has "helm.toolkit.fluxcd.io/v2" }}
-{{- $tenantRoot = lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" "tenant-root" "tenant-root" }}
-{{- end }}
-{{- if and $tenantRoot $tenantRoot.spec $tenantRoot.spec.values $tenantRoot.spec.values.host }}
-{{- $host = $tenantRoot.spec.values.host }}
-{{- else }}
-{{- end }}
-
-{{- if .Values.dashboard }}
+{{- if and (has "dashboard" $exposeServices) }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -21,16 +12,16 @@ metadata:
     cert-manager.io/cluster-issuer: letsencrypt-prod
     {{- if eq $issuerType "cloudflare" }}
     {{- else }}
-    acme.cert-manager.io/http01-ingress-class: {{ .Release.Namespace }}
+    acme.cert-manager.io/http01-ingress-class: {{ $exposeIngress }}
+    {{- end }}
     nginx.ingress.kubernetes.io/proxy-body-size: 100m
     nginx.ingress.kubernetes.io/proxy-buffer-size: 100m
     nginx.ingress.kubernetes.io/proxy-buffers-number: "4"
     nginx.ingress.kubernetes.io/client-max-body-size: 100m
-    {{- end }}
-  name: dashboard-{{ .Release.Namespace }}
+  name: dashboard
   namespace: cozy-dashboard
 spec:
-  ingressClassName: {{ .Release.Namespace }}
+  ingressClassName: {{ $exposeIngress }}
   rules:
   - host: dashboard.{{ $host }}
     http:
@@ -45,5 +36,5 @@ spec:
   tls:
   - hosts:
     - dashboard.{{ $host }}
-    secretName: dashboard-{{ .Release.Namespace }}-tls
+    secretName: dashboard-tls
 {{- end }}

packages/system/keycloak-configure/templates/configure-kk.yaml

@@ -4,15 +4,6 @@
 {{- $rootSaConfigMap := lookup "v1" "ConfigMap" "kube-system" "kube-root-ca.crt" }}
 {{- $k8sCa := index $rootSaConfigMap.data "ca.crt" | b64enc }}
 
-{{- $tenantRoot := dict }}
-{{- if .Capabilities.APIVersions.Has "helm.toolkit.fluxcd.io/v2" }}
-{{- $tenantRoot = lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" "tenant-root" "tenant-root" }}
-{{- end }}
-{{- if and $tenantRoot $tenantRoot.spec $tenantRoot.spec.values $tenantRoot.spec.values.host }}
-{{- $host = $tenantRoot.spec.values.host }}
-{{- else }}
-{{- end }}
-
 {{- $existingK8sSecret := lookup "v1" "Secret" .Release.Namespace "k8s-client" }}
 {{- $existingKubeappsSecret := lookup "v1" "Secret" .Release.Namespace "kubeapps-client" }}
 {{- $existingAuthConfig := lookup "v1" "Secret" "cozy-dashboard" "kubeapps-auth-config" }}

packages/system/keycloak/templates/ingress.yaml

@@ -1,18 +1,7 @@
 {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
 {{- $host := index $cozyConfig.data "root-host" }}
 {{- $issuerType := (index $cozyConfig.data "clusterissuer") | default "http01" }}
-
-{{- $rootns := lookup "v1" "Namespace" "" "tenant-root" }}
-{{- $ingress := index $rootns.metadata.annotations "namespace.cozystack.io/ingress" }}
-
-{{- $tenantRoot := dict }}
-{{- if .Capabilities.APIVersions.Has "helm.toolkit.fluxcd.io/v2" }}
-{{- $tenantRoot = lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" "tenant-root" "tenant-root" }}
-{{- end }}
-{{- if and $tenantRoot $tenantRoot.spec $tenantRoot.spec.values $tenantRoot.spec.values.host }}
-{{- $host = $tenantRoot.spec.values.host }}
-{{- else }}
-{{- end }}
+{{- $exposeIngress := index $cozyConfig.data "expose-ingress" | default "tenant-root" }}
 
 apiVersion: networking.k8s.io/v1
 kind: Ingress
@@ -21,13 +10,13 @@ metadata:
   {{- with .Values.ingress.annotations }}
   annotations:
     {{- if ne $issuerType "cloudflare" }}
-    acme.cert-manager.io/http01-ingress-class: {{ $ingress }}
+    acme.cert-manager.io/http01-ingress-class: {{ $exposeIngress }}
     {{- end }}
     cert-manager.io/cluster-issuer: letsencrypt-prod
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
-  ingressClassName: {{ $ingress }}
+  ingressClassName: {{ $exposeIngress }}
   tls:
   - hosts:
       - keycloak.{{ $host }}

packages/system/keycloak/templates/sts.yaml

@@ -7,15 +7,6 @@
   {{- $password = index $existingPassword.data "password" | b64dec }}
 {{- end }}
 
-{{- $tenantRoot := dict }}
-{{- if .Capabilities.APIVersions.Has "helm.toolkit.fluxcd.io/v2" }}
-{{- $tenantRoot = lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" "tenant-root" "tenant-root" }}
-{{- end }}
-{{- if and $tenantRoot $tenantRoot.spec $tenantRoot.spec.values $tenantRoot.spec.values.host }}
-{{- $host = $tenantRoot.spec.values.host }}
-{{- else }}
-{{- end }}
-
 apiVersion: v1
 kind: Secret
 metadata:

packages/system/kubevirt-cdi/templates/cdi-uploadproxy-ingress.yaml

@@ -0,0 +1,29 @@
+{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
+{{- $host := index $cozyConfig.data "root-host" }}
+{{- $exposeServices := splitList "," ((index $cozyConfig.data "expose-services") | default "") }}
+{{- $exposeIngress := index $cozyConfig.data "expose-ingress" | default "tenant-root" }}
+
+
+{{- if and (has "cdi-uploadproxy" $exposeServices) }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  name: cdi-uploadproxy
+  namespace: cozy-kubevirt-cdi
+spec:
+  ingressClassName: {{ $exposeIngress }}
+  rules:
+  - host: cdi-uploadproxy.{{ $host }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: cdi-uploadproxy
+            port:
+              number: 443
+        path: /
+        pathType: Prefix
+{{- end }}

packages/system/kubevirt/templates/vm-exportproxy-ingress.yaml

@@ -0,0 +1,28 @@
+{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
+{{- $host := index $cozyConfig.data "root-host" }}
+{{- $exposeServices := splitList "," ((index $cozyConfig.data "expose-services") | default "") }}
+{{- $exposeIngress := index $cozyConfig.data "expose-ingress" | default "tenant-root" }}
+
+{{- if and (has "vm-exportproxy" $exposeServices) }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+  name: vm-exportproxy
+  namespace: cozy-kubevirt
+spec:
+  ingressClassName: {{ $exposeIngress }}
+  rules:
+  - host: vm-exportproxy.{{ $host }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: vm-exportproxy
+            port:
+              number: 443
+        path: /
+        pathType: ImplementationSpecific
+{{- end }}

scripts/migrations/12

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Migration 12 --> 13
+
+# Copy configuration from ingress to cozystack configmap
+if kubectl get hr -n tenant-root tenant-root > /dev/null; then
+  expose_services=$(
+    kubectl get hr -n tenant-root ingress -o go-template='{{ with .spec }}{{ with .values }}{{ if .dashboard  }}dashboard,{{ end }}{{ if .cdiUploadProxy }}cdi-uploadproxy,{{ end }}{{ if .virtExportProxy }}vm-exportproxy,{{ end }}{{ end }}{{ end }}'
+  )
+  expose_services=$(echo "$expose_services" | awk '{sub(/,$/,""); print}')
+  
+  expose_external_ips=$(
+    kubectl get hr -n tenant-root ingress -o go-template='{{ with .spec }}{{ with .values }}{{ if .externalIPs  }}{{ range .externalIPs }}{{ . }},{{ end }}{{ end }}{{ end }}{{ end }}'
+  )
+  expose_external_ips=$(echo "$expose_external_ips" | awk '{sub(/,$/,""); print}')
+
+  existing_expose_external_ips=$(kubectl get cm -n cozy-system cozystack -o go-template='{{ index .data "expose-external-ips" }}')
+  existing_expose_services=$(kubectl get cm -n cozy-system cozystack -o go-template='{{ index .data "expose-services" }}')
+
+  if [ "$existing_expose_external_ips" == "<no value>" ]; then
+    kubectl patch cm -n cozy-system cozystack --type merge -p="{\"data\":{\"expose-external-ips\":\"$expose_external_ips\"}}"
+  fi
+
+  if [ "$existing_expose_services" == "<no value>" ]; then
+    kubectl patch cm -n cozy-system cozystack --type merge -p="{\"data\":{\"expose-services\":\"$expose_services\"}}"
+  fi
+
+  kubectl patch hr -n tenant-root ingress --type json -p='[{"op": "remove", "path": "/spec/values/dashboard"}]' || true
+  kubectl patch hr -n tenant-root ingress --type json -p='[{"op": "remove", "path": "/spec/values/cdiUploadProxy"}]' || true
+  kubectl patch hr -n tenant-root ingress --type json -p='[{"op": "remove", "path": "/spec/values/virtExportProxy"}]' || true
+  kubectl patch hr -n tenant-root ingress --type json -p='[{"op": "remove", "path": "/spec/values/externalIPs"}]' || true
+  kubectl patch hr -n tenant-root ingress --type merge -p='{"spec":{"chart":{"spec":{"version":"1.6.0"}}}}'
+fi
+
+# Write version to cozystack-version config
+kubectl create configmap -n cozy-system cozystack-version --from-literal=version=13 --dry-run=client -o yaml | kubectl apply -f-