Revert "[platform] Separate assets server into dedicated deployment (#1705)"

This reverts commit 680f70c03a, reversing
changes made to e3b96e12be.

Makefile

@@ -28,7 +28,6 @@ build: build-deps
 	make -C packages/system/objectstorage-controller image
 	make -C packages/core/testing image
 	make -C packages/core/talos image
-	make -C packages/core/platform image
 	make -C packages/core/installer image
 	make manifests
 

packages/core/flux-aio/Chart.yaml

@@ -1,3 +0,0 @@
-apiVersion: v2
-name: cozy-fluxcd
-version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process

packages/core/flux-aio/Makefile

@@ -1,34 +0,0 @@
-NAME=flux-aio
-NAMESPACE=cozy-$(NAME)
-
-include ../../../scripts/common-envs.mk
-
-show:
-	cozyhr show -n $(NAMESPACE) $(NAME) --plain
-
-apply:
-	cozyhr show -n $(NAMESPACE) $(NAME) --plain | kubectl apply -f- --server-side --force-conflicts
-
-diff:
-	cozyhr show -n $(NAMESPACE) $(NAME) --plain | kubectl diff -f-
-
-update: update-old update-new
-
-# TODO: remove old manifest after migration to cozystack-operator
-update-old:
-	timoni bundle build -f flux-aio.cue > templates/fluxcd.yaml
-	yq eval '(select(.kind == "Namespace") | .metadata.labels."pod-security.kubernetes.io/enforce") = "privileged"' -i templates/fluxcd.yaml
-	sed -i templates/fluxcd.yaml \
-		-e '/timoni/d' \
-		-e 's|\.cluster\.local\.,||g' -e 's|\.cluster\.local\,||g' -e 's|\.cluster\.local\.||g' \
-		-e '/value: .svc/a \            {{- include "cozy.kubernetes_envs" . | nindent 12 }}' \
-		-e '/hostNetwork: true/i \      dnsPolicy: ClusterFirstWithHostNet'
-
-update-new:
-	timoni bundle build -f flux-aio.cue > ../../../internal/fluxinstall/manifests/fluxcd.yaml
-	yq eval '(select(.kind == "Namespace") | .metadata.labels."pod-security.kubernetes.io/enforce") = "privileged"' -i ../../../internal/fluxinstall/manifests/fluxcd.yaml
-	sed -i ../../../internal/fluxinstall/manifests/fluxcd.yaml \
-		-e '/timoni/d' \
-		-e 's|\.cluster\.local\.,||g' -e 's|\.cluster\.local\,||g' -e 's|\.cluster\.local\.||g'
-	# TODO: solve dns issue with hostNetwork for installing helmreleases in tenant k8s clusters
-	#-e '/hostNetwork: true/i \      dnsPolicy: ClusterFirstWithHostNet' 

packages/core/flux-aio/flux-aio.cue

@@ -1,16 +0,0 @@
-bundle: {
-	apiVersion: "v1alpha1"
-	name:       "flux-aio"
-	instances: {
-		"flux": {
-			module: {
-				url:     "oci://ghcr.io/stefanprodan/modules/flux-aio"
-				version: "latest"
-			}
-			namespace: "cozy-fluxcd"
-			values: {
-				securityProfile: "privileged"
-			}
-		}
-	}
-}

packages/core/flux-aio/templates/_helpers.tpl

@@ -1,13 +0,0 @@
-{{- define "cozy.kubernetes_envs" }}
-{{- $cozyDeployment := lookup "apps/v1" "Deployment" "cozy-system" "cozystack" }}
-{{- $cozyContainers := dig "spec" "template" "spec" "containers" dict $cozyDeployment }}
-{{- range $cozyContainers }}
-{{- if eq .name "cozystack" }}
-{{- range .env }}
-{{- if has .name (list "KUBERNETES_SERVICE_HOST" "KUBERNETES_SERVICE_PORT") }}
-- {{ toJson . }}
-{{- end }}
-{{- end }}
-{{- end }}
-{{- end }}
-{{- end }}

packages/core/installer/images/cozystack/Dockerfile

@@ -26,16 +26,23 @@ WORKDIR /src
 
 RUN go mod download
 
+RUN go build -o /cozystack-assets-server -ldflags '-extldflags "-static" -w -s' ./cmd/cozystack-assets-server
+
+RUN make repos
+
 FROM alpine:3.22
 
 RUN wget -O- https://github.com/cozystack/cozyhr/raw/refs/heads/main/hack/install.sh | sh -s -- -v 1.5.0
 
-RUN apk add --no-cache make kubectl helm coreutils git jq openssl
+RUN apk add --no-cache make kubectl helm coreutils git jq
 
 COPY --from=builder /src/scripts /cozystack/scripts
 COPY --from=builder /src/packages/core /cozystack/packages/core
 COPY --from=builder /src/packages/system /cozystack/packages/system
+COPY --from=builder /src/_out/repos /cozystack/assets/repos
+COPY --from=builder /cozystack-assets-server /usr/bin/cozystack-assets-server
 COPY --from=k8s-await-election-builder /k8s-await-election /usr/bin/k8s-await-election
+COPY --from=builder /src/dashboards /cozystack/assets/dashboards
 
 WORKDIR /cozystack
 ENTRYPOINT ["/usr/bin/k8s-await-election", "/cozystack/scripts/installer.sh" ]

packages/core/installer/templates/cozystack.yaml

@@ -55,8 +55,6 @@ spec:
         env:
         - name: KUBERNETES_SERVICE_HOST
           value: localhost
-        - name: INSTALL_FLUX
-          value: "true"
         - name: KUBERNETES_SERVICE_PORT
           value: "7445"
         - name: K8S_AWAIT_ELECTION_ENABLED
@@ -71,6 +69,15 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+      - name: assets
+        image: "{{ .Values.cozystack.image }}"
+        command:
+        - /usr/bin/cozystack-assets-server
+        - "-dir=/cozystack/assets"
+        - "-address=:8123"
+        ports:
+        - name: http
+          containerPort: 8123
       tolerations:
       - key: "node.kubernetes.io/not-ready"
         operator: "Exists"
@@ -78,4 +85,18 @@ spec:
       - key: "node.cilium.io/agent-not-ready"
         operator: "Exists"
         effect: "NoSchedule"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: cozystack
+  namespace: cozy-system
+spec:
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8123
+  selector:
+    app: cozystack
+  type: ClusterIP
 {{- end }}

packages/core/platform/Makefile

@@ -1,8 +1,6 @@
 NAME=platform
 NAMESPACE=cozy-system
 
-include ../../../scripts/common-envs.mk
-
 show:
 	cozyhr show -n $(NAMESPACE) $(NAME) --plain
 
@@ -20,15 +18,3 @@ namespaces-apply:
 
 diff:
 	cozyhr show -n $(NAMESPACE) $(NAME) --plain | kubectl diff -f-
-
-image: image-assets
-image-assets:
-	docker buildx build -f images/cozystack-assets/Dockerfile ../../.. \
-		--tag $(REGISTRY)/cozystack-assets:$(call settag,$(TAG)) \
-		--cache-from type=registry,ref=$(REGISTRY)/cozystack-assets:latest \
-		--cache-to type=inline \
-		--metadata-file images/cozystack-assets.json \
-		$(BUILDX_ARGS)
-	IMAGE="$(REGISTRY)/cozystack-assets:$(call settag,$(TAG))@$$(yq e '."containerimage.digest"' images/cozystack-assets.json -o json -r)" \
-		yq -i '.assets.image = strenv(IMAGE)' values.yaml
-	rm -f images/cozystack-assets.json

packages/core/platform/bundles/distro-full.yaml

@@ -2,6 +2,24 @@
 {{- $clusterDomain := (index $cozyConfig.data "cluster-domain") | default "cozy.local" }}
 
 releases:
+- name: fluxcd-operator
+  releaseName: fluxcd-operator
+  chart: cozy-fluxcd-operator
+  namespace: cozy-fluxcd
+  privileged: true
+  dependsOn: []
+
+- name: fluxcd
+  releaseName: fluxcd
+  chart: cozy-fluxcd
+  namespace: cozy-fluxcd
+  dependsOn: [fluxcd-operator,cilium]
+  values:
+    flux-instance:
+      instance:
+        cluster:
+          domain: {{ $clusterDomain }}
+
 - name: cilium
   releaseName: cilium
   chart: cozy-cilium

packages/core/platform/bundles/distro-hosted.yaml

@@ -2,6 +2,24 @@
 {{- $clusterDomain := (index $cozyConfig.data "cluster-domain") | default "cozy.local" }}
 
 releases:
+- name: fluxcd-operator
+  releaseName: fluxcd-operator
+  chart: cozy-fluxcd-operator
+  namespace: cozy-fluxcd
+  privileged: true
+  dependsOn: []
+
+- name: fluxcd
+  releaseName: fluxcd
+  chart: cozy-fluxcd
+  namespace: cozy-fluxcd
+  dependsOn: [fluxcd-operator]
+  values:
+    flux-instance:
+      instance:
+        cluster:
+          domain: {{ $clusterDomain }}
+
 - name: cert-manager-crds
   releaseName: cert-manager-crds
   chart: cozy-cert-manager-crds

packages/core/platform/bundles/paas-full.yaml

@@ -11,6 +11,24 @@
 {{- end }}
 
 releases:
+- name: fluxcd-operator
+  releaseName: fluxcd-operator
+  chart: cozy-fluxcd-operator
+  namespace: cozy-fluxcd
+  privileged: true
+  dependsOn: []
+
+- name: fluxcd
+  releaseName: fluxcd
+  chart: cozy-fluxcd
+  namespace: cozy-fluxcd
+  dependsOn: [fluxcd-operator,cilium,kubeovn]
+  values:
+    flux-instance:
+      instance:
+        cluster:
+          domain: {{ $clusterDomain }}
+
 - name: cilium
   releaseName: cilium
   chart: cozy-cilium

packages/core/platform/bundles/paas-hosted.yaml

@@ -11,6 +11,24 @@
 {{- end }}
 
 releases:
+- name: fluxcd-operator
+  releaseName: fluxcd-operator
+  chart: cozy-fluxcd-operator
+  namespace: cozy-fluxcd
+  privileged: true
+  dependsOn: []
+
+- name: fluxcd
+  releaseName: fluxcd
+  chart: cozy-fluxcd
+  namespace: cozy-fluxcd
+  dependsOn: [fluxcd-operator]
+  values:
+    flux-instance:
+      instance:
+        cluster:
+          domain: {{ $clusterDomain }}
+
 - name: cert-manager-crds
   releaseName: cert-manager-crds
   chart: cozy-cert-manager-crds

packages/core/platform/images/cozystack-assets/Dockerfile

@@ -1,25 +0,0 @@
-FROM golang:1.25-alpine AS builder
-
-ARG TARGETOS
-ARG TARGETARCH
-
-RUN apk add --no-cache make git
-RUN apk add helm --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
-
-COPY . /src/
-WORKDIR /src
-
-RUN go mod download
-
-RUN go build -o /cozystack-assets-server -ldflags '-extldflags "-static" -w -s' ./cmd/cozystack-assets-server
-
-RUN make repos
-
-FROM alpine:3.22
-
-COPY --from=builder /src/_out/repos /cozystack/assets/repos
-COPY --from=builder /cozystack-assets-server /usr/bin/cozystack-assets-server
-COPY --from=builder /src/dashboards /cozystack/assets/dashboards
-
-WORKDIR /cozystack
-ENTRYPOINT ["/usr/bin/cozystack-assets-server"]

packages/core/platform/templates/_helpers.tpl

@@ -17,36 +17,6 @@ Get IP-addresses of master nodes
 {{ join "," $ips }}
 {{- end -}}
 
-{{/*
-Get Kubernetes API Endpoint from cozystack deployment
-Returns host:port format
-*/}}
-{{- define "cozystack.kubernetesAPIEndpoint" -}}
-{{- $cozyDeployment := lookup "apps/v1" "Deployment" "cozy-system" "cozystack" }}
-{{- $cozyContainers := dig "spec" "template" "spec" "containers" list $cozyDeployment }}
-{{- $kubernetesServiceHost := "" }}
-{{- $kubernetesServicePort := "" }}
-{{- range $cozyContainers }}
-{{- if eq .name "cozystack" }}
-{{- range .env }}
-{{- if eq .name "KUBERNETES_SERVICE_HOST" }}
-{{- $kubernetesServiceHost = .value }}
-{{- end }}
-{{- if eq .name "KUBERNETES_SERVICE_PORT" }}
-{{- $kubernetesServicePort = .value }}
-{{- end }}
-{{- end }}
-{{- end }}
-{{- end }}
-{{- if eq $kubernetesServiceHost "" }}
-{{- $kubernetesServiceHost = "kubernetes.default.svc" }}
-{{- end }}
-{{- if eq $kubernetesServicePort "" }}
-{{- $kubernetesServicePort = "443" }}
-{{- end }}
-{{- printf "%s:%s" $kubernetesServiceHost $kubernetesServicePort }}
-{{- end -}}
-
 {{- define "cozystack.defaultDashboardValues" -}}
 kubeapps:
 {{- if .Capabilities.APIVersions.Has "source.toolkit.fluxcd.io/v1" }}

packages/core/platform/templates/cozystack-assets.yaml

@@ -1,73 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: cozystack-assets
-  namespace: cozy-system
-  labels:
-    app: cozystack-assets
-spec:
-  serviceName: cozystack-assets
-  replicas: 1
-  selector:
-    matchLabels:
-      app: cozystack-assets
-  template:
-    metadata:
-      labels:
-        app: cozystack-assets
-    spec:
-      hostNetwork: true
-      containers:
-      - name: assets-server
-        image: "{{ .Values.assets.image }}"
-        args:
-        - "-dir=/cozystack/assets"
-        - "-address=:8123"
-        ports:
-        - name: http
-          containerPort: 8123
-          hostPort: 8123
-      tolerations:
-      - operator: Exists
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: cozystack-assets-reader
-  namespace: cozy-system
-rules:
-  - apiGroups: [""]
-    resources:
-      - pods/proxy
-    resourceNames:
-      - cozystack-assets-0
-    verbs:
-      - get
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: cozystack-assets-reader
-  namespace: cozy-system
-subjects:
-  - kind: User
-    name: cozystack-assets-reader
-    apiGroup: rbac.authorization.k8s.io
-roleRef:
-  kind: Role
-  name: cozystack-assets-reader
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: cozystack-assets
-  namespace: cozy-system
-spec:
-  ports:
-  - name: http
-    port: 80
-    targetPort: 8123
-  selector:
-    app: cozystack-assets
-  type: ClusterIP

packages/core/platform/templates/helmrepos.yaml

@@ -8,9 +8,7 @@ metadata:
     cozystack.io/repository: system
 spec:
   interval: 5m0s
-  url: https://{{ include "cozystack.kubernetesAPIEndpoint" . }}/api/v1/namespaces/cozy-system/pods/cozystack-assets-0/proxy/repos/system
-  certSecretRef:
-    name: cozystack-assets-tls
+  url: http://cozystack.cozy-system.svc/repos/system
 ---
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
@@ -22,9 +20,7 @@ metadata:
     cozystack.io/repository: apps
 spec:
   interval: 5m0s
-  url: https://{{ include "cozystack.kubernetesAPIEndpoint" . }}/api/v1/namespaces/cozy-system/pods/cozystack-assets-0/proxy/repos/apps
-  certSecretRef:
-    name: cozystack-assets-tls
+  url: http://cozystack.cozy-system.svc/repos/apps
 ---
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
@@ -35,6 +31,4 @@ metadata:
     cozystack.io/repository: extra
 spec:
   interval: 5m0s
-  url: https://{{ include "cozystack.kubernetesAPIEndpoint" . }}/api/v1/namespaces/cozy-system/pods/cozystack-assets-0/proxy/repos/extra
-  certSecretRef:
-    name: cozystack-assets-tls
+  url: http://cozystack.cozy-system.svc/repos/extra

packages/core/platform/values.yaml

@@ -1,2 +0,0 @@
-assets:
-  image: ghcr.io/cozystack/cozystack/cozystack-assets:latest@sha256:19b166819d0205293c85d8351a3e038dc4c146b876a8e2ae21dce1d54f0b9e33

packages/extra/monitoring/templates/dashboards.yaml

@@ -11,6 +11,6 @@ spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
-  url: http://cozystack-assets.cozy-system.svc/dashboards/{{ . }}.json
+  url: http://cozystack.cozy-system.svc/dashboards/{{ . }}.json
 {{- end }}
 {{- end }}

scripts/installer.sh

@@ -19,11 +19,29 @@ run_migrations() {
   done
 }
 
-install_flux() {
-  if [ "$INSTALL_FLUX" != "true" ]; then
+flux_is_ok() {
+  kubectl wait --for=condition=available -n cozy-fluxcd deploy/source-controller deploy/helm-controller --timeout=1s
+  kubectl wait --for=condition=ready -n cozy-fluxcd helmrelease/fluxcd --timeout=1s # to call "apply resume" below
+}
+
+ensure_fluxcd() {
+  if flux_is_ok; then
     return
   fi
-  make -C packages/core/flux-aio apply
+  # Install fluxcd-operator
+  if kubectl get helmreleases.helm.toolkit.fluxcd.io  -n cozy-fluxcd fluxcd-operator; then
+    make -C packages/system/fluxcd-operator apply resume
+  else
+    make -C packages/system/fluxcd-operator apply-locally
+  fi
+  wait_for_crds fluxinstances.fluxcd.controlplane.io
+
+   # Install fluxcd
+  if kubectl get helmreleases.helm.toolkit.fluxcd.io  -n cozy-fluxcd fluxcd; then
+    make -C packages/system/fluxcd apply resume
+  else
+    make -C packages/system/fluxcd apply-locally
+  fi
   wait_for_crds helmreleases.helm.toolkit.fluxcd.io helmrepositories.source.toolkit.fluxcd.io
 }
 
@@ -31,6 +49,15 @@ wait_for_crds() {
   timeout 60 sh -c "until kubectl get crd $*; do sleep 1; done"
 }
 
+install_basic_charts() {
+  if [ "$BUNDLE" = "paas-full" ] || [ "$BUNDLE" = "distro-full" ]; then
+  make -C packages/system/cilium apply resume
+  fi
+  if [ "$BUNDLE" = "paas-full" ]; then
+    make -C packages/system/kubeovn apply resume
+  fi
+}
+
 cd "$(dirname "$0")/.."
 
 # Run migrations
@@ -40,14 +67,16 @@ run_migrations
 make -C packages/core/platform namespaces-apply
 
 # Install fluxcd
-install_flux
-
-# Install fluxcd certificates
-./scripts/issue-flux-certificates.sh
+ensure_fluxcd
 
 # Install platform chart
 make -C packages/core/platform reconcile
 
+# Install basic charts
+if ! flux_is_ok; then
+  install_basic_charts
+fi
+
 # Reconcile Helm repositories
 kubectl annotate helmrepositories.source.toolkit.fluxcd.io -A -l cozystack.io/repository reconcile.fluxcd.io/requestedAt=$(date +"%Y-%m-%dT%H:%M:%SZ") --overwrite
 

scripts/issue-flux-certificates.sh

@@ -1,63 +0,0 @@
-#!/bin/sh
-set -e
-
-if kubectl get secret -n cozy-system cozystack-assets-tls >/dev/null 2>&1 && kubectl get secret -n cozy-public cozystack-assets-tls >/dev/null 2>&1; then
-  echo "Secret cozystack-assets-tls already exists in both cozy-system and cozy-public namespaces. Exiting."
-  exit 0
-fi
-
-USER_CN="cozystack-assets-reader"
-CSR_NAME="csr-${USER_CN}-$(date +%s)"
-
-# make temp directory and cleanup handler
-TMPDIR=$(mktemp -d)
-trap 'rm -rf "$TMPDIR"' EXIT
-
-# move into tmpdir
-cd "$TMPDIR"
-
-openssl genrsa -out tls.key 2048
-openssl req -new -key tls.key -subj "/CN=${USER_CN}" -out tls.csr
-
-CSR_B64=$(base64 < tls.csr | tr -d '\n')
-
-cat <<EOF | kubectl apply -f -
-apiVersion: certificates.k8s.io/v1
-kind: CertificateSigningRequest
-metadata:
-  name: ${CSR_NAME}
-spec:
-  signerName: kubernetes.io/kube-apiserver-client
-  request: ${CSR_B64}
-  usages:
-    - client auth
-EOF
-
-kubectl certificate approve "${CSR_NAME}"
-
-echo "Waiting for .status.certificate..."
-kubectl wait csr "${CSR_NAME}" \
-  --for=jsonpath='{.status.certificate}' \
-  --timeout=120s
-
-kubectl get csr "${CSR_NAME}" \
-  -o jsonpath='{.status.certificate}' | base64 -d > tls.crt
-
-kubectl get -n kube-public configmap kube-root-ca.crt \
-  -o jsonpath='{.data.ca\.crt}' > ca.crt
-
-kubectl create secret generic "cozystack-assets-tls" \
-  --namespace='cozy-system' \
-  --type='kubernetes.io/tls' \
-  --from-file=tls.crt \
-  --from-file=tls.key \
-  --from-file=ca.crt \
-  --dry-run=client -o yaml | kubectl apply -f -
-
-kubectl create secret generic "cozystack-assets-tls" \
-  --namespace='cozy-public' \
-  --type='kubernetes.io/tls' \
-  --from-file=tls.crt \
-  --from-file=tls.key \
-  --from-file=ca.crt \
-  --dry-run=client -o yaml | kubectl apply -f -

scripts/migrations/21

@@ -1,10 +0,0 @@
-#!/bin/sh
-# Migration 21 --> 22
-
-set -euo pipefail
-
-kubectl delete hr -n cozy-fluxcd fluxcd --ignore-not-found
-
-# Stamp version
-kubectl create configmap -n cozy-system cozystack-version \
-  --from-literal=version=22 --dry-run=client -o yaml | kubectl apply -f-