move fluxcd and operator back to system (#188)

Separate and move fluxcd and fluxcd-operator from `core` to `system`.

It should not be problem with self-update now, since we correctly set
dependsOn option, it ensures ordered update of flux instance right after
flux-operator.

As part of https://github.com/aenix-io/cozystack/issues/184 and
https://github.com/aenix-io/cozystack/issues/185
fixes https://github.com/aenix-io/cozystack/issues/169

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

hack/e2e.sh

@@ -294,22 +294,22 @@ kubectl patch -n tenant-root hr/tenant-root --type=merge -p '{"spec":{ "values":
 timeout 60 sh -c 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root; do sleep 1; done'
 
 # Wait for HelmReleases be installed
-kubectl wait --timeout=2m --for=condition=ready hr -n tenant-root etcd ingress monitoring tenant-root
+kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr etcd ingress monitoring tenant-root
 
 # Wait for nginx-ingress-controller
 timeout 60 sh -c 'until kubectl get deploy -n tenant-root root-ingress-controller; do sleep 1; done'
-kubectl wait deploy -n tenant-root --timeout=5m --for=condition=available root-ingress-controller
+kubectl wait --timeout=5m --for=condition=available -n tenant-root deploy root-ingress-controller
 
 # Wait for etcd
-kubectl wait --for=jsonpath=.status.readyReplicas=3 -n tenant-root --timeout=5m sts etcd
+kubectl wait --timeout=5m --for=jsonpath=.status.readyReplicas=3 -n tenant-root sts etcd
 
 # Wait for Victoria metrics
-kubectl wait deploy -n tenant-root --timeout=5m --for=condition=available vmalert-vmalert vminsert-longterm vminsert-shortterm
-kubectl wait --for=jsonpath=.status.readyReplicas=2 -n tenant-root --timeout=5m sts vmalertmanager-alertmanager vmselect-longterm vmselect-shortterm vmstorage-longterm vmstorage-shortterm
+kubectl wait --timeout=5m --for=condition=available deploy -n tenant-root vmalert-vmalert vminsert-longterm vminsert-shortterm
+kubectl wait --timeout=5m --for=jsonpath=.status.readyReplicas=2 -n tenant-root sts vmalertmanager-alertmanager vmselect-longterm vmselect-shortterm vmstorage-longterm vmstorage-shortterm
 
 # Wait for grafana
-kubectl wait --for=condition=ready clusters.postgresql.cnpg.io -n tenant-root grafana-db
-kubectl wait deploy -n tenant-root --timeout=5m --for=condition=available grafana-deployment 
+kubectl wait --timeout=5m --for=condition=ready -n tenant-root clusters.postgresql.cnpg.io grafana-db
+kubectl wait --timeout=5m --for=condition=available -n tenant-root deploy grafana-deployment 
 
 # Get IP of nginx-ingress
 ip=$(kubectl get svc -n tenant-root root-ingress-controller -o jsonpath='{.status.loadBalancer.ingress..ip}')

packages/core/fluxcd/Makefile

@@ -1,20 +0,0 @@
-NAME=fluxcd
-NAMESPACE=cozy-$(NAME)
-
-API_VERSIONS_FLAGS=$(addprefix -a ,$(shell kubectl api-versions))
-
-show:
-	helm template -n $(NAMESPACE) $(NAME) . --no-hooks --dry-run=server $(API_VERSIONS_FLAGS)
-
-apply:
-	helm template -n $(NAMESPACE) $(NAME) . --no-hooks \
-		--dry-run=server $(API_VERSIONS_FLAGS) | kubectl apply -n $(NAMESPACE) -f-
-
-diff:
-	helm template -n $(NAMESPACE) $(NAME) . --no-hooks \
-		--dry-run=server $(API_VERSIONS_FLAGS) | kubectl diff -n $(NAMESPACE) -f-
-
-update:
-	rm -rf charts
-	helm pull oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator --untar --untardir charts
-	patch --no-backup-if-mismatch -p1 < patches/kubernetesEnvs.diff

packages/core/fluxcd/templates/flux-instance.yaml

@@ -1,25 +0,0 @@
-apiVersion: fluxcd.controlplane.io/v1
-kind: FluxInstance
-metadata:
-  name: flux
-spec:
-  {{- with .Values.fluxInstance.cluster }}
-  cluster:
-    {{- with .networkPolicy }}
-    networkPolicy: {{ . }}
-    {{- end }}
-    {{- with .domain }}
-    domain: {{ . }}
-    {{- end }}
-  {{- end }}
-  distribution:
-    version: {{ .Values.fluxInstance.distribution.version }}
-    registry: {{ .Values.fluxInstance.distribution.registry }}
-  components:
-  {{- if .Values.fluxInstance.components }}
-    {{- toYaml .Values.fluxInstance.components | nindent 4 }}
-  {{- end }}
-  kustomize:
-  {{- if .Values.fluxInstance.kustomize }}
-    {{- toYaml .Values.fluxInstance.kustomize | nindent 4 }}
-  {{- end }}

packages/core/fluxcd/values.yaml

@@ -1,55 +0,0 @@
-flux-operator:
-  tolerations:
-    - key: node.kubernetes.io/not-ready
-      operator: Exists
-      effect: NoSchedule
-  hostNetwork: true
-
-fluxInstance:
-  cluster:
-    networkPolicy: true
-  #  domain: cozy.local
-  distribution:
-    version: 2.3.x
-    registry: ghcr.io/fluxcd
-  components:
-    - source-controller
-    - kustomize-controller
-    - helm-controller
-    - notification-controller
-    - image-reflector-controller
-    - image-automation-controller
-  kustomize:
-    patches:
-      - target:
-          kind: Deployment
-          name: "(kustomize-controller|helm-controller|source-controller)"
-        patch: |
-          - op: add
-            path: /spec/template/spec/containers/0/args/-
-            value: --concurrent=20
-          - op: add
-            path: /spec/template/spec/containers/0/args/-
-            value: --requeue-dependency=5s
-          - op: replace
-            path: /spec/template/spec/containers/0/resources/limits
-            value:
-              cpu: 2000m
-              memory: 2048Mi
-      - target:
-          kind: Deployment
-          name: source-controller
-        patch: |
-          - op: add
-            path: /spec/template/spec/containers/0/args/-
-            value: --storage-adv-addr=source-controller.cozy-fluxcd.svc
-          - op: add
-            path: /spec/template/spec/containers/0/args/-
-            value: --events-addr=http://notification-controller.cozy-fluxcd.svc/
-      - target:
-          kind: Deployment
-          name: (kustomize-controller|helm-controller|image-reflector-controller|image-automation-controller)
-        patch: |
-          - op: add
-            path: /spec/template/spec/containers/0/args/-
-            value: --events-addr=http://notification-controller.cozy-fluxcd.svc/

packages/core/installer/images/cozystack.json

@@ -1,10 +1,10 @@
 {
-  "buildx.build.ref": "priceless_leavitt/priceless_leavitt0/d75hbe5lm96nutwocaw0h8ohc",
+  "buildx.build.ref": "priceless_leavitt/priceless_leavitt0/h4dfyd134l9durh9d02r2u2uu",
   "containerimage.descriptor": {
     "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
-    "digest": "sha256:a5544e0cf76b09b421345906d2e85282ba8c2187e9db814cfe5c08ddd9ee491a",
+    "digest": "sha256:b5cbbb921a240d05c6775aab15c99432180c5547801ef6cb6ca51303eeb489e1",
     "size": 685
   },
-  "containerimage.digest": "sha256:a5544e0cf76b09b421345906d2e85282ba8c2187e9db814cfe5c08ddd9ee491a",
+  "containerimage.digest": "sha256:b5cbbb921a240d05c6775aab15c99432180c5547801ef6cb6ca51303eeb489e1",
   "image.name": "ghcr.io/aenix-io/cozystack/cozystack:latest"
 }

packages/core/platform/bundles/distro-full.yaml

@@ -1,6 +1,19 @@
 {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
 
 releases:
+- name: fluxcd-operator
+  releaseName: fluxcd-operator
+  chart: cozy-fluxcd-operator
+  namespace: cozy-fluxcd
+  privileged: true
+  dependsOn: []
+
+- name: fluxcd
+  releaseName: fluxcd
+  chart: cozy-fluxcd
+  namespace: cozy-fluxcd
+  dependsOn: [fluxcd-operator,cilium]
+
 - name: cilium
   releaseName: cilium
   chart: cozy-cilium

packages/core/platform/bundles/distro-hosted.yaml

@@ -1,6 +1,19 @@
 {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
 
 releases:
+- name: fluxcd-operator
+  releaseName: fluxcd-operator
+  chart: cozy-fluxcd-operator
+  namespace: cozy-fluxcd
+  privileged: true
+  dependsOn: []
+
+- name: fluxcd
+  releaseName: fluxcd
+  chart: cozy-fluxcd
+  namespace: cozy-fluxcd
+  dependsOn: [fluxcd-operator]
+
 - name: cert-manager
   releaseName: cert-manager
   chart: cozy-cert-manager

packages/core/platform/bundles/paas-full.yaml

@@ -1,6 +1,19 @@
 {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
 
 releases:
+- name: fluxcd-operator
+  releaseName: fluxcd-operator
+  chart: cozy-fluxcd-operator
+  namespace: cozy-fluxcd
+  privileged: true
+  dependsOn: []
+
+- name: fluxcd
+  releaseName: fluxcd
+  chart: cozy-fluxcd
+  namespace: cozy-fluxcd
+  dependsOn: [fluxcd-operator,cilium,kubeovn]
+
 - name: cilium
   releaseName: cilium
   chart: cozy-cilium

packages/core/platform/bundles/paas-hosted.yaml

@@ -1,6 +1,19 @@
 {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }}
 
 releases:
+- name: fluxcd-operator
+  releaseName: fluxcd-operator
+  chart: cozy-fluxcd-operator
+  namespace: cozy-fluxcd
+  privileged: true
+  dependsOn: []
+
+- name: fluxcd
+  releaseName: fluxcd
+  chart: cozy-fluxcd
+  namespace: cozy-fluxcd
+  dependsOn: [fluxcd-operator]
+
 - name: cert-manager
   releaseName: cert-manager
   chart: cozy-cert-manager

packages/core/platform/templates/namespaces.yaml

@@ -16,7 +16,6 @@
 
 {{/* Add extra namespaces */}}
 {{- $_ := set $namespaces "cozy-public" false }}
-{{- $_ := set $namespaces "cozy-fluxcd" true }}
 
 {{- range $namespace, $privileged := $namespaces }}
 ---

packages/system/fluxcd-operator/Chart.yaml

@@ -0,0 +1,3 @@
+apiVersion: v2
+name: cozy-fluxcd-operator
+version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process

packages/system/fluxcd-operator/Makefile

@@ -0,0 +1,12 @@
+NAME=fluxcd-operator
+NAMESPACE=cozy-fluxcd
+
+include ../../../scripts/package-system.mk
+
+apply-locally:
+	helm upgrade -i -n $(NAMESPACE) $(NAME) .
+
+update:
+	rm -rf charts
+	helm pull oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator --untar --untardir charts
+	patch --no-backup-if-mismatch -p1 < patches/kubernetesEnvs.diff

packages/system/fluxcd-operator/values.yaml

@@ -0,0 +1,7 @@
+flux-operator:
+  fullnameOverride: flux-operator
+  tolerations:
+    - key: node.kubernetes.io/not-ready
+      operator: Exists
+      effect: NoSchedule
+  hostNetwork: true

packages/system/fluxcd/Makefile

@@ -0,0 +1,7 @@
+NAME=fluxcd
+NAMESPACE=cozy-$(NAME)
+
+apply-locally:
+	helm upgrade -i -n $(NAMESPACE) $(NAME) .
+
+include ../../../scripts/package-system.mk

packages/system/fluxcd/templates/flux-instance.yaml

@@ -0,0 +1,25 @@
+apiVersion: fluxcd.controlplane.io/v1
+kind: FluxInstance
+metadata:
+  name: flux
+spec:
+  {{- with .Values.cluster }}
+  cluster:
+    {{- with .networkPolicy }}
+    networkPolicy: {{ . }}
+    {{- end }}
+    {{- with .domain }}
+    domain: {{ . }}
+    {{- end }}
+  {{- end }}
+  distribution:
+    version: {{ .Values.distribution.version }}
+    registry: {{ .Values.distribution.registry }}
+  components:
+  {{- if .Values.components }}
+    {{- toYaml .Values.components | nindent 4 }}
+  {{- end }}
+  kustomize:
+  {{- if .Values.kustomize }}
+    {{- toYaml .Values.kustomize | nindent 4 }}
+  {{- end }}

packages/system/fluxcd/values.yaml

@@ -0,0 +1,47 @@
+cluster:
+  networkPolicy: true
+#  domain: cozy.local
+distribution:
+  version: 2.3.x
+  registry: ghcr.io/fluxcd
+components:
+  - source-controller
+  - kustomize-controller
+  - helm-controller
+  - notification-controller
+  - image-reflector-controller
+  - image-automation-controller
+kustomize:
+  patches:
+    - target:
+        kind: Deployment
+        name: "(kustomize-controller|helm-controller|source-controller)"
+      patch: |
+        - op: add
+          path: /spec/template/spec/containers/0/args/-
+          value: --concurrent=20
+        - op: add
+          path: /spec/template/spec/containers/0/args/-
+          value: --requeue-dependency=5s
+        - op: replace
+          path: /spec/template/spec/containers/0/resources/limits
+          value:
+            cpu: 2000m
+            memory: 2048Mi
+    - target:
+        kind: Deployment
+        name: source-controller
+      patch: |
+        - op: add
+          path: /spec/template/spec/containers/0/args/-
+          value: --storage-adv-addr=source-controller.cozy-fluxcd.svc
+        - op: add
+          path: /spec/template/spec/containers/0/args/-
+          value: --events-addr=http://notification-controller.cozy-fluxcd.svc/
+    - target:
+        kind: Deployment
+        name: (kustomize-controller|helm-controller|image-reflector-controller|image-automation-controller)
+      patch: |
+        - op: add
+          path: /spec/template/spec/containers/0/args/-
+          value: --events-addr=http://notification-controller.cozy-fluxcd.svc/

scripts/installer.sh

@@ -18,17 +18,27 @@ run_migrations() {
   done
 }
 
-
-flux_operator_is_ok() {
-  kubectl wait --for=condition=available -n cozy-fluxcd deploy/fluxcd-flux-operator --timeout=1m
+flux_is_ok() {
+  kubectl wait --for=condition=available -n cozy-fluxcd deploy/source-controller deploy/helm-controller --timeout=1s
 }
 
-flux_instance_is_ok() {
-  kubectl wait --for=condition=ready -n cozy-fluxcd fluxinstance/flux --timeout=5m
+ensure_fluxcd() {
+  if flux_is_ok; then
+    return
+  fi
+  if kubectl get crd helmreleases.helm.toolkit.fluxcd.io helmrepositories.source.toolkit.fluxcd.io; then
+    targets="apply resume"
+  else
+    targets="apply-locally"
+  fi
+  make -C packages/system/fluxcd-operator $targets
+  wait_for_crds fluxinstances.fluxcd.controlplane.io
+  make -C packages/system/fluxcd $targets
+  wait_for_crds helmreleases.helm.toolkit.fluxcd.io helmrepositories.source.toolkit.fluxcd.io
 }
 
-flux_controllers_ok() {
-  kubectl wait --for=condition=available -n cozy-fluxcd deploy/source-controller deploy/helm-controller --timeout=10s 
+wait_for_crds() {
+  timeout 60 sh -c "until kubectl get crd $*; do sleep 1; done"
 }
 
 install_basic_charts() {
@@ -48,18 +58,14 @@ run_migrations
 # Install namespaces
 make -C packages/core/platform namespaces-apply
 
-# Install fluxcd twice (once it will fail, since CRDs can't be ordered)
-make -C packages/core/fluxcd apply || make -C packages/core/fluxcd apply
-
-if flux_operator_is_ok; then
-  echo "Flux operator is installed and FluxInstance CRD is ready"
-fi
+# Install fluxcd
+ensure_fluxcd
 
 # Install platform chart
 make -C packages/core/platform apply
 
-# Install basic system charts (should be after platform chart applied)
-if ! flux_controllers_ok; then
+# Install basic charts
+if ! flux_is_ok; then
   install_basic_charts
 fi