add ccm csi cluster-autoscaler

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

TODO

@@ -28,3 +28,5 @@ recursive namespace deletion
 move icons to repo
 install 'monitoring-system' app
 reconcile system helm releases
+remove cluster and other namespace resources from apps charts, eg extension-apiserver-authentication-reader
+spawn etcd per tenant / per cluster

packages/apps/kubernetes/templates/cloud-config.yaml

@@ -1,11 +1,10 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: cluster1-cloud-config
-  namespace: tenant-foo
+  name: {{ .Release.Name }}-cloud-config
 data:
   cloud-config: |
     loadBalancer:
       creationPollInterval: 5
       creationPollTimeout: 60
-    namespace: tenant-foo
+    namespace: {{ .Release.Namespace }}

packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml

@@ -0,0 +1,86 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-cluster-autoscaler
+  labels:
+    app: {{ .Release.Name }}-cluster-autoscaler
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-cluster-autoscaler
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-cluster-autoscaler
+    spec:
+      containers:
+      - image: ghcr.io/kvaps/test:cluster-autoscaller
+        name: cluster-autoscaler
+        command:
+        - /cluster-autoscaler
+        args:
+        - --cloud-provider=clusterapi
+        - --kubeconfig=/etc/kubernetes/kubeconfig/value
+        - --clusterapi-cloud-config-authoritative
+        - --node-group-auto-discovery=clusterapi:namespace={{ .Release.Namespace }},clusterName={{ .Release.Name }}
+        volumeMounts:
+        - mountPath: /etc/kubernetes/kubeconfig
+          name: kubeconfig
+          readOnly: true
+      volumes:
+      - configMap:
+          name: {{ .Release.Name }}-cloud-config
+        name: cloud-config
+      - secret:
+          secretName: {{ .Release.Name }}-kubeconfig
+        name: kubeconfig
+      serviceAccountName: {{ .Release.Name }}-cluster-autoscaler
+      terminationGracePeriodSeconds: 10
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Release.Name }}-cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-cluster-autoscaler
+subjects:
+- kind: ServiceAccount
+  name: {{ .Release.Name }}-cluster-autoscaler
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-cluster-autoscaler
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Release.Name }}-cluster-autoscaler
+rules:
+  - apiGroups:
+    - cluster.x-k8s.io
+    resources:
+    - machinedeployments
+    - machinedeployments/scale
+    - machines
+    - machinesets
+    - machinepools
+    verbs:
+    - get
+    - list
+    - update
+    - watch
+  - apiGroups:
+    - infrastructure.cluster.x-k8s.io
+    resources:
+    - kubevirtmachinetemplates
+    verbs:
+    - get
+    - list
+    - update
+    - watch

packages/apps/kubernetes/templates/cluster.yaml

@@ -65,6 +65,9 @@ spec:
       joinConfiguration:
         nodeRegistration:
           kubeletExtraArgs: {}
+      initConfiguration:
+        skipPhases:
+        - addon/kube-proxy
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: KubevirtMachineTemplate
@@ -107,6 +110,11 @@ kind: MachineDeployment
 metadata:
   name: {{ .Release.Name }}-md-0
   namespace: {{ .Release.Namespace }}
+  annotations:
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "2"
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0"
+    capacity.cluster-autoscaler.kubernetes.io/memory: "1024Mi"
+    capacity.cluster-autoscaler.kubernetes.io/cpu: "2"
 spec:
   clusterName: {{ .Release.Name }}
   selector:

packages/apps/kubernetes/templates/csi/deploy.yaml

@@ -0,0 +1,126 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: {{ .Release.Name }}-kcsi-controller
+  labels:
+    app: {{ .Release.Name }}-kcsi-driver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-kcsi-driver
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-kcsi-driver
+    spec:
+      serviceAccountName: {{ .Release.Name }}-kcsi
+      priorityClassName: system-cluster-critical
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: "NoSchedule"
+      containers:
+        - name: csi-driver
+          imagePullPolicy: Always
+          image: ghcr.io/kvaps/test:kubevirt-csi-driver
+          args:
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--infra-cluster-namespace=$(INFRACLUSTER_NAMESPACE)"
+            - "--infra-cluster-labels=$(INFRACLUSTER_LABELS)"
+            - "--v=5"
+          ports:
+            - name: healthz
+              containerPort: 10301
+              protocol: TCP
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: INFRACLUSTER_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: INFRACLUSTER_LABELS
+              value: "csi-driver/cluster=test"
+            - name: INFRA_STORAGE_CLASS_ENFORCEMENT
+              valueFrom:
+                configMapKeyRef:
+                  name: driver-config
+                  key: infraStorageClassEnforcement
+                  optional: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: kubeconfig
+              mountPath: /etc/kubernetes/kubeconfig
+              readOnly: true
+          resources:
+            requests:
+              memory: 50Mi
+              cpu: 10m
+        - name: csi-provisioner
+          image: quay.io/openshift/origin-csi-external-provisioner:latest
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--default-fstype=ext4"
+            - "--kubeconfig=/etc/kubernetes/kubeconfig/value"
+            - "--v=5"
+            - "--timeout=3m"
+            - "--retry-interval-max=1m"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: kubeconfig
+              mountPath: /etc/kubernetes/kubeconfig
+              readOnly: true
+        - name: csi-attacher
+          image: quay.io/openshift/origin-csi-external-attacher:latest
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--kubeconfig=/etc/kubernetes/kubeconfig/value"
+            - "--v=5"
+            - "--timeout=3m"
+            - "--retry-interval-max=1m"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: kubeconfig
+              mountPath: /etc/kubernetes/kubeconfig
+              readOnly: true
+          resources:
+            requests:
+              memory: 50Mi
+              cpu: 10m
+        - name: csi-liveness-probe
+          image: quay.io/openshift/origin-csi-livenessprobe:latest
+          args:
+            - "--csi-address=/csi/csi.sock"
+            - "--probe-timeout=3s"
+            - "--health-port=10301"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+            requests:
+              memory: 50Mi
+              cpu: 10m
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - secret:
+            secretName: {{ .Release.Name }}-kubeconfig
+          name: kubeconfig

packages/apps/kubernetes/templates/csi/infra-cluster-service-account.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-kcsi
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-kcsi
+rules:
+- apiGroups: ["cdi.kubevirt.io"]
+  resources: ["datavolumes"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["kubevirt.io"]
+  resources: ["virtualmachineinstances"]
+  verbs: ["list", "get"]
+- apiGroups: ["subresources.kubevirt.io"]
+  resources: ["virtualmachineinstances/addvolume", "virtualmachineinstances/removevolume"]
+  verbs: ["update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}-kcsi
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-kcsi
+subjects:
+- kind: ServiceAccount
+  name: {{ .Release.Name }}-kcsi

packages/apps/kubernetes/templates/kccm/kccm_cluster_role.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-kccm
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get

packages/apps/kubernetes/templates/kccm/kccm_cluster_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata: 
+  name: {{ .Release.Name }}-kccm
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}-kccm
+subjects:
+- kind: ServiceAccount
+  name: {{ .Release.Name }}-kccm
+  namespace: {{ .Release.Namespace }}

packages/apps/kubernetes/templates/kccm/kccm_role.yaml

@@ -1,8 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  namespace: tenant-foo
-  name: kccm
+  name: {{ .Release.Name }}-kccm
 rules:
 - apiGroups: 
   - kubevirt.io

packages/apps/kubernetes/templates/kccm/kccm_role_binding.yaml

@@ -1,30 +1,27 @@
-apiVersion: v1
-items:
-- apiVersion: rbac.authorization.k8s.io/v1
-  kind: RoleBinding
-  metadata: 
-    name: kccm
-    namespace: kube-system
-  roleRef:
-    apiGroup: rbac.authorization.k8s.io
-    kind: Role
-    name: extension-apiserver-authentication-reader
-  subjects:
-  - kind: ServiceAccount
-    name: cloud-controller-manager
-    namespace: tenant-foo
-- apiVersion: rbac.authorization.k8s.io/v1
-  kind: RoleBinding
-  metadata: 
-    name: kccm-sa
-    namespace: tenant-foo
-  roleRef:
-    apiGroup: rbac.authorization.k8s.io
-    kind: Role
-    name: kccm
-  subjects:
-  - kind: ServiceAccount
-    name: cloud-controller-manager
-    namespace: test
-kind: List
-metadata: {}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata: 
+  name: {{ .Release.Name }}-kccm
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: {{ .Release.Name }}-kccm
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata: 
+  name: {{ .Release.Name }}-kccm
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-kccm
+subjects:
+- kind: ServiceAccount
+  name: {{ .Release.Name }}-kccm
+  namespace: {{ .Release.Namespace }}

packages/apps/kubernetes/templates/kccm/manager.yaml

@@ -1,21 +1,19 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: kubevirt-cloud-controller-manager
-  namespace: tenant-foo
+  name: {{ .Release.Name }}-kccm
   labels:
-    k8s-app: kubevirt-cloud-controller-manager
+    k8s-app: {{ .Release.Name }}-kccm
 spec:
   replicas: 1
   selector:
     matchLabels:
-      k8s-app: kubevirt-cloud-controller-manager
+      k8s-app: {{ .Release.Name }}-kccm
   template:
     metadata:
       labels:
-        k8s-app: kubevirt-cloud-controller-manager
+        k8s-app: {{ .Release.Name }}-kccm
     spec:
-      #hostNetwork: true
       containers:
       - name: kubevirt-cloud-controller-manager
         args:
@@ -40,11 +38,11 @@ spec:
           readOnly: true
       volumes:
       - configMap:
-          name: cluster1-cloud-config
+          name: {{ .Release.Name }}-cloud-config
         name: cloud-config
       - secret:
-          secretName: cluster1-kubeconfig
+          secretName: {{ .Release.Name }}-kubeconfig
         name: kubeconfig
       tolerations:
       - operator: Exists
-      serviceAccountName: cloud-controller-manager
+      serviceAccountName: {{ .Release.Name }}-kccm

packages/apps/kubernetes/templates/kccm/service_account.yaml

@@ -1,5 +1,4 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: cloud-controller-manager
-  namespace: tenant-foo
+  name: {{ .Release.Name }}-kccm