diff --git a/TODO b/TODO index 9ce25476..dfe6a931 100644 --- a/TODO +++ b/TODO @@ -28,3 +28,5 @@ recursive namespace deletion move icons to repo install 'monitoring-system' app reconcile system helm releases +remove cluster and other namespace resources from apps charts, eg extension-apiserver-authentication-reader +spawn etcd per tenant / per cluster diff --git a/packages/apps/kubernetes/templates/kccm/config.yaml b/packages/apps/kubernetes/templates/cloud-config.yaml similarity index 63% rename from packages/apps/kubernetes/templates/kccm/config.yaml rename to packages/apps/kubernetes/templates/cloud-config.yaml index 6c9aae83..9f16548a 100644 --- a/packages/apps/kubernetes/templates/kccm/config.yaml +++ b/packages/apps/kubernetes/templates/cloud-config.yaml @@ -1,11 +1,10 @@ apiVersion: v1 kind: ConfigMap metadata: - name: cluster1-cloud-config - namespace: tenant-foo + name: {{ .Release.Name }}-cloud-config data: cloud-config: | loadBalancer: creationPollInterval: 5 creationPollTimeout: 60 - namespace: tenant-foo + namespace: {{ .Release.Namespace }} diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml new file mode 100644 index 00000000..6c8d10b5 --- /dev/null +++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-cluster-autoscaler + labels: + app: {{ .Release.Name }}-cluster-autoscaler +spec: + selector: + matchLabels: + app: {{ .Release.Name }}-cluster-autoscaler + replicas: 1 + template: + metadata: + labels: + app: {{ .Release.Name }}-cluster-autoscaler + spec: + containers: + - image: ghcr.io/kvaps/test:cluster-autoscaller + name: cluster-autoscaler + command: + - /cluster-autoscaler + args: + - --cloud-provider=clusterapi + - --kubeconfig=/etc/kubernetes/kubeconfig/value + - --clusterapi-cloud-config-authoritative + - --node-group-auto-discovery=clusterapi:namespace={{ .Release.Namespace }},clusterName={{ .Release.Name }} + volumeMounts: + - mountPath: /etc/kubernetes/kubeconfig + name: kubeconfig + readOnly: true + volumes: + - configMap: + name: {{ .Release.Name }}-cloud-config + name: cloud-config + - secret: + secretName: {{ .Release.Name }}-kubeconfig + name: kubeconfig + serviceAccountName: {{ .Release.Name }}-cluster-autoscaler + terminationGracePeriodSeconds: 10 +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: {{ .Release.Name }}-cluster-autoscaler + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-cluster-autoscaler +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-cluster-autoscaler +rules: + - apiGroups: + - cluster.x-k8s.io + resources: + - machinedeployments + - machinedeployments/scale + - machines + - machinesets + - machinepools + verbs: + - get + - list + - update + - watch + - apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtmachinetemplates + verbs: + - get + - list + - update + - watch diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 3bb646b5..ee4503e5 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -65,6 +65,9 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: {} + initConfiguration: + skipPhases: + - addon/kube-proxy --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: KubevirtMachineTemplate @@ -107,6 +110,11 @@ kind: MachineDeployment metadata: name: {{ .Release.Name }}-md-0 namespace: {{ .Release.Namespace }} + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "2" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0" + capacity.cluster-autoscaler.kubernetes.io/memory: "1024Mi" + capacity.cluster-autoscaler.kubernetes.io/cpu: "2" spec: clusterName: {{ .Release.Name }} selector: diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml new file mode 100644 index 00000000..1cab4955 --- /dev/null +++ b/packages/apps/kubernetes/templates/csi/deploy.yaml @@ -0,0 +1,126 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-kcsi-controller + labels: + app: {{ .Release.Name }}-kcsi-driver +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-kcsi-driver + template: + metadata: + labels: + app: {{ .Release.Name }}-kcsi-driver + spec: + serviceAccountName: {{ .Release.Name }}-kcsi + priorityClassName: system-cluster-critical + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + effect: "NoSchedule" + containers: + - name: csi-driver + imagePullPolicy: Always + image: ghcr.io/kvaps/test:kubevirt-csi-driver + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--infra-cluster-namespace=$(INFRACLUSTER_NAMESPACE)" + - "--infra-cluster-labels=$(INFRACLUSTER_LABELS)" + - "--v=5" + ports: + - name: healthz + containerPort: 10301 + protocol: TCP + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: INFRACLUSTER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INFRACLUSTER_LABELS + value: "csi-driver/cluster=test" + - name: INFRA_STORAGE_CLASS_ENFORCEMENT + valueFrom: + configMapKeyRef: + name: driver-config + key: infraStorageClassEnforcement + optional: true + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-provisioner + image: quay.io/openshift/origin-csi-external-provisioner:latest + args: + - "--csi-address=$(ADDRESS)" + - "--default-fstype=ext4" + - "--kubeconfig=/etc/kubernetes/kubeconfig/value" + - "--v=5" + - "--timeout=3m" + - "--retry-interval-max=1m" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + - name: csi-attacher + image: quay.io/openshift/origin-csi-external-attacher:latest + args: + - "--csi-address=$(ADDRESS)" + - "--kubeconfig=/etc/kubernetes/kubeconfig/value" + - "--v=5" + - "--timeout=3m" + - "--retry-interval-max=1m" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-liveness-probe + image: quay.io/openshift/origin-csi-livenessprobe:latest + args: + - "--csi-address=/csi/csi.sock" + - "--probe-timeout=3s" + - "--health-port=10301" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + memory: 50Mi + cpu: 10m + volumes: + - name: socket-dir + emptyDir: {} + - secret: + secretName: {{ .Release.Name }}-kubeconfig + name: kubeconfig diff --git a/packages/apps/kubernetes/templates/csi/infra-cluster-service-account.yaml b/packages/apps/kubernetes/templates/csi/infra-cluster-service-account.yaml new file mode 100644 index 00000000..d70ea04a --- /dev/null +++ b/packages/apps/kubernetes/templates/csi/infra-cluster-service-account.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-kcsi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-kcsi +rules: +- apiGroups: ["cdi.kubevirt.io"] + resources: ["datavolumes"] + verbs: ["get", "create", "delete"] +- apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["list", "get"] +- apiGroups: ["subresources.kubevirt.io"] + resources: ["virtualmachineinstances/addvolume", "virtualmachineinstances/removevolume"] + verbs: ["update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-kcsi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-kcsi +subjects: +- kind: ServiceAccount + name: {{ .Release.Name }}-kcsi diff --git a/packages/apps/kubernetes/templates/kccm/kccm_cluster_role.yaml b/packages/apps/kubernetes/templates/kccm/kccm_cluster_role.yaml new file mode 100644 index 00000000..93ee6971 --- /dev/null +++ b/packages/apps/kubernetes/templates/kccm/kccm_cluster_role.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-kccm +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get diff --git a/packages/apps/kubernetes/templates/kccm/kccm_cluster_role_binding.yaml b/packages/apps/kubernetes/templates/kccm/kccm_cluster_role_binding.yaml new file mode 100644 index 00000000..564d37f5 --- /dev/null +++ b/packages/apps/kubernetes/templates/kccm/kccm_cluster_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-kccm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-kccm +subjects: +- kind: ServiceAccount + name: {{ .Release.Name }}-kccm + namespace: {{ .Release.Namespace }} diff --git a/packages/apps/kubernetes/templates/kccm/kccm_role.yaml b/packages/apps/kubernetes/templates/kccm/kccm_role.yaml index 1c394a3b..fee06003 100644 --- a/packages/apps/kubernetes/templates/kccm/kccm_role.yaml +++ b/packages/apps/kubernetes/templates/kccm/kccm_role.yaml @@ -1,8 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - namespace: tenant-foo - name: kccm + name: {{ .Release.Name }}-kccm rules: - apiGroups: - kubevirt.io diff --git a/packages/apps/kubernetes/templates/kccm/kccm_role_binding.yaml b/packages/apps/kubernetes/templates/kccm/kccm_role_binding.yaml index 2295989d..1bc6f96a 100644 --- a/packages/apps/kubernetes/templates/kccm/kccm_role_binding.yaml +++ b/packages/apps/kubernetes/templates/kccm/kccm_role_binding.yaml @@ -1,30 +1,27 @@ -apiVersion: v1 -items: -- apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: kccm - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: tenant-foo -- apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: kccm-sa - namespace: tenant-foo - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kccm - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: test -kind: List -metadata: {} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-kccm + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ .Release.Name }}-kccm + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-kccm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-kccm +subjects: +- kind: ServiceAccount + name: {{ .Release.Name }}-kccm + namespace: {{ .Release.Namespace }} diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml index 03008138..5e211251 100644 --- a/packages/apps/kubernetes/templates/kccm/manager.yaml +++ b/packages/apps/kubernetes/templates/kccm/manager.yaml @@ -1,21 +1,19 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: kubevirt-cloud-controller-manager - namespace: tenant-foo + name: {{ .Release.Name }}-kccm labels: - k8s-app: kubevirt-cloud-controller-manager + k8s-app: {{ .Release.Name }}-kccm spec: replicas: 1 selector: matchLabels: - k8s-app: kubevirt-cloud-controller-manager + k8s-app: {{ .Release.Name }}-kccm template: metadata: labels: - k8s-app: kubevirt-cloud-controller-manager + k8s-app: {{ .Release.Name }}-kccm spec: - #hostNetwork: true containers: - name: kubevirt-cloud-controller-manager args: @@ -40,11 +38,11 @@ spec: readOnly: true volumes: - configMap: - name: cluster1-cloud-config + name: {{ .Release.Name }}-cloud-config name: cloud-config - secret: - secretName: cluster1-kubeconfig + secretName: {{ .Release.Name }}-kubeconfig name: kubeconfig tolerations: - operator: Exists - serviceAccountName: cloud-controller-manager + serviceAccountName: {{ .Release.Name }}-kccm diff --git a/packages/apps/kubernetes/templates/kccm/service_account.yaml b/packages/apps/kubernetes/templates/kccm/service_account.yaml index 27fd4745..10a690b2 100644 --- a/packages/apps/kubernetes/templates/kccm/service_account.yaml +++ b/packages/apps/kubernetes/templates/kccm/service_account.yaml @@ -1,5 +1,4 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: cloud-controller-manager - namespace: tenant-foo + name: {{ .Release.Name }}-kccm