From b366dc252a9a87b2b6678add751631cd17e59270 Mon Sep 17 00:00:00 2001 From: Alan Meadows Date: Tue, 3 Jan 2017 12:23:25 -0800 Subject: [PATCH] Refactor ceph to allow monitors to leverage a stateful set. The IPs of ceph monitors cannot (or should not) change. This refactor allows the ceph monitors to act as statefulsets. It also persists their on disk data to nodeDirs, to allow cluster wide restarts (in parallel), where previously this would lose data. This is accompanied by some docker images changes that ensure that auth for the OSDs is restored/reinserted as auth data is somehow not persisted to disk at this time. --- .../{daemonset.yaml => daemonset-osd.yaml} | 8 +- ceph/templates/deployment-mds.yaml | 72 ++++ ceph/templates/deployment-moncheck.yaml | 64 ++++ ceph/templates/deployment-rgw.yaml | 77 +++++ ceph/templates/deployment.yaml | 310 ------------------ ceph/templates/service.yaml | 3 + ceph/templates/statefulset-mon.yaml | 105 ++++++ ceph/templates/storage.yaml | 3 +- ceph/values.yaml | 52 ++- 9 files changed, 378 insertions(+), 316 deletions(-) rename ceph/templates/{daemonset.yaml => daemonset-osd.yaml} (89%) create mode 100644 ceph/templates/deployment-mds.yaml create mode 100644 ceph/templates/deployment-moncheck.yaml create mode 100644 ceph/templates/deployment-rgw.yaml delete mode 100644 ceph/templates/deployment.yaml create mode 100644 ceph/templates/statefulset-mon.yaml diff --git a/ceph/templates/daemonset.yaml b/ceph/templates/daemonset-osd.yaml similarity index 89% rename from ceph/templates/daemonset.yaml rename to ceph/templates/daemonset-osd.yaml index 6c350cbe..1ad4b24f 100644 --- a/ceph/templates/daemonset.yaml +++ b/ceph/templates/daemonset-osd.yaml @@ -78,8 +78,8 @@ spec: timeoutSeconds: 5 resources: requests: - memory: "512Mi" - cpu: "1000m" + memory: {{ .Values.resources.osd.requests.memory | quote }} + cpu: {{ .Values.resources.osd.requests.cpu | quote }} limits: - memory: "1024Mi" - cpu: "2000m" + memory: {{ .Values.resources.osd.limits.memory | quote }} + cpu: {{ .Values.resources.osd.limits.cpu | quote }} diff --git a/ceph/templates/deployment-mds.yaml b/ceph/templates/deployment-mds.yaml new file mode 100644 index 00000000..9a4f5ead --- /dev/null +++ b/ceph/templates/deployment-mds.yaml @@ -0,0 +1,72 @@ +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: mds + name: ceph-mds +spec: + replicas: 1 + template: + metadata: + name: ceph-mds + labels: + app: ceph + daemon: mds + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + ports: + - containerPort: 6800 + env: + - name: CEPH_DAEMON + value: MDS + - name: CEPHFS_CREATE + value: "1" + - name: KV_TYPE + value: k8s + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + livenessProbe: + tcpSocket: + port: 6800 + initialDelaySeconds: 60 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: 6800 + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.mds.requests.memory | quote }} + cpu: {{ .Values.resources.mds.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mds.limits.memory | quote }} + cpu: {{ .Values.resources.mds.limits.cpu | quote }} diff --git a/ceph/templates/deployment-moncheck.yaml b/ceph/templates/deployment-moncheck.yaml new file mode 100644 index 00000000..aa829b09 --- /dev/null +++ b/ceph/templates/deployment-moncheck.yaml @@ -0,0 +1,64 @@ +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: moncheck + name: ceph-mon-check +spec: + replicas: {{ .Values.replicas.mon_check }} + template: + metadata: + name: ceph-mon + labels: + app: ceph + daemon: moncheck + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + imagePullPolicy: Always + ports: + - containerPort: 6789 + env: + - name: CEPH_DAEMON + value: MON_HEALTH + - name: KV_TYPE + value: k8s + - name: MON_IP_AUTO_DETECT + value: "1" + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + resources: + requests: + memory: {{ .Values.resources.mon_check.requests.memory | quote }} + cpu: {{ .Values.resources.mon_check.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon_check.limits.memory | quote }} + cpu: {{ .Values.resources.mon_check.limits.cpu | quote }} \ No newline at end of file diff --git a/ceph/templates/deployment-rgw.yaml b/ceph/templates/deployment-rgw.yaml new file mode 100644 index 00000000..a22c2ad3 --- /dev/null +++ b/ceph/templates/deployment-rgw.yaml @@ -0,0 +1,77 @@ +{{- if .Values.rgw.enabled }} +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: rgw + name: ceph-rgw +spec: + replicas: {{ .Values.replicas.rgw }} + template: + metadata: + name: ceph-rgw + labels: + app: ceph + daemon: rgw + spec: + hostNetwork: true + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-rgw + image: {{ .Values.images.daemon }} + ports: + - containerPort: {{ .Values.network.port.rgw_target }} + env: + - name: RGW_CIVETWEB_PORT + value: "{{ .Values.network.port.rgw_target }}" + - name: CEPH_DAEMON + value: RGW + - name: KV_TYPE + value: k8s + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + livenessProbe: + httpGet: + path: / + port: {{ .Values.network.port.rgw_target }} + initialDelaySeconds: 120 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: {{ .Values.network.port.rgw_target }} + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.rgw.requests.memory | quote }} + cpu: {{ .Values.resources.rgwrequests.cpu | quote }} + limits: + memory: {{ .Values.resources.rgw.limits.memory | quote }} + cpu: {{ .Values.resources.rgw.limits.cpu | quote }} +{{- end }} diff --git a/ceph/templates/deployment.yaml b/ceph/templates/deployment.yaml deleted file mode 100644 index 26012c11..00000000 --- a/ceph/templates/deployment.yaml +++ /dev/null @@ -1,310 +0,0 @@ ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: mds - name: ceph-mds -spec: - replicas: 1 - template: - metadata: - name: ceph-mds - labels: - app: ceph - daemon: mds - spec: - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} - ports: - - containerPort: 6800 - env: - - name: CEPH_DAEMON - value: MDS - - name: CEPHFS_CREATE - value: "1" - - name: KV_TYPE - value: k8s - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - tcpSocket: - port: 6800 - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 6800 - timeoutSeconds: 5 - resources: - requests: - memory: "10Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: moncheck - name: ceph-mon-check -spec: - replicas: 1 - template: - metadata: - name: ceph-mon - labels: - app: ceph - daemon: moncheck - spec: - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} - imagePullPolicy: Always - ports: - - containerPort: 6789 - env: - - name: CEPH_DAEMON - value: MON_HEALTH - - name: KV_TYPE - value: k8s - - name: MON_IP_AUTO_DETECT - value: "1" - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - resources: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: mon - name: ceph-mon -spec: - replicas: 3 - template: - metadata: - name: ceph-mon - labels: - app: ceph - daemon: mon - annotations: - # alanmeadows: this soft requirement allows single - # host deployments to spawn several ceph-mon - # containers - scheduler.alpha.kubernetes.io/affinity: > - { - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [{ - "labelSelector": { - "matchExpressions": [{ - "key": "daemon", - "operator": "In", - "values":["mon"] - }] - }, - "topologyKey": "kubernetes.io/hostname", - "weight": 10 - }] - } - } - spec: - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} -# imagePullPolicy: Always - lifecycle: - preStop: - exec: - # remove the mon on Pod stop. - command: - - "/remove-mon.sh" - ports: - - containerPort: 6789 - env: - - name: CEPH_DAEMON - value: MON - - name: KV_TYPE - value: k8s - - name: NETWORK_AUTO_DETECT - value: "1" - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - tcpSocket: - port: 6789 - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 6789 - timeoutSeconds: 5 - resources: - requests: - memory: "50Mi" - cpu: "1000m" - limits: - memory: "100Mi" - cpu: "2000m" ---- -# rgw not required: using if statement for deployment -{{- if .Values.rgw.enabled }} -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: rgw - name: ceph-rgw -spec: - replicas: 3 - template: - metadata: - name: ceph-rgw - labels: - app: ceph - daemon: rgw - spec: - hostNetwork: true - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-rgw - image: {{ .Values.images.daemon }} - ports: - - containerPort: {{ .Values.network.port.rgw_target }} - env: - - name: RGW_CIVETWEB_PORT - value: "{{ .Values.network.port.rgw_target }}" - - name: CEPH_DAEMON - value: RGW - - name: KV_TYPE - value: k8s - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - httpGet: - path: / - port: {{ .Values.network.port.rgw_target }} - initialDelaySeconds: 120 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: / - port: {{ .Values.network.port.rgw_target }} - timeoutSeconds: 5 - resources: - requests: - memory: "500Mi" - cpu: ".5" - limits: - memory: "500Mi" - cpu: ".5" -{{ end }} -# end: rgw removed optionally diff --git a/ceph/templates/service.yaml b/ceph/templates/service.yaml index 49a9afb6..cdfd4f55 100644 --- a/ceph/templates/service.yaml +++ b/ceph/templates/service.yaml @@ -15,6 +15,8 @@ spec: app: ceph daemon: mon clusterIP: None + +{{- if .Values.rgw.enabled }} --- apiVersion: v1 kind: Service @@ -32,3 +34,4 @@ spec: app: ceph daemon: rgw type: LoadBalancer +{{- end }} diff --git a/ceph/templates/statefulset-mon.yaml b/ceph/templates/statefulset-mon.yaml new file mode 100644 index 00000000..5ef33cd8 --- /dev/null +++ b/ceph/templates/statefulset-mon.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + labels: + app: ceph + daemon: mon + name: ceph-mon +spec: + serviceName: {{ .Values.service.mon.name | quote }} + replicas: {{ .Values.replicas.mon }} + template: + metadata: + name: ceph-mon + labels: + app: ceph + daemon: mon + annotations: + # alanmeadows: this soft requirement allows single + # host deployments to spawn several ceph-mon + # containers + scheduler.alpha.kubernetes.io/affinity: > + { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [{ + "labelSelector": { + "matchExpressions": [{ + "key": "daemon", + "operator": "In", + "values":["mon"] + }] + }, + "topologyKey": "kubernetes.io/hostname", + "weight": 10 + }] + } + } + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + - name: ceph-monfs + hostPath: + path: {{ .Values.storage.mon_directory }} + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + imagePullPolicy: Always + lifecycle: + preStop: + exec: + # remove the mon on Pod stop. + command: + - "/remove-mon.sh" + ports: + - containerPort: 6789 + env: + - name: CEPH_DAEMON + value: MON + - name: KV_TYPE + value: k8s + - name: NETWORK_AUTO_DETECT + value: "1" + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + - name: ceph-monfs + mountPath: /var/lib/ceph/mon + livenessProbe: + tcpSocket: + port: 6789 + initialDelaySeconds: 60 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: 6789 + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.mon.requests.memory | quote }} + cpu: {{ .Values.resources.mon.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon.limits.memory | quote }} + cpu: {{ .Values.resources.mon.limits.cpu | quote }} diff --git a/ceph/templates/storage.yaml b/ceph/templates/storage.yaml index 9ef1eae2..b60221c1 100644 --- a/ceph/templates/storage.yaml +++ b/ceph/templates/storage.yaml @@ -10,7 +10,8 @@ parameters: monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }} adminId: {{ .Values.storageclass.admin_id }} adminSecretName: {{ .Values.storageclass.admin_secret_name }} - ## forcing namespace due to issue with -- default "{{ .Release.Namespace }}" }} -- + # forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }} + # during helm lint adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }} pool: {{ .Values.storageclass.pool }} userId: {{ .Values.storageclass.user_id }} diff --git a/ceph/values.yaml b/ceph/values.yaml index 76b4ac46..33c6da24 100644 --- a/ceph/values.yaml +++ b/ceph/values.yaml @@ -7,6 +7,15 @@ # tunables available - parameterizing more of the elements # in the manifests is a work in progress +replicas: + mon: 3 + rgw: 3 + mon_check: 1 + +service: + mon: + name: ceph-mon + images: daemon: quay.io/attcomdev/ceph-daemon:latest @@ -23,11 +32,52 @@ network: storage: osd_directory: /var/lib/openstack-helm/ceph/osd var_directory: /var/lib/openstack-helm/ceph/ceph + mon_directory: /var/lib/openstack-helm/ceph/mon # rgw is optionall disabled rgw: enabled: false +rgw: + enabled: false + +resources: + osd: + requests: + memory: "512Mi" + cpu: "1000m" + limits: + memory: "1024Mi" + cpu: "2000m" + mds: + requests: + memory: "10Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + mon: + requests: + memory: "50Mi" + cpu: "1000m" + limits: + memory: "100Mi" + cpu: "2000m" + mon_check: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + rgw: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + # Setting this to false will assume you will # setup and orchestrate your own secrets and # configmaps outside of this helm chart @@ -63,7 +113,7 @@ secrets: storageclass: provision_storage_class: true name: general - monitors: null + monitors: null pool: rbd admin_id: admin admin_secret_name: pvc-ceph-conf-combined-storageclass