diff --git a/ceph/templates/daemonset.yaml b/ceph/templates/daemonset-osd.yaml similarity index 89% rename from ceph/templates/daemonset.yaml rename to ceph/templates/daemonset-osd.yaml index 6c350cbe..1ad4b24f 100644 --- a/ceph/templates/daemonset.yaml +++ b/ceph/templates/daemonset-osd.yaml @@ -78,8 +78,8 @@ spec: timeoutSeconds: 5 resources: requests: - memory: "512Mi" - cpu: "1000m" + memory: {{ .Values.resources.osd.requests.memory | quote }} + cpu: {{ .Values.resources.osd.requests.cpu | quote }} limits: - memory: "1024Mi" - cpu: "2000m" + memory: {{ .Values.resources.osd.limits.memory | quote }} + cpu: {{ .Values.resources.osd.limits.cpu | quote }} diff --git a/ceph/templates/deployment-mds.yaml b/ceph/templates/deployment-mds.yaml new file mode 100644 index 00000000..9a4f5ead --- /dev/null +++ b/ceph/templates/deployment-mds.yaml @@ -0,0 +1,72 @@ +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: mds + name: ceph-mds +spec: + replicas: 1 + template: + metadata: + name: ceph-mds + labels: + app: ceph + daemon: mds + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + ports: + - containerPort: 6800 + env: + - name: CEPH_DAEMON + value: MDS + - name: CEPHFS_CREATE + value: "1" + - name: KV_TYPE + value: k8s + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + livenessProbe: + tcpSocket: + port: 6800 + initialDelaySeconds: 60 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: 6800 + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.mds.requests.memory | quote }} + cpu: {{ .Values.resources.mds.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mds.limits.memory | quote }} + cpu: {{ .Values.resources.mds.limits.cpu | quote }} diff --git a/ceph/templates/deployment-moncheck.yaml b/ceph/templates/deployment-moncheck.yaml new file mode 100644 index 00000000..aa829b09 --- /dev/null +++ b/ceph/templates/deployment-moncheck.yaml @@ -0,0 +1,64 @@ +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: moncheck + name: ceph-mon-check +spec: + replicas: {{ .Values.replicas.mon_check }} + template: + metadata: + name: ceph-mon + labels: + app: ceph + daemon: moncheck + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + imagePullPolicy: Always + ports: + - containerPort: 6789 + env: + - name: CEPH_DAEMON + value: MON_HEALTH + - name: KV_TYPE + value: k8s + - name: MON_IP_AUTO_DETECT + value: "1" + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + resources: + requests: + memory: {{ .Values.resources.mon_check.requests.memory | quote }} + cpu: {{ .Values.resources.mon_check.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon_check.limits.memory | quote }} + cpu: {{ .Values.resources.mon_check.limits.cpu | quote }} \ No newline at end of file diff --git a/ceph/templates/deployment-rgw.yaml b/ceph/templates/deployment-rgw.yaml new file mode 100644 index 00000000..a22c2ad3 --- /dev/null +++ b/ceph/templates/deployment-rgw.yaml @@ -0,0 +1,77 @@ +{{- if .Values.rgw.enabled }} +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: rgw + name: ceph-rgw +spec: + replicas: {{ .Values.replicas.rgw }} + template: + metadata: + name: ceph-rgw + labels: + app: ceph + daemon: rgw + spec: + hostNetwork: true + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-rgw + image: {{ .Values.images.daemon }} + ports: + - containerPort: {{ .Values.network.port.rgw_target }} + env: + - name: RGW_CIVETWEB_PORT + value: "{{ .Values.network.port.rgw_target }}" + - name: CEPH_DAEMON + value: RGW + - name: KV_TYPE + value: k8s + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + livenessProbe: + httpGet: + path: / + port: {{ .Values.network.port.rgw_target }} + initialDelaySeconds: 120 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: {{ .Values.network.port.rgw_target }} + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.rgw.requests.memory | quote }} + cpu: {{ .Values.resources.rgwrequests.cpu | quote }} + limits: + memory: {{ .Values.resources.rgw.limits.memory | quote }} + cpu: {{ .Values.resources.rgw.limits.cpu | quote }} +{{- end }} diff --git a/ceph/templates/deployment.yaml b/ceph/templates/deployment.yaml deleted file mode 100644 index 26012c11..00000000 --- a/ceph/templates/deployment.yaml +++ /dev/null @@ -1,310 +0,0 @@ ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: mds - name: ceph-mds -spec: - replicas: 1 - template: - metadata: - name: ceph-mds - labels: - app: ceph - daemon: mds - spec: - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} - ports: - - containerPort: 6800 - env: - - name: CEPH_DAEMON - value: MDS - - name: CEPHFS_CREATE - value: "1" - - name: KV_TYPE - value: k8s - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - tcpSocket: - port: 6800 - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 6800 - timeoutSeconds: 5 - resources: - requests: - memory: "10Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: moncheck - name: ceph-mon-check -spec: - replicas: 1 - template: - metadata: - name: ceph-mon - labels: - app: ceph - daemon: moncheck - spec: - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} - imagePullPolicy: Always - ports: - - containerPort: 6789 - env: - - name: CEPH_DAEMON - value: MON_HEALTH - - name: KV_TYPE - value: k8s - - name: MON_IP_AUTO_DETECT - value: "1" - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - resources: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: mon - name: ceph-mon -spec: - replicas: 3 - template: - metadata: - name: ceph-mon - labels: - app: ceph - daemon: mon - annotations: - # alanmeadows: this soft requirement allows single - # host deployments to spawn several ceph-mon - # containers - scheduler.alpha.kubernetes.io/affinity: > - { - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [{ - "labelSelector": { - "matchExpressions": [{ - "key": "daemon", - "operator": "In", - "values":["mon"] - }] - }, - "topologyKey": "kubernetes.io/hostname", - "weight": 10 - }] - } - } - spec: - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} -# imagePullPolicy: Always - lifecycle: - preStop: - exec: - # remove the mon on Pod stop. - command: - - "/remove-mon.sh" - ports: - - containerPort: 6789 - env: - - name: CEPH_DAEMON - value: MON - - name: KV_TYPE - value: k8s - - name: NETWORK_AUTO_DETECT - value: "1" - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - tcpSocket: - port: 6789 - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 6789 - timeoutSeconds: 5 - resources: - requests: - memory: "50Mi" - cpu: "1000m" - limits: - memory: "100Mi" - cpu: "2000m" ---- -# rgw not required: using if statement for deployment -{{- if .Values.rgw.enabled }} -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: rgw - name: ceph-rgw -spec: - replicas: 3 - template: - metadata: - name: ceph-rgw - labels: - app: ceph - daemon: rgw - spec: - hostNetwork: true - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-rgw - image: {{ .Values.images.daemon }} - ports: - - containerPort: {{ .Values.network.port.rgw_target }} - env: - - name: RGW_CIVETWEB_PORT - value: "{{ .Values.network.port.rgw_target }}" - - name: CEPH_DAEMON - value: RGW - - name: KV_TYPE - value: k8s - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - httpGet: - path: / - port: {{ .Values.network.port.rgw_target }} - initialDelaySeconds: 120 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: / - port: {{ .Values.network.port.rgw_target }} - timeoutSeconds: 5 - resources: - requests: - memory: "500Mi" - cpu: ".5" - limits: - memory: "500Mi" - cpu: ".5" -{{ end }} -# end: rgw removed optionally diff --git a/ceph/templates/service.yaml b/ceph/templates/service.yaml index 49a9afb6..cdfd4f55 100644 --- a/ceph/templates/service.yaml +++ b/ceph/templates/service.yaml @@ -15,6 +15,8 @@ spec: app: ceph daemon: mon clusterIP: None + +{{- if .Values.rgw.enabled }} --- apiVersion: v1 kind: Service @@ -32,3 +34,4 @@ spec: app: ceph daemon: rgw type: LoadBalancer +{{- end }} diff --git a/ceph/templates/statefulset-mon.yaml b/ceph/templates/statefulset-mon.yaml new file mode 100644 index 00000000..5ef33cd8 --- /dev/null +++ b/ceph/templates/statefulset-mon.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + labels: + app: ceph + daemon: mon + name: ceph-mon +spec: + serviceName: {{ .Values.service.mon.name | quote }} + replicas: {{ .Values.replicas.mon }} + template: + metadata: + name: ceph-mon + labels: + app: ceph + daemon: mon + annotations: + # alanmeadows: this soft requirement allows single + # host deployments to spawn several ceph-mon + # containers + scheduler.alpha.kubernetes.io/affinity: > + { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [{ + "labelSelector": { + "matchExpressions": [{ + "key": "daemon", + "operator": "In", + "values":["mon"] + }] + }, + "topologyKey": "kubernetes.io/hostname", + "weight": 10 + }] + } + } + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + - name: ceph-monfs + hostPath: + path: {{ .Values.storage.mon_directory }} + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + imagePullPolicy: Always + lifecycle: + preStop: + exec: + # remove the mon on Pod stop. + command: + - "/remove-mon.sh" + ports: + - containerPort: 6789 + env: + - name: CEPH_DAEMON + value: MON + - name: KV_TYPE + value: k8s + - name: NETWORK_AUTO_DETECT + value: "1" + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + - name: ceph-monfs + mountPath: /var/lib/ceph/mon + livenessProbe: + tcpSocket: + port: 6789 + initialDelaySeconds: 60 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: 6789 + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.mon.requests.memory | quote }} + cpu: {{ .Values.resources.mon.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon.limits.memory | quote }} + cpu: {{ .Values.resources.mon.limits.cpu | quote }} diff --git a/ceph/templates/storage.yaml b/ceph/templates/storage.yaml index 9ef1eae2..b60221c1 100644 --- a/ceph/templates/storage.yaml +++ b/ceph/templates/storage.yaml @@ -10,7 +10,8 @@ parameters: monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }} adminId: {{ .Values.storageclass.admin_id }} adminSecretName: {{ .Values.storageclass.admin_secret_name }} - ## forcing namespace due to issue with -- default "{{ .Release.Namespace }}" }} -- + # forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }} + # during helm lint adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }} pool: {{ .Values.storageclass.pool }} userId: {{ .Values.storageclass.user_id }} diff --git a/ceph/values.yaml b/ceph/values.yaml index 76b4ac46..33c6da24 100644 --- a/ceph/values.yaml +++ b/ceph/values.yaml @@ -7,6 +7,15 @@ # tunables available - parameterizing more of the elements # in the manifests is a work in progress +replicas: + mon: 3 + rgw: 3 + mon_check: 1 + +service: + mon: + name: ceph-mon + images: daemon: quay.io/attcomdev/ceph-daemon:latest @@ -23,11 +32,52 @@ network: storage: osd_directory: /var/lib/openstack-helm/ceph/osd var_directory: /var/lib/openstack-helm/ceph/ceph + mon_directory: /var/lib/openstack-helm/ceph/mon # rgw is optionall disabled rgw: enabled: false +rgw: + enabled: false + +resources: + osd: + requests: + memory: "512Mi" + cpu: "1000m" + limits: + memory: "1024Mi" + cpu: "2000m" + mds: + requests: + memory: "10Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + mon: + requests: + memory: "50Mi" + cpu: "1000m" + limits: + memory: "100Mi" + cpu: "2000m" + mon_check: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + rgw: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + # Setting this to false will assume you will # setup and orchestrate your own secrets and # configmaps outside of this helm chart @@ -63,7 +113,7 @@ secrets: storageclass: provision_storage_class: true name: general - monitors: null + monitors: null pool: rbd admin_id: admin admin_secret_name: pvc-ceph-conf-combined-storageclass