From a5e5556e46ee143363557dc18c89e6c811c2e2c0 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Fri, 6 Jul 2018 12:33:12 -0500 Subject: [PATCH] Ceph: Split out ceph provsioners from client chart This PS splits out the ceph provisioners from the client chart allowing OSH to be used with external (and multiple) cephs more simply. Change-Id: I0c5d516c7a9a10bc8435e2423eb1dc05460d2c92 Signed-off-by: Pete Birley --- ceph-client/templates/configmap-bin.yaml | 10 +- .../templates/configmap-etc-client.yaml | 2 +- ceph-client/templates/deployment-mds.yaml | 6 +- ceph-client/templates/deployment-mgr.yaml | 6 +- ceph-client/templates/deployment-rgw.yaml | 8 +- ceph-client/templates/job-bootstrap.yaml | 6 +- ceph-client/templates/job-rbd-pool.yaml | 6 +- ceph-client/values.yaml | 54 ---- ceph-provisioners/Chart.yaml | 18 ++ ceph-provisioners/requirements.yaml | 18 ++ .../templates/bin/_bootstrap.sh.tpl | 20 ++ .../cephfs/_client-key-manager.sh.tpl | 0 .../bin/provisioner/cephfs/_start.sh.tpl | 0 .../rbd/_namespace-client-key-cleaner.sh.tpl | 0 .../rbd/_namespace-client-key-manager.sh.tpl | 0 .../bin/provisioner/rbd/_start.sh.tpl | 0 .../templates/configmap-bin-provisioner.yaml | 2 +- .../templates/configmap-bin.yaml | 43 ++++ .../templates/configmap-etc-client.yaml | 51 ++++ .../deployment-cephfs-provisioner.yaml | 6 +- .../templates/deployment-rbd-provisioner.yaml | 6 +- .../templates/job-bootstrap.yaml | 70 ++++++ .../templates/job-cephfs-client-key.yaml | 6 +- .../job-namespace-client-key-cleaner.yaml | 6 +- .../templates/job-namespace-client-key.yaml | 6 +- .../templates/storageclass-cephfs.yaml | 0 .../templates/storageclass-rbd.yaml | 0 ceph-provisioners/values.yaml | 234 ++++++++++++++++++ .../armada/multinode/armada-ceph.yaml | 74 ++++++ .../armada/multinode/armada-lma.yaml | 2 +- .../armada/multinode/armada-osh.yaml | 2 +- tools/deployment/developer/ceph/040-ceph.sh | 4 +- .../developer/ceph/045-ceph-ns-activate.sh | 4 +- tools/deployment/multinode/030-ceph.sh | 2 +- .../multinode/040-ceph-ns-activate.sh | 2 +- 35 files changed, 570 insertions(+), 104 deletions(-) create mode 100644 ceph-provisioners/Chart.yaml create mode 100644 ceph-provisioners/requirements.yaml create mode 100644 ceph-provisioners/templates/bin/_bootstrap.sh.tpl rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/cephfs/_start.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/bin/provisioner/rbd/_start.sh.tpl (100%) rename {ceph-client => ceph-provisioners}/templates/configmap-bin-provisioner.yaml (96%) create mode 100644 ceph-provisioners/templates/configmap-bin.yaml create mode 100644 ceph-provisioners/templates/configmap-etc-client.yaml rename {ceph-client => ceph-provisioners}/templates/deployment-cephfs-provisioner.yaml (97%) rename {ceph-client => ceph-provisioners}/templates/deployment-rbd-provisioner.yaml (97%) create mode 100644 ceph-provisioners/templates/job-bootstrap.yaml rename {ceph-client => ceph-provisioners}/templates/job-cephfs-client-key.yaml (97%) rename {ceph-client => ceph-provisioners}/templates/job-namespace-client-key-cleaner.yaml (95%) rename {ceph-client => ceph-provisioners}/templates/job-namespace-client-key.yaml (96%) rename {ceph-client => ceph-provisioners}/templates/storageclass-cephfs.yaml (100%) rename {ceph-client => ceph-provisioners}/templates/storageclass-rbd.yaml (100%) create mode 100644 ceph-provisioners/values.yaml diff --git a/ceph-client/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml index 5b8e5bc7..0edfe1f1 100644 --- a/ceph-client/templates/configmap-bin.yaml +++ b/ceph-client/templates/configmap-bin.yaml @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }} +{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }} {{- $envAll := . }} --- apiVersion: v1 @@ -48,14 +48,6 @@ data: mgr-check.sh: | {{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - provisioner-cephfs-start.sh: | -{{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - provisioner-cephfs-client-key-manager.sh: | -{{ tuple "bin/provisioner/cephfs/_client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - - provisioner-rbd-start.sh: | -{{ tuple "bin/provisioner/rbd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - rgw-start.sh: | {{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} rgw-init-keystone.sh: | diff --git a/ceph-client/templates/configmap-etc-client.yaml b/ceph-client/templates/configmap-etc-client.yaml index 88d05cf9..0e1610b9 100644 --- a/ceph-client/templates/configmap-etc-client.yaml +++ b/ceph-client/templates/configmap-etc-client.yaml @@ -51,5 +51,5 @@ data: {{- end }} {{- end }} {{- if .Values.manifests.configmap_etc }} -{{- list "ceph-etc" . | include "ceph.configmap.etc" }} +{{- list "ceph-client-etc" . | include "ceph.configmap.etc" }} {{- end }} diff --git a/ceph-client/templates/deployment-mds.yaml b/ceph-client/templates/deployment-mds.yaml index a680b282..2118048e 100644 --- a/ceph-client/templates/deployment-mds.yaml +++ b/ceph-client/templates/deployment-mds.yaml @@ -89,7 +89,7 @@ spec: mountPath: /tmp/mds-start.sh subPath: mds-start.sh readOnly: true - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -108,9 +108,9 @@ spec: mountPath: /run readOnly: false volumes: - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: ceph-client-bin configMap: diff --git a/ceph-client/templates/deployment-mgr.yaml b/ceph-client/templates/deployment-mgr.yaml index e36df725..d81f7fda 100644 --- a/ceph-client/templates/deployment-mgr.yaml +++ b/ceph-client/templates/deployment-mgr.yaml @@ -123,7 +123,7 @@ spec: readOnly: true - name: pod-etc-ceph mountPath: /etc/ceph - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -148,9 +148,9 @@ spec: configMap: name: ceph-client-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: pod-var-lib-ceph emptyDir: {} diff --git a/ceph-client/templates/deployment-rgw.yaml b/ceph-client/templates/deployment-rgw.yaml index 22c48af6..0f0d1db8 100644 --- a/ceph-client/templates/deployment-rgw.yaml +++ b/ceph-client/templates/deployment-rgw.yaml @@ -89,7 +89,7 @@ spec: mountPath: /tmp/rgw-init-keystone.sh subPath: rgw-init-keystone.sh readOnly: true - - name: ceph-etc + - name: ceph-client-etc mountPath: /tmp/ceph.conf subPath: ceph.conf readOnly: true @@ -126,7 +126,7 @@ spec: - name: pod-etc-ceph mountPath: /etc/ceph {{- if not .Values.conf.rgw_ks.enabled }} - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -152,9 +152,9 @@ spec: configMap: name: ceph-client-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: pod-var-lib-ceph emptyDir: {} diff --git a/ceph-client/templates/job-bootstrap.yaml b/ceph-client/templates/job-bootstrap.yaml index 6e558e2b..72a93597 100644 --- a/ceph-client/templates/job-bootstrap.yaml +++ b/ceph-client/templates/job-bootstrap.yaml @@ -47,7 +47,7 @@ spec: mountPath: /tmp/bootstrap.sh subPath: bootstrap.sh readOnly: true - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -60,9 +60,9 @@ spec: configMap: name: ceph-client-bin defaultMode: 0555 - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: ceph-client-admin-keyring secret: diff --git a/ceph-client/templates/job-rbd-pool.yaml b/ceph-client/templates/job-rbd-pool.yaml index c978519b..96132125 100644 --- a/ceph-client/templates/job-rbd-pool.yaml +++ b/ceph-client/templates/job-rbd-pool.yaml @@ -57,7 +57,7 @@ spec: mountPath: /tmp/pool-calc.py subPath: pool-calc.py readOnly: true - - name: ceph-etc + - name: ceph-client-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true @@ -72,9 +72,9 @@ spec: mountPath: /run readOnly: false volumes: - - name: ceph-etc + - name: ceph-client-etc configMap: - name: ceph-etc + name: ceph-client-etc defaultMode: 0444 - name: ceph-client-bin configMap: diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 144388e9..3f6a0d62 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -19,9 +19,6 @@ deployment: ceph: true - client_secrets: false - rbd_provisioner: true - cephfs_provisioner: true rgw_keystone_user_and_endpoints: false release_group: null @@ -30,12 +27,10 @@ images: pull_policy: IfNotPresent tags: ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' - ceph_cephfs_provisioner: 'quay.io/external_storage/cephfs-provisioner:v0.1.1' ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3' ceph_mds: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' ceph_mgr: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' ceph_rbd_pool: 'docker.io/port/ceph-config-helper:v1.10.3' - ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1' ceph_rgw: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1' ks_endpoints: 'docker.io/openstackhelm/heat:newton' @@ -68,8 +63,6 @@ labels: pod: dns_policy: "ClusterFirstWithHostNet" replicas: - cephfs_provisioner: 2 - rbd_provisioner: 2 mds: 2 mgr: 2 rgw: 2 @@ -95,20 +88,6 @@ pod: limits: memory: "50Mi" cpu: "500m" - rbd_provisioner: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" - cephfs_provisioner: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" mgr: requests: memory: "5Mi" @@ -433,32 +412,6 @@ ceph_mgr_enabled_modules: # num_rep: "3" # min_size: "2" -# if you change provision_storage_class to false -# it is presumed you manage your own storage -# class definition externally -storageclass: - rbd: - provision_storage_class: true - provisioner: ceph.com/rbd - name: general - monitors: null - pool: rbd - admin_id: admin - admin_secret_name: pvc-ceph-conf-combined-storageclass - admin_secret_namespace: ceph - user_id: admin - user_secret_name: pvc-ceph-client-key - image_format: "2" - image_features: layering - cephfs: - provision_storage_class: true - provisioner: ceph.com/cephfs - name: cephfs - admin_id: admin - user_secret_name: pvc-ceph-cephfs-client-key - admin_secret_name: pvc-ceph-conf-combined-storageclass - admin_secret_namespace: ceph - endpoints: cluster_domain_suffix: cluster.local local_image_registry: @@ -563,11 +516,8 @@ monitoring: manifests: configmap_bin: true configmap_bin_ks: true - configmap_bin_common: true configmap_etc: true deployment_mds: true - deployment_rbd_provisioner: true - deployment_cephfs_provisioner: true deployment_rgw: true deployment_mgr: true ingress_rgw: true @@ -577,8 +527,6 @@ manifests: job_ks_endpoints: true job_ks_service: true job_ks_user: true - job_namespace_client_key_cleaner: true - job_namespace_client_key: true job_rbd_pool: true secret_ingress_tls: true secret_keystone_rgw: true @@ -586,5 +534,3 @@ manifests: service_ingress_rgw: true service_mgr: true service_rgw: true - storageclass_cephfs: true - storageclass_rbd: true diff --git a/ceph-provisioners/Chart.yaml b/ceph-provisioners/Chart.yaml new file mode 100644 index 00000000..a0d25ad4 --- /dev/null +++ b/ceph-provisioners/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Ceph Client +name: ceph-provisioners +version: 0.1.0 diff --git a/ceph-provisioners/requirements.yaml b/ceph-provisioners/requirements.yaml new file mode 100644 index 00000000..53782e69 --- /dev/null +++ b/ceph-provisioners/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/ceph-provisioners/templates/bin/_bootstrap.sh.tpl b/ceph-provisioners/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 00000000..533c0a5a --- /dev/null +++ b/ceph-provisioners/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,20 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/ceph-client/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl b/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl diff --git a/ceph-client/templates/bin/provisioner/cephfs/_start.sh.tpl b/ceph-provisioners/templates/bin/provisioner/cephfs/_start.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/cephfs/_start.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/cephfs/_start.sh.tpl diff --git a/ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl b/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-cleaner.sh.tpl diff --git a/ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl b/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl diff --git a/ceph-client/templates/bin/provisioner/rbd/_start.sh.tpl b/ceph-provisioners/templates/bin/provisioner/rbd/_start.sh.tpl similarity index 100% rename from ceph-client/templates/bin/provisioner/rbd/_start.sh.tpl rename to ceph-provisioners/templates/bin/provisioner/rbd/_start.sh.tpl diff --git a/ceph-client/templates/configmap-bin-provisioner.yaml b/ceph-provisioners/templates/configmap-bin-provisioner.yaml similarity index 96% rename from ceph-client/templates/configmap-bin-provisioner.yaml rename to ceph-provisioners/templates/configmap-bin-provisioner.yaml index 8a52855e..d34870fb 100644 --- a/ceph-client/templates/configmap-bin-provisioner.yaml +++ b/ceph-provisioners/templates/configmap-bin-provisioner.yaml @@ -20,7 +20,7 @@ limitations under the License. apiVersion: v1 kind: ConfigMap metadata: - name: ceph-client-bin-provisioner + name: ceph-provisioners-bin-clients data: provisioner-rbd-namespace-client-key-manager.sh: | {{ tuple "bin/provisioner/rbd/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph-provisioners/templates/configmap-bin.yaml b/ceph-provisioners/templates/configmap-bin.yaml new file mode 100644 index 00000000..ed735d79 --- /dev/null +++ b/ceph-provisioners/templates/configmap-bin.yaml @@ -0,0 +1,43 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_bin_common .Values.deployment.ceph }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-provisioners-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + +{{- if .Values.bootstrap.enabled }} + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + + provisioner-cephfs-start.sh: | +{{ tuple "bin/provisioner/cephfs/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + provisioner-cephfs-client-key-manager.sh: | +{{ tuple "bin/provisioner/cephfs/_client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + provisioner-rbd-start.sh: | +{{ tuple "bin/provisioner/rbd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + +{{- end }} diff --git a/ceph-provisioners/templates/configmap-etc-client.yaml b/ceph-provisioners/templates/configmap-etc-client.yaml new file mode 100644 index 00000000..35929905 --- /dev/null +++ b/ceph-provisioners/templates/configmap-etc-client.yaml @@ -0,0 +1,51 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} +{{- end -}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} + +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_etc }} +{{- list "ceph-etc" . | include "ceph.configmap.etc" }} +{{- end }} diff --git a/ceph-client/templates/deployment-cephfs-provisioner.yaml b/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml similarity index 97% rename from ceph-client/templates/deployment-cephfs-provisioner.yaml rename to ceph-provisioners/templates/deployment-cephfs-provisioner.yaml index a3cbe1af..6548a317 100644 --- a/ceph-client/templates/deployment-cephfs-provisioner.yaml +++ b/ceph-provisioners/templates/deployment-cephfs-provisioner.yaml @@ -166,13 +166,13 @@ spec: command: - /tmp/provisioner-cephfs-start.sh volumeMounts: - - name: ceph-client-bin + - name: ceph-provisioners-bin mountPath: /tmp/provisioner-cephfs-start.sh subPath: provisioner-cephfs-start.sh readOnly: true volumes: - - name: ceph-client-bin + - name: ceph-provisioners-bin configMap: - name: ceph-client-bin + name: ceph-provisioners-bin defaultMode: 0555 {{- end }} diff --git a/ceph-client/templates/deployment-rbd-provisioner.yaml b/ceph-provisioners/templates/deployment-rbd-provisioner.yaml similarity index 97% rename from ceph-client/templates/deployment-rbd-provisioner.yaml rename to ceph-provisioners/templates/deployment-rbd-provisioner.yaml index fc29d592..ad21d36a 100644 --- a/ceph-client/templates/deployment-rbd-provisioner.yaml +++ b/ceph-provisioners/templates/deployment-rbd-provisioner.yaml @@ -156,13 +156,13 @@ spec: command: - /tmp/provisioner-rbd-start.sh volumeMounts: - - name: ceph-client-bin + - name: ceph-provisioners-bin mountPath: /tmp/provisioner-rbd-start.sh subPath: provisioner-rbd-start.sh readOnly: true volumes: - - name: ceph-client-bin + - name: ceph-provisioners-bin configMap: - name: ceph-client-bin + name: ceph-provisioners-bin defaultMode: 0555 {{- end }} diff --git a/ceph-provisioners/templates/job-bootstrap.yaml b/ceph-provisioners/templates/job-bootstrap.yaml new file mode 100644 index 00000000..00c47373 --- /dev/null +++ b/ceph-provisioners/templates/job-bootstrap.yaml @@ -0,0 +1,70 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-client-bootstrap"}} +{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-client-bootstrap +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-client-bootstrap +{{ tuple $envAll "ceph_bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/bootstrap.sh + volumeMounts: + - name: ceph-provisioners-bin + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + volumes: + - name: ceph-provisioners-bin + configMap: + name: ceph-provisioners-bin + defaultMode: 0555 + - name: ceph-etc + configMap: + name: ceph-etc + defaultMode: 0444 + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} diff --git a/ceph-client/templates/job-cephfs-client-key.yaml b/ceph-provisioners/templates/job-cephfs-client-key.yaml similarity index 97% rename from ceph-client/templates/job-cephfs-client-key.yaml rename to ceph-provisioners/templates/job-cephfs-client-key.yaml index d1df477c..2b0bee5a 100644 --- a/ceph-client/templates/job-cephfs-client-key.yaml +++ b/ceph-provisioners/templates/job-cephfs-client-key.yaml @@ -112,13 +112,13 @@ spec: command: - /tmp/provisioner-cephfs-client-key-manager.sh volumeMounts: - - name: ceph-client-bin + - name: ceph-provisioners-bin mountPath: /tmp/provisioner-cephfs-client-key-manager.sh subPath: provisioner-cephfs-client-key-manager.sh readOnly: true volumes: - - name: ceph-client-bin + - name: ceph-provisioners-bin configMap: - name: ceph-client-bin + name: ceph-provisioners-bin defaultMode: 0555 {{- end }} diff --git a/ceph-client/templates/job-namespace-client-key-cleaner.yaml b/ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml similarity index 95% rename from ceph-client/templates/job-namespace-client-key-cleaner.yaml rename to ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml index 56c74981..f0691fc5 100644 --- a/ceph-client/templates/job-namespace-client-key-cleaner.yaml +++ b/ceph-provisioners/templates/job-namespace-client-key-cleaner.yaml @@ -81,13 +81,13 @@ spec: command: - /tmp/provisioner-rbd-namespace-client-key-cleaner.sh volumeMounts: - - name: ceph-client-bin-provisioner + - name: ceph-provisioners-bin-clients mountPath: /tmp/provisioner-rbd-namespace-client-key-cleaner.sh subPath: provisioner-rbd-namespace-client-key-cleaner.sh readOnly: true volumes: - - name: ceph-client-bin-provisioner + - name: ceph-provisioners-bin-clients configMap: - name: ceph-client-bin-provisioner + name: ceph-provisioners-bin-clients defaultMode: 0555 {{- end }} diff --git a/ceph-client/templates/job-namespace-client-key.yaml b/ceph-provisioners/templates/job-namespace-client-key.yaml similarity index 96% rename from ceph-client/templates/job-namespace-client-key.yaml rename to ceph-provisioners/templates/job-namespace-client-key.yaml index cba3ebb4..75fd0687 100644 --- a/ceph-client/templates/job-namespace-client-key.yaml +++ b/ceph-provisioners/templates/job-namespace-client-key.yaml @@ -112,13 +112,13 @@ spec: command: - /tmp/provisioner-rbd-namespace-client-key-manager.sh volumeMounts: - - name: ceph-client-bin-provisioner + - name: ceph-provisioners-bin-clients mountPath: /tmp/provisioner-rbd-namespace-client-key-manager.sh subPath: provisioner-rbd-namespace-client-key-manager.sh readOnly: true volumes: - - name: ceph-client-bin-provisioner + - name: ceph-provisioners-bin-clients configMap: - name: ceph-client-bin-provisioner + name: ceph-provisioners-bin-clients defaultMode: 0555 {{- end }} diff --git a/ceph-client/templates/storageclass-cephfs.yaml b/ceph-provisioners/templates/storageclass-cephfs.yaml similarity index 100% rename from ceph-client/templates/storageclass-cephfs.yaml rename to ceph-provisioners/templates/storageclass-cephfs.yaml diff --git a/ceph-client/templates/storageclass-rbd.yaml b/ceph-provisioners/templates/storageclass-rbd.yaml similarity index 100% rename from ceph-client/templates/storageclass-rbd.yaml rename to ceph-provisioners/templates/storageclass-rbd.yaml diff --git a/ceph-provisioners/values.yaml b/ceph-provisioners/values.yaml new file mode 100644 index 00000000..f72b18a3 --- /dev/null +++ b/ceph-provisioners/values.yaml @@ -0,0 +1,234 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for ceph-client. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +deployment: + ceph: true + client_secrets: false + rbd_provisioner: true + cephfs_provisioner: true + +release_group: null + +images: + pull_policy: IfNotPresent + tags: + ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04' + ceph_cephfs_provisioner: 'quay.io/external_storage/cephfs-provisioner:v0.1.1' + ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3' + ceph_rbd_provisioner: 'quay.io/external_storage/rbd-provisioner:v0.1.1' + dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1' + image_repo_sync: docker.io/docker:17.07.0 + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + provisioner: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + dns_policy: "ClusterFirstWithHostNet" + replicas: + cephfs_provisioner: 2 + rbd_provisioner: 2 + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + resources: + enabled: false + rbd_provisioner: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + cephfs_provisioner: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + jobs: + bootstrap: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +secrets: + keyrings: + admin: ceph-client-admin-keyring + +network: + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + port: + mon: 6789 + mgr: 7000 + +conf: + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + osd: + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_max_object_name_len: 256 + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - ceph-client-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + bootstrap: + jobs: null + services: + - endpoint: internal + service: ceph_mon + cephfs_client_key_generator: + jobs: null + cephfs_provisioner: + jobs: + - ceph-rbd-pool + services: + - endpoint: internal + service: ceph_mon + namespace_client_key_cleaner: + jobs: null + namespace_client_key_generator: + jobs: null + rbd_provisioner: + jobs: + - ceph-rbd-pool + services: + - endpoint: internal + service: ceph_mon + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +bootstrap: + enabled: false + script: | + ceph -s + function ensure_pool () { + ceph osd pool stats $1 || ceph osd pool create $1 $2 + local test_luminous=$(ceph tell osd.* version | egrep -c "12.2|luminous") + if [[ ${test_luminous} -gt 0 ]]; then + ceph osd pool application enable $1 $3 + fi + } + #ensure_pool volumes 8 cinder + +# if you change provision_storage_class to false +# it is presumed you manage your own storage +# class definition externally +storageclass: + rbd: + provision_storage_class: true + provisioner: ceph.com/rbd + name: general + monitors: null + pool: rbd + admin_id: admin + admin_secret_name: pvc-ceph-conf-combined-storageclass + admin_secret_namespace: ceph + user_id: admin + user_secret_name: pvc-ceph-client-key + image_format: "2" + image_features: layering + cephfs: + provision_storage_class: true + provisioner: ceph.com/cephfs + name: cephfs + admin_id: admin + user_secret_name: pvc-ceph-cephfs-client-key + admin_secret_name: pvc-ceph-conf-combined-storageclass + admin_secret_namespace: ceph + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + ceph_mon: + namespace: null + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + +manifests: + configmap_bin: true + configmap_bin_common: true + configmap_etc: true + deployment_rbd_provisioner: true + deployment_cephfs_provisioner: true + job_bootstrap: false + job_cephfs_client_key: true + job_image_repo_sync: true + job_namespace_client_key_cleaner: true + job_namespace_client_key: true + storageclass_cephfs: true + storageclass_rbd: true diff --git a/tools/deployment/armada/multinode/armada-ceph.yaml b/tools/deployment/armada/multinode/armada-ceph.yaml index 7d986945..d3d7d17c 100644 --- a/tools/deployment/armada/multinode/armada-ceph.yaml +++ b/tools/deployment/armada/multinode/armada-ceph.yaml @@ -273,6 +273,79 @@ data: dependencies: - helm-toolkit --- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: ceph-provisioners +data: + chart_name: ceph-provisioners + release: ceph-provisioners + namespace: ceph + wait: + timeout: 1800 + labels: + release_group: osh-ceph-provisioners + install: + no_hooks: False + upgrade: + no_hooks: False + pre: + delete: + - type: job + labels: + release_group: osh-ceph-provisioners + values: + endpoints: + identity: + namespace: openstack + auth: + swift: + password: ${SWIFT_USER_PASSWORD} + object_store: + namespace: ceph + ceph_mon: + namespace: ceph + network: + public: ${CEPH_NETWORK} + cluster: ${CEPH_NETWORK} + deployment: + storage_secrets: true + ceph: true + rbd_provisioner: true + cephfs_provisioner: true + client_secrets: false + rgw_keystone_user_and_endpoints: false + bootstrap: + enabled: true + conf: + ceph: + global: + fsid: ${CEPH_FS_ID} + rgw_ks: + enabled: true + pool: + crush: + tunables: ${CRUSH_TUNABLES} + target: + # NOTE(portdirect): 5 nodes, with one osd per node + osd: 5 + pg_per_osd: 100 + storage: + osd: + - data: + type: directory + location: /var/lib/openstack-helm/ceph/osd/osd-one + journal: + type: directory + location: /var/lib/openstack-helm/ceph/osd/journal-one + source: + type: local + location: ${OSH_PATH} + subpath: ceph-provisioners + reference: master + dependencies: + - helm-toolkit +--- schema: armada/ChartGroup/v1 metadata: schema: metadata/Document/v1 @@ -285,6 +358,7 @@ data: - ceph-mon - ceph-osd - ceph-client + - ceph-provisioners --- schema: armada/Manifest/v1 metadata: diff --git a/tools/deployment/armada/multinode/armada-lma.yaml b/tools/deployment/armada/multinode/armada-lma.yaml index b319bf20..a40ae044 100644 --- a/tools/deployment/armada/multinode/armada-lma.yaml +++ b/tools/deployment/armada/multinode/armada-lma.yaml @@ -103,7 +103,7 @@ data: source: type: local location: ${OSH_PATH} - subpath: ceph-client + subpath: ceph-provisioners reference: master dependencies: - helm-toolkit diff --git a/tools/deployment/armada/multinode/armada-osh.yaml b/tools/deployment/armada/multinode/armada-osh.yaml index 1f11efc5..8c5b560e 100644 --- a/tools/deployment/armada/multinode/armada-osh.yaml +++ b/tools/deployment/armada/multinode/armada-osh.yaml @@ -107,7 +107,7 @@ data: source: type: local location: ${OSH_PATH} - subpath: ceph-client + subpath: ceph-provisioners reference: master dependencies: - helm-toolkit diff --git a/tools/deployment/developer/ceph/040-ceph.sh b/tools/deployment/developer/ceph/040-ceph.sh index cccc7ec9..f389d06c 100755 --- a/tools/deployment/developer/ceph/040-ceph.sh +++ b/tools/deployment/developer/ceph/040-ceph.sh @@ -17,7 +17,7 @@ set -xe #NOTE: Lint and package chart -for CHART in ceph-mon ceph-osd ceph-client; do +for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do make "${CHART}" done @@ -163,7 +163,7 @@ pod: rgw: 1 EOF -for CHART in ceph-mon ceph-osd ceph-client; do +for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do helm upgrade --install ${CHART} ./${CHART} \ --namespace=ceph \ --values=/tmp/ceph.yaml \ diff --git a/tools/deployment/developer/ceph/045-ceph-ns-activate.sh b/tools/deployment/developer/ceph/045-ceph-ns-activate.sh index 9b96d380..edd5b69d 100755 --- a/tools/deployment/developer/ceph/045-ceph-ns-activate.sh +++ b/tools/deployment/developer/ceph/045-ceph-ns-activate.sh @@ -17,7 +17,7 @@ set -xe #NOTE: Lint and package chart -make ceph-client +make ceph-provisioners #NOTE: Deploy command : ${OSH_EXTRA_HELM_ARGS:=""} @@ -45,7 +45,7 @@ conf: rgw_ks: enabled: true EOF -helm upgrade --install ceph-openstack-config ./ceph-client \ +helm upgrade --install ceph-openstack-config ./ceph-provisioners \ --namespace=openstack \ --values=/tmp/ceph-openstack-config.yaml \ ${OSH_EXTRA_HELM_ARGS} \ diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh index 70573162..043c7c79 100755 --- a/tools/deployment/multinode/030-ceph.sh +++ b/tools/deployment/multinode/030-ceph.sh @@ -76,7 +76,7 @@ conf: location: /var/lib/openstack-helm/ceph/osd/journal-one EOF -for CHART in ceph-mon ceph-osd ceph-client; do +for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do helm upgrade --install ${CHART} ./${CHART} \ --namespace=ceph \ --values=/tmp/ceph.yaml \ diff --git a/tools/deployment/multinode/040-ceph-ns-activate.sh b/tools/deployment/multinode/040-ceph-ns-activate.sh index c4fb8ff5..fc4fb87f 100755 --- a/tools/deployment/multinode/040-ceph-ns-activate.sh +++ b/tools/deployment/multinode/040-ceph-ns-activate.sh @@ -43,7 +43,7 @@ conf: rgw_ks: enabled: true EOF -helm upgrade --install ceph-openstack-config ./ceph-client \ +helm upgrade --install ceph-openstack-config ./ceph-provisioners \ --namespace=openstack \ --values=/tmp/ceph-openstack-config.yaml \ ${OSH_EXTRA_HELM_ARGS} \