From f9df30acbd4f1a58d9e09bdbc964ebb835cb4ebc Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Mon, 25 Jun 2018 09:21:53 -0500 Subject: [PATCH] Ceph: use ingress controller for RGW This PS updates the ceph chart to use the ingress controller for the radosgw. Change-Id: Ie0ff4d9dd91d8228170e0ed4f2793599132077a7 Signed-off-by: Pete Birley --- .../templates/configmap-etc-client.yaml | 4 ++ ceph-client/templates/deployment-rgw.yaml | 10 ++--- ceph-client/templates/ingress-rgw.yaml | 20 ++++++++++ .../templates/service-ingress-rgw.yaml | 20 ++++++++++ ceph-client/templates/service-rgw.yaml | 14 ++++++- ceph-client/values.yaml | 20 +++++++++- glance/values.yaml | 5 +++ .../armada/multinode/armada-ceph.yaml | 37 +++++++++++++++++++ .../developer/common/030-ingress.sh | 22 +++++++---- tools/deployment/multinode/020-ingress.sh | 29 +++++++++------ 10 files changed, 154 insertions(+), 27 deletions(-) create mode 100644 ceph-client/templates/ingress-rgw.yaml create mode 100644 ceph-client/templates/service-ingress-rgw.yaml diff --git a/ceph-client/templates/configmap-etc-client.yaml b/ceph-client/templates/configmap-etc-client.yaml index 35929905..88d05cf9 100644 --- a/ceph-client/templates/configmap-etc-client.yaml +++ b/ceph-client/templates/configmap-etc-client.yaml @@ -34,6 +34,10 @@ limitations under the License. {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} {{- end -}} +{{- if empty .Values.conf.rgw_ks.config.rgw_swift_url -}} +{{- $_ := tuple "object_store" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.object_store.path.default | set .Values.conf.rgw_ks.config "rgw_swift_url" -}} +{{- end -}} + --- apiVersion: v1 kind: ConfigMap diff --git a/ceph-client/templates/deployment-rgw.yaml b/ceph-client/templates/deployment-rgw.yaml index a355b7f4..22c48af6 100644 --- a/ceph-client/templates/deployment-rgw.yaml +++ b/ceph-client/templates/deployment-rgw.yaml @@ -79,7 +79,7 @@ spec: - name: KEYSTONE_URL value: {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.identity.path.default | quote }} - name: RGW_CIVETWEB_PORT - value: "{{ .Values.network.port.rgw }}" + value: "{{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" command: - /tmp/rgw-init-keystone.sh volumeMounts: @@ -102,21 +102,21 @@ spec: - name: CLUSTER value: "ceph" - name: RGW_CIVETWEB_PORT - value: "{{ .Values.network.port.rgw }}" + value: "{{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" command: - /tmp/rgw-start.sh ports: - - containerPort: {{ .Values.network.port.rgw }} + - containerPort: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} livenessProbe: httpGet: path: / - port: {{ .Values.network.port.rgw }} + port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} initialDelaySeconds: 120 timeoutSeconds: 5 readinessProbe: httpGet: path: / - port: {{ .Values.network.port.rgw }} + port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} timeoutSeconds: 5 volumeMounts: - name: ceph-client-bin diff --git a/ceph-client/templates/ingress-rgw.yaml b/ceph-client/templates/ingress-rgw.yaml new file mode 100644 index 00000000..aa6ff278 --- /dev/null +++ b/ceph-client/templates/ingress-rgw.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw ) ) }} +{{- $ingressOpts := dict "envAll" . "backendServiceType" "object_store" "backendPort" "ceph-rgw" -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/ceph-client/templates/service-ingress-rgw.yaml b/ceph-client/templates/service-ingress-rgw.yaml new file mode 100644 index 00000000..aec67016 --- /dev/null +++ b/ceph-client/templates/service-ingress-rgw.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw ) ) }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "object_store" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/ceph-client/templates/service-rgw.yaml b/ceph-client/templates/service-rgw.yaml index 30810909..f986a0b1 100644 --- a/ceph-client/templates/service-rgw.yaml +++ b/ceph-client/templates/service-rgw.yaml @@ -23,9 +23,19 @@ metadata: name: ceph-rgw spec: ports: - - port: {{ .Values.network.port.rgw }} + - name: ceph-rgw + port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} protocol: TCP - targetPort: {{ .Values.network.port.rgw }} + targetPort: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.api.node_port.enabled }} + nodePort: {{ .Values.network.api.node_port.port }} + {{ end }} selector: {{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.api.node_port.enabled }} + type: NodePort + {{ if .Values.network.api.external_policy_local }} + externalTrafficPolicy: Local + {{ end }} + {{ end }} {{- end }} diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 7ca61b80..fdc54230 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -162,11 +162,24 @@ secrets: user_rgw: ceph-keystone-user-rgw network: + api: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.org/proxy-max-temp-file-size: "0" + external_policy_local: false + node_port: + enabled: false + port: 30004 public: 192.168.0.0/16 cluster: 192.168.0.0/16 port: mon: 6789 - rgw: 8088 mgr: 7000 conf: @@ -280,6 +293,7 @@ conf: rgw_keystone_make_new_tenants: true rgw_s3_auth_use_keystone: true rgw_swift_account_in_url: true + rgw_swift_url: null ceph: global: # auth @@ -488,6 +502,7 @@ endpoints: namespace: null hosts: default: ceph-rgw + public: radosgw host_fqdn_override: default: null path: @@ -497,6 +512,7 @@ endpoints: port: api: default: 8088 + public: 80 ceph_mon: namespace: null hosts: @@ -538,6 +554,7 @@ manifests: deployment_cephfs_provisioner: true deployment_rgw: true deployment_mgr: true + ingress_rgw: true job_bootstrap: false job_cephfs_client_key: true job_image_repo_sync: true @@ -549,6 +566,7 @@ manifests: job_rbd_pool: true secret_keystone_rgw: true secret_keystone: true + service_ingress_rgw: true service_mgr: true service_rgw: true storageclass_cephfs: true diff --git a/glance/values.yaml b/glance/values.yaml index 70b46994..4ff296c7 100644 --- a/glance/values.yaml +++ b/glance/values.yaml @@ -239,6 +239,7 @@ conf: swift_store_container: glance swift_store_create_container_on_put: true swift_store_config_file: /etc/glance/swift-store.conf + swift_store_endpoint_type: internalURL paste_deploy: flavor: keystone database: @@ -588,6 +589,7 @@ endpoints: tmpurlkey: supersecret hosts: default: ceph-rgw + public: radosgw host_fqdn_override: default: null path: @@ -597,6 +599,7 @@ endpoints: port: api: default: 8088 + public: 80 ceph_object_store: name: radosgw namespace: ceph @@ -607,6 +610,7 @@ endpoints: tmpurlkey: supersecret hosts: default: ceph-rgw + public: radosgw host_fqdn_override: default: null path: @@ -616,6 +620,7 @@ endpoints: port: api: default: 8088 + public: 80 pod: user: diff --git a/tools/deployment/armada/multinode/armada-ceph.yaml b/tools/deployment/armada/multinode/armada-ceph.yaml index 5c62b60c..ec821e4b 100644 --- a/tools/deployment/armada/multinode/armada-ceph.yaml +++ b/tools/deployment/armada/multinode/armada-ceph.yaml @@ -16,6 +16,42 @@ data: dependencies: [] --- schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: ceph-ingress-controller +data: + chart_name: ceph-ingress-controller + release: ceph-ingress-controller + namespace: ceph + wait: + timeout: 1800 + labels: + release_group: osh-ceph-ingress-controller + install: + no_hooks: False + upgrade: + no_hooks: False + values: + labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + error_server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + pod: + replicas: + error_page: 2 + ingress: 2 + source: + type: local + location: ${OSH_PATH} + subpath: ingress + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ceph-mon @@ -270,6 +306,7 @@ data: description: "Ceph Storage" sequenced: True chart_group: + - ceph-ingress-controller - ceph-mon - ceph-osd - ceph-client diff --git a/tools/deployment/developer/common/030-ingress.sh b/tools/deployment/developer/common/030-ingress.sh index 3a3d9be8..3c854314 100755 --- a/tools/deployment/developer/common/030-ingress.sh +++ b/tools/deployment/developer/common/030-ingress.sh @@ -34,16 +34,22 @@ helm upgrade --install ingress-kube-system ./ingress \ ${OSH_EXTRA_HELM_ARGS} \ ${OSH_EXTRA_HELM_ARGS_INGRESS_KUBE_SYSTEM} -#NOTE: Deploy namespace ingress -helm upgrade --install ingress-openstack ./ingress \ - --namespace=openstack \ - ${OSH_EXTRA_HELM_ARGS} \ - ${OSH_EXTRA_HELM_ARGS_INGRESS_OPENSTACK} - #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system -./tools/deployment/common/wait-for-pods.sh openstack #NOTE: Display info helm status ingress-kube-system -helm status ingress-openstack + +#NOTE: Deploy namespace ingress +for NAMESPACE in openstack ceph; do + helm upgrade --install ingress-${NAMESPACE} ./ingress \ + --namespace=${NAMESPACE} \ + ${OSH_EXTRA_HELM_ARGS} \ + ${OSH_EXTRA_HELM_ARGS_INGRESS_OPENSTACK} + + #NOTE: Wait for deploy + ./tools/deployment/common/wait-for-pods.sh ${NAMESPACE} + + #NOTE: Display info + helm status ingress-openstack +done diff --git a/tools/deployment/multinode/020-ingress.sh b/tools/deployment/multinode/020-ingress.sh index 35353dc8..4192b229 100755 --- a/tools/deployment/multinode/020-ingress.sh +++ b/tools/deployment/multinode/020-ingress.sh @@ -33,21 +33,28 @@ helm upgrade --install ingress-kube-system ./ingress \ ${OSH_EXTRA_HELM_ARGS} \ ${OSH_EXTRA_HELM_ARGS_INGRESS_KUBE_SYSTEM} -#NOTE: Deploy namespace ingress -tee /tmp/ingress-openstack.yaml << EOF +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Display info +helm status ingress-kube-system + +#NOTE: Deploy namespaced ingress controllers +for NAMESPACE in openstack ceph; do + #NOTE: Deploy namespace ingress + tee /tmp/ingress-${NAMESPACE}.yaml << EOF pod: replicas: ingress: 2 error_page: 2 EOF -helm upgrade --install ingress-openstack ./ingress \ - --namespace=openstack \ - --values=/tmp/ingress-openstack.yaml + helm upgrade --install ingress-${NAMESPACE} ./ingress \ + --namespace=${NAMESPACE} \ + --values=/tmp/ingress-${NAMESPACE}.yaml -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system -./tools/deployment/common/wait-for-pods.sh openstack + #NOTE: Wait for deploy + ./tools/deployment/common/wait-for-pods.sh ${NAMESPACE} -#NOTE: Display info -helm status ingress-kube-system -helm status ingress-openstack + #NOTE: Display info + helm status ingress-${NAMESPACE} +done