From bcf23c7399ded7dfdd61f7cfdb95facbef892f71 Mon Sep 17 00:00:00 2001 From: Alan Meadows Date: Tue, 3 Jan 2017 12:20:26 -0800 Subject: [PATCH 1/2] Endpoint lookup fixes for consistent keystone catalog endpoints This introduces changes across several charts, for consistent endpoint lookup routines. Today, only the internal endpoints are defined (and copied to admin and public). This can later be expanded to handle unique public and admin endpoints. This ensures the catalog is populated with consistent URLs for all services. --- common/templates/_endpoints.tpl | 87 ++++++++++++++++++- common/templates/_hosts.tpl | 19 ++++ common/values.yaml | 3 + glance/templates/_helpers.tpl | 5 -- .../templates/ceph.client.glance.keyring.yaml | 6 +- glance/templates/ceph.conf.yaml | 5 ++ glance/templates/glance-api.conf.yaml | 3 +- glance/templates/post.sh.yaml | 8 +- glance/templates/post.yaml | 3 + glance/values.yaml | 35 +++++++- horizon/values.yaml | 2 +- keystone/templates/_helpers.tpl | 3 - keystone/templates/bin/_db-sync.sh.tpl | 7 +- keystone/templates/etc/_keystone.conf.tpl | 6 +- keystone/values.yaml | 17 +++- 15 files changed, 183 insertions(+), 26 deletions(-) delete mode 100644 glance/templates/_helpers.tpl delete mode 100644 keystone/templates/_helpers.tpl diff --git a/common/templates/_endpoints.tpl b/common/templates/_endpoints.tpl index 0170b4eb..68914d6d 100644 --- a/common/templates/_endpoints.tpl +++ b/common/templates/_endpoints.tpl @@ -1,9 +1,94 @@ #----------------------------------------- # endpoints #----------------------------------------- + +# this should be a generic function leveraging a tuple +# for input, e.g. { endpoint keystone internal . } +# however, constructing this appears to be a +# herculean effort in gotpl + {{- define "endpoint_keystone_internal" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} {{- with .Values.endpoints.keystone -}} - {{.scheme}}://{{.hosts.internal | default .hosts.default}}:{{.port.public}}{{.path}} + {{.scheme}}://{{.hosts.internal | default .hosts.default}}.{{ $fqdn }}:{{.port.public}}{{.path}} {{- end -}} {{- end -}} +{{- define "endpoint_keystone_admin" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} +{{- with .Values.endpoints.keystone -}} + {{.scheme}}://{{.hosts.internal | default .hosts.default}}.{{ $fqdn }}:{{.port.admin}}{{.path}} +{{- end -}} +{{- end -}} + +{{- define "endpoint_nova_api_internal" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} +{{- with .Values.endpoints.nova -}} + {{.scheme}}://{{.hosts.internal | default .hosts.default}}.{{ $fqdn }}:{{.port.api}}{{.path}} +{{- end -}} +{{- end -}} + +{{- define "endpoint_nova_metadata_internal" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} +{{- with .Values.endpoints.nova -}} + {{.scheme}}://{{.hosts.internal | default .hosts.default}}.{{ $fqdn }}:{{.port.metadata}}{{.path}} +{{- end -}} +{{- end -}} + +{{- define "endpoint_nova_novncproxy_internal" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} +{{- with .Values.endpoints.nova -}} + {{.scheme}}://{{.hosts.internal | default .hosts.default}}.{{ $fqdn }}:{{.port.novncproxy}}{{.path}} +{{- end -}} +{{- end -}} + +{{- define "endpoint_glance_api_internal" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} +{{- with .Values.endpoints.glance -}} + {{.scheme}}://{{.hosts.internal | default .hosts.default}}.{{ $fqdn }}:{{.port.api}}{{.path}} +{{- end -}} +{{- end -}} + +{{- define "endpoint_glance_registry_internal" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} +{{- with .Values.endpoints.glance -}} + {{.scheme}}://{{.hosts.internal | default .hosts.default}}.{{ $fqdn }}:{{.port.registry}}{{.path}} +{{- end -}} +{{- end -}} + +{{- define "endpoint_neutron_api_internal" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} +{{- with .Values.endpoints.neutron -}} + {{.scheme}}://{{.hosts.internal | default .hosts.default}}.{{ $fqdn }}:{{.port.api}}{{.path}} +{{- end -}} +{{- end -}} + +#------------------------------- +# kolla helpers +#------------------------------- +{{ define "keystone_auth" }}{'auth_url':'{{ include "endpoint_keystone_internal" . }}', 'username':'{{ .Values.keystone.admin_user }}','password':'{{ .Values.keystone.admin_password }}','project_name':'{{ .Values.keystone.admin_project_name }}','domain_name':'default'}{{end}} + diff --git a/common/templates/_hosts.tpl b/common/templates/_hosts.tpl index 6655a887..941ece39 100644 --- a/common/templates/_hosts.tpl +++ b/common/templates/_hosts.tpl @@ -2,6 +2,14 @@ {{- define "region"}}cluster{{- end}} {{- define "tld"}}local{{- end}} +{{- define "fqdn" -}} +{{- $fqdn := .Release.Namespace -}} +{{- if .Values.endpoints.fqdn -}} +{{- $fqdn := .Values.endpoints.fqdn -}} +{{- end -}} +{{- $fqdn -}} +{{- end -}} + #----------------------------------------- # hosts #----------------------------------------- @@ -17,3 +25,14 @@ {{- define "keystone_api_endpoint_host_internal"}}keystone-api.{{.Release.Namespace}}.svc.{{ include "region" . }}.{{ include "tld" . }}{{- end}} {{- define "keystone_api_endpoint_host_public"}}keystone-api.{{ include "region" . }}.{{ include "tld" . }}{{- end}} {{- define "keystone_api_endpoint_host_admin_ext"}}keystone-api.{{ include "region" . }}.{{ include "tld" . }}{{- end}} + +# glance defaults +{{- define "glance_registry_host"}}glance-registry.{{ include "fqdn" . }}{{- end}} + +# nova defaults +{{- define "nova_metadata_host"}}nova-api.{{ include "fqdn" . }}{{- end}} + +# neutron defaults +{{- define "neutron_db_host"}}{{ include "mariadb_host" . }}{{- end}} +{{- define "neutron_rabbit_host"}}{{- include "rabbitmq_host" .}}{{- end}} + diff --git a/common/values.yaml b/common/values.yaml index da62e632..20eb7c7c 100644 --- a/common/values.yaml +++ b/common/values.yaml @@ -7,3 +7,6 @@ global: region: cluster tld: local +endpoints: + fqdn: null + diff --git a/glance/templates/_helpers.tpl b/glance/templates/_helpers.tpl deleted file mode 100644 index 932d1900..00000000 --- a/glance/templates/_helpers.tpl +++ /dev/null @@ -1,5 +0,0 @@ -{{- define "joinListWithColon" -}} -{{ range $k, $v := . }}{{ if $k }},{{ end }}{{ $v }}{{ end }} -{{- end -}} - -{{ define "keystone_auth" }}{'auth_url':'{{ .Values.keystone.auth_url }}', 'username':'{{ .Values.keystone.admin_user }}','password':'{{ .Values.keystone.admin_password }}','project_name':'{{ .Values.keystone.admin_project_name }}','domain_name':'default'}{{end}} diff --git a/glance/templates/ceph.client.glance.keyring.yaml b/glance/templates/ceph.client.glance.keyring.yaml index 27eeac7e..91532480 100644 --- a/glance/templates/ceph.client.glance.keyring.yaml +++ b/glance/templates/ceph.client.glance.keyring.yaml @@ -5,5 +5,9 @@ metadata: data: ceph.client.{{ .Values.ceph.glance_user }}.keyring: |+ [client.{{ .Values.ceph.glance_user }}] + {{- if .Values.ceph.glance_keyring }} key = {{ .Values.ceph.glance_keyring }} - + {{- else }} + key = {{- include "secrets/ceph-client-key" . -}} + {{- end }} + diff --git a/glance/templates/ceph.conf.yaml b/glance/templates/ceph.conf.yaml index 28982f32..3c3aed30 100644 --- a/glance/templates/ceph.conf.yaml +++ b/glance/templates/ceph.conf.yaml @@ -7,12 +7,17 @@ data: [global] rgw_thread_pool_size = 1024 rgw_num_rados_handles = 100 + {{- if .Values.ceph.monitors }} [mon] {{ range .Values.ceph.monitors }} [mon.{{ . }}] host = {{ . }} mon_addr = {{ . }} {{ end }} + {{- else }} + mon_host = ceph-mon.ceph + {{- end }} [client] rbd_cache_enabled = true rbd_cache_writethrough_until_flush = true + diff --git a/glance/templates/glance-api.conf.yaml b/glance/templates/glance-api.conf.yaml index b7e45d36..ee61d333 100644 --- a/glance/templates/glance-api.conf.yaml +++ b/glance/templates/glance-api.conf.yaml @@ -12,7 +12,7 @@ data: bind_port = {{ .Values.network.port.api }} workers = {{ .Values.misc.workers }} - registry_host = glance-registry + registry_host = {{ include "glance_registry_host" . }} # Enable Copy-on-Write show_image_direct_url = True @@ -45,3 +45,4 @@ data: rbd_store_user = {{ .Values.ceph.glance_user }} rbd_store_ceph_conf = /etc/ceph/ceph.conf rbd_store_chunk_size = 8 + diff --git a/glance/templates/post.sh.yaml b/glance/templates/post.sh.yaml index 2415cc8e..156b60d6 100644 --- a/glance/templates/post.sh.yaml +++ b/glance/templates/post.sh.yaml @@ -6,12 +6,13 @@ data: post.sh: |+ #!/bin/bash set -ex + export HOME=/tmp ansible localhost -vvv -m kolla_keystone_service -a "service_name=glance \ service_type=image \ description='Openstack Image' \ endpoint_region='{{ .Values.keystone.glance_region_name }}' \ - url='http://glance-api:{{ .Values.network.port.api }}' \ + url='{{ include "endpoint_glance_api_internal" . }}' \ interface=admin \ region_name='{{ .Values.keystone.admin_region_name }}' \ auth='{{ include "keystone_auth" . }}'" \ @@ -21,7 +22,7 @@ data: service_type=image \ description='Openstack Image' \ endpoint_region='{{ .Values.keystone.glance_region_name }}' \ - url='http://glance-api:{{ .Values.network.port.api }}' \ + url='{{ include "endpoint_glance_api_internal" . }}' \ interface=internal \ region_name='{{ .Values.keystone.admin_region_name }}' \ auth='{{ include "keystone_auth" . }}'" \ @@ -31,7 +32,7 @@ data: service_type=image \ description='Openstack Image' \ endpoint_region='{{ .Values.keystone.glance_region_name }}' \ - url='http://glance-api:{{ .Values.network.port.api }}' \ + url='{{ include "endpoint_glance_api_internal" . }}' \ interface=public \ region_name='{{ .Values.keystone.admin_region_name }}' \ auth='{{ include "keystone_auth" . }}'" \ @@ -44,3 +45,4 @@ data: region_name={{ .Values.keystone.admin_region_name }} \ auth='{{ include "keystone_auth" . }}'" \ -e "{ 'openstack_glance_auth': {{ include "keystone_auth" . }} }" + diff --git a/glance/templates/post.yaml b/glance/templates/post.yaml index 111f9bdc..4fe1c195 100644 --- a/glance/templates/post.yaml +++ b/glance/templates/post.yaml @@ -32,6 +32,8 @@ spec: } ]' spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} restartPolicy: OnFailure containers: - name: glance-post @@ -51,3 +53,4 @@ spec: - name: postsh configMap: name: glance-postsh + diff --git a/glance/values.yaml b/glance/values.yaml index 6e2b4cd3..a94a1bcc 100644 --- a/glance/values.yaml +++ b/glance/values.yaml @@ -33,7 +33,7 @@ network: port: api: 9292 registry: 9191 - ip_address: "{{ .IP }}" + ip_address: "0.0.0.0" database: address: mariadb @@ -47,9 +47,12 @@ database: ceph: enabled: true monitors: [] - glance_user: "glance" + glance_user: "admin" glance_pool: "images" - glance_keyring: "" + # a null value for the keyring will + # attempt to use the key from + # common/secrets/ceph-client-key + glance_keyring: null misc: workers: 8 @@ -97,4 +100,28 @@ dependencies: - mariadb - keystone-api - glance-api - - glance-registry \ No newline at end of file + - glance-registry + +# typically overriden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + glance: + hosts: + default: glance-api + type: image + path: null + scheme: 'http' + port: + api: 9292 + registry: 9191 + keystone: + hosts: + default: keystone-api + path: /v3 + type: identity + scheme: 'http' + port: + admin: 35357 + public: 5000 + diff --git a/horizon/values.yaml b/horizon/values.yaml index f1dd9a1a..6de7d8e1 100644 --- a/horizon/values.yaml +++ b/horizon/values.yaml @@ -38,6 +38,6 @@ endpoints: type: identity scheme: 'http' port: - admin: 35356 + admin: 35357 public: 5000 diff --git a/keystone/templates/_helpers.tpl b/keystone/templates/_helpers.tpl deleted file mode 100644 index d2f33bc8..00000000 --- a/keystone/templates/_helpers.tpl +++ /dev/null @@ -1,3 +0,0 @@ -{{- define "joinListWithColon" -}} -{{ range $k, $v := . }}{{ if $k }},{{ end }}{{ $v }}{{ end }} -{{- end -}} diff --git a/keystone/templates/bin/_db-sync.sh.tpl b/keystone/templates/bin/_db-sync.sh.tpl index b6679c73..89c4c5de 100644 --- a/keystone/templates/bin/_db-sync.sh.tpl +++ b/keystone/templates/bin/_db-sync.sh.tpl @@ -15,7 +15,8 @@ set -ex keystone-manage db_sync kolla_keystone_bootstrap {{ .Values.keystone.admin_user }} {{ .Values.keystone.admin_password }} \ {{ .Values.keystone.admin_project_name }} admin \ - {{ .Values.keystone.scheme }}://{{ include "keystone_api_endpoint_host_admin" . }}:{{ .Values.network.port.admin }}/{{ .Values.keystone.version }} \ - {{ .Values.keystone.scheme }}://{{ include "keystone_api_endpoint_host_internal" . }}:{{ .Values.network.port.public }}/{{ .Values.keystone.version }} \ - {{ .Values.keystone.scheme }}://{{ include "keystone_api_endpoint_host_public" . }}:{{ .Values.network.port.public }}/{{ .Values.keystone.version }} \ + {{ include "endpoint_keystone_admin" . }} \ + {{ include "endpoint_keystone_internal" . }} \ + {{ include "endpoint_keystone_internal" . }} \ {{ .Values.keystone.admin_region_name }} + diff --git a/keystone/templates/etc/_keystone.conf.tpl b/keystone/templates/etc/_keystone.conf.tpl index c62d5245..58603001 100644 --- a/keystone/templates/etc/_keystone.conf.tpl +++ b/keystone/templates/etc/_keystone.conf.tpl @@ -9,11 +9,11 @@ connection = mysql+pymysql://{{ .Values.database.keystone_user }}:{{ .Values.dat max_retries = -1 [memcache] -servers = {{ include "memcached_host" . }} +servers = {{ include "memcached_host" . }}:11211 [cache] backend = dogpile.cache.memcached -memcache_servers = {{ include "memcached_host" . }} +memcache_servers = {{ include "memcached_host" . }}:11211 config_prefix = cache.keystone -distributed_lock = True enabled = True + diff --git a/keystone/values.yaml b/keystone/values.yaml index 3b7b8e67..80801204 100644 --- a/keystone/values.yaml +++ b/keystone/values.yaml @@ -17,7 +17,7 @@ images: pull_policy: "IfNotPresent" keystone: - version: v2.0 + version: v3 scheme: http admin_region_name: RegionOne admin_user: admin @@ -67,3 +67,18 @@ dependencies: - mariadb-seed service: - mariadb + +# typically overriden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + keystone: + hosts: + default: keystone-api + path: /v3 + type: identity + scheme: 'http' + port: + admin: 35357 + public: 5000 + From b366dc252a9a87b2b6678add751631cd17e59270 Mon Sep 17 00:00:00 2001 From: Alan Meadows Date: Tue, 3 Jan 2017 12:23:25 -0800 Subject: [PATCH 2/2] Refactor ceph to allow monitors to leverage a stateful set. The IPs of ceph monitors cannot (or should not) change. This refactor allows the ceph monitors to act as statefulsets. It also persists their on disk data to nodeDirs, to allow cluster wide restarts (in parallel), where previously this would lose data. This is accompanied by some docker images changes that ensure that auth for the OSDs is restored/reinserted as auth data is somehow not persisted to disk at this time. --- .../{daemonset.yaml => daemonset-osd.yaml} | 8 +- ceph/templates/deployment-mds.yaml | 72 ++++ ceph/templates/deployment-moncheck.yaml | 64 ++++ ceph/templates/deployment-rgw.yaml | 77 +++++ ceph/templates/deployment.yaml | 310 ------------------ ceph/templates/service.yaml | 3 + ceph/templates/statefulset-mon.yaml | 105 ++++++ ceph/templates/storage.yaml | 3 +- ceph/values.yaml | 52 ++- 9 files changed, 378 insertions(+), 316 deletions(-) rename ceph/templates/{daemonset.yaml => daemonset-osd.yaml} (89%) create mode 100644 ceph/templates/deployment-mds.yaml create mode 100644 ceph/templates/deployment-moncheck.yaml create mode 100644 ceph/templates/deployment-rgw.yaml delete mode 100644 ceph/templates/deployment.yaml create mode 100644 ceph/templates/statefulset-mon.yaml diff --git a/ceph/templates/daemonset.yaml b/ceph/templates/daemonset-osd.yaml similarity index 89% rename from ceph/templates/daemonset.yaml rename to ceph/templates/daemonset-osd.yaml index 6c350cbe..1ad4b24f 100644 --- a/ceph/templates/daemonset.yaml +++ b/ceph/templates/daemonset-osd.yaml @@ -78,8 +78,8 @@ spec: timeoutSeconds: 5 resources: requests: - memory: "512Mi" - cpu: "1000m" + memory: {{ .Values.resources.osd.requests.memory | quote }} + cpu: {{ .Values.resources.osd.requests.cpu | quote }} limits: - memory: "1024Mi" - cpu: "2000m" + memory: {{ .Values.resources.osd.limits.memory | quote }} + cpu: {{ .Values.resources.osd.limits.cpu | quote }} diff --git a/ceph/templates/deployment-mds.yaml b/ceph/templates/deployment-mds.yaml new file mode 100644 index 00000000..9a4f5ead --- /dev/null +++ b/ceph/templates/deployment-mds.yaml @@ -0,0 +1,72 @@ +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: mds + name: ceph-mds +spec: + replicas: 1 + template: + metadata: + name: ceph-mds + labels: + app: ceph + daemon: mds + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + ports: + - containerPort: 6800 + env: + - name: CEPH_DAEMON + value: MDS + - name: CEPHFS_CREATE + value: "1" + - name: KV_TYPE + value: k8s + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + livenessProbe: + tcpSocket: + port: 6800 + initialDelaySeconds: 60 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: 6800 + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.mds.requests.memory | quote }} + cpu: {{ .Values.resources.mds.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mds.limits.memory | quote }} + cpu: {{ .Values.resources.mds.limits.cpu | quote }} diff --git a/ceph/templates/deployment-moncheck.yaml b/ceph/templates/deployment-moncheck.yaml new file mode 100644 index 00000000..aa829b09 --- /dev/null +++ b/ceph/templates/deployment-moncheck.yaml @@ -0,0 +1,64 @@ +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: moncheck + name: ceph-mon-check +spec: + replicas: {{ .Values.replicas.mon_check }} + template: + metadata: + name: ceph-mon + labels: + app: ceph + daemon: moncheck + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + imagePullPolicy: Always + ports: + - containerPort: 6789 + env: + - name: CEPH_DAEMON + value: MON_HEALTH + - name: KV_TYPE + value: k8s + - name: MON_IP_AUTO_DETECT + value: "1" + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + resources: + requests: + memory: {{ .Values.resources.mon_check.requests.memory | quote }} + cpu: {{ .Values.resources.mon_check.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon_check.limits.memory | quote }} + cpu: {{ .Values.resources.mon_check.limits.cpu | quote }} \ No newline at end of file diff --git a/ceph/templates/deployment-rgw.yaml b/ceph/templates/deployment-rgw.yaml new file mode 100644 index 00000000..a22c2ad3 --- /dev/null +++ b/ceph/templates/deployment-rgw.yaml @@ -0,0 +1,77 @@ +{{- if .Values.rgw.enabled }} +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + labels: + app: ceph + daemon: rgw + name: ceph-rgw +spec: + replicas: {{ .Values.replicas.rgw }} + template: + metadata: + name: ceph-rgw + labels: + app: ceph + daemon: rgw + spec: + hostNetwork: true + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + containers: + - name: ceph-rgw + image: {{ .Values.images.daemon }} + ports: + - containerPort: {{ .Values.network.port.rgw_target }} + env: + - name: RGW_CIVETWEB_PORT + value: "{{ .Values.network.port.rgw_target }}" + - name: CEPH_DAEMON + value: RGW + - name: KV_TYPE + value: k8s + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + livenessProbe: + httpGet: + path: / + port: {{ .Values.network.port.rgw_target }} + initialDelaySeconds: 120 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: {{ .Values.network.port.rgw_target }} + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.rgw.requests.memory | quote }} + cpu: {{ .Values.resources.rgwrequests.cpu | quote }} + limits: + memory: {{ .Values.resources.rgw.limits.memory | quote }} + cpu: {{ .Values.resources.rgw.limits.cpu | quote }} +{{- end }} diff --git a/ceph/templates/deployment.yaml b/ceph/templates/deployment.yaml deleted file mode 100644 index 26012c11..00000000 --- a/ceph/templates/deployment.yaml +++ /dev/null @@ -1,310 +0,0 @@ ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: mds - name: ceph-mds -spec: - replicas: 1 - template: - metadata: - name: ceph-mds - labels: - app: ceph - daemon: mds - spec: - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} - ports: - - containerPort: 6800 - env: - - name: CEPH_DAEMON - value: MDS - - name: CEPHFS_CREATE - value: "1" - - name: KV_TYPE - value: k8s - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - tcpSocket: - port: 6800 - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 6800 - timeoutSeconds: 5 - resources: - requests: - memory: "10Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: moncheck - name: ceph-mon-check -spec: - replicas: 1 - template: - metadata: - name: ceph-mon - labels: - app: ceph - daemon: moncheck - spec: - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} - imagePullPolicy: Always - ports: - - containerPort: 6789 - env: - - name: CEPH_DAEMON - value: MON_HEALTH - - name: KV_TYPE - value: k8s - - name: MON_IP_AUTO_DETECT - value: "1" - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - resources: - requests: - memory: "5Mi" - cpu: "250m" - limits: - memory: "50Mi" - cpu: "500m" ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: mon - name: ceph-mon -spec: - replicas: 3 - template: - metadata: - name: ceph-mon - labels: - app: ceph - daemon: mon - annotations: - # alanmeadows: this soft requirement allows single - # host deployments to spawn several ceph-mon - # containers - scheduler.alpha.kubernetes.io/affinity: > - { - "podAntiAffinity": { - "preferredDuringSchedulingIgnoredDuringExecution": [{ - "labelSelector": { - "matchExpressions": [{ - "key": "daemon", - "operator": "In", - "values":["mon"] - }] - }, - "topologyKey": "kubernetes.io/hostname", - "weight": 10 - }] - } - } - spec: - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-mon - image: {{ .Values.images.daemon }} -# imagePullPolicy: Always - lifecycle: - preStop: - exec: - # remove the mon on Pod stop. - command: - - "/remove-mon.sh" - ports: - - containerPort: 6789 - env: - - name: CEPH_DAEMON - value: MON - - name: KV_TYPE - value: k8s - - name: NETWORK_AUTO_DETECT - value: "1" - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - tcpSocket: - port: 6789 - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - tcpSocket: - port: 6789 - timeoutSeconds: 5 - resources: - requests: - memory: "50Mi" - cpu: "1000m" - limits: - memory: "100Mi" - cpu: "2000m" ---- -# rgw not required: using if statement for deployment -{{- if .Values.rgw.enabled }} -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - app: ceph - daemon: rgw - name: ceph-rgw -spec: - replicas: 3 - template: - metadata: - name: ceph-rgw - labels: - app: ceph - daemon: rgw - spec: - hostNetwork: true - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - containers: - - name: ceph-rgw - image: {{ .Values.images.daemon }} - ports: - - containerPort: {{ .Values.network.port.rgw_target }} - env: - - name: RGW_CIVETWEB_PORT - value: "{{ .Values.network.port.rgw_target }}" - - name: CEPH_DAEMON - value: RGW - - name: KV_TYPE - value: k8s - - name: CLUSTER - value: ceph - volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - livenessProbe: - httpGet: - path: / - port: {{ .Values.network.port.rgw_target }} - initialDelaySeconds: 120 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: / - port: {{ .Values.network.port.rgw_target }} - timeoutSeconds: 5 - resources: - requests: - memory: "500Mi" - cpu: ".5" - limits: - memory: "500Mi" - cpu: ".5" -{{ end }} -# end: rgw removed optionally diff --git a/ceph/templates/service.yaml b/ceph/templates/service.yaml index 49a9afb6..cdfd4f55 100644 --- a/ceph/templates/service.yaml +++ b/ceph/templates/service.yaml @@ -15,6 +15,8 @@ spec: app: ceph daemon: mon clusterIP: None + +{{- if .Values.rgw.enabled }} --- apiVersion: v1 kind: Service @@ -32,3 +34,4 @@ spec: app: ceph daemon: rgw type: LoadBalancer +{{- end }} diff --git a/ceph/templates/statefulset-mon.yaml b/ceph/templates/statefulset-mon.yaml new file mode 100644 index 00000000..5ef33cd8 --- /dev/null +++ b/ceph/templates/statefulset-mon.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + labels: + app: ceph + daemon: mon + name: ceph-mon +spec: + serviceName: {{ .Values.service.mon.name | quote }} + replicas: {{ .Values.replicas.mon }} + template: + metadata: + name: ceph-mon + labels: + app: ceph + daemon: mon + annotations: + # alanmeadows: this soft requirement allows single + # host deployments to spawn several ceph-mon + # containers + scheduler.alpha.kubernetes.io/affinity: > + { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [{ + "labelSelector": { + "matchExpressions": [{ + "key": "daemon", + "operator": "In", + "values":["mon"] + }] + }, + "topologyKey": "kubernetes.io/hostname", + "weight": 10 + }] + } + } + spec: + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + serviceAccount: default + volumes: + - name: ceph-conf + secret: + secretName: ceph-conf-combined + - name: ceph-bootstrap-osd-keyring + secret: + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + secretName: ceph-bootstrap-mds-keyring + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: ceph-bootstrap-rgw-keyring + - name: ceph-monfs + hostPath: + path: {{ .Values.storage.mon_directory }} + containers: + - name: ceph-mon + image: {{ .Values.images.daemon }} + imagePullPolicy: Always + lifecycle: + preStop: + exec: + # remove the mon on Pod stop. + command: + - "/remove-mon.sh" + ports: + - containerPort: 6789 + env: + - name: CEPH_DAEMON + value: MON + - name: KV_TYPE + value: k8s + - name: NETWORK_AUTO_DETECT + value: "1" + - name: CLUSTER + value: ceph + volumeMounts: + - name: ceph-conf + mountPath: /etc/ceph + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw + - name: ceph-monfs + mountPath: /var/lib/ceph/mon + livenessProbe: + tcpSocket: + port: 6789 + initialDelaySeconds: 60 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: 6789 + timeoutSeconds: 5 + resources: + requests: + memory: {{ .Values.resources.mon.requests.memory | quote }} + cpu: {{ .Values.resources.mon.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon.limits.memory | quote }} + cpu: {{ .Values.resources.mon.limits.cpu | quote }} diff --git a/ceph/templates/storage.yaml b/ceph/templates/storage.yaml index 9ef1eae2..b60221c1 100644 --- a/ceph/templates/storage.yaml +++ b/ceph/templates/storage.yaml @@ -10,7 +10,8 @@ parameters: monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }} adminId: {{ .Values.storageclass.admin_id }} adminSecretName: {{ .Values.storageclass.admin_secret_name }} - ## forcing namespace due to issue with -- default "{{ .Release.Namespace }}" }} -- + # forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }} + # during helm lint adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }} pool: {{ .Values.storageclass.pool }} userId: {{ .Values.storageclass.user_id }} diff --git a/ceph/values.yaml b/ceph/values.yaml index 76b4ac46..33c6da24 100644 --- a/ceph/values.yaml +++ b/ceph/values.yaml @@ -7,6 +7,15 @@ # tunables available - parameterizing more of the elements # in the manifests is a work in progress +replicas: + mon: 3 + rgw: 3 + mon_check: 1 + +service: + mon: + name: ceph-mon + images: daemon: quay.io/attcomdev/ceph-daemon:latest @@ -23,11 +32,52 @@ network: storage: osd_directory: /var/lib/openstack-helm/ceph/osd var_directory: /var/lib/openstack-helm/ceph/ceph + mon_directory: /var/lib/openstack-helm/ceph/mon # rgw is optionall disabled rgw: enabled: false +rgw: + enabled: false + +resources: + osd: + requests: + memory: "512Mi" + cpu: "1000m" + limits: + memory: "1024Mi" + cpu: "2000m" + mds: + requests: + memory: "10Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + mon: + requests: + memory: "50Mi" + cpu: "1000m" + limits: + memory: "100Mi" + cpu: "2000m" + mon_check: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + rgw: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + # Setting this to false will assume you will # setup and orchestrate your own secrets and # configmaps outside of this helm chart @@ -63,7 +113,7 @@ secrets: storageclass: provision_storage_class: true name: general - monitors: null + monitors: null pool: rbd admin_id: admin admin_secret_name: pvc-ceph-conf-combined-storageclass