From 1f8933da4610154e28a9a32f711f17d98585dfbe Mon Sep 17 00:00:00 2001 From: Georg Kunz Date: Thu, 14 Mar 2019 11:54:41 +0100 Subject: [PATCH] Extending Neutron chart with support for OVS DPDK Extending the Neutron with configuration parameters and scripts for deploying OVS with DPDK support enabled. The new functionality takes care of binding NICs to DPDK and adding those to OVS bridges of type 'netdev'. Co-Authored-By: Rihab Banday Change-Id: I9932123986a0b723d7523136940d325bcfde983d --- ...tron-openvswitch-agent-init-modules.sh.tpl | 4 + .../_neutron-openvswitch-agent-init.sh.tpl | 165 +++++++++++++++++- neutron/templates/configmap-etc.yaml | 9 + neutron/templates/daemonset-ovs-agent.yaml | 16 ++ neutron/values.yaml | 17 ++ neutron/values_overrides/dpdk.yaml | 23 +++ neutron/values_overrides/gate.yaml | 22 +++ .../developer/dpdk/000-install-packages.sh | 1 + .../developer/dpdk/010-deploy-k8s.sh | 1 + .../developer/dpdk/020-setup-client.sh | 1 + .../deployment/developer/dpdk/030-ingress.sh | 1 + .../developer/dpdk/040-nfs-provisioner.sh | 1 + .../deployment/developer/dpdk/050-mariadb.sh | 1 + .../deployment/developer/dpdk/060-rabbitmq.sh | 1 + .../developer/dpdk/070-memcached.sh | 1 + .../deployment/developer/dpdk/080-keystone.sh | 1 + tools/deployment/developer/dpdk/090-heat.sh | 1 + .../deployment/developer/dpdk/100-horizon.sh | 1 + tools/deployment/developer/dpdk/120-glance.sh | 1 + .../developer/dpdk/140-openvswitch.sh | 22 +++ .../deployment/developer/dpdk/150-libvirt.sh | 1 + .../developer/dpdk/160-compute-kit.sh | 22 +++ 22 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 neutron/values_overrides/dpdk.yaml create mode 100644 neutron/values_overrides/gate.yaml create mode 120000 tools/deployment/developer/dpdk/000-install-packages.sh create mode 120000 tools/deployment/developer/dpdk/010-deploy-k8s.sh create mode 120000 tools/deployment/developer/dpdk/020-setup-client.sh create mode 120000 tools/deployment/developer/dpdk/030-ingress.sh create mode 120000 tools/deployment/developer/dpdk/040-nfs-provisioner.sh create mode 120000 tools/deployment/developer/dpdk/050-mariadb.sh create mode 120000 tools/deployment/developer/dpdk/060-rabbitmq.sh create mode 120000 tools/deployment/developer/dpdk/070-memcached.sh create mode 120000 tools/deployment/developer/dpdk/080-keystone.sh create mode 120000 tools/deployment/developer/dpdk/090-heat.sh create mode 120000 tools/deployment/developer/dpdk/100-horizon.sh create mode 120000 tools/deployment/developer/dpdk/120-glance.sh create mode 100755 tools/deployment/developer/dpdk/140-openvswitch.sh create mode 120000 tools/deployment/developer/dpdk/150-libvirt.sh create mode 100755 tools/deployment/developer/dpdk/160-compute-kit.sh diff --git a/neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl b/neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl index 3321a9f7..83dc3f20 100644 --- a/neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl +++ b/neutron/templates/bin/_neutron-openvswitch-agent-init-modules.sh.tpl @@ -18,3 +18,7 @@ limitations under the License. set -ex chroot /mnt/host-rootfs modprobe ip6_tables + +{{- if .Values.conf.ovs_dpdk.enabled }} +chroot /mnt/host-rootfs modprobe {{ .Values.conf.ovs_dpdk.driver | quote }} +{{- end }} diff --git a/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl b/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl index b9c4b78d..ce4734da 100644 --- a/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl +++ b/neutron/templates/bin/_neutron-openvswitch-agent-init.sh.tpl @@ -18,7 +18,162 @@ limitations under the License. set -ex -chown neutron: /run/openvswitch/db.sock +OVS_SOCKET=/run/openvswitch/db.sock +chown neutron: ${OVS_SOCKET} + + +function get_dpdk_config_value { + values=$1 + filter=$2 + value=$(echo ${values} | jq -r ${filter}) + if [[ "${value}" == "null" ]]; then + echo "" + else + echo "${value}" + fi +} + + +DPDK_CONFIG_FILE=/tmp/dpdk.conf +DPDK_CONFIG="" +DPDK_ENABLED=false +if [ -f ${DPDK_CONFIG_FILE} ]; then + DPDK_CONFIG=$(cat ${DPDK_CONFIG_FILE}) + if [[ $(get_dpdk_config_value ${DPDK_CONFIG} '.enabled') == "true" ]]; then + DPDK_ENABLED=true + fi +fi + +function bind_nic { + echo $2 > /sys/bus/pci/devices/$1/driver_override + echo $1 > /sys/bus/pci/drivers/$2/bind +} + +function unbind_nic { + echo $1 > /sys/bus/pci/drivers/$2/unbind + echo > /sys/bus/pci/devices/$1/driver_override +} + +function get_name_by_pci_id { + path=$(find /sys/bus/pci/devices/$1/ -name net) + if [ -n "${path}" ] ; then + echo $(ls -1 $path/) + fi +} + +function get_ip_address_from_interface { + local interface=$1 + local ip=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $1}') + if [ -z "${ip}" ] ; then + echo "Interface ${interface} has no valid IP address." + exit 1 + fi + echo ${ip} +} + +function get_ip_prefix_from_interface { + local interface=$1 + local prefix=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $2}') + if [ -z "${prefix}" ] ; then + echo "Interface ${interface} has no valid IP address." + exit 1 + fi + echo ${prefix} +} + +function bind_dpdk_nics { + target_driver=$(get_dpdk_config_value ${DPDK_CONFIG} '.driver') + + # loop over all nics + echo $DPDK_CONFIG | jq -r -c '.nics[]' | \ + while IFS= read -r nic; do + local port_name=$(get_dpdk_config_value ${nic} '.name') + local pci_id=$(get_dpdk_config_value ${nic} '.pci_id') + local bridge=$(get_dpdk_config_value ${nic} '.bridge') + + if [[ $(get_dpdk_config_value ${nic} '.migrate_ip') == true ]] ; then + local src_nic=$(get_name_by_pci_id ${pci_id}) + if [ -n "${src_nic}" ] ; then + ip=$(get_ip_address_from_interface ${src_nic}) + prefix=$(get_ip_prefix_from_interface ${src_nic}) + + # Enabling explicit error handling: We must avoid to lose the IP + # address in the migration process. Hence, on every error, we + # attempt to assign the IP back to the original NIC and exit. + set +e + ip addr flush dev ${src_nic} + if [ $? -ne 0 ] ; then + ip addr add ${ip}/${prefix} dev ${src_nic} + echo "Error while flushing IP from ${src_nic}." + exit 1 + fi + + bridge_exists=$(ip a s ${bridge} 2> /dev/null) + if [ -z "${bridge_exists}" ] ; then + echo "Bridge ${bridge} does not exist. Creating it on demand." + init_ovs_dpdk_bridge ${bridge} + fi + + ip addr add ${ip}/${prefix} dev ${bridge} + if [ $? -ne 0 ] ; then + echo "Error assigning IP to bridge ${bridge}." + ip addr add ${ip}/${prefix} dev ${src_nic} + exit 1 + fi + set -e + fi + fi + + current_driver="$(get_driver_by_address ${pci_id} )" + if [ "$current_driver" != "$target_driver" ]; then + if [ "$current_driver" != "" ]; then + unbind_nic ${pci_id} ${current_driver} + fi + bind_nic ${pci_id} ${target_driver} + fi + + ovs-vsctl --db=unix:${OVS_SOCKET} --if-exists del-port ${port_name} + + dpdk_options="" + ofport_request=$(get_dpdk_config_value ${nic} '.ofport_request') + if [ -n "${ofport_request}" ]; then + dpdk_options+='ofport_request=${ofport_request} ' + fi + n_rxq=$(get_dpdk_config_value ${nic} '.n_rxq') + if [ -n "${n_rxq}" ]; then + dpdk_options+='options:n_rxq=${n_rxq} ' + fi + pmd_rxq_affinity=$(get_dpdk_config_value ${nic} '.pmd_rxq_affinity') + if [ -n "${pmd_rxq_affinity}" ]; then + dpdk_options+='other_config:pmd-rxq-affinity=${pmd_rxq_affinity} ' + fi + + ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-port ${bridge} ${port_name} \ + -- set Interface ${port_name} type=dpdk options:dpdk-devargs=${pci_id} ${dpdk_options} + + done +} + +function get_driver_by_address { + if [[ -e /sys/bus/pci/devices/$1/driver ]]; then + echo $(ls /sys/bus/pci/devices/$1/driver -al | awk '{n=split($NF,a,"/"); print a[n]}') + fi +} + +function init_ovs_dpdk_bridge { + bridge=$1 + ovs-vsctl --db=unix:${OVS_SOCKET} --may-exist add-br ${bridge} \ + -- set Bridge ${bridge} datapath_type=netdev + ip link set ${bridge} up +} + +# create all additional bridges defined in the DPDK section +function init_ovs_dpdk_bridges { + for br in $(get_dpdk_config_value ${DPDK_CONFIG} '.bridges[].name'); do + init_ovs_dpdk_bridge ${br} + done +} + # FIXME(portdirect): There is a neutron bug in Queens that needs resolved # for now, if we cannot even get the version of neutron-sanity-check, skip @@ -61,8 +216,13 @@ if [ -z "${tunnel_interface}" ] ; then | awk '{ print $1 }') || exit 1 fi +if [[ "${DPDK_ENABLED}" == "true" ]]; then + init_ovs_dpdk_bridges + bind_dpdk_nics +fi + # determine local-ip dynamically based on interface provided but only if tunnel_types is not null -LOCAL_IP=$(ip a s $tunnel_interface | grep 'inet ' | awk '{print $2}' | awk -F "/" '{print $1}') +LOCAL_IP=$(get_ip_address_from_interface ${tunnel_interface}) if [ -z "${LOCAL_IP}" ] ; then echo "Var LOCAL_IP is empty" exit 1 @@ -72,3 +232,4 @@ tee > /tmp/pod-shared/ml2-local-ip.ini << EOF [ovs] local_ip = "${LOCAL_IP}" EOF + diff --git a/neutron/templates/configmap-etc.yaml b/neutron/templates/configmap-etc.yaml index b07f16fb..8dd37d2f 100644 --- a/neutron/templates/configmap-etc.yaml +++ b/neutron/templates/configmap-etc.yaml @@ -169,6 +169,14 @@ just set it along with nova_metadata_host. {{- $formatter_fluent := dict "class" "oslo_log.formatters.FluentFormatter" -}} {{- $_ := set .Values.conf.logging "formatter_fluent" $formatter_fluent -}} {{- end -}} + +{{- if .Values.conf.ovs_dpdk.enabled -}} +{{- $_ := set $envAll.Values.conf.plugins.openvswitch_agent.ovs "datapath_type" "netdev" -}} +{{- if empty $envAll.Values.conf.plugins.openvswitch_agent.ovs.vhostuser_socket_dir -}} +{{- $_ := set $envAll.Values.conf.plugins.openvswitch_agent.ovs "vhostuser_socket_dir" "/run/openvswitch/vhostuser" -}} +{{- end -}} +{{- end -}} + --- apiVersion: v1 kind: Secret @@ -201,6 +209,7 @@ data: neutron_sudoers: {{ $envAll.Values.conf.neutron_sudoers | b64enc }} rootwrap.conf: {{ $envAll.Values.conf.rootwrap | b64enc }} auto_bridge_add: {{ toJson $envAll.Values.conf.auto_bridge_add | b64enc }} + dpdk.conf: {{ toJson $envAll.Values.conf.ovs_dpdk | b64enc }} {{- range $key, $value := $envAll.Values.conf.rootwrap_filters }} {{- $filePrefix := replace "_" "-" $key }} {{ printf "%s.filters" $filePrefix }}: {{ $value.content | b64enc }} diff --git a/neutron/templates/daemonset-ovs-agent.yaml b/neutron/templates/daemonset-ovs-agent.yaml index 4a3e71de..cd2e9001 100644 --- a/neutron/templates/daemonset-ovs-agent.yaml +++ b/neutron/templates/daemonset-ovs-agent.yaml @@ -102,6 +102,10 @@ spec: - name: host-rootfs mountPath: /mnt/host-rootfs readOnly: true +{{- if .Values.conf.ovs_dpdk.enabled }} + - name: pci-devices + mountPath: /sys/bus/pci/devices +{{- end }} - name: neutron-ovs-agent-init {{ tuple $envAll "neutron_openvswitch_agent" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.agent.ovs | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} @@ -158,6 +162,12 @@ spec: mountPath: /etc/neutron/rootwrap.conf subPath: rootwrap.conf readOnly: true + {{- if .Values.conf.ovs_dpdk.enabled }} + - name: neutron-etc + mountPath: /tmp/dpdk.conf + subPath: dpdk.conf + readOnly: true + {{- end }} {{- range $key, $value := $envAll.Values.conf.rootwrap_filters }} {{- if ( has "ovs_agent" $value.pods ) }} {{- if and ( eq "taas" $key ) (not $envAll.Values.conf.plugins.taas.taas.enabled) }} @@ -271,6 +281,12 @@ spec: - name: host-rootfs hostPath: path: / +{{- if .Values.conf.ovs_dpdk.enabled }} + - name: pci-devices + hostPath: + path: /sys/bus/pci/devices + type: Directory +{{- end }} {{ if $mounts_neutron_ovs_agent.volumes }}{{ toYaml $mounts_neutron_ovs_agent.volumes | indent 8 }}{{ end }} {{- end }} {{- end }} diff --git a/neutron/values.yaml b/neutron/values.yaml index 6c04b9fc..14b03f0a 100644 --- a/neutron/values.yaml +++ b/neutron/values.yaml @@ -1945,6 +1945,23 @@ conf: auto_bridge_add: br-ex: null + # configuration of OVS DPDK bridges and NICs + # this is a separate section and not part of the auto_bridge_add section + # because additional parameters are needed + ovs_dpdk: + enabled: false + # driver: uio_pci_generic + # nics: + # - name: dpdk0 + # pci_id: '0000:05:00.0' + # bridge: br-phy + # migrate_ip: true + # n_rxq: 2 + # pmd_rxq_affinity: "0:3,1:27" + # ofport_request: 1 + # bridges: + # - name: br-phy + # Names of secrets used by bootstrap and environmental checks secrets: identity: diff --git a/neutron/values_overrides/dpdk.yaml b/neutron/values_overrides/dpdk.yaml new file mode 100644 index 00000000..ff1a5419 --- /dev/null +++ b/neutron/values_overrides/dpdk.yaml @@ -0,0 +1,23 @@ +network: + interface: + tunnel: br-phy +conf: + plugins: + openvswitch_agent: + agent: + tunnel_types: vxlan + ovs: + bridge_mappings: public:br-ex + datapath_type: netdev + vhostuser_socket_dir: /var/run/openvswitch/vhostuser + ovs_dpdk: + enabled: true + driver: uio_pci_generic + nics: + # CHANGE-ME: modify pci_id according to your hardware + - name: dpdk0 + pci_id: '0000:05:00.0' + bridge: br-phy + migrate_ip: true + bridges: + - name: br-phy diff --git a/neutron/values_overrides/gate.yaml b/neutron/values_overrides/gate.yaml new file mode 100644 index 00000000..5f812791 --- /dev/null +++ b/neutron/values_overrides/gate.yaml @@ -0,0 +1,22 @@ +network: + interface: + tunnel: docker0 +conf: + neutron: + DEFAULT: + l3_ha: False + max_l3_agents_per_router: 1 + l3_ha_network_type: vxlan + dhcp_agents_per_network: 1 + plugins: + ml2_conf: + ml2_type_flat: + flat_networks: public + openvswitch_agent: + agent: + tunnel_types: vxlan + ovs: + bridge_mappings: public:br-ex + linuxbridge_agent: + linux_bridge: + bridge_mappings: public:br-ex diff --git a/tools/deployment/developer/dpdk/000-install-packages.sh b/tools/deployment/developer/dpdk/000-install-packages.sh new file mode 120000 index 00000000..8ad28d13 --- /dev/null +++ b/tools/deployment/developer/dpdk/000-install-packages.sh @@ -0,0 +1 @@ +../../common/install-packages.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/010-deploy-k8s.sh b/tools/deployment/developer/dpdk/010-deploy-k8s.sh new file mode 120000 index 00000000..3cb1bb90 --- /dev/null +++ b/tools/deployment/developer/dpdk/010-deploy-k8s.sh @@ -0,0 +1 @@ +../../common/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/020-setup-client.sh b/tools/deployment/developer/dpdk/020-setup-client.sh new file mode 120000 index 00000000..5ba93361 --- /dev/null +++ b/tools/deployment/developer/dpdk/020-setup-client.sh @@ -0,0 +1 @@ +../../common/setup-client.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/030-ingress.sh b/tools/deployment/developer/dpdk/030-ingress.sh new file mode 120000 index 00000000..70976836 --- /dev/null +++ b/tools/deployment/developer/dpdk/030-ingress.sh @@ -0,0 +1 @@ +../../component/common/ingress.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/040-nfs-provisioner.sh b/tools/deployment/developer/dpdk/040-nfs-provisioner.sh new file mode 120000 index 00000000..cb2307f8 --- /dev/null +++ b/tools/deployment/developer/dpdk/040-nfs-provisioner.sh @@ -0,0 +1 @@ +../../component/nfs-provisioner/nfs-provisioner.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/050-mariadb.sh b/tools/deployment/developer/dpdk/050-mariadb.sh new file mode 120000 index 00000000..4e0232db --- /dev/null +++ b/tools/deployment/developer/dpdk/050-mariadb.sh @@ -0,0 +1 @@ +../../component/common/mariadb.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/060-rabbitmq.sh b/tools/deployment/developer/dpdk/060-rabbitmq.sh new file mode 120000 index 00000000..a4588fc2 --- /dev/null +++ b/tools/deployment/developer/dpdk/060-rabbitmq.sh @@ -0,0 +1 @@ +../../component/common/rabbitmq.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/070-memcached.sh b/tools/deployment/developer/dpdk/070-memcached.sh new file mode 120000 index 00000000..cb6ef9a0 --- /dev/null +++ b/tools/deployment/developer/dpdk/070-memcached.sh @@ -0,0 +1 @@ +../../component/common/memcached.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/080-keystone.sh b/tools/deployment/developer/dpdk/080-keystone.sh new file mode 120000 index 00000000..b48fa216 --- /dev/null +++ b/tools/deployment/developer/dpdk/080-keystone.sh @@ -0,0 +1 @@ +../../component/keystone/keystone.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/090-heat.sh b/tools/deployment/developer/dpdk/090-heat.sh new file mode 120000 index 00000000..0566a0df --- /dev/null +++ b/tools/deployment/developer/dpdk/090-heat.sh @@ -0,0 +1 @@ +../../component/heat/heat.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/100-horizon.sh b/tools/deployment/developer/dpdk/100-horizon.sh new file mode 120000 index 00000000..4d19e48d --- /dev/null +++ b/tools/deployment/developer/dpdk/100-horizon.sh @@ -0,0 +1 @@ +../../component/horizon/horizon.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/120-glance.sh b/tools/deployment/developer/dpdk/120-glance.sh new file mode 120000 index 00000000..dc8088d3 --- /dev/null +++ b/tools/deployment/developer/dpdk/120-glance.sh @@ -0,0 +1 @@ +../../component/glance/glance.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/140-openvswitch.sh b/tools/deployment/developer/dpdk/140-openvswitch.sh new file mode 100755 index 00000000..7c0f4ee3 --- /dev/null +++ b/tools/deployment/developer/dpdk/140-openvswitch.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -xe + +# enabling DPDK feature +export FEATURE_GATES="dpdk" + +# Deploy openvswitch +./tools/deployment/component/compute-kit/openvswitch.sh diff --git a/tools/deployment/developer/dpdk/150-libvirt.sh b/tools/deployment/developer/dpdk/150-libvirt.sh new file mode 120000 index 00000000..89509157 --- /dev/null +++ b/tools/deployment/developer/dpdk/150-libvirt.sh @@ -0,0 +1 @@ +../../component/compute-kit/libvirt.sh \ No newline at end of file diff --git a/tools/deployment/developer/dpdk/160-compute-kit.sh b/tools/deployment/developer/dpdk/160-compute-kit.sh new file mode 100755 index 00000000..62e1707a --- /dev/null +++ b/tools/deployment/developer/dpdk/160-compute-kit.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -xe + +# enabling DPDK feature +export FEATURE_GATES="dpdk,gate" + +# Deploy nova and neutron charts +./tools/deployment/component/compute-kit/compute-kit.sh