Merge pull request #23600 from mikedanese/addon-manager

Automatic merge from submit-queue

run kube-addon-manager in a static pod

Depends on https://github.com/kubernetes/kubernetes/pull/23605 and https://github.com/kubernetes/kubernetes/pull/24108

Ref #23233

build/common.sh

@@ -912,9 +912,7 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${dst_dir}"
   cp "${salt_dir}/kube-apiserver/abac-authz-policy.jsonl" "${dst_dir}"
   cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}"
-  cp "${salt_dir}/kube-addons/namespace.yaml" "${dst_dir}"
-  cp "${salt_dir}/kube-addons/kube-addons.sh" "${dst_dir}"
-  cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${dst_dir}"
+  cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${dst_dir}"
   cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}"
   cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
   local objects

cluster/addons/addon-manager/.gitignore

@@ -0,0 +1 @@
+kubectl

cluster/addons/addon-manager/Dockerfile

@@ -0,0 +1,24 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:2.7-slim
+
+RUN pip install pyyaml
+
+ADD kube-addons.sh /opt/
+ADD kube-addon-update.sh /opt/
+ADD namespace.yaml /opt/
+ADD kubectl /usr/local/bin/kubectl
+
+CMD /opt/kube-addons.sh

cluster/addons/addon-manager/Makefile

@@ -0,0 +1,34 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+IMAGE=gcr.io/google-containers/kube-addon-manager
+VERSION=v1
+KUBECTL_VERSION=v1.2.3
+
+.PHONY: build push container
+
+build: kubectl
+	docker build -t "$(IMAGE):$(VERSION)" .
+
+kubectl:
+	curl "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
+		-o kubectl
+	chmod +x kubectl
+
+push: build
+	gcloud docker push "$(IMAGE):$(VERSION)"
+
+clean:
+	rm kubectl
+	docker rmi -f "$(IMAGE):$(VERSION)"

cluster/addons/addon-manager/kube-addon-update.sh

@@ -98,7 +98,7 @@ function log() {
 function get-object-kind-from-file() {
     # prints to stdout, so log cannot be used
     #WARNING: only yaml is supported
-    cat $1 | ${PYTHON} -c '''
+    cat $1 | python -c '''
 try:
         import pipes,sys,yaml
         y = yaml.load(sys.stdin)
@@ -120,7 +120,7 @@ function get-object-nsname-from-file() {
     # prints to stdout, so log cannot be used
     #WARNING: only yaml is supported
     #addons that do not specify a namespace are assumed to be in "default".
-    cat $1 | ${PYTHON} -c '''
+    cat $1 | python -c '''
 try:
         import pipes,sys,yaml
         y = yaml.load(sys.stdin)

cluster/addons/addon-manager/kube-addons.sh

@@ -19,21 +19,11 @@
 # managed result is of that. Start everything below that directory.
 KUBECTL=${KUBECTL_BIN:-/usr/local/bin/kubectl}
 
-ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-600}
+ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-10}
 
 SYSTEM_NAMESPACE=kube-system
 trusty_master=${TRUSTY_MASTER:-false}
 
-function ensure_python() {
-  if ! python --version > /dev/null 2>&1; then    
-    echo "No python on the machine, will use a python image"
-    local -r PYTHON_IMAGE=gcr.io/google_containers/python:v1
-    export PYTHON="docker run --interactive --rm --net=none ${PYTHON_IMAGE} python"
-  else
-    export PYTHON=python
-  fi
-}
-
 # $1 filename of addon to start.
 # $2 count of tries to start the addon.
 # $3 delay in seconds between two consecutive tries
@@ -74,10 +64,8 @@ function create-resource-from-string() {
 # managed result is of that. Start everything below that directory.
 echo "== Kubernetes addon manager started at $(date -Is) with ADDON_CHECK_INTERVAL_SEC=${ADDON_CHECK_INTERVAL_SEC} =="
 
-ensure_python
-
 # Create the namespace that will be used to host the cluster-level add-ons.
-start_addon /etc/kubernetes/addons/namespace.yaml 100 10 "" &
+start_addon /opt/namespace.yaml 100 10 "" &
 
 # Wait for the default service account to be created in the kube-system namespace.
 token_found=""

cluster/gce/trusty/configure-helper.sh

@@ -662,13 +662,6 @@ prepare_kube_addons() {
     setup_addon_manifests "admission-controls" "limit-range"
   fi
 
-  # Prepare the scripts for running addons.
-  addon_script_dir="/var/lib/cloud/scripts/kubernetes"
-  mkdir -p "${addon_script_dir}"
-  cp "${addon_src_dir}/kube-addons.sh" "${addon_script_dir}"
-  cp "${addon_src_dir}/kube-addon-update.sh" "${addon_script_dir}"
-  chmod 544 "${addon_script_dir}/"*.sh
-  # In case that some GCE customized trusty may have a read-only /root.
-  mount -t tmpfs tmpfs /root
-  mount --bind -o remount,rw,noexec /root
+  # Place addon manager pod manifest
+  cp "${addon_src_dir}/kube-addon-manager.yaml" /etc/kubernetes/manifests
 }

cluster/gce/trusty/master.yaml

@@ -180,38 +180,6 @@ script
 	prepare_kube_addons
 end script
 
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-addons.conf"
-
-#upstart-job
-
-description "Run kubernetes addon pods"
-
-start on stopped kube-master-components
-
-respawn
-
-script
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-env
-	export HOME="/root"
-	export KUBECTL_BIN="/usr/bin/kubectl"
-	export TRUSTY_MASTER="true"
-	if [ -n "${TEST_ADDON_CHECK_INTERVAL_SEC:-}" ]; then
-		export TEST_ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC}
-	fi
-	# Run the script to start and monitoring addon manifest changes.
-	exec /var/lib/cloud/scripts/kubernetes/kube-addons.sh 1>>/var/log/kube-addons.log 2>&1
-end script
-
-# Wait for 10s to start it again.
-post-stop exec sleep 10
-
 --====================================
 MIME-Version: 1.0
 Content-Type: text/upstart-job; charset="us-ascii"

cluster/saltbase/salt/kube-addons/init.sls

@@ -11,13 +11,6 @@ addon-dir-create:
     - require:
         - file: addon-dir-delete
 
-/etc/kubernetes/addons/namespace.yaml:
-  file.managed:
-    - source: salt://kube-addons/namespace.yaml
-    - user: root
-    - group: root
-    - file_mode: 644
-
 {% if pillar.get('enable_cluster_monitoring', '').lower() == 'influxdb' %}
 /etc/kubernetes/addons/cluster-monitoring/influxdb:
   file.recurse:
@@ -157,63 +150,9 @@ addon-dir-create:
     - file_mode: 644
 {% endif %}
 
-/etc/kubernetes/kube-addons.sh:
+/etc/kubernetes/manifests/kube-addon-manager.yaml:
   file.managed:
-    - source: salt://kube-addons/kube-addons.sh
+    - source: salt://kube-addons/kube-addon-manager.yaml
     - user: root
     - group: root
     - mode: 755
-
-/etc/kubernetes/kube-addon-update.sh:
-  file.managed:
-    - source: salt://kube-addons/kube-addon-update.sh
-    - user: root
-    - group: root
-    - mode: 755
-
-{% if pillar.get('is_systemd') %}
-
-{{ pillar.get('systemd_system_path') }}/kube-addons.service:
-  file.managed:
-    - source: salt://kube-addons/kube-addons.service
-    - user: root
-    - group: root
-  cmd.wait:
-    - name: /opt/kubernetes/helpers/services bounce kube-addons
-    - watch:
-      - file: {{ pillar.get('systemd_system_path') }}/kube-addons.service
-
-{% else %}
-
-/etc/init.d/kube-addons:
-  file.managed:
-    - source: salt://kube-addons/initd
-    - user: root
-    - group: root
-    - mode: 755
-
-{% endif %}
-
-# Stop kube-addons service each time salt is executed, just in case
-# there was a modification of addons.
-# Actually, this should be handled by watching file changes, but
-# somehow it doesn't work.
-service-kube-addon-stop:
-  service.dead:
-    - name: kube-addons
-
-kube-addons:
-  service.running:
-    - enable: True
-    - require:
-        - service: service-kube-addon-stop
-    - watch:
-{% if pillar.get('is_systemd') %}
-      - file: {{ pillar.get('systemd_system_path') }}/kube-addons.service
-{% else %}
-      - file: /etc/init.d/kube-addons
-{% endif %}
-{% if pillar.get('is_systemd') %}
-    - provider:
-      - service: systemd
-{%- endif %}

cluster/saltbase/salt/kube-addons/initd

@@ -1,120 +0,0 @@
-#!/bin/bash
-#
-### BEGIN INIT INFO
-# Provides:   kube-addons
-# Required-Start:    $local_fs $network $syslog kube-apiserver
-# Required-Stop:
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Kubernetes Addon Object Manager
-# Description:
-#   Enforces installation of Kubernetes Addon Objects
-### END INIT INFO
-
-
-# PATH should only include /usr/* if it runs after the mountnfs.sh script
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-DESC="Kubernetes Addon Object Manager"
-NAME=kube-addons
-DAEMON_LOG_FILE=/var/log/${NAME}.log
-PIDFILE=/var/run/${NAME}.pid
-SCRIPTNAME=/etc/init.d/${NAME}
-KUBE_ADDONS_SH=/etc/kubernetes/kube-addons.sh
-
-# Define LSB log_* functions.
-# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
-# and status_of_proc is working.
-. /lib/lsb/init-functions
-
-
-
-
-#
-# Function that starts the daemon/service
-#
-do_start()
-{
-    # use setsid to make sure the new daemon has its own group (I suppose
-    # start-stop-daemon does create a process group, but let's stay on the
-    # safe side).
-    setsid start-stop-daemon --start --verbose --background --no-close --make-pidfile --pidfile "${PIDFILE}" --startas "${KUBE_ADDONS_SH}" </dev/null >> ${DAEMON_LOG_FILE} 2>&1
-}
-
-#
-# Function that stops the daemon/service
-#
-do_stop()
-{
-    # start-stop-daemon is not used because we have to stop all children
-    # limitations:
-    #  - stop does not work if the pid file is missing
-    #  - stop does not work if the daemon process is missing (children will not
-    #    be killed)
-    # This is sufficient - remaining processes will end after a while.
-
-    local pid
-    pid=$(cat "${PIDFILE}" 2> /dev/null)
-    if [[ $? != 0 ]]; then
-        return 1
-    fi
-    local pgrp
-    # find the process group for the service and kill entire group
-    #  o - output format: pgpg - process group
-    pgrp=$(ps --no-headers --pid "${pid}" -o pgrp 2>/dev/null)
-    if [[ $? != 0 ]] || [[ "${pgrp}" == "" ]]; then
-        return 1
-    fi
-    pgrp=$(echo -e ${pgrp})   # strip whitespaces (that's why there are no quotes around pgrp)
-    # negative pid is for killing entire group
-    kill -- -${pgrp} 2> /dev/null
-    if [[ $? != 0 ]]; then
-        return 2
-    fi
-    rm -f "${PIDFILE}"
-    return
-}
-
-case "$1" in
-  start)
-        log_daemon_msg "Starting ${DESC}" "${NAME}"
-        do_start
-        case "$?" in
-                0|1) log_end_msg 0 || exit 0 ;;
-                2) log_end_msg 1 || exit 1 ;;
-        esac
-        ;;
-  stop)
-        log_daemon_msg "Stopping ${DESC}" "${NAME}"
-        do_stop
-        case "$?" in
-                0|1) log_end_msg 0 || exit 0 ;;
-                2) log_end_msg 1 || exit 1 ;;
-        esac
-        ;;
-  status)
-        status_of_proc -p "${PIDFILE}" "${KUBE_ADDONS_SH}" "${NAME}"
-        ;;
-
-  restart|force-reload)
-        log_daemon_msg "Restarting ${DESC}" "${NAME}"
-        do_stop
-        case "$?" in
-          0|1)
-                do_start
-                case "$?" in
-                        0) log_end_msg 0 ;;
-                        1) log_end_msg 1 ;; # Old process is still running
-                        *) log_end_msg 1 ;; # Failed to start
-                esac
-                ;;
-          *)
-                # Failed to stop
-                log_end_msg 1
-                ;;
-        esac
-        ;;
-  *)
-        echo "Usage: ${SCRIPTNAME} {start|stop|status|restart|force-reload}" >&2
-        exit 3
-        ;;
-esac

cluster/saltbase/salt/kube-addons/kube-addon-manager.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-addon-manager
+  namespace: kube-system
+  version: v1
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-addon-manager
+    image: gcr.io/google-containers/kube-addon-manager:v1
+    resources:
+      requests:
+        cpu: 5m
+        memory: 50Mi
+    volumeMounts:
+    - mountPath: /etc/kubernetes/
+      name: addons
+      readOnly: true
+  volumes:
+  - hostPath:
+      path: /etc/kubernetes/
+    name: addons

cluster/saltbase/salt/kube-addons/kube-addons.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Kubernetes Addon Object Manager
-Documentation=https://github.com/kubernetes/kubernetes
-
-[Service]
-Environment=HOME=/root
-ExecStart=/etc/kubernetes/kube-addons.sh
-
-[Install]
-WantedBy=multi-user.target

hack/verify-flags/exceptions.txt

@@ -1,4 +1,5 @@
 Vagrantfile:      node_ip = $node_ips[n]
+cluster/addons/addon-manager/kube-addons.sh:# Create admission_control objects if defined before any other addon services. If the limits
 cluster/addons/registry/images/Dockerfile:ADD run_proxy.sh /usr/bin/run_proxy
 cluster/addons/registry/images/Dockerfile:CMD ["/usr/bin/run_proxy"]
 cluster/aws/templates/configure-vm-aws.sh:  # We set the hostname_override to the full EC2 private dns name
@@ -31,7 +32,6 @@ cluster/photon-controller/util.sh:    node_ip=$(${PHOTON} vm networks "${node_id
 cluster/photon-controller/util.sh:  local cert_dir="/srv/kubernetes"
 cluster/photon-controller/util.sh:  node_name=${1}
 cluster/rackspace/util.sh:    local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \
-cluster/saltbase/salt/kube-addons/kube-addons.sh:# Create admission_control objects if defined before any other addon services. If the limits
 cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %}
 cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout -%}
 cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + terminated_pod_gc + " " + cloud_provider  + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%}

test/e2e/addon_update.go

@@ -207,37 +207,10 @@ var _ = framework.KubeDescribe("Addon update", func() {
 		var err error
 		sshClient, err = getMasterSSHClient()
 		Expect(err).NotTo(HaveOccurred())
-
-		// Reduce the addon update intervals so that we have faster response
-		// to changes in the addon directory.
-		// do not use "service" command because it clears the environment variables
-		switch framework.TestContext.OSDistro {
-		case "debian":
-			sshExecAndVerify(sshClient, "sudo TEST_ADDON_CHECK_INTERVAL_SEC=1 /etc/init.d/kube-addons restart")
-		case "trusty", "gci":
-			sshExecAndVerify(sshClient, "sudo initctl restart kube-addons TEST_ADDON_CHECK_INTERVAL_SEC=1")
-		case "coreos":
-			sshExecAndVerify(sshClient, "sudo systemctl set-environment TEST_ADDON_CHECK_INTERVAL_SEC=1")
-			sshExecAndVerify(sshClient, "sudo systemctl restart kubernetes-addons")
-		default:
-			framework.Failf("Unsupported OS distro type %s", framework.TestContext.OSDistro)
-		}
 	})
 
 	AfterEach(func() {
 		if sshClient != nil {
-			// restart addon_update with the default options
-			switch framework.TestContext.OSDistro {
-			case "debian":
-				sshExec(sshClient, "sudo /etc/init.d/kube-addons restart")
-			case "trusty", "gci":
-				sshExec(sshClient, "sudo initctl restart kube-addons")
-			case "coreos":
-				sshExec(sshClient, "sudo systemctl unset-environment TEST_ADDON_CHECK_INTERVAL_SEC")
-				sshExec(sshClient, "sudo systemctl restart kubernetes-addons")
-			default:
-				framework.Failf("Unsupported OS distro type %s", framework.TestContext.OSDistro)
-			}
 			sshClient.Close()
 		}
 	})