Kill docker daemon after configing cbr0 if flag --babysit-daemon is true so that babysitter process can restart it again with proper configurations and checkpoint file.

cluster/saltbase/salt/kubelet/default

@@ -164,6 +164,11 @@
   {% set hairpin_mode = "--hairpin-mode=" + pillar['hairpin_mode'] -%}
 {% endif -%}
 
+{% set babysit_daemons = "" -%}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce' ]  %}
+  {% set babysit_daemons = "--babysit-daemons=true" -%}
+{% endif -%}
+
 {% set kubelet_port = "" -%}
 {% if pillar['kubelet_port'] is defined -%}
   {% set kubelet_port="--port=" + pillar['kubelet_port'] %}

cmd/kubelet/app/options/options.go

@@ -126,6 +126,7 @@ func NewKubeletServer() *KubeletServer {
 			ExperimentalFlannelOverlay:     experimentalFlannelOverlay,
 			OutOfDiskTransitionFrequency:   unversioned.Duration{5 * time.Minute},
 			HairpinMode:                    componentconfig.PromiscuousBridge,
+			BabysitDaemons:                 false,
 		},
 	}
 }
@@ -208,6 +209,8 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.RktStage1Image, "rkt-stage1-image", s.RktStage1Image, "image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used")
 	fs.BoolVar(&s.ConfigureCBR0, "configure-cbr0", s.ConfigureCBR0, "If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.")
 	fs.StringVar(&s.HairpinMode, "hairpin-mode", s.HairpinMode, "How should the kubelet setup hairpin NAT. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service. Valid values are \"promiscuous-bridge\", \"hairpin-veth\" and \"none\".")
+	fs.BoolVar(&s.BabysitDaemons, "babysit-daemons", s.BabysitDaemons, "If true, the node has babysitter process monitoring docker and kubelet.")
+	fs.MarkDeprecated("babysit-daemons", "Will be removed in a future version.")
 	fs.IntVar(&s.MaxPods, "max-pods", s.MaxPods, "Number of Pods that can run on this Kubelet.")
 	fs.StringVar(&s.DockerExecHandlerName, "docker-exec-handler", s.DockerExecHandlerName, "Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.")
 	fs.StringVar(&s.NonMasqueradeCIDR, "non-masquerade-cidr", s.NonMasqueradeCIDR, "Traffic to IPs outside this range will use IP masquerade.")

cmd/kubelet/app/server.go

@@ -253,8 +253,8 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 		VolumePlugins:                  ProbeVolumePlugins(s.VolumePluginDir),
 		OutOfDiskTransitionFrequency:   s.OutOfDiskTransitionFrequency.Duration,
 		HairpinMode:                    s.HairpinMode,
-
-		ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay,
+		BabysitDaemons:                 s.BabysitDaemons,
+		ExperimentalFlannelOverlay:     s.ExperimentalFlannelOverlay,
 		NodeIP: net.ParseIP(s.NodeIP),
 	}, nil
 }
@@ -766,6 +766,7 @@ type KubeletConfig struct {
 	NodeIP                     net.IP
 	ContainerRuntimeOptions    []kubecontainer.Option
 	HairpinMode                string
+	BabysitDaemons             bool
 	Options                    []kubelet.Option
 }
 
@@ -853,6 +854,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.VolumeStatsAggPeriod,
 		kc.ContainerRuntimeOptions,
 		kc.HairpinMode,
+		kc.BabysitDaemons,
 		kc.Options,
 	)
 

docs/admin/kubelet.md

@@ -152,7 +152,7 @@ kubelet
       --volume-stats-agg-period=1m0s: Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes.  To disable volume calculations, set to 0.  Default: '1m'
 ```
 
-###### Auto generated by spf13/cobra on 24-Feb-2016
+###### Auto generated by spf13/cobra on 1-Mar-2016
 
 
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

hack/verify-flags/known-flags.txt

@@ -22,6 +22,7 @@ authorization-mode
 authorization-policy-file
 authorization-webhook-config-file
 basic-auth-file
+babysit-daemons
 bench-pods
 bench-quiet
 bench-tasks

pkg/apis/componentconfig/types.go

@@ -280,6 +280,8 @@ type KubeletConfiguration struct {
 	// one must set --hairpin-mode=veth-flag, because bridge assumes the
 	// existence of a container bridge named cbr0.
 	HairpinMode string `json:"hairpinMode"`
+	// The node has babysitter process monitoring docker and kubelet.
+	BabysitDaemons bool `json:"babysitDaemons"`
 	// maxPods is the number of pods that can run on this Kubelet.
 	MaxPods int `json:"maxPods"`
 	// dockerExecHandlerName is the handler to use when executing a command

pkg/kubelet/container_bridge.go

@@ -29,7 +29,7 @@ import (
 
 var cidrRegexp = regexp.MustCompile(`inet ([0-9a-fA-F.:]*/[0-9]*)`)
 
-func createCBR0(wantCIDR *net.IPNet) error {
+func createCBR0(wantCIDR *net.IPNet, babysitDaemons bool) error {
 	// recreate cbr0 with wantCIDR
 	if err := exec.Command("brctl", "addbr", "cbr0").Run(); err != nil {
 		glog.Error(err)
@@ -43,10 +43,19 @@ func createCBR0(wantCIDR *net.IPNet) error {
 		glog.Error(err)
 		return err
 	}
-	// restart docker
+	// Stop docker so that babysitter process can restart it again with proper configurations and
+	// checkpoint file (https://github.com/docker/docker/issues/18283). It is safe to kill docker
+	// process here since CIDR can be changed only once for a given node object, and node is marked
+	// as NotReady until the docker daemon is restarted with the newly configured custom bridge.
+	// TODO (dawnchen): Remove this once corrupted checkpoint issue is fixed.
+	//
 	// For now just log the error. The containerRuntime check will catch docker failures.
 	// TODO (dawnchen) figure out what we should do for rkt here.
-	if util.UsingSystemdInitSystem() {
+	if babysitDaemons {
+		if err := exec.Command("pkill", "-KILL", "docker").Run(); err != nil {
+			glog.Error(err)
+		}
+	} else if util.UsingSystemdInitSystem() {
 		if err := exec.Command("systemctl", "restart", "docker").Run(); err != nil {
 			glog.Error(err)
 		}
@@ -59,14 +68,14 @@ func createCBR0(wantCIDR *net.IPNet) error {
 	return nil
 }
 
-func ensureCbr0(wantCIDR *net.IPNet, promiscuous bool) error {
+func ensureCbr0(wantCIDR *net.IPNet, promiscuous, babysitDaemons bool) error {
 	exists, err := cbr0Exists()
 	if err != nil {
 		return err
 	}
 	if !exists {
 		glog.V(2).Infof("CBR0 doesn't exist, attempting to create it with range: %s", wantCIDR)
-		return createCBR0(wantCIDR)
+		return createCBR0(wantCIDR, babysitDaemons)
 	}
 	if !cbr0CidrCorrect(wantCIDR) {
 		glog.V(2).Infof("Attempting to recreate cbr0 with address range: %s", wantCIDR)
@@ -80,7 +89,7 @@ func ensureCbr0(wantCIDR *net.IPNet, promiscuous bool) error {
 			glog.Error(err)
 			return err
 		}
-		if err := createCBR0(wantCIDR); err != nil {
+		if err := createCBR0(wantCIDR, babysitDaemons); err != nil {
 			glog.Error(err)
 			return err
 		}

pkg/kubelet/kubelet.go

@@ -208,6 +208,7 @@ func NewMainKubelet(
 	volumeStatsAggPeriod time.Duration,
 	containerRuntimeOptions []kubecontainer.Option,
 	hairpinMode string,
+	babysitDaemons bool,
 	kubeOptions []Option,
 ) (*Kubelet, error) {
 	if rootDirectory == "" {
@@ -331,6 +332,7 @@ func NewMainKubelet(
 		reservation:                  reservation,
 		enableCustomMetrics:          enableCustomMetrics,
 		hairpinMode:                  componentconfig.HairpinMode(hairpinMode),
+		babysitDaemons:               babysitDaemons,
 	}
 	// TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency
 	klet.resourceAnalyzer = stats.NewResourceAnalyzer(klet, volumeStatsAggPeriod)
@@ -731,6 +733,9 @@ type Kubelet struct {
 	// or "none" (do nothing).
 	hairpinMode componentconfig.HairpinMode
 
+	// The node has babysitter process monitoring docker and kubelet
+	babysitDaemons bool
+
 	// handlers called during the tryUpdateNodeStatus cycle
 	setNodeStatusFuncs []func(*api.Node) error
 }
@@ -2666,7 +2671,7 @@ func (kl *Kubelet) reconcileCBR0(podCIDR string) error {
 	}
 	// Set cbr0 interface address to first address in IPNet
 	cidr.IP.To4()[3] += 1
-	if err := ensureCbr0(cidr, kl.hairpinMode == componentconfig.PromiscuousBridge); err != nil {
+	if err := ensureCbr0(cidr, kl.hairpinMode == componentconfig.PromiscuousBridge, kl.babysitDaemons); err != nil {
 		return err
 	}
 	if kl.shaper == nil {