Configured resource-only container /docker-daemon with 70% of node memory

capacity. This is a workaround to docker memory leakage issue.

pkg/kubelet/container_manager_linux.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors"
 	"github.com/docker/libcontainer/cgroups"
@@ -36,6 +37,13 @@ import (
 	"github.com/golang/glog"
 )
 
+const (
+	// The percent of the machine memory capacity. The value is used to calculate
+	// docker memory resource container's hardlimit to workaround docker memory
+	// leakage issue. Please see kubernetes/issues/9881 for more detail.
+	DockerMemoryLimitThresholdPercent = 70
+)
+
 // A non-user container tracked by the Kubelet.
 type systemContainer struct {
 	// Absolute name of the container.
@@ -69,13 +77,31 @@ var _ containerManager = &containerManagerImpl{}
 // TODO(vmarmol): Add limits to the system containers.
 // Takes the absolute name of the specified containers.
 // Empty container name disables use of the specified container.
-func newContainerManager(dockerDaemonContainerName, systemContainerName, kubeletContainerName string) (containerManager, error) {
+func newContainerManager(cadvisorInterface cadvisor.Interface, dockerDaemonContainerName, systemContainerName, kubeletContainerName string) (containerManager, error) {
 	systemContainers := []*systemContainer{}
 
 	if dockerDaemonContainerName != "" {
 		cont := newSystemContainer(dockerDaemonContainerName)
+
+		info, err := cadvisorInterface.MachineInfo()
+		var capacity = api.ResourceList{}
+		if err != nil {
+		} else {
+			capacity = CapacityFromMachineInfo(info)
+		}
+		memoryLimit := (int64(capacity.Memory().Value() * DockerMemoryLimitThresholdPercent / 100))
+		glog.Infof("Configure resource-only container %s with memory limit: %d", dockerDaemonContainerName, memoryLimit)
+
+		dockerContainer := &fs.Manager{
+			Cgroups: &configs.Cgroup{
+				Name:            dockerDaemonContainerName,
+				Memory:          memoryLimit,
+				MemorySwap:      -1,
+				AllowAllDevices: true,
+			},
+		}
 		cont.ensureStateFunc = func(manager *fs.Manager) error {
-			return ensureDockerInContainer(-900, createManager(dockerDaemonContainerName))
+			return ensureDockerInContainer(cadvisorInterface, -900, dockerContainer)
 		}
 		systemContainers = append(systemContainers, cont)
 	}
@@ -162,7 +188,7 @@ func (cm *containerManagerImpl) SystemContainersLimit() api.ResourceList {
 }
 
 // Ensures that the Docker daemon is in the desired container.
-func ensureDockerInContainer(oomScoreAdj int, manager *fs.Manager) error {
+func ensureDockerInContainer(cadvisor cadvisor.Interface, oomScoreAdj int, manager *fs.Manager) error {
 	// What container is Docker in?
 	out, err := exec.Command("pidof", "docker").Output()
 	if err != nil {

pkg/kubelet/container_manager_unsupported.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor"
 )
 
 type unsupportedContainerManager struct {
@@ -37,6 +38,6 @@ func (unsupportedContainerManager) SystemContainersLimit() api.ResourceList {
 	return api.ResourceList{}
 }
 
-func newContainerManager(dockerDaemonContainer, systemContainer, kubeletContainer string) (containerManager, error) {
+func newContainerManager(cadvisorInterface cadvisor.Interface, dockerDaemonContainer, systemContainer, kubeletContainer string) (containerManager, error) {
 	return &unsupportedContainerManager{}, nil
 }

pkg/kubelet/kubelet.go

@@ -309,7 +309,7 @@ func NewMainKubelet(
 
 	// Setup container manager, can fail if the devices hierarchy is not mounted
 	// (it is required by Docker however).
-	containerManager, err := newContainerManager(dockerDaemonContainer, systemContainer, resourceContainer)
+	containerManager, err := newContainerManager(cadvisorInterface, dockerDaemonContainer, systemContainer, resourceContainer)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create the Container Manager: %v", err)
 	}

pkg/kubelet/kubelet_test.go

@@ -133,7 +133,7 @@ func newTestKubelet(t *testing.T) *TestKubelet {
 		t:            t,
 	}
 	kubelet.volumeManager = newVolumeManager()
-	kubelet.containerManager, _ = newContainerManager("", "", "")
+	kubelet.containerManager, _ = newContainerManager(mockCadvisor, "", "", "")
 	return &TestKubelet{kubelet, fakeDocker, mockCadvisor, fakeKubeClient, fakeMirrorClient}
 }
 
@@ -243,7 +243,7 @@ func newTestKubeletWithFakeRuntime(t *testing.T) *TestKubeletWithFakeRuntime {
 		t:            t,
 	}
 	kubelet.volumeManager = newVolumeManager()
-	kubelet.containerManager, _ = newContainerManager("", "", "")
+	kubelet.containerManager, _ = newContainerManager(mockCadvisor, "", "", "")
 	return &TestKubeletWithFakeRuntime{kubelet, fakeRuntime, mockCadvisor, fakeKubeClient, fakeMirrorClient}
 }
 

pkg/kubelet/runonce_test.go

@@ -89,7 +89,7 @@ func TestRunOnce(t *testing.T) {
 		os:                  kubecontainer.FakeOS{},
 		volumeManager:       newVolumeManager(),
 	}
-	kb.containerManager, _ = newContainerManager("", "", "")
+	kb.containerManager, _ = newContainerManager(cadvisor, "", "", "")
 
 	kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", network.NewFakeHost(nil))
 	if err := kb.setupDataDirs(); err != nil {