Inject top level QoS cgroup creation in the Kubelet

test/e2e/framework/test_context.go

@@ -67,6 +67,8 @@ type TestContextType struct {
 	DumpLogsOnFailure bool
 	// Name of the node to run tests on (node e2e suite only).
 	NodeName string
+	// Whether to enable the QoS Cgroup Hierarchy or not
+	CgroupsPerQOS bool
 }
 
 type CloudConfig struct {
@@ -148,4 +150,5 @@ func RegisterClusterFlags() {
 // Register flags specific to the node e2e test suite.
 func RegisterNodeFlags() {
 	flag.StringVar(&TestContext.NodeName, "node-name", "", "Name of the node to run tests on (node e2e suite only).")
+	flag.BoolVar(&TestContext.CgroupsPerQOS, "cgroups-per-qos", false, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.")
 }

test/e2e_node/cgroup_manager_test.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/util"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = framework.KubeDescribe("Kubelet Cgroup Manager", func() {
+	f := NewDefaultFramework("kubelet-cgroup-manager")
+	Describe("QOS containers", func() {
+		Context("On enabling QOS cgroup hierarchy", func() {
+			It("Top level QoS containers should have been created", func() {
+				if framework.TestContext.CgroupsPerQOS {
+					podName := "qos-pod" + string(util.NewUUID())
+					contName := "qos-container" + string(util.NewUUID())
+					pod := &api.Pod{
+						ObjectMeta: api.ObjectMeta{
+							Name:      podName,
+							Namespace: f.Namespace.Name,
+						},
+						Spec: api.PodSpec{
+							// Don't restart the Pod since it is expected to exit
+							RestartPolicy: api.RestartPolicyNever,
+							Containers: []api.Container{
+								{
+									Image:   "gcr.io/google_containers/busybox:1.24",
+									Name:    contName,
+									Command: []string{"sh", "-c", "if [ -d /tmp/memory/Burstable ] && [ -d /tmp/memory/BestEffort ]; then exit 0; else exit 1; fi"},
+									VolumeMounts: []api.VolumeMount{
+										{
+											Name:      "sysfscgroup",
+											MountPath: "/tmp",
+										},
+									},
+								},
+							},
+							Volumes: []api.Volume{
+								{
+									Name: "sysfscgroup",
+									VolumeSource: api.VolumeSource{
+										HostPath: &api.HostPathVolumeSource{Path: "/sys/fs/cgroup"},
+									},
+								},
+							},
+						},
+					}
+					f.MungePodSpec(pod)
+					podClient := f.Client.Pods(f.Namespace.Name)
+					_, err := podClient.Create(pod)
+					Expect(err).NotTo(HaveOccurred())
+					err = framework.WaitForPodSuccessInNamespace(f.Client, podName, contName, f.Namespace.Name)
+					Expect(err).NotTo(HaveOccurred())
+				}
+			})
+		})
+	})
+})

test/e2e_node/e2e_node_suite_test.go

@@ -96,7 +96,7 @@ var _ = BeforeSuite(func() {
 	maskLocksmithdOnCoreos()
 
 	if *startServices {
-		e2es = newE2eService(framework.TestContext.NodeName)
+		e2es = newE2eService(framework.TestContext.NodeName, framework.TestContext.CgroupsPerQOS)
 		if err := e2es.start(); err != nil {
 			Fail(fmt.Sprintf("Unable to start node services.\n%v", err))
 		}

test/e2e_node/e2e_remote.go

@@ -146,7 +146,7 @@ func CreateTestArchive() (string, error) {
 }
 
 // Returns the command output, whether the exit was ok, and any errors
-func RunRemote(archive string, host string, cleanup bool, junitFileNumber int, setupNode bool) (string, bool, error) {
+func RunRemote(archive string, host string, cleanup bool, junitFileNumber int, setupNode bool, testArgs string) (string, bool, error) {
 	if setupNode {
 		uname, err := user.Current()
 		if err != nil {
@@ -211,11 +211,10 @@ func RunRemote(archive string, host string, cleanup bool, junitFileNumber int, s
 		// Exit failure with the error
 		return "", false, err
 	}
-
 	// Run the tests
 	cmd = getSshCommand(" && ",
 		fmt.Sprintf("cd %s", tmp),
-		fmt.Sprintf("timeout -k 30s %ds ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --junit-file-number=%d", *testTimeoutSeconds, *ginkgoFlags, cleanup, host, tmp, junitFileNumber),
+		fmt.Sprintf("timeout -k 30s %ds ./ginkgo %s ./e2e_node.test -- --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --junit-file-number=%d %s", *testTimeoutSeconds, *ginkgoFlags, cleanup, host, tmp, junitFileNumber, testArgs),
 	)
 	aggErrs := []error{}
 

test/e2e_node/e2e_service.go

@@ -46,6 +46,7 @@ type e2eService struct {
 	kubeletStaticPodDir string
 	nodeName            string
 	logFiles            map[string]logFileData
+	cgroupsPerQOS       bool
 }
 
 type logFileData struct {
@@ -58,14 +59,18 @@ const (
 	LOG_VERBOSITY_LEVEL = "4"
 )
 
-func newE2eService(nodeName string) *e2eService {
+func newE2eService(nodeName string, cgroupsPerQOS bool) *e2eService {
 	// Special log files that need to be collected for additional debugging.
 	var logFiles = map[string]logFileData{
 		"kern.log":   {[]string{"/var/log/kern.log"}, []string{"-k"}},
 		"docker.log": {[]string{"/var/log/docker.log", "/var/log/upstart/docker.log"}, []string{"-u", "docker"}},
 	}
 
-	return &e2eService{nodeName: nodeName, logFiles: logFiles}
+	return &e2eService{
+		nodeName:      nodeName,
+		logFiles:      logFiles,
+		cgroupsPerQOS: cgroupsPerQOS,
+	}
 }
 
 func (es *e2eService) start() error {
@@ -236,6 +241,12 @@ func (es *e2eService) startKubeletServer() (*killCmd, error) {
 		"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
 		"--pod-cidr=10.180.0.0/24", // Assign a fixed CIDR to the node because there is no node controller.
 	)
+	if es.cgroupsPerQOS {
+		cmdArgs = append(cmdArgs,
+			"--cgroups-per-qos", "true",
+			"--cgroup-root", "/",
+		)
+	}
 	if !*disableKubenet {
 		cwd, err := os.Getwd()
 		if err != nil {
@@ -245,6 +256,7 @@ func (es *e2eService) startKubeletServer() (*killCmd, error) {
 			"--network-plugin=kubenet",
 			"--network-plugin-dir", filepath.Join(cwd, CNIDirectory, "bin")) // Enable kubenet
 	}
+
 	cmd := exec.Command("sudo", cmdArgs...)
 	hcc := newHealthCheckCommand(
 		"http://127.0.0.1:10255/healthz",

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -39,4 +39,4 @@ go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="g
   --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \
   --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
   --results-dir="$ARTIFACTS" --ginkgo-flags="$GINKGO_FLAGS" \
-  --setup-node="$SETUP_NODE" --instance-metadata="$GCE_INSTANCE_METADATA"
+  --setup-node="$SETUP_NODE" --test_args="$TEST_ARGS" --instance-metadata="$GCE_INSTANCE_METADATA"

test/e2e_node/jenkins/jenkins-ci.properties

@@ -5,4 +5,4 @@ GCE_PROJECT=kubernetes-jenkins
 CLEANUP=true
 GINKGO_FLAGS=--skip=FLAKY
 SETUP_NODE=false
-
+TEST_ARGS=--cgroups-per-qos=true

test/e2e_node/jenkins/jenkins-pull.properties

@@ -5,3 +5,4 @@ GCE_PROJECT=kubernetes-jenkins-pull
 CLEANUP=true
 GINKGO_FLAGS=--skip=FLAKY
 SETUP_NODE=false
+TEST_ARGS=--cgroups-per-qos=true

test/e2e_node/jenkins/template.properties

@@ -17,3 +17,5 @@ GCE_IMAGE_PROJECT=
 CLEANUP=true
 # If true, current user will be added to the docker group on test node
 SETUP_NODE=false
+# If true QoS Cgroup Hierarchy is created and tests specifc to the cgroup hierarchy run
+TEST_ARGS=--cgroups-per-qos=true

test/e2e_node/runner/run_e2e.go

@@ -41,6 +41,7 @@ import (
 	"google.golang.org/api/compute/v1"
 )
 
+var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
 var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
 var zone = flag.String("zone", "", "gce zone the hosts live in")
 var project = flag.String("project", "", "gce project the hosts live in")
@@ -254,7 +255,7 @@ func testHost(host string, deleteFiles bool, junitFileNum int, setupNode bool) *
 		}
 	}
 
-	output, exitOk, err := e2e_node.RunRemote(path, host, deleteFiles, junitFileNum, setupNode)
+	output, exitOk, err := e2e_node.RunRemote(path, host, deleteFiles, junitFileNum, setupNode, *testArgs)
 	return &TestResult{
 		output: output,
 		err:    err,