scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-11-04 04:08:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extract commonly used functions from e2e/resize_nodes.go to e2e/framework

Browse Source
This commit is contained in:
Marcin Wielgus
2017-04-03 19:37:32 +02:00
parent dac7b3e268
commit 3f7bf0206f
8 changed files with 148 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
test/e2e/BUILD
View File

@@ -112,7 +112,6 @@ go_library(
"//pkg/client/clientset_generated/clientset/typed/extensions/v1beta1:go_default_library",
"//pkg/client/clientset_generated/internalclientset:go_default_library",
"//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers/aws:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/cloudprovider/providers/vsphere:go_default_library",
"//pkg/controller:go_default_library",
@@ -149,7 +148,6 @@ go_library(
"//test/utils:go_default_library",
"//vendor:github.com/aws/aws-sdk-go/aws",
"//vendor:github.com/aws/aws-sdk-go/aws/session",
"//vendor:github.com/aws/aws-sdk-go/service/autoscaling",
"//vendor:github.com/aws/aws-sdk-go/service/ec2",
"//vendor:github.com/elazarl/goproxy",
"//vendor:github.com/ghodss/yaml",
@@ -211,6 +209,7 @@ go_test(
"//pkg/api/v1:go_default_library",
"//pkg/client/clientset_generated/clientset:go_default_library",
"//pkg/metrics:go_default_library",
"//test/e2e/autoscaling:go_default_library",
"//test/e2e/cluster-logging:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/perf:go_default_library",

1
test/e2e/autoscaling/BUILD
View File

@@ -18,7 +18,6 @@ go_library(
deps = [
"//pkg/api/v1:go_default_library",
"//pkg/client/clientset_generated/clientset:go_default_library",
"//test/e2e:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/scheduling:go_default_library",

13
test/e2e/autoscaling/cluster_size_autoscaling.go
View File

@@ -36,7 +36,6 @@ import (
policy "k8s.io/client-go/pkg/apis/policy/v1beta1"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/scheduling"
testutils "k8s.io/kubernetes/test/utils"
@@ -83,7 +82,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
originalSizes = make(map[string]int)
sum := 0
for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") {
size, err := e2e.GroupSize(mig)
size, err := framework.GroupSize(mig)
framework.ExpectNoError(err)
By(fmt.Sprintf("Initial size of %s: %d", mig, size))
originalSizes[mig] = size
@@ -211,7 +210,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
}
}
nodes, err := e2e.GetGroupNodes(minMig)
nodes, err := framework.GetGroupNodes(minMig)
framework.ExpectNoError(err)
nodesSet := sets.NewString(nodes...)
defer removeLabels(nodesSet)
@@ -224,12 +223,12 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
CreateNodeSelectorPods(f, "node-selector", minSize+1, map[string]string{labelKey: labelValue}, false)
By("Waiting for new node to appear and annotating it")
e2e.WaitForGroupSize(minMig, int32(minSize+1))
framework.WaitForGroupSize(minMig, int32(minSize+1))
// Verify, that cluster size is increased
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout))
newNodes, err := e2e.GetGroupNodes(minMig)
newNodes, err := framework.GetGroupNodes(minMig)
framework.ExpectNoError(err)
newNodesSet := sets.NewString(newNodes...)
newNodesSet.Delete(nodes...)
@@ -655,11 +654,11 @@ func waitForAllCaPodsReadyInNamespace(f *framework.Framework, c clientset.Interf
func setMigSizes(sizes map[string]int) {
for mig, desiredSize := range sizes {
currentSize, err := e2e.GroupSize(mig)
currentSize, err := framework.GroupSize(mig)
framework.ExpectNoError(err)
if desiredSize != currentSize {
By(fmt.Sprintf("Setting size of %s to %d", mig, desiredSize))
err = e2e.ResizeGroup(mig, int32(desiredSize))
err = framework.ResizeGroup(mig, int32(desiredSize))
framework.ExpectNoError(err)
}
}

3
test/e2e/autoscaling/dns_autoscaling.go
View File

@@ -28,7 +28,6 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
@@ -120,7 +119,7 @@ var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() {
originalSizes := make(map[string]int)
sum := 0
for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") {
size, err := e2e.GroupSize(mig)
size, err := framework.GroupSize(mig)
Expect(err).NotTo(HaveOccurred())
By(fmt.Sprintf("Initial size of %s: %d", mig, size))
originalSizes[mig] = size

2
test/e2e/framework/BUILD
View File

@@ -30,6 +30,7 @@ go_library(
"pv_util.go",
"resource_usage_gatherer.go",
"service_util.go",
"size.go",
"statefulset_utils.go",
"test_context.go",
"upgrade_util.go",
@@ -88,6 +89,7 @@ go_library(
"//vendor:github.com/aws/aws-sdk-go/aws",
"//vendor:github.com/aws/aws-sdk-go/aws/awserr",
"//vendor:github.com/aws/aws-sdk-go/aws/session",
"//vendor:github.com/aws/aws-sdk-go/service/autoscaling",
"//vendor:github.com/aws/aws-sdk-go/service/ec2",
"//vendor:github.com/golang/glog",
"//vendor:github.com/google/cadvisor/info/v1",

128
test/e2e/framework/size.go Normal file
View File

@@ -0,0 +1,128 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
import (
"fmt"
"os/exec"
"regexp"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/autoscaling"
awscloud "k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
)
const (
serveHostnameImage = "gcr.io/google_containers/serve_hostname:v1.4"
resizeNodeReadyTimeout = 2 * time.Minute
resizeNodeNotReadyTimeout = 2 * time.Minute
nodeReadinessTimeout = 3 * time.Minute
podNotReadyTimeout = 1 * time.Minute
podReadyTimeout = 2 * time.Minute
testPort = 9376
)
func ResizeGroup(group string, size int32) error {
if TestContext.ReportDir != "" {
CoreDump(TestContext.ReportDir)
defer CoreDump(TestContext.ReportDir)
}
if TestContext.Provider == "gce" || TestContext.Provider == "gke" {
// TODO: make this hit the compute API directly instead of shelling out to gcloud.
// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "resize",
group, fmt.Sprintf("--size=%v", size),
"--project="+TestContext.CloudConfig.ProjectID, "--zone="+TestContext.CloudConfig.Zone).CombinedOutput()
if err != nil {
Logf("Failed to resize node instance group: %v", string(output))
}
return err
} else if TestContext.Provider == "aws" {
client := autoscaling.New(session.New())
return awscloud.ResizeInstanceGroup(client, group, int(size))
} else {
return fmt.Errorf("Provider does not support InstanceGroups")
}
}
func GetGroupNodes(group string) ([]string, error) {
if TestContext.Provider == "gce" || TestContext.Provider == "gke" {
// TODO: make this hit the compute API directly instead of shelling out to gcloud.
// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
output, err := exec.Command("gcloud", "compute", "instance-groups", "managed",
"list-instances", group, "--project="+TestContext.CloudConfig.ProjectID,
"--zone="+TestContext.CloudConfig.Zone).CombinedOutput()
if err != nil {
return nil, err
}
re := regexp.MustCompile(".*RUNNING")
lines := re.FindAllString(string(output), -1)
for i, line := range lines {
lines[i] = line[:strings.Index(line, " ")]
}
return lines, nil
} else {
return nil, fmt.Errorf("provider does not support InstanceGroups")
}
}
func GroupSize(group string) (int, error) {
if TestContext.Provider == "gce" || TestContext.Provider == "gke" {
// TODO: make this hit the compute API directly instead of shelling out to gcloud.
// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
output, err := exec.Command("gcloud", "compute", "instance-groups", "managed",
"list-instances", group, "--project="+TestContext.CloudConfig.ProjectID,
"--zone="+TestContext.CloudConfig.Zone).CombinedOutput()
if err != nil {
return -1, err
}
re := regexp.MustCompile("RUNNING")
return len(re.FindAllString(string(output), -1)), nil
} else if TestContext.Provider == "aws" {
client := autoscaling.New(session.New())
instanceGroup, err := awscloud.DescribeInstanceGroup(client, group)
if err != nil {
return -1, fmt.Errorf("error describing instance group: %v", err)
}
if instanceGroup == nil {
return -1, fmt.Errorf("instance group not found: %s", group)
}
return instanceGroup.CurrentSize()
} else {
return -1, fmt.Errorf("provider does not support InstanceGroups")
}
}
func WaitForGroupSize(group string, size int32) error {
timeout := 30 * time.Minute
for start := time.Now(); time.Since(start) < timeout; time.Sleep(20 * time.Second) {
currentSize, err := GroupSize(group)
if err != nil {
Logf("Failed to get node instance group size: %v", err)
continue
}
if currentSize != int(size) {
Logf("Waiting for node instance group size %d, current size %d", size, currentSize)
continue
}
Logf("Node instance group has reached the desired size %d", size)
return nil
}
return fmt.Errorf("timeout waiting %v for node instance group size to be %d", timeout, size)
}

8
test/e2e/pd.go
View File

@@ -413,7 +413,7 @@ var _ = framework.KubeDescribe("Pod Disks", func() {
It("should be able to detach from a node which was deleted [Slow] [Disruptive] [Volume]", func() {
framework.SkipUnlessProviderIs("gce")
initialGroupSize, err := GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup)
initialGroupSize, err := framework.GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup)
framework.ExpectNoError(err, "Error getting group size")
By("Creating a pd")
@@ -461,13 +461,13 @@ var _ = framework.KubeDescribe("Pod Disks", func() {
// The disk should be detached from host0 on it's deletion
By("Waiting for pd to detach from host0")
waitForPDDetach(diskName, host0Name)
framework.ExpectNoError(WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get back the cluster to inital size")
framework.ExpectNoError(framework.WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get back the cluster to inital size")
return
})
It("should be able to detach from a node whose api object was deleted [Slow] [Disruptive] [Volume]", func() {
framework.SkipUnlessProviderIs("gce")
initialGroupSize, err := GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup)
initialGroupSize, err := framework.GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup)
framework.ExpectNoError(err, "Error getting group size")
By("Creating a pd")
diskName, err := framework.CreatePDWithRetry()
@@ -484,7 +484,7 @@ var _ = framework.KubeDescribe("Pod Disks", func() {
// need to set the resource version or else the Create() fails
_, err := nodeClient.Create(nodeToDelete)
framework.ExpectNoError(err, "Unable to re-create the deleted node")
framework.ExpectNoError(WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get the node group back to the original size")
framework.ExpectNoError(framework.WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get the node group back to the original size")
framework.WaitForNodeToBeReady(f.ClientSet, nodeToDelete.Name, nodeStatusTimeout)
if len(nodes.Items) != originalCount {
return fmt.Errorf("The node count is not back to original count")

106
test/e2e/resize_nodes.go
View File

@@ -18,8 +18,6 @@ package e2e
import (
"fmt"
"os/exec"
"regexp"
"strings"
"time"
@@ -29,11 +27,8 @@ import (
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/test/e2e/framework"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/autoscaling"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
awscloud "k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
)
const (
@@ -46,95 +41,6 @@ const (
testPort = 9376
)
func ResizeGroup(group string, size int32) error {
if framework.TestContext.ReportDir != "" {
framework.CoreDump(framework.TestContext.ReportDir)
defer framework.CoreDump(framework.TestContext.ReportDir)
}
if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
// TODO: make this hit the compute API directly instead of shelling out to gcloud.
// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "resize",
group, fmt.Sprintf("--size=%v", size),
"--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput()
if err != nil {
framework.Logf("Failed to resize node instance group: %v", string(output))
}
return err
} else if framework.TestContext.Provider == "aws" {
client := autoscaling.New(session.New())
return awscloud.ResizeInstanceGroup(client, group, int(size))
} else {
return fmt.Errorf("Provider does not support InstanceGroups")
}
}
func GetGroupNodes(group string) ([]string, error) {
if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
// TODO: make this hit the compute API directly instead of shelling out to gcloud.
// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
output, err := exec.Command("gcloud", "compute", "instance-groups", "managed",
"list-instances", group, "--project="+framework.TestContext.CloudConfig.ProjectID,
"--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput()
if err != nil {
return nil, err
}
re := regexp.MustCompile(".*RUNNING")
lines := re.FindAllString(string(output), -1)
for i, line := range lines {
lines[i] = line[:strings.Index(line, " ")]
}
return lines, nil
} else {
return nil, fmt.Errorf("provider does not support InstanceGroups")
}
}
func GroupSize(group string) (int, error) {
if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
// TODO: make this hit the compute API directly instead of shelling out to gcloud.
// TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic
output, err := exec.Command("gcloud", "compute", "instance-groups", "managed",
"list-instances", group, "--project="+framework.TestContext.CloudConfig.ProjectID,
"--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput()
if err != nil {
return -1, err
}
re := regexp.MustCompile("RUNNING")
return len(re.FindAllString(string(output), -1)), nil
} else if framework.TestContext.Provider == "aws" {
client := autoscaling.New(session.New())
instanceGroup, err := awscloud.DescribeInstanceGroup(client, group)
if err != nil {
return -1, fmt.Errorf("error describing instance group: %v", err)
}
if instanceGroup == nil {
return -1, fmt.Errorf("instance group not found: %s", group)
}
return instanceGroup.CurrentSize()
} else {
return -1, fmt.Errorf("provider does not support InstanceGroups")
}
}
func WaitForGroupSize(group string, size int32) error {
timeout := 30 * time.Minute
for start := time.Now(); time.Since(start) < timeout; time.Sleep(20 * time.Second) {
currentSize, err := GroupSize(group)
if err != nil {
framework.Logf("Failed to get node instance group size: %v", err)
continue
}
if currentSize != int(size) {
framework.Logf("Waiting for node instance group size %d, current size %d", size, currentSize)
continue
}
framework.Logf("Node instance group has reached the desired size %d", size)
return nil
}
return fmt.Errorf("timeout waiting %v for node instance group size to be %d", timeout, size)
}
func svcByName(name string, port int) *v1.Service {
return &v1.Service{
ObjectMeta: metav1.ObjectMeta{
@@ -213,7 +119,7 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() {
}
By("restoring the original node instance group size")
if err := ResizeGroup(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil {
if err := framework.ResizeGroup(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil {
framework.Failf("Couldn't restore the original node instance group size: %v", err)
}
// In GKE, our current tunneling setup has the potential to hold on to a broken tunnel (from a
@@ -228,7 +134,7 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() {
By("waiting 5 minutes for all dead tunnels to be dropped")
time.Sleep(5 * time.Minute)
}
if err := WaitForGroupSize(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil {
if err := framework.WaitForGroupSize(group, int32(framework.TestContext.CloudConfig.NumNodes)); err != nil {
framework.Failf("Couldn't restore the original node instance group size: %v", err)
}
if err := framework.WaitForClusterSize(c, framework.TestContext.CloudConfig.NumNodes, 10*time.Minute); err != nil {
@@ -253,9 +159,9 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() {
Expect(err).NotTo(HaveOccurred())
By(fmt.Sprintf("decreasing cluster size to %d", replicas-1))
err = ResizeGroup(group, replicas-1)
err = framework.ResizeGroup(group, replicas-1)
Expect(err).NotTo(HaveOccurred())
err = WaitForGroupSize(group, replicas-1)
err = framework.WaitForGroupSize(group, replicas-1)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForClusterSize(c, int(replicas-1), 10*time.Minute)
Expect(err).NotTo(HaveOccurred())
@@ -281,9 +187,9 @@ var _ = framework.KubeDescribe("Nodes [Disruptive]", func() {
Expect(err).NotTo(HaveOccurred())
By(fmt.Sprintf("increasing cluster size to %d", replicas+1))
err = ResizeGroup(group, replicas+1)
err = framework.ResizeGroup(group, replicas+1)
Expect(err).NotTo(HaveOccurred())
err = WaitForGroupSize(group, replicas+1)
err = framework.WaitForGroupSize(group, replicas+1)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForClusterSize(c, int(replicas+1), 10*time.Minute)
Expect(err).NotTo(HaveOccurred())