Merge pull request #37998 from ymqytw/add_test_for_concurrent_evictions

Automatic merge from submit-queue (batch tested with PRs 36071, 32752, 37998, 38350, 38401)

Add test for concurrent evictions requests

This is a followup PR after #37668.
Add a test case to make sure concurrent eviction requests can be handled.

@davidopp @lavalamp

pkg/apis/policy/v1beta1/types.go

@@ -89,6 +89,9 @@ type PodDisruptionBudgetList struct {
 	Items           []PodDisruptionBudget `json:"items" protobuf:"bytes,2,rep,name=items"`
 }
 
+// +genclient=true
+// +noMethods=true
+
 // Eviction evicts a pod from its node subject to certain policies and safety constraints.
 // This is a subresource of Pod.  A request to cause such an eviction is
 // created by POSTing to .../pods/<pod name>/evictions.

pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/BUILD

@@ -14,6 +14,8 @@ go_library(
     name = "go_default_library",
     srcs = [
         "doc.go",
+        "eviction.go",
+        "eviction_expansion.go",
         "generated_expansion.go",
         "poddisruptionbudget.go",
         "policy_client.go",

pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/eviction.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	restclient "k8s.io/kubernetes/pkg/client/restclient"
+)
+
+// EvictionsGetter has a method to return a EvictionInterface.
+// A group's client should implement this interface.
+type EvictionsGetter interface {
+	Evictions(namespace string) EvictionInterface
+}
+
+// EvictionInterface has methods to work with Eviction resources.
+type EvictionInterface interface {
+	EvictionExpansion
+}
+
+// evictions implements EvictionInterface
+type evictions struct {
+	client restclient.Interface
+	ns     string
+}
+
+// newEvictions returns a Evictions
+func newEvictions(c *PolicyV1beta1Client, namespace string) *evictions {
+	return &evictions{
+		client: c.RESTClient(),
+		ns:     namespace,
+	}
+}

pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/eviction_expansion.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	policy "k8s.io/kubernetes/pkg/apis/policy/v1beta1"
+)
+
+// The EvictionExpansion interface allows manually adding extra methods to the ScaleInterface.
+type EvictionExpansion interface {
+	Evict(eviction *policy.Eviction) error
+}
+
+func (c *evictions) Evict(eviction *policy.Eviction) error {
+	return c.client.Post().
+		AbsPath("/api/v1").
+		Namespace(eviction.Namespace).
+		Resource("pods").
+		Name(eviction.Name).
+		SubResource("eviction").
+		Body(eviction).
+		Do().
+		Error()
+}

pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/BUILD

@@ -14,6 +14,8 @@ go_library(
     name = "go_default_library",
     srcs = [
         "doc.go",
+        "fake_eviction.go",
+        "fake_eviction_expansion.go",
         "fake_poddisruptionbudget.go",
         "fake_policy_client.go",
     ],

pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_eviction.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fake
+
+// FakeEvictions implements EvictionInterface
+type FakeEvictions struct {
+	Fake *FakePolicyV1beta1
+	ns   string
+}

pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_eviction_expansion.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fake
+
+import (
+	policy "k8s.io/kubernetes/pkg/apis/policy/v1beta1"
+	core "k8s.io/kubernetes/pkg/client/testing/core"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+)
+
+func (c *FakeEvictions) Evict(eviction *policy.Eviction) error {
+	action := core.GetActionImpl{}
+	action.Verb = "post"
+	action.Namespace = c.ns
+	action.Resource = schema.GroupVersionResource{Group: "", Version: "", Resource: "pods"}
+	action.Subresource = "eviction"
+	_, err := c.Fake.Invokes(action, eviction)
+	return err
+}

pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/fake/fake_policy_client.go

@@ -26,6 +26,10 @@ type FakePolicyV1beta1 struct {
 	*core.Fake
 }
 
+func (c *FakePolicyV1beta1) Evictions(namespace string) v1beta1.EvictionInterface {
+	return &FakeEvictions{c, namespace}
+}
+
 func (c *FakePolicyV1beta1) PodDisruptionBudgets(namespace string) v1beta1.PodDisruptionBudgetInterface {
 	return &FakePodDisruptionBudgets{c, namespace}
 }

pkg/client/clientset_generated/release_1_5/typed/policy/v1beta1/policy_client.go

@@ -27,6 +27,7 @@ import (
 
 type PolicyV1beta1Interface interface {
 	RESTClient() restclient.Interface
+	EvictionsGetter
 	PodDisruptionBudgetsGetter
 }
 
@@ -35,6 +36,10 @@ type PolicyV1beta1Client struct {
 	restClient restclient.Interface
 }
 
+func (c *PolicyV1beta1Client) Evictions(namespace string) EvictionInterface {
+	return newEvictions(c, namespace)
+}
+
 func (c *PolicyV1beta1Client) PodDisruptionBudgets(namespace string) PodDisruptionBudgetInterface {
 	return newPodDisruptionBudgets(c, namespace)
 }

pkg/client/listers/policy/v1beta1/BUILD

@@ -13,6 +13,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "eviction.go",
         "expansion_generated.go",
         "poddisruptionbudget.go",
     ],

pkg/client/listers/policy/v1beta1/eviction.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This file was automatically generated by lister-gen with arguments: --input-dirs=[k8s.io/kubernetes/pkg/api,k8s.io/kubernetes/pkg/api/v1,k8s.io/kubernetes/pkg/apis/abac,k8s.io/kubernetes/pkg/apis/abac/v0,k8s.io/kubernetes/pkg/apis/abac/v1beta1,k8s.io/kubernetes/pkg/apis/apps,k8s.io/kubernetes/pkg/apis/apps/v1beta1,k8s.io/kubernetes/pkg/apis/authentication,k8s.io/kubernetes/pkg/apis/authentication/v1beta1,k8s.io/kubernetes/pkg/apis/authorization,k8s.io/kubernetes/pkg/apis/authorization/v1beta1,k8s.io/kubernetes/pkg/apis/autoscaling,k8s.io/kubernetes/pkg/apis/autoscaling/v1,k8s.io/kubernetes/pkg/apis/batch,k8s.io/kubernetes/pkg/apis/batch/v1,k8s.io/kubernetes/pkg/apis/batch/v2alpha1,k8s.io/kubernetes/pkg/apis/certificates,k8s.io/kubernetes/pkg/apis/certificates/v1alpha1,k8s.io/kubernetes/pkg/apis/componentconfig,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,k8s.io/kubernetes/pkg/apis/extensions,k8s.io/kubernetes/pkg/apis/extensions/v1beta1,k8s.io/kubernetes/pkg/apis/imagepolicy,k8s.io/kubernetes/pkg/apis/imagepolicy/v1alpha1,k8s.io/kubernetes/pkg/apis/meta/v1,k8s.io/kubernetes/pkg/apis/policy,k8s.io/kubernetes/pkg/apis/policy/v1alpha1,k8s.io/kubernetes/pkg/apis/policy/v1beta1,k8s.io/kubernetes/pkg/apis/rbac,k8s.io/kubernetes/pkg/apis/rbac/v1alpha1,k8s.io/kubernetes/pkg/apis/storage,k8s.io/kubernetes/pkg/apis/storage/v1beta1]
+
+package v1beta1
+
+import (
+	"k8s.io/kubernetes/pkg/api/errors"
+	policy "k8s.io/kubernetes/pkg/apis/policy"
+	v1beta1 "k8s.io/kubernetes/pkg/apis/policy/v1beta1"
+	"k8s.io/kubernetes/pkg/client/cache"
+	"k8s.io/kubernetes/pkg/labels"
+)
+
+// EvictionLister helps list Evictions.
+type EvictionLister interface {
+	// List lists all Evictions in the indexer.
+	List(selector labels.Selector) (ret []*v1beta1.Eviction, err error)
+	// Evictions returns an object that can list and get Evictions.
+	Evictions(namespace string) EvictionNamespaceLister
+	EvictionListerExpansion
+}
+
+// evictionLister implements the EvictionLister interface.
+type evictionLister struct {
+	indexer cache.Indexer
+}
+
+// NewEvictionLister returns a new EvictionLister.
+func NewEvictionLister(indexer cache.Indexer) EvictionLister {
+	return &evictionLister{indexer: indexer}
+}
+
+// List lists all Evictions in the indexer.
+func (s *evictionLister) List(selector labels.Selector) (ret []*v1beta1.Eviction, err error) {
+	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1beta1.Eviction))
+	})
+	return ret, err
+}
+
+// Evictions returns an object that can list and get Evictions.
+func (s *evictionLister) Evictions(namespace string) EvictionNamespaceLister {
+	return evictionNamespaceLister{indexer: s.indexer, namespace: namespace}
+}
+
+// EvictionNamespaceLister helps list and get Evictions.
+type EvictionNamespaceLister interface {
+	// List lists all Evictions in the indexer for a given namespace.
+	List(selector labels.Selector) (ret []*v1beta1.Eviction, err error)
+	// Get retrieves the Eviction from the indexer for a given namespace and name.
+	Get(name string) (*v1beta1.Eviction, error)
+	EvictionNamespaceListerExpansion
+}
+
+// evictionNamespaceLister implements the EvictionNamespaceLister
+// interface.
+type evictionNamespaceLister struct {
+	indexer   cache.Indexer
+	namespace string
+}
+
+// List lists all Evictions in the indexer for a given namespace.
+func (s evictionNamespaceLister) List(selector labels.Selector) (ret []*v1beta1.Eviction, err error) {
+	err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1beta1.Eviction))
+	})
+	return ret, err
+}
+
+// Get retrieves the Eviction from the indexer for a given namespace and name.
+func (s evictionNamespaceLister) Get(name string) (*v1beta1.Eviction, error) {
+	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(policy.Resource("eviction"), name)
+	}
+	return obj.(*v1beta1.Eviction), nil
+}

pkg/client/listers/policy/v1beta1/expansion_generated.go

@@ -18,6 +18,14 @@ limitations under the License.
 
 package v1beta1
 
+// EvictionListerExpansion allows custom methods to be added to
+// EvictionLister.
+type EvictionListerExpansion interface{}
+
+// EvictionNamespaceListerExpansion allows custom methods to be added to
+// EvictionNamespaeLister.
+type EvictionNamespaceListerExpansion interface{}
+
 // PodDisruptionBudgetListerExpansion allows custom methods to be added to
 // PodDisruptionBudgetLister.
 type PodDisruptionBudgetListerExpansion interface{}

test/integration/evictions/evictions_test.go

@@ -0,0 +1,257 @@
+// +build integration,!no-etcd
+
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package evictions
+
+import (
+	"fmt"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api/errors"
+	"k8s.io/kubernetes/pkg/api/v1"
+	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/apis/policy/v1beta1"
+	"k8s.io/kubernetes/pkg/client/cache"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
+	"k8s.io/kubernetes/pkg/client/restclient"
+	"k8s.io/kubernetes/pkg/controller/disruption"
+	"k8s.io/kubernetes/pkg/controller/informers"
+	"k8s.io/kubernetes/pkg/util/intstr"
+	"k8s.io/kubernetes/pkg/util/wait"
+	"k8s.io/kubernetes/test/integration/framework"
+)
+
+const (
+	defaultTimeout = 10 * time.Minute
+)
+
+func newPod(podName string) *v1.Pod {
+	return &v1.Pod{
+		ObjectMeta: v1.ObjectMeta{
+			Name:   podName,
+			Labels: map[string]string{"app": "test-evictions"},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "fake-name",
+					Image: "fakeimage",
+				},
+			},
+		},
+	}
+}
+
+func addPodConditionReady(pod *v1.Pod) {
+	pod.Status = v1.PodStatus{
+		Phase: v1.PodRunning,
+		Conditions: []v1.PodCondition{
+			{
+				Type:   v1.PodReady,
+				Status: v1.ConditionTrue,
+			},
+		},
+	}
+}
+
+func newPDB() *v1beta1.PodDisruptionBudget {
+	return &v1beta1.PodDisruptionBudget{
+		ObjectMeta: v1.ObjectMeta{
+			Name: "test-pdb",
+		},
+		Spec: v1beta1.PodDisruptionBudgetSpec{
+			MinAvailable: intstr.IntOrString{
+				Type:   intstr.Int,
+				IntVal: 0,
+			},
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"app": "test-evictions"},
+			},
+		},
+	}
+}
+
+func newEviction(ns, evictionName string, deleteOption *v1.DeleteOptions) *v1beta1.Eviction {
+	return &v1beta1.Eviction{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "Policy/v1beta1",
+			Kind:       "Eviction",
+		},
+		ObjectMeta: v1.ObjectMeta{
+			Name:      evictionName,
+			Namespace: ns,
+		},
+		DeleteOptions: deleteOption,
+	}
+}
+
+func rmSetup(t *testing.T) (*httptest.Server, *disruption.DisruptionController, cache.SharedIndexInformer, clientset.Interface) {
+	masterConfig := framework.NewIntegrationTestMasterConfig()
+	_, s := framework.RunAMaster(masterConfig)
+
+	config := restclient.Config{Host: s.URL}
+	clientSet, err := clientset.NewForConfig(&config)
+	if err != nil {
+		t.Fatalf("Error in create clientset: %v", err)
+	}
+	resyncPeriod := 12 * time.Hour
+	informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pdb-informers")), nil, resyncPeriod)
+
+	rm := disruption.NewDisruptionController(
+		informers.Pods().Informer(),
+		clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "disruption-controller")),
+	)
+	return s, rm, informers.Pods().Informer(), clientSet
+}
+
+func TestConcurrentEvictionRequests(t *testing.T) {
+	podNameFormat := "test-pod-%d"
+
+	s, rm, podInformer, clientSet := rmSetup(t)
+	defer s.Close()
+
+	ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t)
+	defer framework.DeleteTestingNamespace(ns, s, t)
+
+	stopCh := make(chan struct{})
+	go podInformer.Run(stopCh)
+	go rm.Run(stopCh)
+
+	config := restclient.Config{Host: s.URL}
+	clientSet, err := clientset.NewForConfig(&config)
+
+	var gracePeriodSeconds int64 = 30
+	deleteOption := &v1.DeleteOptions{
+		GracePeriodSeconds: &gracePeriodSeconds,
+	}
+
+	// Generate 10 pods to evict
+	for i := 0; i < 10; i++ {
+		podName := fmt.Sprintf(podNameFormat, i)
+		pod := newPod(podName)
+
+		if _, err := clientSet.Core().Pods(ns.Name).Create(pod); err != nil {
+			t.Errorf("Failed to create pod: %v", err)
+		}
+
+		addPodConditionReady(pod)
+		if _, err := clientSet.Core().Pods(ns.Name).UpdateStatus(pod); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	waitToObservePods(t, podInformer, 10)
+
+	pdb := newPDB()
+	if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil {
+		t.Errorf("Failed to create PodDisruptionBudget: %v", err)
+	}
+
+	waitPDBStable(t, clientSet, 10, ns.Name, pdb.Name)
+
+	doneCh := make(chan bool, 10)
+	errCh := make(chan error, 1)
+	// spawn 10 goroutine to concurrently evict the pods
+	for i := 0; i < 10; i++ {
+		go func(id int, doneCh chan bool, errCh chan error) {
+			evictionName := fmt.Sprintf(podNameFormat, id)
+			eviction := newEviction(ns.Name, evictionName, deleteOption)
+
+			var e error
+			for {
+				e = clientSet.Policy().Evictions(ns.Name).Evict(eviction)
+				if errors.IsTooManyRequests(e) {
+					time.Sleep(5 * time.Second)
+				} else {
+					break
+				}
+			}
+			if e != nil {
+				if errors.IsConflict(err) {
+					fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
+				} else {
+					errCh <- e
+				}
+				return
+			}
+			doneCh <- true
+		}(i, doneCh, errCh)
+	}
+
+	doneCount := 0
+	for {
+		select {
+		case err := <-errCh:
+			t.Errorf("%v", err)
+			return
+		case <-doneCh:
+			doneCount++
+			if doneCount == 10 {
+				return
+			}
+		case <-time.After(defaultTimeout):
+			t.Errorf("Eviction did not complete within %v", defaultTimeout)
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		podName := fmt.Sprintf(podNameFormat, i)
+		_, err := clientSet.Core().Pods(ns.Name).Get(podName)
+		if !errors.IsNotFound(err) {
+			t.Errorf("Pod %q is expected to be evicted", podName)
+		}
+	}
+
+	if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil {
+		t.Errorf("Failed to delete PodDisruptionBudget: %v", err)
+	}
+
+	close(stopCh)
+}
+
+// wait for the podInformer to observe the pods. Call this function before
+// running the RS controller to prevent the rc manager from creating new pods
+// rather than adopting the existing ones.
+func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int) {
+	if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) {
+		objects := podInformer.GetIndexer().List()
+		if len(objects) == podNum {
+			return true, nil
+		}
+		return false, nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func waitPDBStable(t *testing.T, clientSet clientset.Interface, podNum int32, ns, pdbName string) {
+	if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) {
+		pdb, err := clientSet.Policy().PodDisruptionBudgets(ns).Get(pdbName)
+		if err != nil {
+			return false, err
+		}
+		if pdb.Status.CurrentHealthy != podNum {
+			return false, nil
+		}
+		return true, nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}