implement taints and tolerations

plugin/pkg/scheduler/algorithm/predicates/error.go

@@ -32,6 +32,7 @@ var (
 	ErrVolumeZoneConflict        = newPredicateFailureError("NoVolumeZoneConflict")
 	ErrNodeSelectorNotMatch      = newPredicateFailureError("MatchNodeSelector")
 	ErrPodAffinityNotMatch       = newPredicateFailureError("MatchInterPodAffinity")
+	ErrTaintsTolerationsNotMatch = newPredicateFailureError("PodToleratesNodeTaints")
 	ErrPodNotMatchHostName       = newPredicateFailureError("HostName")
 	ErrPodNotFitsHostPorts       = newPredicateFailureError("PodFitsHostPorts")
 	ErrNodeLabelPresenceViolated = newPredicateFailureError("CheckNodeLabelPresence")

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -944,3 +944,58 @@ func (checker *PodAffinityChecker) NodeMatchPodAffinityAntiAffinity(pod *api.Pod
 	}
 	return true
 }
+
+type TolerationMatch struct {
+	info NodeInfo
+}
+
+func NewTolerationMatchPredicate(info NodeInfo) algorithm.FitPredicate {
+	tolerationMatch := &TolerationMatch{
+		info: info,
+	}
+	return tolerationMatch.PodToleratesNodeTaints
+}
+
+func (t *TolerationMatch) PodToleratesNodeTaints(pod *api.Pod, nodeInfo *schedulercache.NodeInfo) (bool, error) {
+	node := nodeInfo.Node()
+
+	taints, err := api.GetTaintsFromNodeAnnotations(node.Annotations)
+	if err != nil {
+		return false, err
+	}
+
+	tolerations, err := api.GetTolerationsFromPodAnnotations(pod.Annotations)
+	if err != nil {
+		return false, err
+	}
+
+	if tolerationsToleratesTaints(tolerations, taints) {
+		return true, nil
+	}
+	return false, ErrTaintsTolerationsNotMatch
+}
+
+func tolerationsToleratesTaints(tolerations []api.Toleration, taints []api.Taint) bool {
+	// If the taint list is nil/empty, it is tolerated by all tolerations by default.
+	if len(taints) == 0 {
+		return true
+	}
+
+	// The taint list isn't nil/empty, a nil/empty toleration list can't tolerate them.
+	if len(tolerations) == 0 {
+		return false
+	}
+
+	for _, taint := range taints {
+		// skip taints that have effect PreferNoSchedule, since it is for priorities
+		if taint.Effect == api.TaintEffectPreferNoSchedule {
+			continue
+		}
+
+		if !api.TaintToleratedByTolerations(taint, tolerations) {
+			return false
+		}
+	}
+
+	return true
+}

plugin/pkg/scheduler/algorithm/predicates/predicates_test.go

@@ -2358,3 +2358,286 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) {
 		}
 	}
 }
+
+func TestPodToleratesTaints(t *testing.T) {
+	podTolerateTaintsTests := []struct {
+		pod  *api.Pod
+		node api.Node
+		fits bool
+		test string
+	}{
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "pod0",
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+			},
+			fits: false,
+			test: "a pod having no tolerations can't be scheduled onto a node with nonempty taints",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "pod1",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod1:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+			},
+			fits: true,
+			test: "a pod which can be scheduled on a dedicated node assgined to user1 with effect NoSchedule",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "pod2",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"operator": "Equal",
+							"value": "user2",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod2:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+			},
+			fits: false,
+			test: "a pod which can't be scheduled on a dedicated node assgined to user2 with effect NoSchedule",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "pod2",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "foo",
+							"operator": "Exists",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod2:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "foo",
+							"value": "bar",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+			},
+			fits: true,
+			test: "a pod can be scheduled onto the node, with a toleration uses operator Exists that tolerates the taints on the node",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "pod2",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"operator": "Equal",
+							"value": "user2",
+							"effect": "NoSchedule"
+						}, {
+							"key": "foo",
+							"operator": "Exists",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod2:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user2",
+							"effect": "NoSchedule"
+						}, {
+							"key": "foo",
+							"value": "bar",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+			},
+			fits: true,
+			test: "a pod has multiple tolerations, node has multiple taints, all the taints are tolerated, pod can be scheduled onto the node",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "pod2",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "foo",
+							"operator": "Equal",
+							"value": "bar",
+							"effect": "PreferNoSchedule"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod2:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "foo",
+							"value": "bar",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+			},
+			fits: false,
+			test: "a pod has a toleration that keys and values match the taint on the node, but (non-empty) effect doesn't match, " +
+				"can't be scheduled onto the node",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "pod2",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "foo",
+							"operator": "Equal",
+							"value": "bar"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod2:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "foo",
+							"value": "bar",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+			},
+			fits: true,
+			test: "The pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " +
+				"and the effect of taint is NoSchedule. Pod can be scheduled onto the node",
+		},
+		{
+			pod: &api.Pod{
+				ObjectMeta: api.ObjectMeta{
+					Name: "pod2",
+					Annotations: map[string]string{
+						api.TolerationsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"operator": "Equal",
+							"value": "user2",
+							"effect": "NoSchedule"
+						}]`,
+					},
+				},
+				Spec: api.PodSpec{
+					Containers: []api.Container{{Image: "pod2:V1"}},
+				},
+			},
+			node: api.Node{
+				ObjectMeta: api.ObjectMeta{
+					Annotations: map[string]string{
+						api.TaintsAnnotationKey: `
+						[{
+							"key": "dedicated",
+							"value": "user1",
+							"effect": "PreferNoSchedule"
+						}]`,
+					},
+				},
+			},
+			fits: true,
+			test: "The pod has a toleration that key and value don't match the taint on the node, " +
+				"but the effect of taint on node is PreferNochedule. Pod can be shceduled onto the node",
+		},
+	}
+
+	for _, test := range podTolerateTaintsTests {
+		tolerationMatch := TolerationMatch{FakeNodeInfo(test.node)}
+		nodeInfo := schedulercache.NewNodeInfo()
+		nodeInfo.SetNode(&test.node)
+		fits, err := tolerationMatch.PodToleratesNodeTaints(test.pod, nodeInfo)
+		if fits == false && !reflect.DeepEqual(err, ErrTaintsTolerationsNotMatch) {
+			t.Errorf("%s, unexpected error: %v", test.test, err)
+		}
+		if fits != test.fits {
+			t.Errorf("%s, expected: %v got %v", test.test, test.fits, fits)
+		}
+	}
+}

plugin/pkg/scheduler/algorithm/priorities/taint_toleration.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package priorities
+
+import (
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
+	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
+	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
+)
+
+// NodeTaints hold the node lister
+type TaintToleration struct {
+	nodeLister algorithm.NodeLister
+}
+
+// NewTaintTolerationPriority
+func NewTaintTolerationPriority(nodeLister algorithm.NodeLister) algorithm.PriorityFunction {
+	taintToleration := &TaintToleration{
+		nodeLister: nodeLister,
+	}
+	return taintToleration.ComputeTaintTolerationPriority
+}
+
+// CountIntolerableTaintsPreferNoSchedule gives the count of intolerable taints of a pod with effect PreferNoSchedule
+func countIntolerableTaintsPreferNoSchedule(taints []api.Taint, tolerations []api.Toleration) (intolerableTaints int) {
+	for _, taint := range taints {
+		// check only on taints that have effect PreferNoSchedule
+		if taint.Effect != api.TaintEffectPreferNoSchedule {
+			continue
+		}
+
+		if !api.TaintToleratedByTolerations(taint, tolerations) {
+			intolerableTaints++
+		}
+	}
+	return
+}
+
+// getAllTolerationEffectPreferNoSchedule gets the list of all Toleration with Effect PreferNoSchedule
+func getAllTolerationPreferNoSchedule(tolerations []api.Toleration) (tolerationList []api.Toleration) {
+	for _, toleration := range tolerations {
+		if len(toleration.Effect) == 0 || toleration.Effect == api.TaintEffectPreferNoSchedule {
+			tolerationList = append(tolerationList, toleration)
+		}
+	}
+	return
+}
+
+// ComputeTaintTolerationPriority prepares the priority list for all the nodes based on the number of intolerable taints on the node
+func (s *TaintToleration) ComputeTaintTolerationPriority(pod *api.Pod, nodeNameToInfo map[string]*schedulercache.NodeInfo, nodeLister algorithm.NodeLister) (schedulerapi.HostPriorityList, error) {
+	// counts hold the count of intolerable taints of a pod for a given node
+	counts := make(map[string]int)
+
+	// the max value of counts
+	var maxCount int
+
+	nodes, err := nodeLister.List()
+	if err != nil {
+		return nil, err
+	}
+
+	tolerations, err := api.GetTolerationsFromPodAnnotations(pod.Annotations)
+	if err != nil {
+		return nil, err
+	}
+	// Fetch a list of all toleration with effect PreferNoSchedule
+	tolerationList := getAllTolerationPreferNoSchedule(tolerations)
+
+	// calculate the intolerable taints for all the nodes
+	for _, node := range nodes.Items {
+		taints, err := api.GetTaintsFromNodeAnnotations(node.Annotations)
+		if err != nil {
+			return nil, err
+		}
+
+		count := countIntolerableTaintsPreferNoSchedule(taints, tolerationList)
+		counts[node.Name] = count
+		if count > maxCount {
+			maxCount = count
+		}
+	}
+
+	// The maximum priority value to give to a node
+	// Priority values range from 0 - maxPriority
+	const maxPriority = 10
+	result := []schedulerapi.HostPriority{}
+	for _, node := range nodes.Items {
+		fScore := float64(maxPriority)
+		if maxCount > 0 {
+			fScore = (1.0 - float64(counts[node.Name])/float64(maxCount)) * 10
+		}
+		result = append(result, schedulerapi.HostPriority{Host: node.Name, Score: int(fScore)})
+	}
+	return result, nil
+}

plugin/pkg/scheduler/algorithm/priorities/taint_toleration_test.go

@@ -0,0 +1,229 @@
+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package priorities
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
+	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
+	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
+)
+
+func nodeWithTaints(nodeName string, taints []api.Taint) api.Node {
+	taintsData, _ := json.Marshal(taints)
+	return api.Node{
+		ObjectMeta: api.ObjectMeta{
+			Name: nodeName,
+			Annotations: map[string]string{
+				api.TaintsAnnotationKey: string(taintsData),
+			},
+		},
+	}
+}
+
+func podWithTolerations(tolerations []api.Toleration) *api.Pod {
+	tolerationData, _ := json.Marshal(tolerations)
+	return &api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			Annotations: map[string]string{
+				api.TolerationsAnnotationKey: string(tolerationData),
+			},
+		},
+	}
+}
+
+// This function will create a set of nodes and pods and test the priority
+// Nodes with zero,one,two,three,four and hundred taints are created
+// Pods with zero,one,two,three,four and hundred tolerations are created
+
+func TestTaintAndToleration(t *testing.T) {
+	tests := []struct {
+		pod          *api.Pod
+		nodes        []api.Node
+		expectedList schedulerapi.HostPriorityList
+		test         string
+	}{
+		// basic test case
+		{
+			test: "node with taints tolerated by the pod, gets a higher score than those node with intolerable taints",
+			pod: podWithTolerations([]api.Toleration{{
+				Key:      "foo",
+				Operator: api.TolerationOpEqual,
+				Value:    "bar",
+				Effect:   api.TaintEffectPreferNoSchedule,
+			}}),
+			nodes: []api.Node{
+				nodeWithTaints("nodeA", []api.Taint{{
+					Key:    "foo",
+					Value:  "bar",
+					Effect: api.TaintEffectPreferNoSchedule,
+				}}),
+				nodeWithTaints("nodeB", []api.Taint{{
+					Key:    "foo",
+					Value:  "blah",
+					Effect: api.TaintEffectPreferNoSchedule,
+				}}),
+			},
+			expectedList: []schedulerapi.HostPriority{
+				{Host: "nodeA", Score: 10},
+				{Host: "nodeB", Score: 0},
+			},
+		},
+		// the count of taints that are tolerated by pod, does not matter.
+		{
+			test: "the nodes that all of their taints are tolerated by the pod, get the same score, no matter how many tolerable taints a node has",
+			pod: podWithTolerations([]api.Toleration{
+				{
+					Key:      "cpu-type",
+					Operator: api.TolerationOpEqual,
+					Value:    "arm64",
+					Effect:   api.TaintEffectPreferNoSchedule,
+				}, {
+					Key:      "disk-type",
+					Operator: api.TolerationOpEqual,
+					Value:    "ssd",
+					Effect:   api.TaintEffectPreferNoSchedule,
+				},
+			}),
+			nodes: []api.Node{
+				nodeWithTaints("nodeA", []api.Taint{}),
+				nodeWithTaints("nodeB", []api.Taint{
+					{
+						Key:    "cpu-type",
+						Value:  "arm64",
+						Effect: api.TaintEffectPreferNoSchedule,
+					},
+				}),
+				nodeWithTaints("nodeC", []api.Taint{
+					{
+						Key:    "cpu-type",
+						Value:  "arm64",
+						Effect: api.TaintEffectPreferNoSchedule,
+					}, {
+						Key:    "disk-type",
+						Value:  "ssd",
+						Effect: api.TaintEffectPreferNoSchedule,
+					},
+				}),
+			},
+			expectedList: []schedulerapi.HostPriority{
+				{Host: "nodeA", Score: 10},
+				{Host: "nodeB", Score: 10},
+				{Host: "nodeC", Score: 10},
+			},
+		},
+		// the count of taints on a node that are not tolerated by pod, matters.
+		{
+			test: "the more intolerable taints a node has, the lower score it gets.",
+			pod: podWithTolerations([]api.Toleration{{
+				Key:      "foo",
+				Operator: api.TolerationOpEqual,
+				Value:    "bar",
+				Effect:   api.TaintEffectPreferNoSchedule,
+			}}),
+			nodes: []api.Node{
+				nodeWithTaints("nodeA", []api.Taint{}),
+				nodeWithTaints("nodeB", []api.Taint{
+					{
+						Key:    "cpu-type",
+						Value:  "arm64",
+						Effect: api.TaintEffectPreferNoSchedule,
+					},
+				}),
+				nodeWithTaints("nodeC", []api.Taint{
+					{
+						Key:    "cpu-type",
+						Value:  "arm64",
+						Effect: api.TaintEffectPreferNoSchedule,
+					}, {
+						Key:    "disk-type",
+						Value:  "ssd",
+						Effect: api.TaintEffectPreferNoSchedule,
+					},
+				}),
+			},
+			expectedList: []schedulerapi.HostPriority{
+				{Host: "nodeA", Score: 10},
+				{Host: "nodeB", Score: 5},
+				{Host: "nodeC", Score: 0},
+			},
+		},
+		// taints-tolerations priority only takes care about the taints and tolerations that have effect PreferNoSchedule
+		{
+			test: "only taints and tolerations that have effect PreferNoSchedule are checked by taints-tolerations priority function",
+			pod: podWithTolerations([]api.Toleration{
+				{
+					Key:      "cpu-type",
+					Operator: api.TolerationOpEqual,
+					Value:    "arm64",
+					Effect:   api.TaintEffectNoSchedule,
+				}, {
+					Key:      "disk-type",
+					Operator: api.TolerationOpEqual,
+					Value:    "ssd",
+					Effect:   api.TaintEffectNoSchedule,
+				},
+			}),
+			nodes: []api.Node{
+				nodeWithTaints("nodeA", []api.Taint{}),
+				nodeWithTaints("nodeB", []api.Taint{
+					{
+						Key:    "cpu-type",
+						Value:  "arm64",
+						Effect: api.TaintEffectNoSchedule,
+					},
+				}),
+				nodeWithTaints("nodeC", []api.Taint{
+					{
+						Key:    "cpu-type",
+						Value:  "arm64",
+						Effect: api.TaintEffectPreferNoSchedule,
+					}, {
+						Key:    "disk-type",
+						Value:  "ssd",
+						Effect: api.TaintEffectPreferNoSchedule,
+					},
+				}),
+			},
+			expectedList: []schedulerapi.HostPriority{
+				{Host: "nodeA", Score: 10},
+				{Host: "nodeB", Score: 10},
+				{Host: "nodeC", Score: 0},
+			},
+		},
+	}
+	for _, test := range tests {
+		nodeNameToInfo := schedulercache.CreateNodeNameToInfoMap([]*api.Pod{{}})
+		taintToleration := TaintToleration{nodeLister: algorithm.FakeNodeLister(api.NodeList{Items: test.nodes})}
+		list, err := taintToleration.ComputeTaintTolerationPriority(
+			test.pod,
+			nodeNameToInfo,
+			algorithm.FakeNodeLister(api.NodeList{Items: test.nodes}))
+		if err != nil {
+			t.Errorf("%s, unexpected error: %v", test.test, err)
+		}
+
+		if !reflect.DeepEqual(test.expectedList, list) {
+			t.Errorf("%s,\nexpected:\n\t%+v,\ngot:\n\t%+v", test.test, test.expectedList, list)
+		}
+	}
+
+}