scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-11-04 04:08:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #86205 from Huang-Wei/eps-pred-prefitler

Browse Source 
Move EvenPodsSpread metadata computation logic as a PreFilter Plugin
This commit is contained in:
Kubernetes Prow Robot
2019-12-13 21:19:36 -08:00
committed by GitHub
parent efe159e8d2 3d353f5122
commit b92d20033e
15 changed files with 411 additions and 741 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
pkg/scheduler/algorithm/predicates/metadata.go
View File

@@ -105,11 +105,11 @@ func (paths *criticalPaths) update(tpVal string, num int32) {
}
}
// evenPodsSpreadMetadata combines tpKeyToCriticalPaths and tpPairToMatchNum
// PodTopologySpreadMetadata combines tpKeyToCriticalPaths and tpPairToMatchNum
// to represent:
// (1) critical paths where the least pods are matched on each spread constraint.
// (2) number of pods matched on each spread constraint.
type evenPodsSpreadMetadata struct {
type PodTopologySpreadMetadata struct {
constraints []topologySpreadConstraint
// We record 2 critical paths instead of all critical paths here.
// criticalPaths[0].matchNum always holds the minimum matching number.
@@ -309,10 +309,6 @@ func (m *podFitsResourcesMetadata) clone() *podFitsResourcesMetadata {
type predicateMetadata struct {
pod *v1.Pod
// evenPodsSpreadMetadata holds info of the minimum match number on each topology spread constraint,
// and the match number of all valid topology pairs.
evenPodsSpreadMetadata *evenPodsSpreadMetadata
serviceAffinityMetadata *serviceAffinityMetadata
podFitsResourcesMetadata *podFitsResourcesMetadata
}
@@ -357,27 +353,8 @@ func (f *MetadataProducerFactory) GetPredicateMetadata(pod *v1.Pod, sharedLister
return nil
}
var allNodes []*schedulernodeinfo.NodeInfo
if sharedLister != nil {
var err error
allNodes, err = sharedLister.NodeInfos().List()
if err != nil {
klog.Errorf("failed to list NodeInfos: %v", err)
return nil
}
}
// evenPodsSpreadMetadata represents how existing pods match "pod"
// on its spread constraints
evenPodsSpreadMetadata, err := getEvenPodsSpreadMetadata(pod, allNodes)
if err != nil {
klog.Errorf("Error calculating spreadConstraintsMap: %v", err)
return nil
}
predicateMetadata := &predicateMetadata{
pod: pod,
evenPodsSpreadMetadata: evenPodsSpreadMetadata,
podFitsResourcesMetadata: getPodFitsResourcesMetedata(pod),
}
for predicateName, precomputeFunc := range predicateMetadataProducers {
@@ -414,7 +391,8 @@ func GetPodAffinityMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo,
}, nil
}
func getEvenPodsSpreadMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo) (*evenPodsSpreadMetadata, error) {
// GetPodTopologySpreadMetadata computes pod topology spread metadata.
func GetPodTopologySpreadMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo) (*PodTopologySpreadMetadata, error) {
// We have feature gating in APIServer to strip the spec
// so don't need to re-check feature gate, just check length of constraints.
constraints, err := filterHardTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints)
@@ -422,14 +400,14 @@ func getEvenPodsSpreadMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeIn
return nil, err
}
if len(constraints) == 0 {
return nil, nil
return &PodTopologySpreadMetadata{}, nil
}
var lock sync.Mutex
// TODO(Huang-Wei): It might be possible to use "make(map[topologyPair]*int32)".
// In that case, need to consider how to init each tpPairToCount[pair] in an atomic fashion.
m := evenPodsSpreadMetadata{
m := PodTopologySpreadMetadata{
constraints: constraints,
tpKeyToCriticalPaths: make(map[string]*criticalPaths, len(constraints)),
tpPairToMatchNum: make(map[topologyPair]int32),
@@ -526,15 +504,17 @@ func (m topologyToMatchedTermCount) clone() topologyToMatchedTermCount {
return copy
}
func (m *evenPodsSpreadMetadata) addPod(addedPod, preemptorPod *v1.Pod, node *v1.Node) {
m.updatePod(addedPod, preemptorPod, node, 1)
// AddPod updates the metadata with addedPod.
func (m *PodTopologySpreadMetadata) AddPod(addedPod, preemptorPod *v1.Pod, node *v1.Node) {
m.updateWithPod(addedPod, preemptorPod, node, 1)
}
func (m *evenPodsSpreadMetadata) removePod(deletedPod, preemptorPod *v1.Pod, node *v1.Node) {
m.updatePod(deletedPod, preemptorPod, node, -1)
// RemovePod updates the metadata with deletedPod.
func (m *PodTopologySpreadMetadata) RemovePod(deletedPod, preemptorPod *v1.Pod, node *v1.Node) {
m.updateWithPod(deletedPod, preemptorPod, node, -1)
}
func (m *evenPodsSpreadMetadata) updatePod(updatedPod, preemptorPod *v1.Pod, node *v1.Node, delta int32) {
func (m *PodTopologySpreadMetadata) updateWithPod(updatedPod, preemptorPod *v1.Pod, node *v1.Node, delta int32) {
if m == nil || updatedPod.Namespace != preemptorPod.Namespace || node == nil {
return
}
@@ -556,12 +536,13 @@ func (m *evenPodsSpreadMetadata) updatePod(updatedPod, preemptorPod *v1.Pod, nod
}
}
func (m *evenPodsSpreadMetadata) clone() *evenPodsSpreadMetadata {
// c could be nil when EvenPodsSpread feature is disabled
// Clone makes a deep copy of PodTopologySpreadMetadata.
func (m *PodTopologySpreadMetadata) Clone() *PodTopologySpreadMetadata {
// m could be nil when EvenPodsSpread feature is disabled
if m == nil {
return nil
}
cp := evenPodsSpreadMetadata{
cp := PodTopologySpreadMetadata{
// constraints are shared because they don't change.
constraints: m.constraints,
tpKeyToCriticalPaths: make(map[string]*criticalPaths, len(m.tpKeyToCriticalPaths)),
@@ -584,7 +565,6 @@ func (meta *predicateMetadata) RemovePod(deletedPod *v1.Pod, node *v1.Node) erro
if deletedPodFullName == schedutil.GetPodFullName(meta.pod) {
return fmt.Errorf("deletedPod and meta.pod must not be the same")
}
meta.evenPodsSpreadMetadata.removePod(deletedPod, meta.pod, node)
meta.serviceAffinityMetadata.removePod(deletedPod, node)
return nil
@@ -601,10 +581,6 @@ func (meta *predicateMetadata) AddPod(addedPod *v1.Pod, node *v1.Node) error {
return fmt.Errorf("node not found")
}
// Update meta.evenPodsSpreadMetadata if meta.pod has hard spread constraints
// and addedPod matches that
meta.evenPodsSpreadMetadata.addPod(addedPod, meta.pod, node)
meta.serviceAffinityMetadata.addPod(addedPod, meta.pod, node)
return nil
@@ -616,7 +592,6 @@ func (meta *predicateMetadata) ShallowCopy() Metadata {
newPredMeta := &predicateMetadata{
pod: meta.pod,
}
newPredMeta.evenPodsSpreadMetadata = meta.evenPodsSpreadMetadata.clone()
newPredMeta.serviceAffinityMetadata = meta.serviceAffinityMetadata.clone()
newPredMeta.podFitsResourcesMetadata = meta.podFitsResourcesMetadata.clone()
return (Metadata)(newPredMeta)

87
pkg/scheduler/algorithm/predicates/metadata_test.go
View File

@@ -235,15 +235,6 @@ func TestPredicateMetadata_ShallowCopy(t *testing.T) {
AllowedPodNumber: 4,
},
},
evenPodsSpreadMetadata: &evenPodsSpreadMetadata{
tpKeyToCriticalPaths: map[string]*criticalPaths{
"name": {{"nodeA", 1}, {"nodeC", 2}},
},
tpPairToMatchNum: map[topologyPair]int32{
{key: "name", value: "nodeA"}: 1,
{key: "name", value: "nodeC"}: 2,
},
},
serviceAffinityMetadata: &serviceAffinityMetadata{
matchingPodList: []*v1.Pod{
{ObjectMeta: metav1.ObjectMeta{Name: "pod1"}},
@@ -485,7 +476,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
pod *v1.Pod
nodes []*v1.Node
existingPods []*v1.Pod
want *evenPodsSpreadMetadata
want *PodTopologySpreadMetadata
}{
{
name: "clean cluster with one spreadConstraint",
@@ -498,7 +489,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 5,
@@ -533,7 +524,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 1,
@@ -570,7 +561,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 1,
@@ -606,7 +597,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-y1").Namespace("ns2").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 1,
@@ -644,7 +635,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 1,
@@ -693,7 +684,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 1,
@@ -734,7 +725,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-a").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b").Node("node-b").Label("bar", "").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 1,
@@ -780,7 +771,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Label("bar", "").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 1,
@@ -828,7 +819,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
{
maxSkew: 1,
@@ -859,7 +850,7 @@ func TestGetTPMapMatchingSpreadConstraints(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
s := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, tt.nodes))
l, _ := s.NodeInfos().List()
got, _ := getEvenPodsSpreadMetadata(tt.pod, l)
got, _ := GetPodTopologySpreadMetadata(tt.pod, l)
got.sortCriticalPaths()
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("getEvenPodsSpreadMetadata() = %#v, want %#v", *got, *tt.want)
@@ -883,7 +874,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
existingPods []*v1.Pod
nodeIdx int // denotes which node 'addedPod' belongs to
nodes []*v1.Node
want *evenPodsSpreadMetadata
want *PodTopologySpreadMetadata
}{
{
name: "node a and b both impact current min match",
@@ -897,7 +888,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{nodeConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"node": {{"node-b", 0}, {"node-a", 1}},
@@ -922,7 +913,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{nodeConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"node": {{"node-a", 1}, {"node-b", 1}},
@@ -947,7 +938,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{nodeConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"node": {{"node-a", 0}, {"node-b", 1}},
@@ -972,7 +963,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{nodeConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"node": {{"node-a", 0}, {"node-b", 2}},
@@ -996,7 +987,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{zoneConstraint, nodeConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"zone": {{"zone2", 0}, {"zone1", 1}},
@@ -1025,7 +1016,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{zoneConstraint, nodeConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"zone": {{"zone1", 1}, {"zone2", 1}},
@@ -1057,7 +1048,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{zoneConstraint, nodeConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"zone": {{"zone2", 1}, {"zone1", 3}},
@@ -1090,7 +1081,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
zoneConstraint,
{
@@ -1130,7 +1121,7 @@ func TestPodSpreadCache_addPod(t *testing.T) {
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
},
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{
zoneConstraint,
{
@@ -1157,11 +1148,11 @@ func TestPodSpreadCache_addPod(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
s := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, tt.nodes))
l, _ := s.NodeInfos().List()
evenPodsSpreadMetadata, _ := getEvenPodsSpreadMetadata(tt.preemptor, l)
evenPodsSpreadMetadata.addPod(tt.addedPod, tt.preemptor, tt.nodes[tt.nodeIdx])
evenPodsSpreadMetadata.sortCriticalPaths()
if !reflect.DeepEqual(evenPodsSpreadMetadata, tt.want) {
t.Errorf("evenPodsSpreadMetadata#addPod() = %v, want %v", evenPodsSpreadMetadata, tt.want)
podTopologySpreadMeta, _ := GetPodTopologySpreadMetadata(tt.preemptor, l)
podTopologySpreadMeta.AddPod(tt.addedPod, tt.preemptor, tt.nodes[tt.nodeIdx])
podTopologySpreadMeta.sortCriticalPaths()
if !reflect.DeepEqual(podTopologySpreadMeta, tt.want) {
t.Errorf("podTopologySpreadMeta#addPod() = %v, want %v", podTopologySpreadMeta, tt.want)
}
})
}
@@ -1183,7 +1174,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
deletedPodIdx int // need to reuse *Pod of existingPods[i]
deletedPod *v1.Pod // this field is used only when deletedPodIdx is -1
nodeIdx int // denotes which node "deletedPod" belongs to
want *evenPodsSpreadMetadata
want *PodTopologySpreadMetadata
}{
{
// A high priority pod may not be scheduled due to node taints or resource shortage.
@@ -1204,7 +1195,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
},
deletedPodIdx: 0, // remove pod "p-a1"
nodeIdx: 0, // node-a
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{zoneConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"zone": {{"zone1", 1}, {"zone2", 1}},
@@ -1234,7 +1225,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
},
deletedPodIdx: 0, // remove pod "p-a1"
nodeIdx: 0, // node-a
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{zoneConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"zone": {{"zone1", 1}, {"zone2", 2}},
@@ -1265,7 +1256,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
},
deletedPodIdx: 0, // remove pod "p-a0"
nodeIdx: 0, // node-a
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{zoneConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"zone": {{"zone1", 2}, {"zone2", 2}},
@@ -1296,7 +1287,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
deletedPodIdx: -1,
deletedPod: st.MakePod().Name("p-a0").Node("node-a").Label("bar", "").Obj(),
nodeIdx: 0, // node-a
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{zoneConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"zone": {{"zone1", 2}, {"zone2", 2}},
@@ -1327,7 +1318,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
},
deletedPodIdx: 3, // remove pod "p-x1"
nodeIdx: 2, // node-x
want: &evenPodsSpreadMetadata{
want: &PodTopologySpreadMetadata{
constraints: []topologySpreadConstraint{zoneConstraint, nodeConstraint},
tpKeyToCriticalPaths: map[string]*criticalPaths{
"zone": {{"zone2", 1}, {"zone1", 3}},
@@ -1347,7 +1338,7 @@ func TestPodSpreadCache_removePod(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
s := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, tt.nodes))
l, _ := s.NodeInfos().List()
evenPodsSpreadMetadata, _ := getEvenPodsSpreadMetadata(tt.preemptor, l)
podTopologySpreadMeta, _ := GetPodTopologySpreadMetadata(tt.preemptor, l)
var deletedPod *v1.Pod
if tt.deletedPodIdx < len(tt.existingPods) && tt.deletedPodIdx >= 0 {
@@ -1355,10 +1346,10 @@ func TestPodSpreadCache_removePod(t *testing.T) {
} else {
deletedPod = tt.deletedPod
}
evenPodsSpreadMetadata.removePod(deletedPod, tt.preemptor, tt.nodes[tt.nodeIdx])
evenPodsSpreadMetadata.sortCriticalPaths()
if !reflect.DeepEqual(evenPodsSpreadMetadata, tt.want) {
t.Errorf("evenPodsSpreadMetadata#removePod() = %v, want %v", evenPodsSpreadMetadata, tt.want)
podTopologySpreadMeta.RemovePod(deletedPod, tt.preemptor, tt.nodes[tt.nodeIdx])
podTopologySpreadMeta.sortCriticalPaths()
if !reflect.DeepEqual(podTopologySpreadMeta, tt.want) {
t.Errorf("podTopologySpreadMeta#removePod() = %v, want %v", podTopologySpreadMeta, tt.want)
}
})
}
@@ -1408,7 +1399,7 @@ func BenchmarkTestGetTPMapMatchingSpreadConstraints(b *testing.B) {
l, _ := s.NodeInfos().List()
b.ResetTimer()
for i := 0; i < b.N; i++ {
getEvenPodsSpreadMetadata(tt.pod, l)
GetPodTopologySpreadMetadata(tt.pod, l)
}
})
}
@@ -1420,7 +1411,7 @@ var (
)
// sortCriticalPaths is only served for testing purpose.
func (m *evenPodsSpreadMetadata) sortCriticalPaths() {
func (m *PodTopologySpreadMetadata) sortCriticalPaths() {
for _, paths := range m.tpKeyToCriticalPaths {
// If two paths both hold minimum matching number, and topologyValue is unordered.
if paths[0].matchNum == paths[1].matchNum && paths[0].topologyValue > paths[1].topologyValue {

31
pkg/scheduler/algorithm/predicates/predicates.go
View File

@@ -1647,28 +1647,33 @@ func (c *VolumeBindingChecker) predicate(pod *v1.Pod, meta Metadata, nodeInfo *s
return true, nil, nil
}
// EvenPodsSpreadPredicate checks if a pod can be scheduled on a node which satisfies
// its topologySpreadConstraints.
// EvenPodsSpreadPredicate is the legacy function using old path of metadata.
// DEPRECATED
func EvenPodsSpreadPredicate(pod *v1.Pod, meta Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) {
return false, nil, fmt.Errorf("this function should never be called")
}
// PodTopologySpreadPredicate checks if a pod can be scheduled on a node which satisfies
// its topologySpreadConstraints.
func PodTopologySpreadPredicate(pod *v1.Pod, meta *PodTopologySpreadMetadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) {
node := nodeInfo.Node()
if node == nil {
return false, nil, fmt.Errorf("node not found")
}
var epsMeta *evenPodsSpreadMetadata
if predicateMeta, ok := meta.(*predicateMetadata); ok {
epsMeta = predicateMeta.evenPodsSpreadMetadata
} else { // We don't have precomputed metadata. We have to follow a slow path to check spread constraints.
// TODO(autoscaler): get it implemented
return false, nil, errors.New("metadata not pre-computed for EvenPodsSpreadPredicate")
// nil meta is illegal.
if meta == nil {
// TODO(autoscaler): get it implemented.
return false, nil, errors.New("metadata not pre-computed for PodTopologySpreadPredicate")
}
if epsMeta == nil || len(epsMeta.tpPairToMatchNum) == 0 || len(epsMeta.constraints) == 0 {
// However, "empty" meta is legit which tolerates every toSchedule Pod.
if len(meta.tpPairToMatchNum) == 0 || len(meta.constraints) == 0 {
return true, nil, nil
}
podLabelSet := labels.Set(pod.Labels)
for _, c := range epsMeta.constraints {
for _, c := range meta.constraints {
tpKey := c.topologyKey
tpVal, ok := node.Labels[c.topologyKey]
if !ok {
@@ -1682,16 +1687,16 @@ func EvenPodsSpreadPredicate(pod *v1.Pod, meta Metadata, nodeInfo *schedulernode
}
pair := topologyPair{key: tpKey, value: tpVal}
paths, ok := epsMeta.tpKeyToCriticalPaths[tpKey]
paths, ok := meta.tpKeyToCriticalPaths[tpKey]
if !ok {
// error which should not happen
klog.Errorf("internal error: get paths from key %q of %#v", tpKey, epsMeta.tpKeyToCriticalPaths)
klog.Errorf("internal error: get paths from key %q of %#v", tpKey, meta.tpKeyToCriticalPaths)
continue
}
// judging criteria:
// 'existing matching num' + 'if self-match (1 or 0)' - 'global min matching num' <= 'maxSkew'
minMatchNum := paths[0].matchNum
matchNum := epsMeta.tpPairToMatchNum[pair]
matchNum := meta.tpPairToMatchNum[pair]
skew := matchNum + selfMatchNum - minMatchNum
if skew > c.maxSkew {
klog.V(5).Infof("node '%s' failed spreadConstraint[%s]: matchNum(%d) + selfMatchNum(%d) - minMatchNum(%d) > maxSkew(%d)", node.Name, tpKey, matchNum, selfMatchNum, minMatchNum, c.maxSkew)

445
pkg/scheduler/algorithm/predicates/predicates_test.go
View File

@@ -37,7 +37,6 @@ import (
fakelisters "k8s.io/kubernetes/pkg/scheduler/listers/fake"
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
nodeinfosnapshot "k8s.io/kubernetes/pkg/scheduler/nodeinfo/snapshot"
st "k8s.io/kubernetes/pkg/scheduler/testing"
)
var (
@@ -2658,447 +2657,3 @@ func TestCheckNodeUnschedulablePredicate(t *testing.T) {
}
}
}
func TestEvenPodsSpreadPredicate_SingleConstraint(t *testing.T) {
tests := []struct {
name string
pod *v1.Pod
nodes []*v1.Node
existingPods []*v1.Pod
fits map[string]bool
}{
{
name: "no existing pods",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
fits: map[string]bool{
"node-a": true,
"node-b": true,
"node-x": true,
"node-y": true,
},
},
{
name: "no existing pods, incoming pod doesn't match itself",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
1, "zone", hardSpread, st.MakeLabelSelector().Exists("bar").Obj(),
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
fits: map[string]bool{
"node-a": true,
"node-b": true,
"node-x": true,
"node-y": true,
},
},
{
name: "existing pods with mis-matched namespace doesn't count",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Namespace("ns1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Namespace("ns2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": true,
"node-b": true,
"node-x": false,
"node-y": false,
},
},
{
name: "pods spread across zones as 3/3, all nodes fit",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": true,
"node-b": true,
"node-x": true,
"node-y": true,
},
},
{
// TODO(Huang-Wei): maybe document this to remind users that typos on node labels
// can cause unexpected behavior
name: "pods spread across zones as 1/2 due to absence of label 'zone' on node-b",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zon", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-x1").Node("node-x").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": true,
"node-b": false,
"node-x": false,
"node-y": false,
},
},
{
name: "pods spread across nodes as 2/1/0/3, only node-x fits",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": false,
"node-b": false,
"node-x": true,
"node-y": false,
},
},
{
name: "pods spread across nodes as 2/1/0/3, maxSkew is 2, node-b and node-x fit",
pod: st.MakePod().Name("p").Label("foo", "").SpreadConstraint(
2, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": false,
"node-b": true,
"node-x": true,
"node-y": false,
},
},
{
// not a desired case, but it can happen
// TODO(Huang-Wei): document this "pod-not-match-itself" case
// in this case, placement of the new pod doesn't change pod distribution of the cluster
// as the incoming pod doesn't have label "foo"
name: "pods spread across nodes as 2/1/0/3, but pod doesn't match itself",
pod: st.MakePod().Name("p").Label("bar", "").SpreadConstraint(
1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj(),
).Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": false,
"node-b": true,
"node-x": true,
"node-y": false,
},
},
{
// only node-a and node-y are considered, so pods spread as 2/~1~/~0~/3
// ps: '~num~' is a markdown symbol to denote a crossline through 'num'
// but in this unit test, we don't run NodeAffinityPredicate, so node-b and node-x are
// still expected to be fits;
// the fact that node-a fits can prove the underlying logic works
name: "incoming pod has nodeAffinity, pods spread as 2/~1~/~0~/3, hence node-a fits",
pod: st.MakePod().Name("p").Label("foo", "").
NodeAffinityIn("node", []string{"node-a", "node-y"}).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": true,
"node-b": true, // in real case, it's false
"node-x": true, // in real case, it's false
"node-y": false,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, tt.nodes))
factory := &MetadataProducerFactory{}
meta := factory.GetPredicateMetadata(tt.pod, s)
for _, node := range tt.nodes {
fits, _, _ := EvenPodsSpreadPredicate(tt.pod, meta, s.NodeInfoMap[node.Name])
if fits != tt.fits[node.Name] {
t.Errorf("[%s]: expected %v got %v", node.Name, tt.fits[node.Name], fits)
}
}
})
}
}
func TestEvenPodsSpreadPredicate_MultipleConstraints(t *testing.T) {
tests := []struct {
name string
pod *v1.Pod
nodes []*v1.Node
existingPods []*v1.Pod
fits map[string]bool
}{
{
// 1. to fulfil "zone" constraint, incoming pod can be placed on any zone (hence any node)
// 2. to fulfil "node" constraint, incoming pod can be placed on node-x
// intersection of (1) and (2) returns node-x
name: "two constraints on zone and node, spreads = [3/3, 2/1/0/3]",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": false,
"node-b": false,
"node-x": true,
"node-y": false,
},
},
{
// 1. to fulfil "zone" constraint, incoming pod can be placed on zone1 (node-a or node-b)
// 2. to fulfil "node" constraint, incoming pod can be placed on node-x
// intersection of (1) and (2) returns no node
name: "two constraints on zone and node, spreads = [3/4, 2/1/0/4]",
pod: st.MakePod().Name("p").Label("foo", "").
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-b1").Node("node-b").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y4").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": false,
"node-b": false,
"node-x": false,
"node-y": false,
},
},
{
// 1. to fulfil "zone" constraint, incoming pod can be placed on zone2 (node-x or node-y)
// 2. to fulfil "node" constraint, incoming pod can be placed on node-b or node-x
// intersection of (1) and (2) returns node-x
name: "constraints hold different labelSelectors, spreads = [1/0, 1/0/0/1]",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("bar", "").Obj(),
},
fits: map[string]bool{
"node-a": false,
"node-b": false,
"node-x": true,
"node-y": false,
},
},
{
// 1. to fulfil "zone" constraint, incoming pod can be placed on zone2 (node-x or node-y)
// 2. to fulfil "node" constraint, incoming pod can be placed on node-a or node-b
// intersection of (1) and (2) returns no node
name: "constraints hold different labelSelectors, spreads = [1/0, 0/0/1/1]",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-x1").Node("node-x").Label("bar", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("bar", "").Obj(),
},
fits: map[string]bool{
"node-a": false,
"node-b": false,
"node-x": false,
"node-y": false,
},
},
{
// 1. to fulfil "zone" constraint, incoming pod can be placed on zone1 (node-a or node-b)
// 2. to fulfil "node" constraint, incoming pod can be placed on node-b or node-x
// intersection of (1) and (2) returns node-b
name: "constraints hold different labelSelectors, spreads = [2/3, 1/0/0/1]",
pod: st.MakePod().Name("p").Label("foo", "").Label("bar", "").
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-a2").Node("node-a").Label("foo", "").Label("bar", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("foo", "").Obj(),
st.MakePod().Name("p-y2").Node("node-y").Label("foo", "").Label("bar", "").Obj(),
st.MakePod().Name("p-y3").Node("node-y").Label("foo", "").Obj(),
},
fits: map[string]bool{
"node-a": false,
"node-b": true,
"node-x": false,
"node-y": false,
},
},
{
// 1. pod doesn't match itself on "zone" constraint, so it can be put onto any zone
// 2. to fulfil "node" constraint, incoming pod can be placed on node-a or node-b
// intersection of (1) and (2) returns node-a and node-b
name: "constraints hold different labelSelectors but pod doesn't match itself on 'zone' constraint",
pod: st.MakePod().Name("p").Label("bar", "").
SpreadConstraint(1, "zone", hardSpread, st.MakeLabelSelector().Exists("foo").Obj()).
SpreadConstraint(1, "node", hardSpread, st.MakeLabelSelector().Exists("bar").Obj()).
Obj(),
nodes: []*v1.Node{
st.MakeNode().Name("node-a").Label("zone", "zone1").Label("node", "node-a").Obj(),
st.MakeNode().Name("node-b").Label("zone", "zone1").Label("node", "node-b").Obj(),
st.MakeNode().Name("node-x").Label("zone", "zone2").Label("node", "node-x").Obj(),
st.MakeNode().Name("node-y").Label("zone", "zone2").Label("node", "node-y").Obj(),
},
existingPods: []*v1.Pod{
st.MakePod().Name("p-a1").Node("node-a").Label("foo", "").Obj(),
st.MakePod().Name("p-x1").Node("node-x").Label("bar", "").Obj(),
st.MakePod().Name("p-y1").Node("node-y").Label("bar", "").Obj(),
},
fits: map[string]bool{
"node-a": true,
"node-b": true,
"node-x": false,
"node-y": false,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, tt.nodes))
factory := &MetadataProducerFactory{}
meta := factory.GetPredicateMetadata(tt.pod, s)
for _, node := range tt.nodes {
fits, _, _ := EvenPodsSpreadPredicate(tt.pod, meta, s.NodeInfoMap[node.Name])
if fits != tt.fits[node.Name] {
t.Errorf("[%s]: expected %v got %v", node.Name, tt.fits[node.Name], fits)
}
}
})
}
}

1
pkg/scheduler/algorithmprovider/defaults/defaults.go
View File

@@ -58,6 +58,7 @@ func defaultPredicates() sets.String {
// The returned function is used to restore the state of registered predicates/priorities
// when this function is called, and should be called in tests which may modify the value
// of a feature gate temporarily.
// TODO(Huang-Wei): refactor this function to have a clean way to disable/enable plugins.
func ApplyFeatureGates() (restore func()) {
snapshot := scheduler.RegisteredPredicatesAndPrioritiesSnapshot()

3
pkg/scheduler/apis/config/testing/compatibility_test.go
View File

@@ -1191,6 +1191,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
features.EvenPodsSpread: true,
},
wantPlugins: map[string][]config.Plugin{
"PreFilterPlugin": {
{Name: "PodTopologySpread"},
},
"FilterPlugin": {
{Name: "NodeUnschedulable"},
{Name: "TaintToleration"},

48
pkg/scheduler/core/extender_test.go
View File

@@ -362,15 +362,16 @@ var _ algorithm.SchedulerExtender = &FakeExtender{}
func TestGenericSchedulerWithExtenders(t *testing.T) {
tests := []struct {
name string
registerFilterPlugin st.RegisterFilterPluginFunc
registerScorePlugin st.RegisterScorePluginFunc
registerPlugins []st.RegisterPluginFunc
extenders []FakeExtender
nodes []string
expectedResult ScheduleResult
expectsErr bool
}{
{
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{truePredicateExtender},
@@ -384,7 +385,9 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
name: "test 1",
},
{
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{truePredicateExtender},
@@ -398,7 +401,9 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
name: "test 2",
},
{
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{truePredicateExtender},
@@ -416,7 +421,9 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
name: "test 3",
},
{
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{machine2PredicateExtender},
@@ -430,7 +437,9 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
name: "test 4",
},
{
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{truePredicateExtender},
@@ -447,7 +456,9 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
name: "test 5",
},
{
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{truePredicateExtender},
@@ -469,8 +480,10 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
name: "test 6",
},
{
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerScorePlugin: st.RegisterScorePlugin("Machine2Prioritizer", newMachine2PrioritizerPlugin(), 20),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
st.RegisterScorePlugin("Machine2Prioritizer", newMachine2PrioritizerPlugin(), 20),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{truePredicateExtender},
@@ -494,8 +507,10 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
// If scheduler sends the pod by mistake, the test would fail
// because of the errors from errorPredicateExtender and/or
// errorPrioritizerExtender.
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerScorePlugin: st.RegisterScorePlugin("Machine2Prioritizer", newMachine2PrioritizerPlugin(), 1),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
st.RegisterScorePlugin("Machine2Prioritizer", newMachine2PrioritizerPlugin(), 1),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{errorPredicateExtender},
@@ -518,7 +533,9 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
//
// If scheduler did not ignore the extender, the test would fail
// because of the errors from errorPredicateExtender.
registerFilterPlugin: st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
extenders: []FakeExtender{
{
predicates: []fitPredicate{errorPredicateExtender},
@@ -560,9 +577,8 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
Score: &schedulerapi.PluginSet{},
}
var pluginConfigs []schedulerapi.PluginConfig
test.registerFilterPlugin(&registry, plugins, pluginConfigs)
if test.registerScorePlugin != nil {
test.registerScorePlugin(&registry, plugins, pluginConfigs)
for _, f := range test.registerPlugins {
f(&registry, plugins, pluginConfigs)
}
fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs)

217
pkg/scheduler/core/generic_scheduler_test.go
View File

@@ -371,19 +371,17 @@ func TestGenericScheduler(t *testing.T) {
tests := []struct {
name string
registerFilterPlugins []st.RegisterFilterPluginFunc
registerScorePlugins []st.RegisterScorePluginFunc
registerPlugins []st.RegisterPluginFunc
alwaysCheckAllPredicates bool
nodes []string
pvcs []v1.PersistentVolumeClaim
pod *v1.Pod
pods []*v1.Pod
buildPredMeta bool // build predicates metadata or not
expectedHosts sets.String
wErr error
}{
{
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -400,7 +398,7 @@ func TestGenericScheduler(t *testing.T) {
},
},
{
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -411,7 +409,7 @@ func TestGenericScheduler(t *testing.T) {
},
{
// Fits on a machine where the pod ID matches the machine name
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -421,10 +419,8 @@ func TestGenericScheduler(t *testing.T) {
wErr: nil,
},
{
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1),
},
nodes: []string{"3", "2", "1"},
@@ -434,10 +430,8 @@ func TestGenericScheduler(t *testing.T) {
wErr: nil,
},
{
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1),
},
nodes: []string{"3", "2", "1"},
@@ -447,10 +441,8 @@ func TestGenericScheduler(t *testing.T) {
wErr: nil,
},
{
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1),
st.RegisterScorePlugin("ReverseNumericMap", newReverseNumericMapPlugin(), 2),
},
@@ -461,11 +453,9 @@ func TestGenericScheduler(t *testing.T) {
wErr: nil,
},
{
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1),
},
nodes: []string{"3", "2", "1"},
@@ -483,11 +473,9 @@ func TestGenericScheduler(t *testing.T) {
},
},
{
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("NoPodsFilter", NewNoPodsFilterPlugin),
st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1),
},
pods: []*v1.Pod{
@@ -516,7 +504,7 @@ func TestGenericScheduler(t *testing.T) {
},
{
// Pod with existing PVC
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -541,7 +529,7 @@ func TestGenericScheduler(t *testing.T) {
},
{
// Pod with non existing PVC
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -564,7 +552,7 @@ func TestGenericScheduler(t *testing.T) {
},
{
// Pod with deleting PVC
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -587,10 +575,8 @@ func TestGenericScheduler(t *testing.T) {
wErr: fmt.Errorf("persistentvolumeclaim \"existingPVC\" is being deleted"),
},
{
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("FalseMap", newFalseMapPlugin(), 1),
st.RegisterScorePlugin("TrueMap", newTrueMapPlugin(), 2),
},
@@ -601,8 +587,14 @@ func TestGenericScheduler(t *testing.T) {
},
{
name: "test even pods spread predicate - 2 nodes with maxskew=1",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
st.RegisterFilterPlugin(podtopologyspread.Name, podtopologyspread.New),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterPluginAsExtensions(
podtopologyspread.Name,
1,
podtopologyspread.New,
"PreFilter",
"Filter",
),
},
nodes: []string{"machine1", "machine2"},
pod: &v1.Pod{
@@ -636,14 +628,19 @@ func TestGenericScheduler(t *testing.T) {
},
},
},
buildPredMeta: true,
expectedHosts: sets.NewString("machine2"),
wErr: nil,
},
{
name: "test even pods spread predicate - 3 nodes with maxskew=2",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
st.RegisterFilterPlugin(podtopologyspread.Name, podtopologyspread.New),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterPluginAsExtensions(
podtopologyspread.Name,
1,
podtopologyspread.New,
"PreFilter",
"Filter",
),
},
nodes: []string{"machine1", "machine2", "machine3"},
pod: &v1.Pod{
@@ -695,19 +692,16 @@ func TestGenericScheduler(t *testing.T) {
},
},
},
buildPredMeta: true,
expectedHosts: sets.NewString("machine2", "machine3"),
wErr: nil,
},
{
name: "test with filter plugin returning Unschedulable status",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(
"FakeFilter",
NewFakeFilterPlugin(map[string]framework.Code{"3": framework.Unschedulable}),
),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1),
},
nodes: []string{"3"},
@@ -724,13 +718,11 @@ func TestGenericScheduler(t *testing.T) {
},
{
name: "test with filter plugin returning UnschedulableAndUnresolvable status",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(
"FakeFilter",
NewFakeFilterPlugin(map[string]framework.Code{"3": framework.UnschedulableAndUnresolvable}),
),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1),
},
nodes: []string{"3"},
@@ -747,13 +739,11 @@ func TestGenericScheduler(t *testing.T) {
},
{
name: "test with partial failed filter plugin",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(
"FakeFilter",
NewFakeFilterPlugin(map[string]framework.Code{"1": framework.Unschedulable}),
),
},
registerScorePlugins: []st.RegisterScorePluginFunc{
st.RegisterScorePlugin("NumericMap", newNumericMapPlugin(), 1),
},
nodes: []string{"1", "2"},
@@ -767,44 +757,42 @@ func TestGenericScheduler(t *testing.T) {
client := clientsetfake.NewSimpleClientset()
informerFactory := informers.NewSharedInformerFactory(client, 0)
registry := framework.Registry{}
plugins := &schedulerapi.Plugins{
Filter: &schedulerapi.PluginSet{},
Score: &schedulerapi.PluginSet{},
}
var pluginConfigs []schedulerapi.PluginConfig
for _, f := range test.registerFilterPlugins {
f(&registry, plugins, pluginConfigs)
}
for _, f := range test.registerScorePlugins {
f(&registry, plugins, pluginConfigs)
}
fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs)
cache := internalcache.New(time.Duration(0), wait.NeverStop)
for _, pod := range test.pods {
cache.AddPod(pod)
}
var nodes []*v1.Node
for _, name := range test.nodes {
cache.AddNode(&v1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"hostname": name}}})
node := &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"hostname": name}}}
nodes = append(nodes, node)
cache.AddNode(node)
}
registry := framework.Registry{}
plugins := &schedulerapi.Plugins{
PreFilter: &schedulerapi.PluginSet{},
Filter: &schedulerapi.PluginSet{},
Score: &schedulerapi.PluginSet{},
}
var pluginConfigs []schedulerapi.PluginConfig
for _, f := range test.registerPlugins {
f(&registry, plugins, pluginConfigs)
}
snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(test.pods, nodes))
fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithSnapshotSharedLister(snapshot))
pvcs := []v1.PersistentVolumeClaim{}
pvcs = append(pvcs, test.pvcs...)
pvcLister := fakelisters.PersistentVolumeClaimLister(pvcs)
predMetaProducer := algorithmpredicates.EmptyMetadataProducer
if test.buildPredMeta {
f := &algorithmpredicates.MetadataProducerFactory{}
predMetaProducer = f.GetPredicateMetadata
}
scheduler := NewGenericScheduler(
cache,
internalqueue.NewSchedulingQueue(nil),
nil,
predMetaProducer,
algorithmpredicates.EmptyMetadataProducer,
// test.prioritizers,
priorities.EmptyMetadataProducer,
emptySnapshot,
snapshot,
fwk,
[]algorithm.SchedulerExtender{},
nil,
@@ -829,7 +817,7 @@ func TestGenericScheduler(t *testing.T) {
}
// makeScheduler makes a simple genericScheduler for testing.
func makeScheduler(nodes []*v1.Node, fns ...st.RegisterFilterPluginFunc) *genericScheduler {
func makeScheduler(nodes []*v1.Node, fns ...st.RegisterPluginFunc) *genericScheduler {
cache := internalcache.New(time.Duration(0), wait.NeverStop)
for _, n := range nodes {
cache.AddNode(n)
@@ -1151,7 +1139,7 @@ func TestZeroRequest(t *testing.T) {
Score: &schedulerapi.PluginSet{},
}
var pluginConfigs []schedulerapi.PluginConfig
pluginRegistrations := []st.RegisterScorePluginFunc{
pluginRegistrations := []st.RegisterPluginFunc{
st.RegisterScorePlugin(noderesources.LeastAllocatedName, noderesources.NewLeastAllocated, 1),
st.RegisterScorePlugin(noderesources.BalancedAllocationName, noderesources.NewBalancedAllocation, 1),
st.RegisterScorePlugin(defaultpodtopologyspread.Name, defaultpodtopologyspread.New, 1),
@@ -1314,7 +1302,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
tests := []struct {
name string
registerFilterPlugins []st.RegisterFilterPluginFunc
registerPlugins []st.RegisterPluginFunc
nodes []string
pod *v1.Pod
pods []*v1.Pod
@@ -1325,7 +1313,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
}{
{
name: "a pod that does not fit on any machine",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("FalseFilter", NewFalseFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -1338,7 +1326,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "a pod that fits with no preemption",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("TrueFilter", NewTrueFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -1351,7 +1339,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "a pod that fits on one machine with no preemption",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin("MatchFilter", NewMatchFilterPlugin),
},
nodes: []string{"machine1", "machine2"},
@@ -1364,7 +1352,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "a pod that fits on both machines when lower priority pods are preempted",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
},
nodes: []string{"machine1", "machine2"},
@@ -1377,7 +1365,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "a pod that would fit on the machines, but other pods running are higher priority",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
},
nodes: []string{"machine1", "machine2"},
@@ -1390,7 +1378,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "medium priority pod is preempted, but lower priority one stays as it is small",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
},
nodes: []string{"machine1", "machine2"},
@@ -1404,7 +1392,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "mixed priority pods are preempted",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
},
nodes: []string{"machine1", "machine2"},
@@ -1420,7 +1408,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "mixed priority pods are preempted, pick later StartTime one when priorities are equal",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
},
nodes: []string{"machine1", "machine2"},
@@ -1436,7 +1424,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "pod with anti-affinity is preempted",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
st.RegisterFilterPlugin(interpodaffinity.Name, interpodaffinity.New),
},
@@ -1471,8 +1459,14 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "preemption to resolve even pods spread FitError",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
st.RegisterFilterPlugin(podtopologyspread.Name, podtopologyspread.New),
registerPlugins: []st.RegisterPluginFunc{
st.RegisterPluginAsExtensions(
podtopologyspread.Name,
1,
podtopologyspread.New,
"PreFilter",
"Filter",
),
},
nodes: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"},
pod: &v1.Pod{
@@ -1547,7 +1541,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
},
{
name: "get Unschedulable in the preemption phase when the filter plugins filtering the nodes",
registerFilterPlugins: []st.RegisterFilterPluginFunc{
registerPlugins: []st.RegisterPluginFunc{
st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
},
nodes: []string{"machine1", "machine2"},
@@ -1591,6 +1585,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
registry := framework.Registry{}
plugins := &schedulerapi.Plugins{
PreFilter: &schedulerapi.PluginSet{},
Filter: &schedulerapi.PluginSet{},
}
var pluginConfigs []schedulerapi.PluginConfig
@@ -1605,7 +1600,7 @@ func TestSelectNodesForPreemption(t *testing.T) {
)
registerFakeFilterFunc(&registry, plugins, pluginConfigs)
// Next, register other filter plugins defined in test struct.
for _, f := range test.registerFilterPlugins {
for _, f := range test.registerPlugins {
f(&registry, plugins, pluginConfigs)
}
// Use a real snapshot since it's needed in some Filter Plugin (e.g., PodAffinity)
@@ -1639,6 +1634,11 @@ func TestSelectNodesForPreemption(t *testing.T) {
newnode.ObjectMeta.Labels = map[string]string{"hostname": "newnode"}
nodes = append(nodes, newnode)
state := framework.NewCycleState()
// Some tests rely on PreFilter plugin to compute its CycleState.
preFilterStatus := fwk.RunPreFilterPlugins(context.Background(), state, test.pod)
if !preFilterStatus.IsSuccess() {
t.Errorf("Unexpected preFilterStatus: %v", preFilterStatus)
}
nodeToPods, err := g.selectNodesForPreemption(context.Background(), state, test.pod, nodes, nil)
if err != nil {
t.Error(err)
@@ -1660,7 +1660,7 @@ func TestPickOneNodeForPreemption(t *testing.T) {
defer algorithmpredicates.SetPredicatesOrderingDuringTest(order)()
tests := []struct {
name string
registerFilterPlugin st.RegisterFilterPluginFunc
registerFilterPlugin st.RegisterPluginFunc
nodes []string
pod *v1.Pod
pods []*v1.Pod
@@ -2052,8 +2052,7 @@ func TestPreempt(t *testing.T) {
extenders []*FakeExtender
failedNodeToStatusMap framework.NodeToStatusMap
nodeNames []string
registerFilterPlugin st.RegisterFilterPluginFunc
buildPredMeta bool
registerPlugin st.RegisterPluginFunc
expectedNode string
expectedPods []string // list of preempted pods
}{
@@ -2070,7 +2069,7 @@ func TestPreempt(t *testing.T) {
{ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}},
{ObjectMeta: metav1.ObjectMeta{Name: "m3.1", UID: types.UID("m3.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine3"}, Status: v1.PodStatus{Phase: v1.PodRunning}},
},
registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
registerPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
expectedNode: "machine1",
expectedPods: []string{"m1.1", "m1.2"},
},
@@ -2087,7 +2086,7 @@ func TestPreempt(t *testing.T) {
{ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}},
},
registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
registerPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
expectedNode: "machine3",
expectedPods: []string{},
},
@@ -2162,9 +2161,14 @@ func TestPreempt(t *testing.T) {
"node-b": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrTopologySpreadConstraintsNotMatch.GetReason()),
"node-x": framework.NewStatus(framework.Unschedulable, algorithmpredicates.ErrTopologySpreadConstraintsNotMatch.GetReason()),
},
buildPredMeta: true,
nodeNames: []string{"node-a/zone1", "node-b/zone1", "node-x/zone2"},
registerFilterPlugin: st.RegisterFilterPlugin(podtopologyspread.Name, podtopologyspread.New),
registerPlugin: st.RegisterPluginAsExtensions(
podtopologyspread.Name,
1,
podtopologyspread.New,
"PreFilter",
"Filter",
),
expectedNode: "node-b",
expectedPods: []string{"pod-b1"},
},
@@ -2189,7 +2193,7 @@ func TestPreempt(t *testing.T) {
predicates: []fitPredicate{machine1PredicateExtender},
},
},
registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
registerPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
expectedNode: "machine1",
expectedPods: []string{"m1.1", "m1.2"},
},
@@ -2211,7 +2215,7 @@ func TestPreempt(t *testing.T) {
predicates: []fitPredicate{falsePredicateExtender},
},
},
registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
registerPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
expectedNode: "",
expectedPods: []string{},
},
@@ -2237,7 +2241,7 @@ func TestPreempt(t *testing.T) {
predicates: []fitPredicate{machine1PredicateExtender},
},
},
registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
registerPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
expectedNode: "machine1",
expectedPods: []string{"m1.1", "m1.2"},
},
@@ -2263,7 +2267,7 @@ func TestPreempt(t *testing.T) {
predicates: []fitPredicate{truePredicateExtender},
},
},
registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
registerPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
expectedNode: "machine3",
expectedPods: []string{},
},
@@ -2280,7 +2284,7 @@ func TestPreempt(t *testing.T) {
{ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}},
{ObjectMeta: metav1.ObjectMeta{Name: "m3.1", UID: types.UID("m3.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine3"}, Status: v1.PodStatus{Phase: v1.PodRunning}},
},
registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
registerPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
expectedNode: "",
expectedPods: nil,
},
@@ -2297,7 +2301,7 @@ func TestPreempt(t *testing.T) {
{ObjectMeta: metav1.ObjectMeta{Name: "m2.1", UID: types.UID("m2.1")}, Spec: v1.PodSpec{Containers: largeContainers, Priority: &highPriority, NodeName: "machine2"}, Status: v1.PodStatus{Phase: v1.PodRunning}},
{ObjectMeta: metav1.ObjectMeta{Name: "m3.1", UID: types.UID("m3.1")}, Spec: v1.PodSpec{Containers: mediumContainers, Priority: &midPriority, NodeName: "machine3"}, Status: v1.PodStatus{Phase: v1.PodRunning}},
},
registerFilterPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
registerPlugin: st.RegisterFilterPlugin(noderesources.FitName, noderesources.NewFit),
expectedNode: "machine1",
expectedPods: []string{"m1.1", "m1.2"},
},
@@ -2319,6 +2323,7 @@ func TestPreempt(t *testing.T) {
if len(test.nodeNames) != 0 {
nodeNames = test.nodeNames
}
var nodes []*v1.Node
for i, name := range nodeNames {
node := makeNode(name, 1000*5, priorityutil.DefaultMemoryRequest*5)
// if possible, split node name by '/' to form labels in a format of
@@ -2329,6 +2334,7 @@ func TestPreempt(t *testing.T) {
}
node.Name = node.ObjectMeta.Labels["hostname"]
cache.AddNode(node)
nodes = append(nodes, node)
nodeNames[i] = node.Name
// Set nodeInfo to extenders to mock extenders' cache for preemption.
@@ -2336,33 +2342,30 @@ func TestPreempt(t *testing.T) {
cachedNodeInfo.SetNode(node)
cachedNodeInfoMap[node.Name] = cachedNodeInfo
}
extenders := []algorithm.SchedulerExtender{}
var extenders []algorithm.SchedulerExtender
for _, extender := range test.extenders {
// Set nodeInfoMap as extenders cached node information.
extender.cachedNodeNameToInfo = cachedNodeInfoMap
extenders = append(extenders, extender)
}
predMetaProducer := algorithmpredicates.EmptyMetadataProducer
if test.buildPredMeta {
f := &algorithmpredicates.MetadataProducerFactory{}
predMetaProducer = f.GetPredicateMetadata
}
registry := framework.Registry{}
plugins := &schedulerapi.Plugins{
PreFilter: &schedulerapi.PluginSet{},
Filter: &schedulerapi.PluginSet{},
}
var pluginConfigs []schedulerapi.PluginConfig
test.registerFilterPlugin(&registry, plugins, pluginConfigs)
fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs)
test.registerPlugin(&registry, plugins, pluginConfigs)
snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(test.pods, nodes))
fwk, _ := framework.NewFramework(registry, plugins, pluginConfigs, framework.WithSnapshotSharedLister(snapshot))
scheduler := NewGenericScheduler(
cache,
internalqueue.NewSchedulingQueue(nil),
nil,
predMetaProducer,
algorithmpredicates.EmptyMetadataProducer,
priorities.EmptyMetadataProducer,
emptySnapshot,
snapshot,
fwk,
extenders,
nil,
@@ -2373,7 +2376,11 @@ func TestPreempt(t *testing.T) {
schedulerapi.DefaultPercentageOfNodesToScore,
true)
state := framework.NewCycleState()
scheduler.Snapshot()
// Some tests rely on PreFilter plugin to compute its CycleState.
preFilterStatus := fwk.RunPreFilterPlugins(context.Background(), state, test.pod)
if !preFilterStatus.IsSuccess() {
t.Errorf("Unexpected preFilterStatus: %v", preFilterStatus)
}
// Call Preempt and check the expected results.
failedNodeToStatusMap := defaultFailedNodeToStatusMap
if test.failedNodeToStatusMap != nil {

1
pkg/scheduler/framework/plugins/default_registry.go
View File

@@ -205,6 +205,7 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
})
registry.RegisterPredicate(predicates.EvenPodsSpreadPred,
func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
plugins.PreFilter = appendToPluginSet(plugins.PreFilter, podtopologyspread.Name, nil)
plugins.Filter = appendToPluginSet(plugins.Filter, podtopologyspread.Name, nil)
return
})

4
pkg/scheduler/framework/plugins/podtopologyspread/BUILD
View File

@@ -10,9 +10,11 @@ go_library(
"//pkg/scheduler/algorithm/priorities:go_default_library",
"//pkg/scheduler/framework/plugins/migration:go_default_library",
"//pkg/scheduler/framework/v1alpha1:go_default_library",
"//pkg/scheduler/listers:go_default_library",
"//pkg/scheduler/nodeinfo:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
)
@@ -21,8 +23,6 @@ go_test(
srcs = ["pod_topology_spread_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/framework/plugins/migration:go_default_library",
"//pkg/scheduler/framework/v1alpha1:go_default_library",
"//pkg/scheduler/nodeinfo/snapshot:go_default_library",
"//pkg/scheduler/testing:go_default_library",

112
pkg/scheduler/framework/plugins/podtopologyspread/pod_topology_spread.go
View File

@@ -22,36 +22,125 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
"k8s.io/kubernetes/pkg/scheduler/algorithm/priorities"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
schedulerlisters "k8s.io/kubernetes/pkg/scheduler/listers"
"k8s.io/kubernetes/pkg/scheduler/nodeinfo"
)
// PodTopologySpread is a plugin that ensures pod's topologySpreadConstraints is satisfied.
type PodTopologySpread struct {
handle framework.FrameworkHandle
snapshotSharedLister schedulerlisters.SharedLister
}
var _ framework.PreFilterPlugin = &PodTopologySpread{}
var _ framework.FilterPlugin = &PodTopologySpread{}
var _ framework.ScorePlugin = &PodTopologySpread{}
// Name is the name of the plugin used in the plugin registry and configurations.
const Name = "PodTopologySpread"
const (
// Name is the name of the plugin used in the plugin registry and configurations.
Name = "PodTopologySpread"
// preFilterStateKey is the key in CycleState to PodTopologySpread pre-computed data.
// Using the name of the plugin will likely help us avoid collisions with other plugins.
preFilterStateKey = "PreFilter" + Name
)
// Name returns name of the plugin. It is used in logs, etc.
func (pl *PodTopologySpread) Name() string {
return Name
}
// preFilterState computed at PreFilter and used at Filter.
type preFilterState struct {
// `nil` represents the meta is not set at all (in PreFilter phase)
// An empty `PodTopologySpreadMetadata` object denotes it's a legit meta and is set in PreFilter phase.
meta *predicates.PodTopologySpreadMetadata
}
// Clone makes a copy of the given state.
func (s *preFilterState) Clone() framework.StateData {
copy := &preFilterState{
meta: s.meta.Clone(),
}
return copy
}
// PreFilter invoked at the prefilter extension point.
func (pl *PodTopologySpread) PreFilter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod) *framework.Status {
var meta *predicates.PodTopologySpreadMetadata
var allNodes []*nodeinfo.NodeInfo
var err error
if allNodes, err = pl.snapshotSharedLister.NodeInfos().List(); err != nil {
return framework.NewStatus(framework.Error, fmt.Sprintf("failed to list NodeInfos: %v", err))
}
if meta, err = predicates.GetPodTopologySpreadMetadata(pod, allNodes); err != nil {
return framework.NewStatus(framework.Error, fmt.Sprintf("Error calculating podTopologySpreadMetadata: %v", err))
}
s := &preFilterState{
meta: meta,
}
cycleState.Write(preFilterStateKey, s)
return nil
}
// PreFilterExtensions returns prefilter extensions, pod add and remove.
func (pl *PodTopologySpread) PreFilterExtensions() framework.PreFilterExtensions {
return pl
}
// AddPod from pre-computed data in cycleState.
func (pl *PodTopologySpread) AddPod(ctx context.Context, cycleState *framework.CycleState, podToSchedule *v1.Pod, podToAdd *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
meta, err := getPodTopologySpreadMetadata(cycleState)
if err != nil {
return framework.NewStatus(framework.Error, err.Error())
}
meta.AddPod(podToAdd, podToSchedule, nodeInfo.Node())
return nil
}
// RemovePod from pre-computed data in cycleState.
func (pl *PodTopologySpread) RemovePod(ctx context.Context, cycleState *framework.CycleState, podToSchedule *v1.Pod, podToRemove *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
meta, err := getPodTopologySpreadMetadata(cycleState)
if err != nil {
return framework.NewStatus(framework.Error, err.Error())
}
meta.RemovePod(podToRemove, podToSchedule, nodeInfo.Node())
return nil
}
func getPodTopologySpreadMetadata(cycleState *framework.CycleState) (*predicates.PodTopologySpreadMetadata, error) {
c, err := cycleState.Read(preFilterStateKey)
if err != nil {
// The metadata wasn't pre-computed in prefilter. We ignore the error for now since
// we are able to handle that by computing it again (e.g. in Filter()).
klog.Error(err)
return nil, nil
}
s, ok := c.(*preFilterState)
if !ok {
return nil, fmt.Errorf("%+v convert to podtopologyspread.state error", c)
}
return s.meta, nil
}
// Filter invoked at the filter extension point.
func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeInfo *nodeinfo.NodeInfo) *framework.Status {
meta, ok := migration.CovertStateRefToPredMeta(migration.PredicateMetadata(cycleState))
if !ok {
return migration.ErrorToFrameworkStatus(fmt.Errorf("%+v convert to predicates.Metadata error", cycleState))
meta, err := getPodTopologySpreadMetadata(cycleState)
if err != nil {
return framework.NewStatus(framework.Error, err.Error())
}
_, reasons, err := predicates.EvenPodsSpreadPredicate(pod, meta, nodeInfo)
_, reasons, err := predicates.PodTopologySpreadPredicate(pod, meta, nodeInfo)
return migration.PredicateResultToFrameworkStatus(reasons, err)
}
@@ -59,7 +148,7 @@ func (pl *PodTopologySpread) Filter(ctx context.Context, cycleState *framework.C
// The "score" returned in this function is the matching number of pods on the `nodeName`,
// it is normalized later.
func (pl *PodTopologySpread) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
nodeInfo, err := pl.handle.SnapshotSharedLister().NodeInfos().Get(nodeName)
nodeInfo, err := pl.snapshotSharedLister.NodeInfos().Get(nodeName)
if err != nil {
return 0, framework.NewStatus(framework.Error, fmt.Sprintf("getting node %q from Snapshot: %v", nodeName, err))
}
@@ -72,7 +161,7 @@ func (pl *PodTopologySpread) Score(ctx context.Context, state *framework.CycleSt
// NormalizeScore invoked after scoring all nodes.
func (pl *PodTopologySpread) NormalizeScore(ctx context.Context, state *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
meta := migration.PriorityMetadata(state)
err := priorities.CalculateEvenPodsSpreadPriorityReduce(pod, meta, pl.handle.SnapshotSharedLister(), scores)
err := priorities.CalculateEvenPodsSpreadPriorityReduce(pod, meta, pl.snapshotSharedLister, scores)
return migration.ErrorToFrameworkStatus(err)
}
@@ -83,5 +172,8 @@ func (pl *PodTopologySpread) ScoreExtensions() framework.ScoreExtensions {
// New initializes a new plugin and returns it.
func New(_ *runtime.Unknown, h framework.FrameworkHandle) (framework.Plugin, error) {
return &PodTopologySpread{handle: h}, nil
if h.SnapshotSharedLister() == nil {
return nil, fmt.Errorf("SnapshotSharedlister is nil")
}
return &PodTopologySpread{snapshotSharedLister: h.SnapshotSharedLister()}, nil
}

30
pkg/scheduler/framework/plugins/podtopologyspread/pod_topology_spread_test.go
View File

@@ -21,8 +21,6 @@ import (
"testing"
"k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/scheduler/algorithm/predicates"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/migration"
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
nodeinfosnapshot "k8s.io/kubernetes/pkg/scheduler/nodeinfo/snapshot"
st "k8s.io/kubernetes/pkg/scheduler/testing"
@@ -32,7 +30,7 @@ var (
hardSpread = v1.DoNotSchedule
)
func TestPodTopologySpread_Filter_SingleConstraint(t *testing.T) {
func TestSingleConstraint(t *testing.T) {
tests := []struct {
name string
pod *v1.Pod
@@ -270,14 +268,16 @@ func TestPodTopologySpread_Filter_SingleConstraint(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, tt.nodes))
factory := &predicates.MetadataProducerFactory{}
meta := factory.GetPredicateMetadata(tt.pod, snapshot)
p := &PodTopologySpread{snapshotSharedLister: snapshot}
state := framework.NewCycleState()
state.Write(migration.PredicatesStateKey, &migration.PredicatesStateData{Reference: meta})
plugin, _ := New(nil, nil)
preFilterStatus := p.PreFilter(context.Background(), state, tt.pod)
if !preFilterStatus.IsSuccess() {
t.Errorf("preFilter failed with status: %v", preFilterStatus)
}
for _, node := range tt.nodes {
nodeInfo, _ := snapshot.NodeInfos().Get(node.Name)
status := plugin.(*PodTopologySpread).Filter(context.Background(), state, tt.pod, nodeInfo)
status := p.Filter(context.Background(), state, tt.pod, nodeInfo)
if status.IsSuccess() != tt.fits[node.Name] {
t.Errorf("[%s]: expected %v got %v", node.Name, tt.fits[node.Name], status.IsSuccess())
}
@@ -286,7 +286,7 @@ func TestPodTopologySpread_Filter_SingleConstraint(t *testing.T) {
}
}
func TestPodTopologySpread_Filter_MultipleConstraints(t *testing.T) {
func TestMultipleConstraints(t *testing.T) {
tests := []struct {
name string
pod *v1.Pod
@@ -468,14 +468,16 @@ func TestPodTopologySpread_Filter_MultipleConstraints(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
snapshot := nodeinfosnapshot.NewSnapshot(nodeinfosnapshot.CreateNodeInfoMap(tt.existingPods, tt.nodes))
factory := &predicates.MetadataProducerFactory{}
meta := factory.GetPredicateMetadata(tt.pod, snapshot)
p := &PodTopologySpread{snapshotSharedLister: snapshot}
state := framework.NewCycleState()
state.Write(migration.PredicatesStateKey, &migration.PredicatesStateData{Reference: meta})
plugin, _ := New(nil, nil)
preFilterStatus := p.PreFilter(context.Background(), state, tt.pod)
if !preFilterStatus.IsSuccess() {
t.Errorf("preFilter failed with status: %v", preFilterStatus)
}
for _, node := range tt.nodes {
nodeInfo, _ := snapshot.NodeInfos().Get(node.Name)
status := plugin.(*PodTopologySpread).Filter(context.Background(), state, tt.pod, nodeInfo)
status := p.Filter(context.Background(), state, tt.pod, nodeInfo)
if status.IsSuccess() != tt.fits[node.Name] {
t.Errorf("[%s]: expected %v got %v", node.Name, tt.fits[node.Name], status.IsSuccess())
}

2
pkg/scheduler/framework/v1alpha1/framework.go
View File

@@ -87,7 +87,7 @@ type extensionPoint struct {
// the set of plugins to be configured at this extension point.
plugins *config.PluginSet
// a pointer to the slice storing plugins implementations that will run at this
// extenstion point.
// extension point.
slicePtr interface{}
}

6
pkg/scheduler/scheduler_test.go
View File

@@ -568,7 +568,7 @@ func TestSchedulerErrorWithLongBinding(t *testing.T) {
// queuedPodStore: pods queued before processing.
// cache: scheduler cache that might contain assumed pods.
func setupTestSchedulerWithOnePodOnNode(t *testing.T, queuedPodStore *clientcache.FIFO, scache internalcache.Cache,
informerFactory informers.SharedInformerFactory, stop chan struct{}, f st.RegisterFilterPluginFunc, pod *v1.Pod, node *v1.Node) (*Scheduler, chan *v1.Binding, chan error) {
informerFactory informers.SharedInformerFactory, stop chan struct{}, f st.RegisterPluginFunc, pod *v1.Pod, node *v1.Node) (*Scheduler, chan *v1.Binding, chan error) {
scheduler, bindingChan, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, f, nil)
@@ -679,7 +679,7 @@ func TestSchedulerFailedSchedulingReasons(t *testing.T) {
// queuedPodStore: pods queued before processing.
// scache: scheduler cache that might contain assumed pods.
func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, f st.RegisterFilterPluginFunc, recorder events.EventRecorder) (*Scheduler, chan *v1.Binding, chan error) {
func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, f st.RegisterPluginFunc, recorder events.EventRecorder) (*Scheduler, chan *v1.Binding, chan error) {
registry := framework.Registry{}
plugins := &schedulerapi.Plugins{
Filter: &schedulerapi.PluginSet{},
@@ -736,7 +736,7 @@ func setupTestScheduler(queuedPodStore *clientcache.FIFO, scache internalcache.C
return sched, bindingChan, errChan
}
func setupTestSchedulerLongBindingWithRetry(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, f st.RegisterFilterPluginFunc, stop chan struct{}, bindingTime time.Duration) (*Scheduler, chan *v1.Binding) {
func setupTestSchedulerLongBindingWithRetry(queuedPodStore *clientcache.FIFO, scache internalcache.Cache, informerFactory informers.SharedInformerFactory, f st.RegisterPluginFunc, stop chan struct{}, bindingTime time.Duration) (*Scheduler, chan *v1.Binding) {
registry := framework.Registry{}
plugins := &schedulerapi.Plugins{
Filter: &schedulerapi.PluginSet{},

54
pkg/scheduler/testing/framework_helpers.go
View File

@@ -21,32 +21,54 @@ import (
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
)
// RegisterFilterPluginFunc is a function signature used in method RegisterFilterPlugin()
// RegisterPluginFunc is a function signature used in method RegisterFilterPlugin()
// to register a Filter Plugin to a given registry.
type RegisterFilterPluginFunc func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig)
type RegisterPluginFunc func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig)
// RegisterFilterPlugin returns a function to register a Filter Plugin to a given registry.
func RegisterFilterPlugin(pluginName string, pluginNewFunc framework.PluginFactory) RegisterFilterPluginFunc {
return func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig) {
reg.Register(pluginName, pluginNewFunc)
plugins.Filter.Enabled = append(plugins.Filter.Enabled, schedulerapi.Plugin{Name: pluginName})
//lint:ignore SA4006 this value of pluginConfigs is never used.
//lint:ignore SA4010 this result of append is never used.
pluginConfigs = append(pluginConfigs, schedulerapi.PluginConfig{Name: pluginName})
}
func RegisterFilterPlugin(pluginName string, pluginNewFunc framework.PluginFactory) RegisterPluginFunc {
return RegisterPluginAsExtensions(pluginName, 1, pluginNewFunc, "Filter")
}
// RegisterScorePluginFunc is a function signature used in method RegisterScorePlugin()
// to register a Score Plugin to a given registry.
type RegisterScorePluginFunc func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig)
// RegisterScorePlugin returns a function to register a Score Plugin to a given registry.
func RegisterScorePlugin(pluginName string, pluginNewFunc framework.PluginFactory, weight int32) RegisterScorePluginFunc {
func RegisterScorePlugin(pluginName string, pluginNewFunc framework.PluginFactory, weight int32) RegisterPluginFunc {
return RegisterPluginAsExtensions(pluginName, weight, pluginNewFunc, "Score")
}
// RegisterPluginAsExtensions returns a function to register a Plugin as given extensionPoints to a given registry.
func RegisterPluginAsExtensions(pluginName string, weight int32, pluginNewFunc framework.PluginFactory, extensions ...string) RegisterPluginFunc {
return func(reg *framework.Registry, plugins *schedulerapi.Plugins, pluginConfigs []schedulerapi.PluginConfig) {
reg.Register(pluginName, pluginNewFunc)
plugins.Score.Enabled = append(plugins.Score.Enabled, schedulerapi.Plugin{Name: pluginName, Weight: weight})
for _, extension := range extensions {
pluginSet := getPluginSetByExtension(plugins, extension)
if pluginSet == nil {
continue
}
pluginSet.Enabled = append(pluginSet.Enabled, schedulerapi.Plugin{Name: pluginName, Weight: weight})
}
//lint:ignore SA4006 this value of pluginConfigs is never used.
//lint:ignore SA4010 this result of append is never used.
pluginConfigs = append(pluginConfigs, schedulerapi.PluginConfig{Name: pluginName})
}
}
func getPluginSetByExtension(plugins *schedulerapi.Plugins, extension string) *schedulerapi.PluginSet {
switch extension {
case "Filter":
return plugins.Filter
case "PreFilter":
return plugins.PreFilter
case "PostFilter":
return plugins.PostFilter
case "Score":
return plugins.Score
case "Bind":
return plugins.Bind
case "Reserve":
return plugins.Reserve
case "Permit":
return plugins.Permit
default:
return nil
}
}