scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-11-04 12:18:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

optionally ignore preferred terms of existing pods unless incoming pod

Browse Source 
has inter-pod affinities
This commit is contained in:
Daniel Vega-Myhre
2022-12-08 01:49:43 +00:00
parent 3f752b5edf
commit 41817b1888
12 changed files with 342 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
cmd/kube-scheduler/app/options/options_test.go
View File

@@ -389,6 +389,27 @@ profiles:
t.Fatal(err)
}
// high throughput profile config
highThroughputProfileConfig := filepath.Join(tmpDir, "high-throughput.yaml")
if err := os.WriteFile(highThroughputProfileConfig, []byte(fmt.Sprintf(`
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
clientConnection:
kubeconfig: '%s'
profiles:
- schedulerName: "high-throughput-profile"
plugins:
preScore:
enabled:
- name: InterPodAffinity
pluginConfig:
- name: InterPodAffinity
args:
ignorePreferredTermsOfExistingPods: true
`, configKubeconfig)), os.FileMode(0600)); err != nil {
t.Fatal(err)
}
// Insulate this test from picking up in-cluster config when run inside a pod
// We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing
originalHost := os.Getenv("KUBERNETES_SERVICE_HOST")
@@ -1525,6 +1546,110 @@ profiles:
expectedError: `key "leaderElect" already set`,
checkErrFn: runtime.IsStrictDecodingError,
},
{
name: "high throughput profile",
options: &Options{
ConfigFile: highThroughputProfileConfig,
Logs: logs.NewOptions(),
},
expectedUsername: "config",
expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: v1.SchemeGroupVersion.String(),
},
Parallelism: 16,
DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{
EnableProfiling: true,
EnableContentionProfiling: true,
},
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: true,
LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
RetryPeriod: metav1.Duration{Duration: 2 * time.Second},
ResourceLock: "leases",
ResourceNamespace: "kube-system",
ResourceName: "kube-scheduler",
},
ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
Kubeconfig: configKubeconfig,
QPS: 50,
Burst: 100,
ContentType: "application/vnd.kubernetes.protobuf",
},
PercentageOfNodesToScore: defaultPercentageOfNodesToScore,
PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds,
PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds,
Profiles: []kubeschedulerconfig.KubeSchedulerProfile{
{
SchedulerName: "high-throughput-profile",
Plugins: &kubeschedulerconfig.Plugins{
QueueSort: defaults.PluginsV1.QueueSort,
PreFilter: defaults.PluginsV1.PreFilter,
Filter: defaults.PluginsV1.Filter,
PostFilter: defaults.PluginsV1.PostFilter,
PreScore: kubeschedulerconfig.PluginSet{
Enabled: []kubeschedulerconfig.Plugin{
{Name: "InterPodAffinity"},
},
},
Score: defaults.PluginsV1.Score,
Bind: defaults.PluginsV1.Bind,
PreBind: defaults.PluginsV1.PreBind,
Reserve: defaults.PluginsV1.Reserve,
MultiPoint: defaults.PluginsV1.MultiPoint,
},
PluginConfig: []kubeschedulerconfig.PluginConfig{
{
Name: "InterPodAffinity",
Args: &kubeschedulerconfig.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
IgnorePreferredTermsOfExistingPods: true,
},
},
{
Name: "DefaultPreemption",
Args: &kubeschedulerconfig.DefaultPreemptionArgs{
MinCandidateNodesPercentage: 10,
MinCandidateNodesAbsolute: 100,
},
},
{
Name: "NodeAffinity",
Args: &kubeschedulerconfig.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &kubeschedulerconfig.NodeResourcesBalancedAllocationArgs{
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &kubeschedulerconfig.NodeResourcesFitArgs{
ScoringStrategy: &kubeschedulerconfig.ScoringStrategy{
Type: kubeschedulerconfig.LeastAllocated,
Resources: []kubeschedulerconfig.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
},
{
Name: "PodTopologySpread",
Args: &kubeschedulerconfig.PodTopologySpreadArgs{
DefaultingType: kubeschedulerconfig.SystemDefaulting,
},
},
{
Name: "VolumeBinding",
Args: &kubeschedulerconfig.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
},
},
},
},
}
for _, tc := range testcases {
@@ -1550,7 +1675,7 @@ profiles:
}
return
}
t.Errorf("unexpected error to create a config: %v", err)
t.Errorf("unexpected error creating config: %v", err)
return
}

27
pkg/generated/openapi/zz_generated.openapi.go generated
View File

@@ -54133,9 +54133,18 @@ func schema_k8sio_kube_scheduler_config_v1_InterPodAffinityArgs(ref common.Refer
Format: "int32",
},
},
"ignorePreferredTermsOfExistingPods": {
SchemaProps: spec.SchemaProps{
Description: "IgnorePreferredTermsOfExistingPods configures the scheduler to ignore existing pods' preferred affinity rules when scoring candidate nodes, unless the incoming pod has inter-pod affinities.",
Default: false,
Type: []string{"boolean"},
Format: "",
},
},
},
Required: []string{"ignorePreferredTermsOfExistingPods"},
},
},
}
}
@@ -55237,9 +55246,18 @@ func schema_k8sio_kube_scheduler_config_v1beta2_InterPodAffinityArgs(ref common.
Format: "int32",
},
},
"ignorePreferredTermsOfExistingPods": {
SchemaProps: spec.SchemaProps{
Description: "IgnorePreferredTermsOfExistingPods configures the scheduler to ignore existing pods' preferred affinity rules when scoring candidate nodes, unless the incoming pod has inter-pod affinities.",
Default: false,
Type: []string{"boolean"},
Format: "",
},
},
},
Required: []string{"ignorePreferredTermsOfExistingPods"},
},
},
}
}
@@ -56348,9 +56366,18 @@ func schema_k8sio_kube_scheduler_config_v1beta3_InterPodAffinityArgs(ref common.
Format: "int32",
},
},
"ignorePreferredTermsOfExistingPods": {
SchemaProps: spec.SchemaProps{
Description: "IgnorePreferredTermsOfExistingPods configures the scheduler to ignore existing pods' preferred affinity rules when scoring candidate nodes, unless the incoming pod has inter-pod affinities.",
Default: false,
Type: []string{"boolean"},
Format: "",
},
},
},
Required: []string{"ignorePreferredTermsOfExistingPods"},
},
},
}
}

122
pkg/scheduler/apis/config/scheme/scheme_test.go
View File

@@ -1134,6 +1134,71 @@ profiles:
},
},
},
{
name: "ignorePreferredTermsOfExistingPods is enabled",
data: []byte(`
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- pluginConfig:
- name: InterPodAffinity
args:
ignorePreferredTermsOfExistingPods: true
`),
wantProfiles: []config.KubeSchedulerProfile{
{
SchedulerName: "default-scheduler",
Plugins: defaults.PluginsV1,
PluginConfig: []config.PluginConfig{
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 1,
IgnorePreferredTermsOfExistingPods: true,
},
},
{
Name: "DefaultPreemption",
Args: &config.DefaultPreemptionArgs{MinCandidateNodesPercentage: 10, MinCandidateNodesAbsolute: 100},
},
{
Name: "NodeAffinity",
Args: &config.NodeAffinityArgs{},
},
{
Name: "NodeResourcesBalancedAllocation",
Args: &config.NodeResourcesBalancedAllocationArgs{
Resources: []config.ResourceSpec{{Name: "cpu", Weight: 1}, {Name: "memory", Weight: 1}},
},
},
{
Name: "NodeResourcesFit",
Args: &config.NodeResourcesFitArgs{
ScoringStrategy: &config.ScoringStrategy{
Type: config.LeastAllocated,
Resources: []config.ResourceSpec{
{Name: "cpu", Weight: 1},
{Name: "memory", Weight: 1},
},
},
},
},
{
Name: "PodTopologySpread",
Args: &config.PodTopologySpreadArgs{
DefaultingType: config.SystemDefaulting,
},
},
{
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 600,
},
},
},
},
},
},
}
decoder := Codecs.UniversalDecoder()
for _, tt := range testCases {
@@ -1255,6 +1320,7 @@ profiles:
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta2
hardPodAffinityWeight: 5
ignorePreferredTermsOfExistingPods: false
kind: InterPodAffinityArgs
name: InterPodAffinity
- args:
@@ -1360,6 +1426,7 @@ profiles:
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta2
hardPodAffinityWeight: 5
ignorePreferredTermsOfExistingPods: false
kind: InterPodAffinityArgs
name: InterPodAffinity
- args:
@@ -1475,6 +1542,7 @@ profiles:
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta3
hardPodAffinityWeight: 5
ignorePreferredTermsOfExistingPods: false
kind: InterPodAffinityArgs
name: InterPodAffinity
- args:
@@ -1578,6 +1646,7 @@ profiles:
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta3
hardPodAffinityWeight: 5
ignorePreferredTermsOfExistingPods: false
kind: InterPodAffinityArgs
name: InterPodAffinity
- args:
@@ -1693,6 +1762,7 @@ profiles:
- args:
apiVersion: kubescheduler.config.k8s.io/v1
hardPodAffinityWeight: 5
ignorePreferredTermsOfExistingPods: false
kind: InterPodAffinityArgs
name: InterPodAffinity
- args:
@@ -1796,6 +1866,7 @@ profiles:
- args:
apiVersion: kubescheduler.config.k8s.io/v1
hardPodAffinityWeight: 5
ignorePreferredTermsOfExistingPods: false
kind: InterPodAffinityArgs
name: InterPodAffinity
- args:
@@ -1820,6 +1891,57 @@ profiles:
foo: bar
name: OutOfTreePlugin
schedulerName: ""
`,
},
{
name: "v1 ignorePreferredTermsOfExistingPods is enabled",
version: v1.SchemeGroupVersion,
obj: &config.KubeSchedulerConfiguration{
Parallelism: 8,
Profiles: []config.KubeSchedulerProfile{
{
PluginConfig: []config.PluginConfig{
{
Name: "InterPodAffinity",
Args: &config.InterPodAffinityArgs{
HardPodAffinityWeight: 5,
IgnorePreferredTermsOfExistingPods: true,
},
},
},
},
},
},
want: `apiVersion: kubescheduler.config.k8s.io/v1
clientConnection:
acceptContentTypes: ""
burst: 0
contentType: ""
kubeconfig: ""
qps: 0
enableContentionProfiling: false
enableProfiling: false
kind: KubeSchedulerConfiguration
leaderElection:
leaderElect: false
leaseDuration: 0s
renewDeadline: 0s
resourceLock: ""
resourceName: ""
resourceNamespace: ""
retryPeriod: 0s
parallelism: 8
podInitialBackoffSeconds: 0
podMaxBackoffSeconds: 0
profiles:
- pluginConfig:
- args:
apiVersion: kubescheduler.config.k8s.io/v1
hardPodAffinityWeight: 5
ignorePreferredTermsOfExistingPods: true
kind: InterPodAffinityArgs
name: InterPodAffinity
schedulerName: ""
`,
},
}

4
pkg/scheduler/apis/config/types_pluginargs.go
View File

@@ -52,6 +52,10 @@ type InterPodAffinityArgs struct {
// HardPodAffinityWeight is the scoring weight for existing pods with a
// matching hard affinity to the incoming pod.
HardPodAffinityWeight int32
// IgnorePreferredTermsOfExistingPods configures the scheduler to ignore existing pods' preferred affinity
// rules when scoring candidate nodes, unless the incoming pod has inter-pod affinities.
IgnorePreferredTermsOfExistingPods bool
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

2
pkg/scheduler/apis/config/v1/zz_generated.conversion.go generated
View File

@@ -375,6 +375,7 @@ func autoConvert_v1_InterPodAffinityArgs_To_config_InterPodAffinityArgs(in *v1.I
if err := metav1.Convert_Pointer_int32_To_int32(&in.HardPodAffinityWeight, &out.HardPodAffinityWeight, s); err != nil {
return err
}
out.IgnorePreferredTermsOfExistingPods = in.IgnorePreferredTermsOfExistingPods
return nil
}
@@ -387,6 +388,7 @@ func autoConvert_config_InterPodAffinityArgs_To_v1_InterPodAffinityArgs(in *conf
if err := metav1.Convert_int32_To_Pointer_int32(&in.HardPodAffinityWeight, &out.HardPodAffinityWeight, s); err != nil {
return err
}
out.IgnorePreferredTermsOfExistingPods = in.IgnorePreferredTermsOfExistingPods
return nil
}

2
pkg/scheduler/apis/config/v1beta2/zz_generated.conversion.go generated
View File

@@ -375,6 +375,7 @@ func autoConvert_v1beta2_InterPodAffinityArgs_To_config_InterPodAffinityArgs(in
if err := v1.Convert_Pointer_int32_To_int32(&in.HardPodAffinityWeight, &out.HardPodAffinityWeight, s); err != nil {
return err
}
out.IgnorePreferredTermsOfExistingPods = in.IgnorePreferredTermsOfExistingPods
return nil
}
@@ -387,6 +388,7 @@ func autoConvert_config_InterPodAffinityArgs_To_v1beta2_InterPodAffinityArgs(in
if err := v1.Convert_int32_To_Pointer_int32(&in.HardPodAffinityWeight, &out.HardPodAffinityWeight, s); err != nil {
return err
}
out.IgnorePreferredTermsOfExistingPods = in.IgnorePreferredTermsOfExistingPods
return nil
}

2
pkg/scheduler/apis/config/v1beta3/zz_generated.conversion.go generated
View File

@@ -375,6 +375,7 @@ func autoConvert_v1beta3_InterPodAffinityArgs_To_config_InterPodAffinityArgs(in
if err := v1.Convert_Pointer_int32_To_int32(&in.HardPodAffinityWeight, &out.HardPodAffinityWeight, s); err != nil {
return err
}
out.IgnorePreferredTermsOfExistingPods = in.IgnorePreferredTermsOfExistingPods
return nil
}
@@ -387,6 +388,7 @@ func autoConvert_config_InterPodAffinityArgs_To_v1beta3_InterPodAffinityArgs(in
if err := v1.Convert_int32_To_Pointer_int32(&in.HardPodAffinityWeight, &out.HardPodAffinityWeight, s); err != nil {
return err
}
out.IgnorePreferredTermsOfExistingPods = in.IgnorePreferredTermsOfExistingPods
return nil
}

9
pkg/scheduler/framework/plugins/interpodaffinity/scoring.go
View File

@@ -142,6 +142,15 @@ func (pl *InterPodAffinity) PreScore(
hasPreferredAffinityConstraints := affinity != nil && affinity.PodAffinity != nil && len(affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution) > 0
hasPreferredAntiAffinityConstraints := affinity != nil && affinity.PodAntiAffinity != nil && len(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) > 0
// Optionally ignore calculating preferences of existing pods' affinity rules
// if the incoming pod has no inter-pod affinities.
if pl.args.IgnorePreferredTermsOfExistingPods && !hasPreferredAffinityConstraints && !hasPreferredAntiAffinityConstraints {
cycleState.Write(preScoreStateKey, &preScoreState{
topologyScore: make(map[string]map[string]int64),
})
return nil
}
// Unless the pod being scheduled has preferred affinity terms, we only
// need to process nodes hosting pods with affinity.
var allNodes []*framework.NodeInfo

31
pkg/scheduler/framework/plugins/interpodaffinity/scoring_test.go
View File

@@ -374,6 +374,7 @@ func TestPreferredAffinity(t *testing.T) {
nodes []*v1.Node
expectedList framework.NodeScoreList
name string
ignorePreferredTermsOfExistingPods bool
wantStatus *framework.Status
}{
{
@@ -736,13 +737,41 @@ func TestPreferredAffinity(t *testing.T) {
},
expectedList: []framework.NodeScore{{Name: "node1", Score: 0}, {Name: "node2", Score: framework.MaxNodeScore}},
},
{
name: "Ignore preferred terms of existing pods",
pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}},
pods: []*v1.Pod{
{Spec: v1.PodSpec{NodeName: "node1", Affinity: stayWithS1InRegionAwayFromS2InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}},
{Spec: v1.PodSpec{NodeName: "node2", Affinity: stayWithS2InRegion}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}},
},
nodes: []*v1.Node{
{ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: labelRgChina}},
{ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: labelRgIndia}},
},
expectedList: []framework.NodeScore{{Name: "node1", Score: 0}, {Name: "node2", Score: 0}},
ignorePreferredTermsOfExistingPods: true,
},
{
name: "Do not ignore preferred terms of existing pods",
pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}},
pods: []*v1.Pod{
{Spec: v1.PodSpec{NodeName: "node1", Affinity: stayWithS1InRegionAwayFromS2InAz}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}},
{Spec: v1.PodSpec{NodeName: "node2", Affinity: stayWithS2InRegion}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS2}},
},
nodes: []*v1.Node{
{ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: labelRgChina}},
{ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: labelRgIndia}},
},
expectedList: []framework.NodeScore{{Name: "node1", Score: 0}, {Name: "node2", Score: framework.MaxNodeScore}},
ignorePreferredTermsOfExistingPods: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
state := framework.NewCycleState()
p := plugintesting.SetupPluginWithInformers(ctx, t, New, &config.InterPodAffinityArgs{HardPodAffinityWeight: 1}, cache.NewSnapshot(test.pods, test.nodes), namespaces)
p := plugintesting.SetupPluginWithInformers(ctx, t, New, &config.InterPodAffinityArgs{HardPodAffinityWeight: 1, IgnorePreferredTermsOfExistingPods: test.ignorePreferredTermsOfExistingPods}, cache.NewSnapshot(test.pods, test.nodes), namespaces)
status := p.(framework.PreScorePlugin).PreScore(ctx, state, test.pod, test.nodes)
if !status.IsSuccess() {
if !strings.Contains(status.Message(), test.wantStatus.Message()) {

4
staging/src/k8s.io/kube-scheduler/config/v1/types_pluginargs.go
View File

@@ -52,6 +52,10 @@ type InterPodAffinityArgs struct {
// HardPodAffinityWeight is the scoring weight for existing pods with a
// matching hard affinity to the incoming pod.
HardPodAffinityWeight *int32 `json:"hardPodAffinityWeight,omitempty"`
// IgnorePreferredTermsOfExistingPods configures the scheduler to ignore existing pods' preferred affinity
// rules when scoring candidate nodes, unless the incoming pod has inter-pod affinities.
IgnorePreferredTermsOfExistingPods bool `json:"ignorePreferredTermsOfExistingPods"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

4
staging/src/k8s.io/kube-scheduler/config/v1beta2/types_pluginargs.go
View File

@@ -52,6 +52,10 @@ type InterPodAffinityArgs struct {
// HardPodAffinityWeight is the scoring weight for existing pods with a
// matching hard affinity to the incoming pod.
HardPodAffinityWeight *int32 `json:"hardPodAffinityWeight,omitempty"`
// IgnorePreferredTermsOfExistingPods configures the scheduler to ignore existing pods' preferred affinity
// rules when scoring candidate nodes, unless the incoming pod has inter-pod affinities.
IgnorePreferredTermsOfExistingPods bool `json:"ignorePreferredTermsOfExistingPods"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

4
staging/src/k8s.io/kube-scheduler/config/v1beta3/types_pluginargs.go
View File

@@ -52,6 +52,10 @@ type InterPodAffinityArgs struct {
// HardPodAffinityWeight is the scoring weight for existing pods with a
// matching hard affinity to the incoming pod.
HardPodAffinityWeight *int32 `json:"hardPodAffinityWeight,omitempty"`
// IgnorePreferredTermsOfExistingPods configures the scheduler to ignore existing pods' preferred affinity
// rules when scoring candidate nodes, unless the incoming pod has inter-pod affinities.
IgnorePreferredTermsOfExistingPods bool `json:"ignorePreferredTermsOfExistingPods"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object