diff --git a/pkg/api/serialization_test.go b/pkg/api/serialization_test.go index 5916e057783..1a3932a151a 100644 --- a/pkg/api/serialization_test.go +++ b/pkg/api/serialization_test.go @@ -332,6 +332,7 @@ func TestEncode_Ptr(t *testing.T) { TerminationGracePeriodSeconds: &grace, SecurityContext: &api.PodSecurityContext{}, + Affinity: &api.Affinity{}, }, } obj := runtime.Object(pod) diff --git a/pkg/api/testing/fuzzer.go b/pkg/api/testing/fuzzer.go index 2c0cd521333..913d2bf026e 100644 --- a/pkg/api/testing/fuzzer.go +++ b/pkg/api/testing/fuzzer.go @@ -130,6 +130,9 @@ func FuzzerFor(t *testing.T, version schema.GroupVersion, src rand.Source) *fuzz if s.SecurityContext == nil { s.SecurityContext = new(api.PodSecurityContext) } + if s.Affinity == nil { + s.Affinity = new(api.Affinity) + } }, func(j *api.PodPhase, c fuzz.Continue) { statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown} diff --git a/pkg/api/testing/pod_specs.go b/pkg/api/testing/pod_specs.go index 87300a3da73..a50e992c793 100644 --- a/pkg/api/testing/pod_specs.go +++ b/pkg/api/testing/pod_specs.go @@ -29,6 +29,7 @@ func DeepEqualSafePodSpec() api.PodSpec { DNSPolicy: api.DNSClusterFirst, TerminationGracePeriodSeconds: &grace, SecurityContext: &api.PodSecurityContext{}, + Affinity: &api.Affinity{}, } } diff --git a/pkg/api/types.go b/pkg/api/types.go index 9dd6ed27ea2..806d7050600 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1824,6 +1824,9 @@ type PodSpec struct { // If not specified, the pod will not have a domainname at all. // +optional Subdomain string + // If specified, the pod's scheduling constraints + // +optional + Affinity *Affinity } // Sysctl defines a kernel parameter to be set diff --git a/pkg/api/v1/defaults.go b/pkg/api/v1/defaults.go index c1ea75c445f..89412479061 100644 --- a/pkg/api/v1/defaults.go +++ b/pkg/api/v1/defaults.go @@ -175,6 +175,9 @@ func SetDefaults_PodSpec(obj *PodSpec) { period := int64(DefaultTerminationGracePeriodSeconds) obj.TerminationGracePeriodSeconds = &period } + if obj.Affinity == nil { + obj.Affinity = &Affinity{} + } } func SetDefaults_Probe(obj *Probe) { if obj.TimeoutSeconds == 0 { diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index d84c4b100f7..168f5994348 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -2107,6 +2107,9 @@ type PodSpec struct { // If not specified, the pod will not have a domainname at all. // +optional Subdomain string `json:"subdomain,omitempty" protobuf:"bytes,17,opt,name=subdomain"` + // If specified, the pod's scheduling constraints + // +optional + Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,18,opt,name=affinity"` } // PodSecurityContext holds pod-level security attributes and common container settings. diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index 03665339bb7..22abd7b5ce2 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -1811,6 +1811,29 @@ func validateImagePullSecrets(imagePullSecrets []api.LocalObjectReference, fldPa return allErrors } +// validateAffinity checks if given affinities are valid +func validateAffinity(affinity *api.Affinity, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if affinity != nil { + if na := affinity.NodeAffinity; na != nil { + // TODO: Uncomment the next three lines once RequiredDuringSchedulingRequiredDuringExecution is implemented. + // if na.RequiredDuringSchedulingRequiredDuringExecution != nil { + // allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingRequiredDuringExecution, fldPath.Child("requiredDuringSchedulingRequiredDuringExecution"))...) + // } + + if na.RequiredDuringSchedulingIgnoredDuringExecution != nil { + allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingIgnoredDuringExecution, fldPath.Child("requiredDuringSchedulingIgnoredDuringExecution"))...) + } + + if len(na.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { + allErrs = append(allErrs, ValidatePreferredSchedulingTerms(na.PreferredDuringSchedulingIgnoredDuringExecution, fldPath.Child("preferredDuringSchedulingIgnoredDuringExecution"))...) + } + } + } + return allErrs +} + func validateTaintEffect(effect *api.TaintEffect, allowEmpty bool, fldPath *field.Path) field.ErrorList { if !allowEmpty && len(*effect) == 0 { return field.ErrorList{field.Required(fldPath, "")} @@ -1890,6 +1913,7 @@ func ValidatePodSpec(spec *api.PodSpec, fldPath *field.Path) field.ErrorList { allErrs = append(allErrs, unversionedvalidation.ValidateLabels(spec.NodeSelector, fldPath.Child("nodeSelector"))...) allErrs = append(allErrs, ValidatePodSecurityContext(spec.SecurityContext, spec, fldPath, fldPath.Child("securityContext"))...) allErrs = append(allErrs, validateImagePullSecrets(spec.ImagePullSecrets, fldPath.Child("imagePullSecrets"))...) + allErrs = append(allErrs, validateAffinity(spec.Affinity, fldPath.Child("affinity"))...) if len(spec.ServiceAccountName) > 0 { for _, msg := range ValidateServiceAccountName(spec.ServiceAccountName, false) { allErrs = append(allErrs, field.Invalid(fldPath.Child("serviceAccountName"), spec.ServiceAccountName, msg)) @@ -2116,22 +2140,6 @@ func ValidateAffinityInPodAnnotations(annotations map[string]string, fldPath *fi } affinityFldPath := fldPath.Child(api.AffinityAnnotationKey) - if affinity.NodeAffinity != nil { - na := affinity.NodeAffinity - naFldPath := affinityFldPath.Child("nodeAffinity") - // TODO: Uncomment the next three lines once RequiredDuringSchedulingRequiredDuringExecution is implemented. - // if na.RequiredDuringSchedulingRequiredDuringExecution != nil { - // allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingRequiredDuringExecution, naFldPath.Child("requiredDuringSchedulingRequiredDuringExecution"))...) - // } - - if na.RequiredDuringSchedulingIgnoredDuringExecution != nil { - allErrs = append(allErrs, ValidateNodeSelector(na.RequiredDuringSchedulingIgnoredDuringExecution, naFldPath.Child("requiredDuringSchedulingIgnoredDuringExecution"))...) - } - - if len(na.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { - allErrs = append(allErrs, ValidatePreferredSchedulingTerms(na.PreferredDuringSchedulingIgnoredDuringExecution, naFldPath.Child("preferredDuringSchedulingIgnoredDuringExecution"))...) - } - } if affinity.PodAffinity != nil { allErrs = append(allErrs, validatePodAffinity(affinity.PodAffinity, affinityFldPath.Child("podAffinity"))...) } diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index 47089acf031..239af9ad811 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -3482,50 +3482,60 @@ func TestValidatePod(t *testing.T) { NodeName: "foobar", }, }, - { // Serialized affinity requirements in annotations. + { // Serialized node affinity requirements. ObjectMeta: api.ObjectMeta{ Name: "123", Namespace: "ns", - // TODO: Uncomment and move this block into Annotations map once - // RequiredDuringSchedulingRequiredDuringExecution is implemented - // "requiredDuringSchedulingRequiredDuringExecution": { - // "nodeSelectorTerms": [{ - // "matchExpressions": [{ - // "key": "key1", - // "operator": "Exists" - // }] - // }] - // }, - Annotations: map[string]string{ - api.AffinityAnnotationKey: ` - {"nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "key2", - "operator": "In", - "values": ["value1", "value2"] - }] - }] - }, - "preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 10, - "preference": {"matchExpressions": [ - { - "key": "foo", - "operator": "In", "values": ["bar"] - } - ]} - } - ] - }}`, - }, }, Spec: api.PodSpec{ Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}}, RestartPolicy: api.RestartPolicyAlways, DNSPolicy: api.DNSClusterFirst, + // TODO: Uncomment and move this block and move inside NodeAffinity once + // RequiredDuringSchedulingRequiredDuringExecution is implemented + // RequiredDuringSchedulingRequiredDuringExecution: &api.NodeSelector{ + // NodeSelectorTerms: []api.NodeSelectorTerm{ + // { + // MatchExpressions: []api.NodeSelectorRequirement{ + // { + // Key: "key1", + // Operator: api.NodeSelectorOpExists + // }, + // }, + // }, + // }, + // }, + Affinity: &api.Affinity{ + NodeAffinity: &api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{ + NodeSelectorTerms: []api.NodeSelectorTerm{ + { + MatchExpressions: []api.NodeSelectorRequirement{ + { + Key: "key2", + Operator: api.NodeSelectorOpIn, + Values: []string{"value1", "value2"}, + }, + }, + }, + }, + }, + PreferredDuringSchedulingIgnoredDuringExecution: []api.PreferredSchedulingTerm{ + { + Weight: 10, + Preference: api.NodeSelectorTerm{ + MatchExpressions: []api.NodeSelectorRequirement{ + { + Key: "foo", + Operator: api.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, + }, }, }, { // Serialized pod affinity in affinity requirements in annotations. @@ -3863,90 +3873,101 @@ func TestValidatePod(t *testing.T) { Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}}, }, }, - "invalid json of node affinity in pod annotations": { + "invalid node selector requirement in node affinity, operator can't be null": { ObjectMeta: api.ObjectMeta{ Name: "123", Namespace: "ns", - Annotations: map[string]string{ - api.AffinityAnnotationKey: ` - {"nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - `, - }, }, - Spec: validPodSpec, - }, - "invalid node selector requirement in node affinity in pod annotations, operator can't be null": { - ObjectMeta: api.ObjectMeta{ - Name: "123", - Namespace: "ns", - Annotations: map[string]string{ - api.AffinityAnnotationKey: ` - {"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "key1", - }] - }] - }}}`, - }, - }, - Spec: validPodSpec, - }, - "invalid preferredSchedulingTerm in node affinity in pod annotations, weight should be in range 1-100": { - ObjectMeta: api.ObjectMeta{ - Name: "123", - Namespace: "ns", - Annotations: map[string]string{ - api.AffinityAnnotationKey: ` - {"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 199, - "preference": {"matchExpressions": [ + Spec: api.PodSpec{ + Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}}, + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSClusterFirst, + Affinity: &api.Affinity{ + NodeAffinity: &api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{ + NodeSelectorTerms: []api.NodeSelectorTerm{ { - "key": "foo", - "operator": "In", - "values": ["bar"] - } - ]} - } - ]}}`, + MatchExpressions: []api.NodeSelectorRequirement{ + { + + Key: "key1", + }, + }, + }, + }, + }, + }, + }, + }, + }, + "invalid preferredSchedulingTerm in node affinity, weight should be in range 1-100": { + ObjectMeta: api.ObjectMeta{ + Name: "123", + Namespace: "ns", + }, + Spec: api.PodSpec{ + Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}}, + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSClusterFirst, + Affinity: &api.Affinity{ + NodeAffinity: &api.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []api.PreferredSchedulingTerm{ + { + Weight: 199, + Preference: api.NodeSelectorTerm{ + MatchExpressions: []api.NodeSelectorRequirement{ + { + Key: "foo", + Operator: api.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, + }, + }, + }, + }, }, }, - Spec: validPodSpec, }, "invalid requiredDuringSchedulingIgnoredDuringExecution node selector, nodeSelectorTerms must have at least one term": { ObjectMeta: api.ObjectMeta{ Name: "123", Namespace: "ns", - Annotations: map[string]string{ - api.AffinityAnnotationKey: ` - {"nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [] + }, + Spec: api.PodSpec{ + Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}}, + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSClusterFirst, + Affinity: &api.Affinity{ + NodeAffinity: &api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{ + NodeSelectorTerms: []api.NodeSelectorTerm{}, }, - }}`, + }, }, }, - Spec: validPodSpec, }, "invalid requiredDuringSchedulingIgnoredDuringExecution node selector term, matchExpressions must have at least one node selector requirement": { ObjectMeta: api.ObjectMeta{ Name: "123", Namespace: "ns", - Annotations: map[string]string{ - api.AffinityAnnotationKey: ` - {"nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [] - }] + }, + Spec: api.PodSpec{ + Containers: []api.Container{{Name: "ctr", Image: "image", ImagePullPolicy: "IfNotPresent"}}, + RestartPolicy: api.RestartPolicyAlways, + DNSPolicy: api.DNSClusterFirst, + Affinity: &api.Affinity{ + NodeAffinity: &api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &api.NodeSelector{ + NodeSelectorTerms: []api.NodeSelectorTerm{ + { + MatchExpressions: []api.NodeSelectorRequirement{}, + }, + }, }, - }}`, + }, }, }, - Spec: validPodSpec, }, "invalid weight in preferredDuringSchedulingIgnoredDuringExecution in pod affinity annotations, weight should be in range 1-100": { ObjectMeta: api.ObjectMeta{ diff --git a/pkg/apis/extensions/v1beta1/defaults_test.go b/pkg/apis/extensions/v1beta1/defaults_test.go index 1bf8046c77d..cfd915a8c16 100644 --- a/pkg/apis/extensions/v1beta1/defaults_test.go +++ b/pkg/apis/extensions/v1beta1/defaults_test.go @@ -40,6 +40,7 @@ func TestSetDefaultDaemonSet(t *testing.T) { RestartPolicy: v1.RestartPolicyAlways, SecurityContext: &v1.PodSecurityContext{}, TerminationGracePeriodSeconds: &period, + Affinity: &v1.Affinity{}, }, ObjectMeta: v1.ObjectMeta{ Labels: defaultLabels, @@ -51,6 +52,7 @@ func TestSetDefaultDaemonSet(t *testing.T) { RestartPolicy: v1.RestartPolicyAlways, SecurityContext: &v1.PodSecurityContext{}, TerminationGracePeriodSeconds: &period, + Affinity: &v1.Affinity{}, }, } tests := []struct { @@ -155,6 +157,7 @@ func TestSetDefaultDeployment(t *testing.T) { RestartPolicy: v1.RestartPolicyAlways, SecurityContext: &v1.PodSecurityContext{}, TerminationGracePeriodSeconds: &period, + Affinity: &v1.Affinity{}, }, } tests := []struct { diff --git a/pkg/controller/daemon/daemoncontroller_test.go b/pkg/controller/daemon/daemoncontroller_test.go index 07dafcfe0a6..26983c3dc18 100644 --- a/pkg/controller/daemon/daemoncontroller_test.go +++ b/pkg/controller/daemon/daemoncontroller_test.go @@ -553,19 +553,23 @@ func TestNodeAffinityDaemonLaunchesPods(t *testing.T) { addNodes(manager.nodeStore.Store, 0, 4, nil) addNodes(manager.nodeStore.Store, 4, 3, simpleNodeLabel) daemon := newDaemonSet("foo") - affinity := map[string]string{ - v1.AffinityAnnotationKey: fmt.Sprintf(` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "color", - "operator": "In", - "values": ["%s"] - }] - }] - }}}`, simpleNodeLabel["color"]), + daemon.Spec.Template.Spec.Affinity = &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "color", + Operator: v1.NodeSelectorOpIn, + Values: []string{simpleNodeLabel["color"]}, + }, + }, + }, + }, + }, + }, } - daemon.Spec.Template.ObjectMeta.Annotations = affinity manager.dsStore.Add(daemon) syncAndValidateDaemonSets(t, manager, daemon, podControl, 3, 0) } diff --git a/pkg/kubectl/cmd/util/helpers_test.go b/pkg/kubectl/cmd/util/helpers_test.go index 256bbb6497f..9c8422d16a0 100644 --- a/pkg/kubectl/cmd/util/helpers_test.go +++ b/pkg/kubectl/cmd/util/helpers_test.go @@ -132,6 +132,7 @@ func TestMerge(t *testing.T) { DNSPolicy: api.DNSClusterFirst, TerminationGracePeriodSeconds: &grace, SecurityContext: &api.PodSecurityContext{}, + Affinity: &api.Affinity{}, }, }, }, diff --git a/pkg/kubelet/config/common_test.go b/pkg/kubelet/config/common_test.go index dc815b9e461..91a50044e43 100644 --- a/pkg/kubelet/config/common_test.go +++ b/pkg/kubelet/config/common_test.go @@ -54,6 +54,7 @@ func TestDecodeSinglePod(t *testing.T) { SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults(), }}, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, }, } json, err := runtime.Encode(testapi.Default.Codec(), pod) @@ -114,6 +115,7 @@ func TestDecodePodList(t *testing.T) { SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults(), }}, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, }, } podList := &v1.PodList{ diff --git a/pkg/kubelet/config/file_linux_test.go b/pkg/kubelet/config/file_linux_test.go index c0af7e5de94..faad8493ce9 100644 --- a/pkg/kubelet/config/file_linux_test.go +++ b/pkg/kubelet/config/file_linux_test.go @@ -188,6 +188,7 @@ func getTestCases(hostname types.NodeName) []*testCase { Spec: v1.PodSpec{ Containers: []v1.Container{{Name: "image", Image: "test/image", SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults()}}, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, }, Status: v1.PodStatus{ Phase: v1.PodPending, @@ -213,6 +214,7 @@ func getTestCases(hostname types.NodeName) []*testCase { ImagePullPolicy: "Always", SecurityContext: securitycontext.ValidSecurityContextWithContainerDefaults()}}, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, }, Status: v1.PodStatus{ Phase: v1.PodPending, diff --git a/pkg/kubelet/config/http_test.go b/pkg/kubelet/config/http_test.go index ebc79dc8620..c8d64a34abb 100644 --- a/pkg/kubelet/config/http_test.go +++ b/pkg/kubelet/config/http_test.go @@ -148,6 +148,7 @@ func TestExtractPodsFromHTTP(t *testing.T) { NodeName: string(nodeName), Containers: []v1.Container{{Name: "1", Image: "foo", ImagePullPolicy: v1.PullAlways}}, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, }, Status: v1.PodStatus{ Phase: v1.PodPending, @@ -169,6 +170,7 @@ func TestExtractPodsFromHTTP(t *testing.T) { DNSPolicy: v1.DNSClusterFirst, SecurityContext: &v1.PodSecurityContext{}, TerminationGracePeriodSeconds: &grace, + Affinity: &v1.Affinity{}, Containers: []v1.Container{{ Name: "1", @@ -199,6 +201,7 @@ func TestExtractPodsFromHTTP(t *testing.T) { NodeName: nodeName, Containers: []v1.Container{{Name: "1", Image: "foo", ImagePullPolicy: v1.PullAlways}}, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, }, Status: v1.PodStatus{ Phase: v1.PodPending, @@ -213,6 +216,7 @@ func TestExtractPodsFromHTTP(t *testing.T) { NodeName: nodeName, Containers: []v1.Container{{Name: "2", Image: "bar:bartag", ImagePullPolicy: ""}}, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, }, Status: v1.PodStatus{ Phase: v1.PodPending, @@ -236,6 +240,7 @@ func TestExtractPodsFromHTTP(t *testing.T) { DNSPolicy: v1.DNSClusterFirst, TerminationGracePeriodSeconds: &grace, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, Containers: []v1.Container{{ Name: "1", @@ -262,6 +267,7 @@ func TestExtractPodsFromHTTP(t *testing.T) { DNSPolicy: v1.DNSClusterFirst, TerminationGracePeriodSeconds: &grace, SecurityContext: &v1.PodSecurityContext{}, + Affinity: &v1.Affinity{}, Containers: []v1.Container{{ Name: "2", diff --git a/pkg/registry/core/pod/etcd/etcd_test.go b/pkg/registry/core/pod/etcd/etcd_test.go index 5da86bf6ac2..80a9a181de4 100644 --- a/pkg/registry/core/pod/etcd/etcd_test.go +++ b/pkg/registry/core/pod/etcd/etcd_test.go @@ -72,6 +72,7 @@ func validNewPod() *api.Pod { }, }, SecurityContext: &api.PodSecurityContext{}, + Affinity: &api.Affinity{}, }, } } @@ -658,6 +659,7 @@ func TestEtcdUpdateScheduled(t *testing.T) { }, }, SecurityContext: &api.PodSecurityContext{}, + Affinity: &api.Affinity{}, }, }, nil, 1) if err != nil { @@ -686,6 +688,7 @@ func TestEtcdUpdateScheduled(t *testing.T) { TerminationGracePeriodSeconds: &grace, SecurityContext: &api.PodSecurityContext{}, + Affinity: &api.Affinity{}, }, } _, _, err = storage.Update(ctx, podIn.Name, rest.DefaultUpdatedObjectInfo(&podIn, api.Scheme)) @@ -726,6 +729,7 @@ func TestEtcdUpdateStatus(t *testing.T) { }, }, SecurityContext: &api.PodSecurityContext{}, + Affinity: &api.Affinity{}, }, } err := storage.Storage.Create(ctx, key, &podStart, nil, 0) @@ -750,6 +754,7 @@ func TestEtcdUpdateStatus(t *testing.T) { }, }, SecurityContext: &api.PodSecurityContext{}, + Affinity: &api.Affinity{}, }, Status: api.PodStatus{ Phase: api.PodRunning, diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates.go b/plugin/pkg/scheduler/algorithm/predicates/predicates.go index 0d1d80eb7ba..e265be984a5 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates.go @@ -582,14 +582,6 @@ func podMatchesNodeLabels(pod *v1.Pod, node *v1.Node) bool { } } - // Parse required node affinity scheduling requirements - // and check if the current node match the requirements. - affinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations) - if err != nil { - glog.V(10).Infof("Failed to get Affinity from Pod %+v, err: %+v", podName(pod), err) - return false - } - // 1. nil NodeSelector matches all nodes (i.e. does not filter out any nodes) // 2. nil []NodeSelectorTerm (equivalent to non-nil empty NodeSelector) matches no nodes // 3. zero-length non-nil []NodeSelectorTerm matches no nodes also, just for simplicity @@ -597,6 +589,7 @@ func podMatchesNodeLabels(pod *v1.Pod, node *v1.Node) bool { // 5. zero-length non-nil []NodeSelectorRequirement matches no nodes also, just for simplicity // 6. non-nil empty NodeSelectorRequirement is not allowed nodeAffinityMatches := true + affinity := pod.Spec.Affinity if affinity != nil && affinity.NodeAffinity != nil { nodeAffinity := affinity.NodeAffinity // if no required NodeAffinity requirements, will do no-op, means select all nodes. diff --git a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go index 05c1502a570..c0627e4612a 100644 --- a/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go +++ b/plugin/pkg/scheduler/algorithm/predicates/predicates_test.go @@ -866,18 +866,23 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "foo", - "operator": "In", - "values": ["bar", "value2"] - }] - }] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar", "value2"}, + }, + }, + }, + }, + }, + }, }, }, }, @@ -889,18 +894,23 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "kernel-version", - "operator": "Gt", - "values": ["0204"] - }] - }] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "kernel-version", + Operator: v1.NodeSelectorOpGt, + Values: []string{"0204"}, + }, + }, + }, + }, + }, + }, }, }, }, @@ -913,18 +923,23 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "mem-type", - "operator": "NotIn", - "values": ["DDR", "DDR2"] - }] - }] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "mem-type", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"DDR", "DDR2"}, + }, + }, + }, + }, + }, + }, }, }, }, @@ -936,17 +951,22 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "GPU", - "operator": "Exists" - }] - }] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "GPU", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, }, }, }, @@ -958,18 +978,23 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "foo", - "operator": "In", - "values": ["value1", "value2"] - }] - }] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value1", "value2"}, + }, + }, + }, + }, + }, + }, }, }, }, @@ -981,12 +1006,13 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": null - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: nil, + }, + }, }, }, }, @@ -998,12 +1024,13 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{}, + }, + }, }, }, }, @@ -1013,31 +1040,54 @@ func TestPodFitsSelector(t *testing.T) { fits: false, test: "Pod with an empty []NodeSelectorTerm in affinity, can't match the node's labels and won't schedule onto the node", }, - { - pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{}, {}] - }}}`, + /* + { + pod: &v1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Annotations: map[string]string{ + v1.AffinityAnnotationKey: ` + {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{}, {}] + }}}`, + }, + }, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: {}, + }, + { + MatchExpressions: {}, + }, + }, + }, + }, + }, }, }, + labels: map[string]string{ + "foo": "bar", + }, + fits: false, + test: "Pod with invalid NodeSelectTerms in affinity will match no objects and won't schedule onto the node", }, - labels: map[string]string{ - "foo": "bar", - }, - fits: false, - test: "Pod with invalid NodeSelectTerms in affinity will match no objects and won't schedule onto the node", - }, + */ { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{"matchExpressions": [{}]}] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{}, + }, + }, + }, + }, }, }, }, @@ -1048,13 +1098,7 @@ func TestPodFitsSelector(t *testing.T) { test: "Pod with empty MatchExpressions is not a valid value will match no objects and won't schedule onto the node", }, { - pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - "some-key": "some-value", - }, - }, - }, + pod: &v1.Pod{}, labels: map[string]string{ "foo": "bar", }, @@ -1063,11 +1107,11 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": null - }}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: nil, + }, }, }, }, @@ -1079,21 +1123,26 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "GPU", - "operator": "Exists" - }, { - "key": "GPU", - "operator": "NotIn", - "values": ["AMD", "INTER"] - }] - }] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "GPU", + Operator: v1.NodeSelectorOpExists, + }, { + Key: "GPU", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"AMD", "INTER"}, + }, + }, + }, + }, + }, + }, }, }, }, @@ -1105,21 +1154,26 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "GPU", - "operator": "Exists" - }, { - "key": "GPU", - "operator": "In", - "values": ["AMD", "INTER"] - }] - }] - }}}`, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "GPU", + Operator: v1.NodeSelectorOpExists, + }, { + Key: "GPU", + Operator: v1.NodeSelectorOpIn, + Values: []string{"AMD", "INTER"}, + }, + }, + }, + }, + }, + }, }, }, }, @@ -1131,27 +1185,32 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [{ - "key": "foo", - "operator": "In", - "values": ["bar", "value2"] - }] + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar", "value2"}, + }, + }, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "diffkey", + Operator: v1.NodeSelectorOpIn, + Values: []string{"wrong", "value2"}, + }, + }, + }, }, - { - "matchExpressions": [{ - "key": "diffkey", - "operator": "In", - "values": ["wrong", "value2"] - }] - } - ] - }}}`, + }, + }, }, }, }, @@ -1199,23 +1258,26 @@ func TestPodFitsSelector(t *testing.T) { // }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "foo", - "operator": "Exists" - }] - }] - }}}`, - }, - }, Spec: v1.PodSpec{ NodeSelector: map[string]string{ "foo": "bar", }, + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + }, }, }, labels: map[string]string{ @@ -1227,23 +1289,26 @@ func TestPodFitsSelector(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "foo", - "operator": "Exists" - }] - }] - }}}`, - }, - }, Spec: v1.PodSpec{ NodeSelector: map[string]string{ "foo": "bar", }, + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + }, }, }, labels: map[string]string{ @@ -2604,17 +2669,6 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) { Annotations: map[string]string{ v1.AffinityAnnotationKey: ` { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "hostname", - "operator": "NotIn", - "values": ["h1"] - }] - }] - } - }, "podAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": [{ "labelSelector": { @@ -2630,6 +2684,25 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) { }`, }, }, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "hostname", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"h1"}, + }, + }, + }, + }, + }, + }, + }, + }, }, pods: []*v1.Pod{ {Spec: v1.PodSpec{NodeName: "nodeA"}, ObjectMeta: v1.ObjectMeta{Labels: map[string]string{"foo": "abc"}}}, @@ -2781,10 +2854,7 @@ func TestInterPodAffinityWithMultipleNodes(t *testing.T) { if !fits && !reflect.DeepEqual(reasons, affinityExpectedFailureReasons) { t.Errorf("%s: unexpected failure reasons: %v", test.test, reasons) } - affinity, err := v1.GetAffinityFromPodAnnotations(test.pod.ObjectMeta.Annotations) - if err != nil { - t.Errorf("%s: unexpected error: %v", test.test, err) - } + affinity := test.pod.Spec.Affinity if affinity != nil && affinity.NodeAffinity != nil { nodeInfo := schedulercache.NewNodeInfo() nodeInfo.SetNode(&node) diff --git a/plugin/pkg/scheduler/algorithm/priorities/metadata.go b/plugin/pkg/scheduler/algorithm/priorities/metadata.go index 1a83826547d..eb3e65f3f45 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/metadata.go +++ b/plugin/pkg/scheduler/algorithm/priorities/metadata.go @@ -38,10 +38,15 @@ func PriorityMetadata(pod *v1.Pod, nodeNameToInfo map[string]*schedulercache.Nod if err != nil { return nil } - affinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations) + affinity := pod.Spec.Affinity + annotationAffinity, err := v1.GetAffinityFromPodAnnotations(pod.Annotations) if err != nil { return nil } + if annotationAffinity != nil { + affinity.PodAffinity = annotationAffinity.PodAffinity + affinity.PodAntiAffinity = annotationAffinity.PodAntiAffinity + } return &priorityMetadata{ nonZeroRequest: getNonZeroRequests(pod), podTolerations: tolerations, diff --git a/plugin/pkg/scheduler/algorithm/priorities/node_affinity.go b/plugin/pkg/scheduler/algorithm/priorities/node_affinity.go index 4c9c151d752..419493fab1b 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/node_affinity.go +++ b/plugin/pkg/scheduler/algorithm/priorities/node_affinity.go @@ -41,12 +41,8 @@ func CalculateNodeAffinityPriorityMap(pod *v1.Pod, meta interface{}, nodeInfo *s if priorityMeta, ok := meta.(*priorityMetadata); ok { affinity = priorityMeta.affinity } else { - // We couldn't parse metadata - fallback to computing it. - var err error - affinity, err = v1.GetAffinityFromPodAnnotations(pod.Annotations) - if err != nil { - return schedulerapi.HostPriority{}, err - } + // We couldn't parse metadata - fallback to the podspec. + affinity = pod.Spec.Affinity } var count int32 diff --git a/plugin/pkg/scheduler/algorithm/priorities/node_affinity_test.go b/plugin/pkg/scheduler/algorithm/priorities/node_affinity_test.go index 64335f37b57..a13b90a7d71 100644 --- a/plugin/pkg/scheduler/algorithm/priorities/node_affinity_test.go +++ b/plugin/pkg/scheduler/algorithm/priorities/node_affinity_test.go @@ -32,62 +32,72 @@ func TestNodeAffinityPriority(t *testing.T) { label4 := map[string]string{"abc": "az11", "def": "az22"} label5 := map[string]string{"foo": "bar", "key": "value", "az": "az1"} - affinity1 := map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 2, - "preference": { - "matchExpressions": [ - { - "key": "foo", - "operator": "In", "values": ["bar"] - } - ] - } - } - ]}}`, + affinity1 := &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{{ + Weight: 2, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }}, + }, + }}, + }, } - affinity2 := map[string]string{ - v1.AffinityAnnotationKey: ` - {"nodeAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [ - { - "weight": 2, - "preference": {"matchExpressions": [ - { - "key": "foo", - "operator": "In", "values": ["bar"] - } - ]} - }, - { - "weight": 4, - "preference": {"matchExpressions": [ - { - "key": "key", - "operator": "In", "values": ["value"] - } - ]} - }, - { - "weight": 5, - "preference": {"matchExpressions": [ - { - "key": "foo", - "operator": "In", "values": ["bar"] + affinity2 := &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 2, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + }, }, - { - "key": "key", - "operator": "In", "values": ["value"] + }, + { + Weight: 4, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "key", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value"}, + }, + }, }, - { - "key": "az", - "operator": "In", "values": ["az1"] - } - ]} - } - ]}}`, + }, + { + Weight: 5, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar"}, + }, + { + Key: "key", + Operator: v1.NodeSelectorOpIn, + Values: []string{"value"}, + }, + { + Key: "az", + Operator: v1.NodeSelectorOpIn, + Values: []string{"az1"}, + }, + }, + }, + }, + }, + }, } tests := []struct { @@ -112,8 +122,8 @@ func TestNodeAffinityPriority(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: affinity1, + Spec: v1.PodSpec{ + Affinity: affinity1, }, }, nodes: []*v1.Node{ @@ -126,8 +136,8 @@ func TestNodeAffinityPriority(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: affinity1, + Spec: v1.PodSpec{ + Affinity: affinity1, }, }, nodes: []*v1.Node{ @@ -140,8 +150,8 @@ func TestNodeAffinityPriority(t *testing.T) { }, { pod: &v1.Pod{ - ObjectMeta: v1.ObjectMeta{ - Annotations: affinity2, + Spec: v1.PodSpec{ + Affinity: affinity2, }, }, nodes: []*v1.Node{ diff --git a/staging/src/k8s.io/client-go/pkg/api/types.go b/staging/src/k8s.io/client-go/pkg/api/types.go index 093f5780b52..21f2db5dd6a 100644 --- a/staging/src/k8s.io/client-go/pkg/api/types.go +++ b/staging/src/k8s.io/client-go/pkg/api/types.go @@ -1824,6 +1824,8 @@ type PodSpec struct { // If not specified, the pod will not have a domainname at all. // +optional Subdomain string `json:"subdomain,omitempty"` + // If specified, the pod's scheduling constraints + Affinity *Affinity `json:"affinity,omitempty"` } // Sysctl defines a kernel parameter to be set diff --git a/staging/src/k8s.io/client-go/pkg/api/v1/defaults.go b/staging/src/k8s.io/client-go/pkg/api/v1/defaults.go index 92ddd384486..1476b9d88b4 100644 --- a/staging/src/k8s.io/client-go/pkg/api/v1/defaults.go +++ b/staging/src/k8s.io/client-go/pkg/api/v1/defaults.go @@ -175,6 +175,9 @@ func SetDefaults_PodSpec(obj *PodSpec) { period := int64(DefaultTerminationGracePeriodSeconds) obj.TerminationGracePeriodSeconds = &period } + if obj.Affinity == nil { + obj.Affinity = &Affinity{} + } } func SetDefaults_Probe(obj *Probe) { if obj.TimeoutSeconds == 0 { diff --git a/staging/src/k8s.io/client-go/pkg/api/v1/types.go b/staging/src/k8s.io/client-go/pkg/api/v1/types.go index 7d64339b887..4ff20a108e8 100644 --- a/staging/src/k8s.io/client-go/pkg/api/v1/types.go +++ b/staging/src/k8s.io/client-go/pkg/api/v1/types.go @@ -2107,6 +2107,8 @@ type PodSpec struct { // If not specified, the pod will not have a domainname at all. // +optional Subdomain string `json:"subdomain,omitempty" protobuf:"bytes,17,opt,name=subdomain"` + // If specified, the pod's scheduling constraints + Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,18,opt,name=affinity"` } // PodSecurityContext holds pod-level security attributes and common container settings. diff --git a/test/e2e/daemon_set.go b/test/e2e/daemon_set.go index 96ed27a12ce..ab37d803fdf 100644 --- a/test/e2e/daemon_set.go +++ b/test/e2e/daemon_set.go @@ -209,17 +209,22 @@ var _ = framework.KubeDescribe("Daemon set [Serial]", func() { complexLabel := map[string]string{daemonsetNameLabel: dsName} nodeSelector := map[string]string{daemonsetColorLabel: "blue"} framework.Logf("Creating daemon with a node affinity %s", dsName) - affinity := map[string]string{ - v1.AffinityAnnotationKey: fmt.Sprintf(` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "%s", - "operator": "In", - "values": ["%s"] - }] - }] - }}}`, daemonsetColorLabel, nodeSelector[daemonsetColorLabel]), + affinity := &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: daemonsetColorLabel, + Operator: v1.NodeSelectorOpIn, + Values: []string{nodeSelector[daemonsetColorLabel]}, + }, + }, + }, + }, + }, + }, } _, err := c.Extensions().DaemonSets(ns).Create(&extensions.DaemonSet{ ObjectMeta: v1.ObjectMeta{ @@ -229,10 +234,10 @@ var _ = framework.KubeDescribe("Daemon set [Serial]", func() { Selector: &metav1.LabelSelector{MatchLabels: complexLabel}, Template: v1.PodTemplateSpec{ ObjectMeta: v1.ObjectMeta{ - Labels: complexLabel, - Annotations: affinity, + Labels: complexLabel, }, Spec: v1.PodSpec{ + Affinity: affinity, Containers: []v1.Container{ { Name: dsName, diff --git a/test/e2e/empty_dir_wrapper.go b/test/e2e/empty_dir_wrapper.go index 28a1e0a0e20..6f3b7b72b42 100644 --- a/test/e2e/empty_dir_wrapper.go +++ b/test/e2e/empty_dir_wrapper.go @@ -314,17 +314,22 @@ func testNoWrappedVolumeRace(f *framework.Framework, volumes []v1.Volume, volume targetNode := nodeList.Items[0] By("Creating RC which spawns configmap-volume pods") - affinity := map[string]string{ - v1.AffinityAnnotationKey: fmt.Sprintf(` - {"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "kubernetes.io/hostname", - "operator": "In", - "values": ["%s"] - }] - }] - }}}`, targetNode.Name), + affinity := &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/hostname", + Operator: v1.NodeSelectorOpIn, + Values: []string{targetNode.Name}, + }, + }, + }, + }, + }, + }, } rc := &v1.ReplicationController{ @@ -338,8 +343,7 @@ func testNoWrappedVolumeRace(f *framework.Framework, volumes []v1.Volume, volume }, Template: &v1.PodTemplateSpec{ ObjectMeta: v1.ObjectMeta{ - Annotations: affinity, - Labels: map[string]string{"name": rcName}, + Labels: map[string]string{"name": rcName}, }, Spec: v1.PodSpec{ Containers: []v1.Container{ @@ -355,6 +359,7 @@ func testNoWrappedVolumeRace(f *framework.Framework, volumes []v1.Volume, volume VolumeMounts: volumeMounts, }, }, + Affinity: affinity, DNSPolicy: v1.DNSDefault, Volumes: volumes, }, diff --git a/test/e2e/scheduler_predicates.go b/test/e2e/scheduler_predicates.go index c6dc4f0f166..e42bc29d548 100644 --- a/test/e2e/scheduler_predicates.go +++ b/test/e2e/scheduler_predicates.go @@ -45,6 +45,7 @@ var masterNodes sets.String type pausePodConfig struct { Name string Affinity string + NodeAffinity *v1.Affinity Annotations, Labels, NodeSelector map[string]string Resources *v1.ResourceRequirements } @@ -240,15 +241,17 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() { podName := "without-label" _, err := cs.Core().Pods(ns).Create(initPausePod(f, pausePodConfig{ Name: podName, - Affinity: `{ - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [] - }] + NodeAffinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{}, + }, + }, }, - } - }`, + }, + }, })) if err == nil || !errors.IsInvalid(err) { @@ -300,28 +303,31 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() { createPausePod(f, pausePodConfig{ Name: podName, - Affinity: `{ - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ + NodeAffinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ { - "matchExpressions": [{ - "key": "foo", - "operator": "In", - "values": ["bar", "value2"] - }] + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar", "value2"}, + }, + }, + }, { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "diffkey", + Operator: v1.NodeSelectorOpIn, + Values: []string{"wrong", "value2"}, + }, + }, }, - { - "matchExpressions": [{ - "key": "diffkey", - "operator": "In", - "values": ["wrong", "value2"] - }] - } - ] - } - } - }`, + }, + }, + }, + }, Labels: map[string]string{"name": "restricted"}, }) waitForScheduler() @@ -344,23 +350,27 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() { labelPodName := "with-labels" pod := createPausePod(f, pausePodConfig{ Name: labelPodName, - Affinity: `{ - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "kubernetes.io/hostname", - "operator": "In", - "values": ["` + nodeName + `"] - },{ - "key": "` + k + `", - "operator": "In", - "values": ["` + v + `"] - }] - }] - } - } - }`, + NodeAffinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/hostname", + Operator: v1.NodeSelectorOpIn, + Values: []string{nodeName}, + }, { + Key: k, + Operator: v1.NodeSelectorOpIn, + Values: []string{v}, + }, + }, + }, + }, + }, + }, + }, }) // check that pod got scheduled. We intentionally DO NOT check that the @@ -374,31 +384,6 @@ var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() { Expect(labelPod.Spec.NodeName).To(Equal(nodeName)) }) - // Verify that an escaped JSON string of NodeAffinity in a YAML PodSpec works. - It("validates that embedding the JSON NodeAffinity setting as a string in the annotation value work", func() { - nodeName := getNodeThatCanRunPod(f) - - By("Trying to apply a label with fake az info on the found node.") - k := "kubernetes.io/e2e-az-name" - v := "e2e-az1" - framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v) - framework.ExpectNodeHasLabel(cs, nodeName, k, v) - defer framework.RemoveLabelOffNode(cs, nodeName, k) - - By("Trying to launch a pod that with NodeAffinity setting as embedded JSON string in the annotation value.") - pod := createPodWithNodeAffinity(f) - - // check that pod got scheduled. We intentionally DO NOT check that the - // pod is running because this will create a race condition with the - // kubelet and the scheduler: the scheduler might have scheduled a pod - // already when the kubelet does not know about its new label yet. The - // kubelet will then refuse to launch the pod. - framework.ExpectNoError(framework.WaitForPodNotPending(cs, ns, pod.Name, "")) - labelPod, err := cs.Core().Pods(ns).Get(pod.Name, metav1.GetOptions{}) - framework.ExpectNoError(err) - Expect(labelPod.Spec.NodeName).To(Equal(nodeName)) - }) - // labelSelector Operator is DoesNotExist but values are there in requiredDuringSchedulingIgnoredDuringExecution // part of podAffinity,so validation fails. It("validates that a pod with an invalid podAffinity is rejected because of the LabelSelectorRequirement is invalid", func() { @@ -777,6 +762,7 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod { }, Spec: v1.PodSpec{ NodeSelector: conf.NodeSelector, + Affinity: conf.NodeAffinity, Containers: []v1.Container{ { Name: podName, @@ -822,19 +808,23 @@ func runPodAndGetNodeName(f *framework.Framework, conf pausePodConfig) string { func createPodWithNodeAffinity(f *framework.Framework) *v1.Pod { return createPausePod(f, pausePodConfig{ Name: "with-nodeaffinity-" + string(uuid.NewUUID()), - Affinity: `{ - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [{ - "matchExpressions": [{ - "key": "kubernetes.io/e2e-az-name", - "operator": "In", - "values": ["e2e-az1", "e2e-az2"] - }] - }] - } - } - }`, + NodeAffinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "kubernetes.io/e2e-az-name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"e2e-az1", "e2e-az2"}, + }, + }, + }, + }, + }, + }, + }, }) }