dual stack services (#91824)

* api: structure change

* api: defaulting, conversion, and validation

* [FIX] validation: auto remove second ip/family when service changes to SingleStack

* [FIX] api: defaulting, conversion, and validation

* api-server: clusterIPs alloc, printers, storage and strategy

* [FIX] clusterIPs default on read

* alloc: auto remove second ip/family when service changes to SingleStack

* api-server: repair loop handling for clusterIPs

* api-server: force kubernetes default service into single stack

* api-server: tie dualstack feature flag with endpoint feature flag

* controller-manager: feature flag, endpoint, and endpointSlice controllers handling multi family service

* [FIX] controller-manager: feature flag, endpoint, and endpointSlicecontrollers handling multi family service

* kube-proxy: feature-flag, utils, proxier, and meta proxier

* [FIX] kubeproxy: call both proxier at the same time

* kubenet: remove forced pod IP sorting

* kubectl: modify describe to include ClusterIPs, IPFamilies, and IPFamilyPolicy

* e2e: fix tests that depends on IPFamily field AND add dual stack tests

* e2e: fix expected error message for ClusterIP immutability

* add integration tests for dualstack

the third phase of dual stack is a very complex change in the API,
basically it introduces Dual Stack services. Main changes are:

- It pluralizes the Service IPFamily field to IPFamilies,
and removes the singular field.
- It introduces a new field IPFamilyPolicyType that can take
3 values to express the "dual-stack(mad)ness" of the cluster:
SingleStack, PreferDualStack and RequireDualStack
- It pluralizes ClusterIP to ClusterIPs.

The goal is to add coverage to the services API operations,
taking into account the 6 different modes a cluster can have:

- single stack: IP4 or IPv6 (as of today)
- dual stack: IPv4 only, IPv6 only, IPv4 - IPv6, IPv6 - IPv4

* [FIX] add integration tests for dualstack

* generated data

* generated files

Co-authored-by: Antonio Ojea <aojea@redhat.com>
This commit is contained in:
Khaled Henidak (Kal)
2020-10-26 13:15:59 -07:00
committed by GitHub
parent d0e06cf3e0
commit 6675eba3ef
84 changed files with 11170 additions and 3514 deletions

View File

@@ -4139,13 +4139,16 @@ var supportedSessionAffinityType = sets.NewString(string(core.ServiceAffinityCli
var supportedServiceType = sets.NewString(string(core.ServiceTypeClusterIP), string(core.ServiceTypeNodePort),
string(core.ServiceTypeLoadBalancer), string(core.ServiceTypeExternalName))
var supportedServiceIPFamily = sets.NewString(string(core.IPv4Protocol), string(core.IPv6Protocol))
var supportedServiceIPFamilyPolicy = sets.NewString(string(core.IPFamilyPolicySingleStack), string(core.IPFamilyPolicyPreferDualStack), string(core.IPFamilyPolicyRequireDualStack))
// ValidateService tests if required fields/annotations of a Service are valid.
func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorList {
allErrs := ValidateObjectMeta(&service.ObjectMeta, true, ValidateServiceName, field.NewPath("metadata"))
specPath := field.NewPath("spec")
isHeadlessService := service.Spec.ClusterIP == core.ClusterIPNone
if len(service.Spec.Ports) == 0 && !isHeadlessService && service.Spec.Type != core.ServiceTypeExternalName {
if len(service.Spec.Ports) == 0 && !isHeadlessService(service) && service.Spec.Type != core.ServiceTypeExternalName {
allErrs = append(allErrs, field.Required(specPath.Child("ports"), ""))
}
switch service.Spec.Type {
@@ -4160,16 +4163,25 @@ func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorLi
allErrs = append(allErrs, field.Invalid(portPath, port.Port, fmt.Sprintf("may not expose port %v externally since it is used by kubelet", ports.KubeletPort)))
}
}
if service.Spec.ClusterIP == "None" {
allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIP"), service.Spec.ClusterIP, "may not be set to 'None' for LoadBalancer services"))
if isHeadlessService(service) {
allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIPs").Index(0), service.Spec.ClusterIPs[0], "may not be set to 'None' for LoadBalancer services"))
}
case core.ServiceTypeNodePort:
if service.Spec.ClusterIP == "None" {
allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIP"), service.Spec.ClusterIP, "may not be set to 'None' for NodePort services"))
if isHeadlessService(service) {
allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIPs").Index(0), service.Spec.ClusterIPs[0], "may not be set to 'None' for NodePort services"))
}
case core.ServiceTypeExternalName:
if service.Spec.ClusterIP != "" {
allErrs = append(allErrs, field.Forbidden(specPath.Child("clusterIP"), "must be empty for ExternalName services"))
// must have len(.spec.ClusterIPs) == 0 // note: strategy sets ClusterIPs based on ClusterIP
if len(service.Spec.ClusterIPs) > 0 {
allErrs = append(allErrs, field.Forbidden(specPath.Child("clusterIPs"), "may not be set for ExternalName services"))
}
// must have nil families and nil policy
if len(service.Spec.IPFamilies) > 0 {
allErrs = append(allErrs, field.Forbidden(specPath.Child("ipFamilies"), "may not be set for ExternalName services"))
}
if service.Spec.IPFamilyPolicy != nil {
allErrs = append(allErrs, field.Forbidden(specPath.Child("ipFamilyPolicy"), "may not be set for ExternalName services"))
}
// The value (a CNAME) may have a trailing dot to denote it as fully qualified
@@ -4185,7 +4197,7 @@ func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorLi
portsPath := specPath.Child("ports")
for i := range service.Spec.Ports {
portPath := portsPath.Index(i)
allErrs = append(allErrs, validateServicePort(&service.Spec.Ports[i], len(service.Spec.Ports) > 1, isHeadlessService, allowAppProtocol, &allPortNames, portPath)...)
allErrs = append(allErrs, validateServicePort(&service.Spec.Ports[i], len(service.Spec.Ports) > 1, isHeadlessService(service), allowAppProtocol, &allPortNames, portPath)...)
}
if service.Spec.Selector != nil {
@@ -4206,11 +4218,8 @@ func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorLi
}
}
if helper.IsServiceIPSet(service) {
if ip := net.ParseIP(service.Spec.ClusterIP); ip == nil {
allErrs = append(allErrs, field.Invalid(specPath.Child("clusterIP"), service.Spec.ClusterIP, "must be empty, 'None', or a valid IP address"))
}
}
// dualstack <-> ClusterIPs <-> ipfamilies
allErrs = append(allErrs, validateServiceClusterIPsRelatedFields(service)...)
ipPath := specPath.Child("externalIPs")
for i, ip := range service.Spec.ExternalIPs {
@@ -4338,6 +4347,7 @@ func ValidateService(service *core.Service, allowAppProtocol bool) field.ErrorLi
}
}
// external traffic fields
allErrs = append(allErrs, validateServiceExternalTrafficFieldsValue(service)...)
return allErrs
}
@@ -4446,13 +4456,16 @@ func ValidateServiceCreate(service *core.Service) field.ErrorList {
func ValidateServiceUpdate(service, oldService *core.Service) field.ErrorList {
allErrs := ValidateObjectMetaUpdate(&service.ObjectMeta, &oldService.ObjectMeta, field.NewPath("metadata"))
// ClusterIP should be immutable for services using it (every type other than ExternalName)
// which do not have ClusterIP assigned yet (empty string value)
if service.Spec.Type != core.ServiceTypeExternalName {
if oldService.Spec.Type != core.ServiceTypeExternalName && oldService.Spec.ClusterIP != "" {
allErrs = append(allErrs, ValidateImmutableField(service.Spec.ClusterIP, oldService.Spec.ClusterIP, field.NewPath("spec", "clusterIP"))...)
}
}
// User can upgrade (add another clusterIP or ipFamily)
// can downgrade (remove secondary clusterIP or ipFamily)
// but *CAN NOT* change primary/secondary clusterIP || ipFamily *UNLESS*
// they are changing from/to/ON ExternalName
upgradeDowngradeClusterIPsErrs := validateUpgradeDowngradeClusterIPs(oldService, service)
allErrs = append(allErrs, upgradeDowngradeClusterIPsErrs...)
upgradeDowngradeIPFamiliesErrs := validateUpgradeDowngradeIPFamilies(oldService, service)
allErrs = append(allErrs, upgradeDowngradeIPFamiliesErrs...)
// allow AppProtocol value if the feature gate is set or the field is
// already set on the resource.
@@ -6094,3 +6107,255 @@ func ValidateSpreadConstraintNotRepeat(fldPath *field.Path, constraint core.Topo
}
return nil
}
// validateServiceClusterIPsRelatedFields validates .spec.ClusterIPs,, .spec.IPFamilies, .spec.ipFamilyPolicy
func validateServiceClusterIPsRelatedFields(service *core.Service) field.ErrorList {
// ClusterIP, ClusterIPs, IPFamilyPolicy and IPFamilies are validated prior (all must be unset) for ExternalName service
if service.Spec.Type == core.ServiceTypeExternalName {
return field.ErrorList{}
}
allErrs := field.ErrorList{}
hasInvalidIPs := false
specPath := field.NewPath("spec")
clusterIPsField := specPath.Child("clusterIPs")
ipFamiliesField := specPath.Child("ipFamilies")
ipFamilyPolicyField := specPath.Child("ipFamilyPolicy")
// Make sure ClusterIP and ClusterIPs are synced. For most cases users can
// just manage one or the other and we'll handle the rest (see PrepareFor*
// in strategy).
if len(service.Spec.ClusterIP) != 0 {
// If ClusterIP is set, ClusterIPs[0] must match.
if len(service.Spec.ClusterIPs) == 0 {
allErrs = append(allErrs, field.Required(clusterIPsField, ""))
} else if service.Spec.ClusterIPs[0] != service.Spec.ClusterIP {
allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "element [0] must match clusterIP"))
}
} else { // ClusterIP == ""
// If ClusterIP is not set, ClusterIPs must also be unset.
if len(service.Spec.ClusterIPs) != 0 {
allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "must be empty when clusterIP is empty"))
}
}
// ipfamilies stand alone validation
// must be either IPv4 or IPv6
seen := sets.String{}
for i, ipFamily := range service.Spec.IPFamilies {
if !supportedServiceIPFamily.Has(string(ipFamily)) {
allErrs = append(allErrs, field.NotSupported(ipFamiliesField.Index(i), ipFamily, supportedServiceIPFamily.List()))
}
// no duplicate check also ensures that ipfamilies is dualstacked, in any order
if seen.Has(string(ipFamily)) {
allErrs = append(allErrs, field.Duplicate(ipFamiliesField.Index(i), ipFamily))
}
seen.Insert(string(ipFamily))
}
// IPFamilyPolicy stand alone validation
//note: nil is ok, defaulted in alloc check registry/core/service/*
if service.Spec.IPFamilyPolicy != nil {
// must have a supported value
if !supportedServiceIPFamilyPolicy.Has(string(*(service.Spec.IPFamilyPolicy))) {
allErrs = append(allErrs, field.NotSupported(ipFamilyPolicyField, service.Spec.IPFamilyPolicy, supportedServiceIPFamilyPolicy.List()))
}
}
// clusterIPs stand alone validation
// valid ips with None and empty string handling
// duplication check is done as part of DualStackvalidation below
for i, clusterIP := range service.Spec.ClusterIPs {
// valid at first location only. if and only if len(clusterIPs) == 1
if i == 0 && clusterIP == core.ClusterIPNone {
if len(service.Spec.ClusterIPs) > 1 {
hasInvalidIPs = true
allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "'None' must be the first and only value"))
}
continue
}
// is it valid ip?
errorMessages := validation.IsValidIP(clusterIP)
hasInvalidIPs = (len(errorMessages) != 0) || hasInvalidIPs
for _, msg := range errorMessages {
allErrs = append(allErrs, field.Invalid(clusterIPsField.Index(i), clusterIP, msg))
}
}
// max two
if len(service.Spec.ClusterIPs) > 2 {
allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "may only hold up to 2 values"))
}
// at this stage if there is an invalid ip or misplaced none/empty string
// it will skew the error messages (bad index || dualstackness of already bad ips). so we
// stop here if there are errors in clusterIPs validation
if hasInvalidIPs {
return allErrs
}
// must be dual stacked ips if they are more than one ip
if len(service.Spec.ClusterIPs) > 1 /* meaning: it does not have a None or empty string */ {
dualStack, err := netutils.IsDualStackIPStrings(service.Spec.ClusterIPs)
if err != nil { // though we check for that earlier. safe > sorry
allErrs = append(allErrs, field.InternalError(clusterIPsField, fmt.Errorf("failed to check for dual stack with error:%v", err)))
}
// We only support one from each IP family (i.e. max two IPs in this list).
if !dualStack {
allErrs = append(allErrs, field.Invalid(clusterIPsField, service.Spec.ClusterIPs, "may specify no more than one IP for each IP family"))
}
}
// match clusterIPs to their families, if they were provided
if !isHeadlessService(service) && len(service.Spec.ClusterIPs) > 0 && len(service.Spec.IPFamilies) > 0 {
for i, ip := range service.Spec.ClusterIPs {
if i > (len(service.Spec.IPFamilies) - 1) {
break // no more families to check
}
// 4=>6
if service.Spec.IPFamilies[i] == core.IPv4Protocol && netutils.IsIPv6String(ip) {
allErrs = append(allErrs, field.Invalid(clusterIPsField.Index(i), ip, fmt.Sprintf("expected an IPv4 value as indicated by `ipFamilies[%v]`", i)))
}
// 6=>4
if service.Spec.IPFamilies[i] == core.IPv6Protocol && !netutils.IsIPv6String(ip) {
allErrs = append(allErrs, field.Invalid(clusterIPsField.Index(i), ip, fmt.Sprintf("expected an IPv6 value as indicated by `ipFamilies[%v]`", i)))
}
}
}
return allErrs
}
// specific validation for clusterIPs in cases of user upgrading or downgrading to/from dualstack
func validateUpgradeDowngradeClusterIPs(oldService, service *core.Service) field.ErrorList {
allErrs := make(field.ErrorList, 0)
// bail out early for ExternalName
if service.Spec.Type == core.ServiceTypeExternalName || oldService.Spec.Type == core.ServiceTypeExternalName {
return allErrs
}
newIsHeadless := isHeadlessService(service)
oldIsHeadless := isHeadlessService(oldService)
if oldIsHeadless && newIsHeadless {
return allErrs
}
switch {
// no change in ClusterIP lengths
// compare each
case len(oldService.Spec.ClusterIPs) == len(service.Spec.ClusterIPs):
for i, ip := range oldService.Spec.ClusterIPs {
if ip != service.Spec.ClusterIPs[i] {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(i), service.Spec.ClusterIPs, "may not change once set"))
}
}
// something has been released (downgraded)
case len(oldService.Spec.ClusterIPs) > len(service.Spec.ClusterIPs):
// primary ClusterIP has been released
if len(service.Spec.ClusterIPs) == 0 {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "primary clusterIP can not be unset"))
}
// test if primary clusterIP has changed
if len(oldService.Spec.ClusterIPs) > 0 &&
len(service.Spec.ClusterIPs) > 0 &&
service.Spec.ClusterIPs[0] != oldService.Spec.ClusterIPs[0] {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "may not change once set"))
}
// test if secondary ClusterIP has been released. has this service been downgraded correctly?
// user *must* set IPFamilyPolicy == SingleStack
if len(service.Spec.ClusterIPs) == 1 {
if service.Spec.IPFamilyPolicy == nil || *(service.Spec.IPFamilyPolicy) != core.IPFamilyPolicySingleStack {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "`ipFamilyPolicy` must be set to 'SingleStack' when releasing the secondary clusterIP"))
}
}
case len(oldService.Spec.ClusterIPs) < len(service.Spec.ClusterIPs):
// something has been added (upgraded)
// test if primary clusterIP has changed
if len(oldService.Spec.ClusterIPs) > 0 &&
service.Spec.ClusterIPs[0] != oldService.Spec.ClusterIPs[0] {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "may not change once set"))
}
// we don't check for Policy == RequireDualStack here since, Validation/Creation func takes care of it
}
return allErrs
}
// specific validation for ipFamilies in cases of user upgrading or downgrading to/from dualstack
func validateUpgradeDowngradeIPFamilies(oldService, service *core.Service) field.ErrorList {
allErrs := make(field.ErrorList, 0)
// bail out early for ExternalName
if service.Spec.Type == core.ServiceTypeExternalName || oldService.Spec.Type == core.ServiceTypeExternalName {
return allErrs
}
oldIsHeadless := isHeadlessService(oldService)
newIsHeadless := isHeadlessService(service)
// if changed to/from headless, then bail out
if newIsHeadless != oldIsHeadless {
return allErrs
}
// headless can change families
if newIsHeadless {
return allErrs
}
switch {
case len(oldService.Spec.IPFamilies) == len(service.Spec.IPFamilies):
// no change in ClusterIP lengths
// compare each
for i, ip := range oldService.Spec.IPFamilies {
if ip != service.Spec.IPFamilies[i] {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilies").Index(0), service.Spec.IPFamilies, "may not change once set"))
}
}
case len(oldService.Spec.IPFamilies) > len(service.Spec.IPFamilies):
// something has been released (downgraded)
// test if primary ipfamily has been released
if len(service.Spec.ClusterIPs) == 0 {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilies").Index(0), service.Spec.IPFamilies, "primary ipFamily can not be unset"))
}
// test if primary ipFamily has changed
if len(service.Spec.IPFamilies) > 0 &&
service.Spec.IPFamilies[0] != oldService.Spec.IPFamilies[0] {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilies").Index(0), service.Spec.ClusterIPs, "may not change once set"))
}
// test if secondary IPFamily has been released. has this service been downgraded correctly?
// user *must* set IPFamilyPolicy == SingleStack
if len(service.Spec.IPFamilies) == 1 {
if service.Spec.IPFamilyPolicy == nil || *(service.Spec.IPFamilyPolicy) != core.IPFamilyPolicySingleStack {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "clusterIPs").Index(0), service.Spec.ClusterIPs, "`ipFamilyPolicy` must be set to 'SingleStack' when releasing the secondary ipFamily"))
}
}
case len(oldService.Spec.IPFamilies) < len(service.Spec.IPFamilies):
// something has been added (upgraded)
// test if primary ipFamily has changed
if len(oldService.Spec.IPFamilies) > 0 &&
len(service.Spec.IPFamilies) > 0 &&
service.Spec.IPFamilies[0] != oldService.Spec.IPFamilies[0] {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "ipFamilies").Index(0), service.Spec.ClusterIPs, "may not change once set"))
}
// we don't check for Policy == RequireDualStack here since, Validation/Creation func takes care of it
}
return allErrs
}
func isHeadlessService(service *core.Service) bool {
return service != nil &&
len(service.Spec.ClusterIPs) == 1 &&
service.Spec.ClusterIPs[0] == core.ClusterIPNone
}