prevent enabling beta by default for new api groups

pkg/controlplane/instance.go

@@ -55,6 +55,7 @@ import (
 	storageapiv1alpha1 "k8s.io/api/storage/v1alpha1"
 	storageapiv1beta1 "k8s.io/api/storage/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -571,7 +572,7 @@ func (m *Instance) InstallAPIs(apiResourceConfigSource serverstorage.APIResource
 		if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 {
 			// If we have no storage for any resource configured, this API group is effectively disabled.
 			// This can happen when an entire API group, version, or development-stage (alpha, beta, GA) is disabled.
-			klog.Warningf("API group %q is not enabled, skipping.", groupName)
+			klog.Infof("API group %q is not enabled, skipping.", groupName)
 			continue
 		}
 
@@ -638,11 +639,9 @@ func (n nodeAddressProvider) externalAddresses() ([]string, error) {
 	return addrs, nil
 }
 
-// DefaultAPIResourceConfigSource returns default configuration for an APIResource.
+var (
-func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
+	// stableAPIGroupVersionsEnabledByDefault is a list of our stable versions.
-	ret := serverstorage.NewResourceConfig()
+	stableAPIGroupVersionsEnabledByDefault = []schema.GroupVersion{
-	// NOTE: GroupVersions listed here will be enabled by default. Don't put alpha versions in the list.
-	ret.EnableVersions(
 		admissionregistrationv1.SchemeGroupVersion,
 		apiv1.SchemeGroupVersion,
 		appsv1.SchemeGroupVersion,
@@ -650,34 +649,73 @@ func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
 		authorizationapiv1.SchemeGroupVersion,
 		autoscalingapiv1.SchemeGroupVersion,
 		autoscalingapiv2.SchemeGroupVersion,
-		autoscalingapiv2beta1.SchemeGroupVersion,
-		autoscalingapiv2beta2.SchemeGroupVersion,
 		batchapiv1.SchemeGroupVersion,
-		batchapiv1beta1.SchemeGroupVersion,
 		certificatesapiv1.SchemeGroupVersion,
 		coordinationapiv1.SchemeGroupVersion,
 		discoveryv1.SchemeGroupVersion,
-		discoveryv1beta1.SchemeGroupVersion,
 		eventsv1.SchemeGroupVersion,
-		eventsv1beta1.SchemeGroupVersion,
 		networkingapiv1.SchemeGroupVersion,
 		nodev1.SchemeGroupVersion,
-		nodev1beta1.SchemeGroupVersion, // remove in 1.26
 		policyapiv1.SchemeGroupVersion,
-		policyapiv1beta1.SchemeGroupVersion,
 		rbacv1.SchemeGroupVersion,
 		storageapiv1.SchemeGroupVersion,
-		storageapiv1beta1.SchemeGroupVersion,
 		schedulingapiv1.SchemeGroupVersion,
-		flowcontrolv1beta2.SchemeGroupVersion,
+	}
+
+	// legacyBetaEnabledByDefaultResources is the list of beta resources we enable.  You may only add to this list
+	// if your resource is already enabled by default in a beta level we still serve AND there is no stable API for it.
+	// see https://github.com/kubernetes/enhancements/tree/master/keps/sig-architecture/3136-beta-apis-off-by-default
+	// for more details.
+	legacyBetaEnabledByDefaultResources = []schema.GroupVersionResource{
+		autoscalingapiv2beta1.SchemeGroupVersion.WithResource("horizontalpodautoscalers"), // remove in 1.25
+		autoscalingapiv2beta2.SchemeGroupVersion.WithResource("horizontalpodautoscalers"), // remove in 1.26
+		batchapiv1beta1.SchemeGroupVersion.WithResource("cronjobs"),                       // remove in 1.25
+		discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices"),                // remove in 1.25
+		eventsv1beta1.SchemeGroupVersion.WithResource("events"),                           // remove in 1.25
+		nodev1beta1.SchemeGroupVersion.WithResource("runtimeclasses"),                     // remove in 1.25
+		policyapiv1beta1.SchemeGroupVersion.WithResource("poddisruptionbudgets"),          // remove in 1.25
+		policyapiv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"),           // remove in 1.25
+		storageapiv1beta1.SchemeGroupVersion.WithResource("csinodes"),                     // remove in 1.25
+		storageapiv1beta1.SchemeGroupVersion.WithResource("csistoragecapacities"),         // remove in 1.27
+		flowcontrolv1beta1.SchemeGroupVersion.WithResource("flowschemas"),                 // remove in 1.26
+		flowcontrolv1beta1.SchemeGroupVersion.WithResource("prioritylevelconfigurations"), // remove in 1.26
+		flowcontrolv1beta2.SchemeGroupVersion.WithResource("flowschemas"),                 // remove in 1.29
+		flowcontrolv1beta2.SchemeGroupVersion.WithResource("prioritylevelconfigurations"), // remove in 1.29
+	}
+	// betaAPIGroupVersionsDisabledByDefault is for all future beta groupVersions.
+	betaAPIGroupVersionsDisabledByDefault = []schema.GroupVersion{
+		autoscalingapiv2beta1.SchemeGroupVersion,
+		autoscalingapiv2beta2.SchemeGroupVersion,
+		batchapiv1beta1.SchemeGroupVersion,
+		discoveryv1beta1.SchemeGroupVersion,
+		eventsv1beta1.SchemeGroupVersion,
+		nodev1beta1.SchemeGroupVersion, // remove in 1.26
+		policyapiv1beta1.SchemeGroupVersion,
+		storageapiv1beta1.SchemeGroupVersion,
 		flowcontrolv1beta1.SchemeGroupVersion,
-	)
+		flowcontrolv1beta2.SchemeGroupVersion,
-	// disable alpha versions explicitly so we have a full list of what's possible to serve
+	}
-	ret.DisableVersions(
+
+	// alphaAPIGroupVersionsDisabledByDefault holds the alpha APIs we have.  They are always disabled by default.
+	alphaAPIGroupVersionsDisabledByDefault = []schema.GroupVersion{
 		apiserverinternalv1alpha1.SchemeGroupVersion,
 		storageapiv1alpha1.SchemeGroupVersion,
 		flowcontrolv1alpha1.SchemeGroupVersion,
-	)
+	}
+)
+
+// DefaultAPIResourceConfigSource returns default configuration for an APIResource.
+func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig {
+	ret := serverstorage.NewResourceConfig()
+	// NOTE: GroupVersions listed here will be enabled by default. Don't put alpha or beta versions in the list.
+	ret.EnableVersions(stableAPIGroupVersionsEnabledByDefault...)
+
+	// disable alpha and beta versions explicitly so we have a full list of what's possible to serve
+	ret.DisableVersions(betaAPIGroupVersionsDisabledByDefault...)
+	ret.DisableVersions(alphaAPIGroupVersionsDisabledByDefault...)
+
+	// enable the legacy beta resources that were present before stopped serving new beta APIs by default.
+	ret.EnableResources(legacyBetaEnabledByDefaultResources...)
 
 	return ret
 }

pkg/controlplane/instance_test.go

@@ -28,8 +28,16 @@ import (
 	"strings"
 	"testing"
 
+	autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1"
+	autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2"
+	batchapiv1beta1 "k8s.io/api/batch/v1beta1"
 	certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1"
 	apiv1 "k8s.io/api/core/v1"
+	discoveryv1beta1 "k8s.io/api/discovery/v1beta1"
+	eventsv1beta1 "k8s.io/api/events/v1beta1"
+	nodev1beta1 "k8s.io/api/node/v1beta1"
+	policyapiv1beta1 "k8s.io/api/policy/v1beta1"
+	storageapiv1beta1 "k8s.io/api/storage/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
@@ -50,6 +58,7 @@ import (
 	kubeversion "k8s.io/component-base/version"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/apis/batch"
+	flowcontrolv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2"
 	"k8s.io/kubernetes/pkg/apis/networking"
 	apisstorage "k8s.io/kubernetes/pkg/apis/storage"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
@@ -460,3 +469,56 @@ func TestNoAlphaVersionsEnabledByDefault(t *testing.T) {
 		}
 	}
 }
+
+func TestNoBetaVersionsEnabledByDefault(t *testing.T) {
+	config := DefaultAPIResourceConfigSource()
+	for gv, enable := range config.GroupVersionConfigs {
+		if enable && strings.Contains(gv.Version, "beta") {
+			t.Errorf("Beta API version %s enabled by default", gv.String())
+		}
+	}
+}
+
+func TestNewBetaResourcesEnabledByDefault(t *testing.T) {
+	// legacyEnabledBetaResources is nearly a duplication from elsewhere.  This is intentional.  These types already have
+	// GA equivalents available and should therefore never have a beta enabled by default again.
+	legacyEnabledBetaResources := map[schema.GroupVersionResource]bool{
+		autoscalingapiv2beta1.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true,
+		autoscalingapiv2beta2.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true,
+		batchapiv1beta1.SchemeGroupVersion.WithResource("cronjobs"):                       true,
+		discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices"):                true,
+		eventsv1beta1.SchemeGroupVersion.WithResource("events"):                           true,
+		nodev1beta1.SchemeGroupVersion.WithResource("runtimeclasses"):                     true,
+		policyapiv1beta1.SchemeGroupVersion.WithResource("poddisruptionbudgets"):          true,
+		policyapiv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"):           true,
+		storageapiv1beta1.SchemeGroupVersion.WithResource("csinodes"):                     true,
+		storageapiv1beta1.SchemeGroupVersion.WithResource("csistoragecapacities"):         true,
+	}
+
+	// legacyBetaResourcesWithoutStableEquivalents contains those groupresources that were enabled by default as beta
+	// before we changed that policy and do not have stable versions. These resources are allowed to have additional
+	// beta versions enabled by default.  Nothing new should be added here.  There are no future exceptions because there
+	// are no more beta resources enabled by default.
+	legacyBetaResourcesWithoutStableEquivalents := map[schema.GroupResource]bool{
+		storageapiv1beta1.SchemeGroupVersion.WithResource("csistoragecapacities").GroupResource():         true,
+		flowcontrolv1beta2.SchemeGroupVersion.WithResource("flowschemas").GroupResource():                 true,
+		flowcontrolv1beta2.SchemeGroupVersion.WithResource("prioritylevelconfigurations").GroupResource(): true,
+	}
+
+	config := DefaultAPIResourceConfigSource()
+	for gvr, enable := range config.ResourceConfigs {
+		if !strings.Contains(gvr.Version, "beta") {
+			continue // only check beta things
+		}
+		if !enable {
+			continue // only check things that are enabled
+		}
+		if legacyEnabledBetaResources[gvr] {
+			continue // this is a legacy beta resource
+		}
+		if legacyBetaResourcesWithoutStableEquivalents[gvr.GroupResource()] {
+			continue // this is another beta of a legacy beta resource with no stable equivalent
+		}
+		t.Errorf("no new beta resources can be enabled by default, see https://github.com/kubernetes/enhancements/blob/0ad0fc8269165ca300d05ca51c7ce190a79976a5/keps/sig-architecture/3136-beta-apis-off-by-default/README.md: %v", gvr)
+	}
+}

staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -240,7 +240,7 @@ func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource {
 
 func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource {
 	for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources {
-		// TODO determine if have ever stored any of our cohabitating resources in a different location on new clusters
+		// TODO deads2k or liggitt determine if have ever stored any of our cohabitating resources in a different location on new clusters
 		if s.APIResourceConfigSource.AnyResourceForGroupEnabled(potentialStorageResource.Group) {
 			return potentialStorageResource
 		}