add emulated-version flag to kube-scheduler to control the feature gate.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

cmd/kube-apiserver/app/options/validation.go

@@ -24,9 +24,9 @@ import (
 
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	netutils "k8s.io/utils/net"
 
-	"k8s.io/apimachinery/pkg/util/version"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	"k8s.io/kubernetes/pkg/features"
@@ -142,13 +142,10 @@ func (s CompletedOptions) Validate() []error {
 		errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
 	}
 
-	// TODO: remove in 1.32
-	// emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31.
+	// TODO(#125980): remove in 1.32
 	effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName)
-	binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor())
-	if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) {
-		errs = append(errs, fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s",
-			binaryVersion.String(), effectiveVersion.EmulationVersion().String()))
+	if err := utilversion.ValidateKubeEffectiveVersion(effectiveVersion); err != nil {
+		errs = append(errs, err)
 	}
 
 	return errs

cmd/kube-scheduler/app/options/options.go

@@ -25,9 +25,11 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/dynamic/dynamicinformer"
 	clientset "k8s.io/client-go/kubernetes"
@@ -72,12 +74,21 @@ type Options struct {
 
 	Master string
 
+	// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
+	ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry
+
 	// Flags hold the parsed CLI flags.
 	Flags *cliflag.NamedFlagSets
 }
 
 // NewOptions returns default scheduler app options.
 func NewOptions() *Options {
+	// make sure DefaultKubeComponent is registered in the DefaultComponentGlobalsRegistry.
+	if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil {
+		featureGate := utilfeature.DefaultMutableFeatureGate
+		effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
+		utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
+	}
 	o := &Options{
 		SecureServing:  apiserveroptions.NewSecureServingOptions().WithLoopback(),
 		Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(),
@@ -96,6 +107,7 @@ func NewOptions() *Options {
 		},
 		Metrics:                  metrics.NewOptions(),
 		Logs:                     logs.NewOptions(),
+		ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 	}
 
 	o.Authentication.TolerateInClusterLookupFailure = true
@@ -189,7 +201,7 @@ func (o *Options) initFlags() {
 	o.Authorization.AddFlags(nfs.FlagSet("authorization"))
 	o.Deprecated.AddFlags(nfs.FlagSet("deprecated"))
 	options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election"))
-	utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
+	o.ComponentGlobalsRegistry.AddFlags(nfs.FlagSet("feature gate"))
 	o.Metrics.AddFlags(nfs.FlagSet("metrics"))
 	logsapi.AddFlags(o.Logs, nfs.FlagSet("logs"))
 
@@ -198,6 +210,9 @@ func (o *Options) initFlags() {
 
 // ApplyTo applies the scheduler options to the given scheduler app configuration.
 func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) error {
+	if err := o.ComponentGlobalsRegistry.SetFallback(); err != nil {
+		return err
+	}
 	if len(o.ConfigFile) == 0 {
 		// If the --config arg is not specified, honor the deprecated as well as leader election CLI args.
 		o.ApplyDeprecated()
@@ -251,7 +266,11 @@ func (o *Options) ApplyTo(logger klog.Logger, c *schedulerappconfig.Config) erro
 // Validate validates all the required options.
 func (o *Options) Validate() []error {
 	var errs []error
-
+	if err := o.ComponentGlobalsRegistry.SetFallback(); err != nil {
+		errs = append(errs, err)
+	} else {
+		errs = append(errs, o.ComponentGlobalsRegistry.Validate()...)
+	}
 	if err := validation.ValidateKubeSchedulerConfiguration(o.ComponentConfig); err != nil {
 		errs = append(errs, err.Errors()...)
 	}
@@ -260,6 +279,12 @@ func (o *Options) Validate() []error {
 	errs = append(errs, o.Authorization.Validate()...)
 	errs = append(errs, o.Metrics.Validate()...)
 
+	// TODO(#125980): remove in 1.32
+	effectiveVersion := o.ComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent)
+	if err := utilversion.ValidateKubeEffectiveVersion(effectiveVersion); err != nil {
+		errs = append(errs, err)
+	}
+
 	return errs
 }
 

cmd/kube-scheduler/app/options/options_test.go

@@ -32,6 +32,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	componentbaseconfig "k8s.io/component-base/config"
 	"k8s.io/component-base/logs"
 	"k8s.io/klog/v2/ktesting"
@@ -322,6 +323,7 @@ profiles:
 					AlwaysAllowGroups:            []string{"system:masters"},
 				},
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedUsername: "config",
 			expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
@@ -373,6 +375,7 @@ profiles:
 					return cfg
 				}(),
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"componentconfig/v1alpha1\"",
 		},
@@ -381,6 +384,7 @@ profiles:
 			options: &Options{
 				ConfigFile:               unknownVersionConfig,
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"kubescheduler.config.k8s.io/unknown\"",
 		},
@@ -389,6 +393,7 @@ profiles:
 			options: &Options{
 				ConfigFile:               noVersionConfig,
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedError: "Object 'apiVersion' is missing",
 		},
@@ -425,6 +430,7 @@ profiles:
 					AlwaysAllowGroups:            []string{"system:masters"},
 				},
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedUsername: "flag",
 			expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
@@ -497,6 +503,7 @@ profiles:
 					AlwaysAllowGroups:            []string{"system:masters"},
 				},
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
 				TypeMeta: metav1.TypeMeta{
@@ -541,6 +548,7 @@ profiles:
 			options: &Options{
 				ConfigFile:               pluginConfigFile,
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedUsername: "config",
 			expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
@@ -661,6 +669,7 @@ profiles:
 			options: &Options{
 				ConfigFile:               multiProfilesConfig,
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedUsername: "config",
 			expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
@@ -775,6 +784,7 @@ profiles:
 			name: "no config",
 			options: &Options{
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedError: "no configuration has been provided",
 		},
@@ -783,6 +793,7 @@ profiles:
 			options: &Options{
 				ConfigFile:               unknownFieldConfig,
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedError: `unknown field "foo"`,
 			checkErrFn:    runtime.IsStrictDecodingError,
@@ -792,6 +803,7 @@ profiles:
 			options: &Options{
 				ConfigFile:               duplicateFieldConfig,
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedError: `key "leaderElect" already set`,
 			checkErrFn:    runtime.IsStrictDecodingError,
@@ -801,6 +813,7 @@ profiles:
 			options: &Options{
 				ConfigFile:               highThroughputProfileConfig,
 				Logs:                     logs.NewOptions(),
+				ComponentGlobalsRegistry: utilversion.DefaultComponentGlobalsRegistry,
 			},
 			expectedUsername: "config",
 			expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{

cmd/kube-scheduler/app/server.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/apiserver/pkg/server/mux"
 	"k8s.io/apiserver/pkg/server/routes"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/events"
@@ -45,6 +46,7 @@ import (
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/configz"
+	"k8s.io/component-base/featuregate"
 	"k8s.io/component-base/logs"
 	logsapi "k8s.io/component-base/logs/api/v1"
 	"k8s.io/component-base/metrics/features"
@@ -74,6 +76,10 @@ type Option func(runtime.Registry) error
 
 // NewSchedulerCommand creates a *cobra.Command object with default parameters and registryOptions
 func NewSchedulerCommand(registryOptions ...Option) *cobra.Command {
+	// explicitly register (if not already registered) the kube effective version and feature gate in DefaultComponentGlobalsRegistry,
+	// which will be used in NewOptions.
+	_, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
+		utilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
 	opts := options.NewOptions()
 
 	cmd := &cobra.Command{
@@ -86,6 +92,10 @@ suitable Node. Multiple different schedulers may be used within a cluster;
 kube-scheduler is the reference implementation.
 See [scheduling](https://kubernetes.io/docs/concepts/scheduling-eviction/)
 for more information about scheduling and the kube-scheduler component.`,
+		PersistentPreRunE: func(*cobra.Command, []string) error {
+			// makes sure feature gates are set before RunE.
+			return opts.ComponentGlobalsRegistry.Set()
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runCommand(cmd, opts, registryOptions...)
 		},
@@ -120,10 +130,10 @@ for more information about scheduling and the kube-scheduler component.`,
 // runCommand runs the scheduler.
 func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Option) error {
 	verflag.PrintAndExitIfRequested()
-
+	fg := opts.ComponentGlobalsRegistry.FeatureGateFor(utilversion.DefaultKubeComponent)
 	// Activate logging as soon as possible, after that
 	// show flags with the final logging configuration.
-	if err := logsapi.ValidateAndApply(opts.Logs, utilfeature.DefaultFeatureGate); err != nil {
+	if err := logsapi.ValidateAndApply(opts.Logs, fg); err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}
@@ -142,7 +152,7 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op
 		return err
 	}
 	// add feature enablement metrics
-	utilfeature.DefaultMutableFeatureGate.AddMetrics()
+	fg.(featuregate.MutableFeatureGate).AddMetrics()
 	return Run(ctx, cc, sched)
 }
 

cmd/kube-scheduler/app/server_test.go

@@ -32,7 +32,10 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apiserver/pkg/util/feature"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	componentbaseconfig "k8s.io/component-base/config"
 	"k8s.io/component-base/featuregate"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
@@ -215,6 +218,8 @@ leaderElection:
 		wantPlugins          map[string]*config.Plugins
 		wantLeaderElection   *componentbaseconfig.LeaderElectionConfiguration
 		wantClientConnection *componentbaseconfig.ClientConnectionConfiguration
+		wantErr              bool
+		wantFeaturesGates    map[string]bool
 	}{
 		{
 			name: "default config with an alpha feature enabled",
@@ -376,6 +381,46 @@ leaderElection:
 				ResourceNamespace: configv1.SchedulerDefaultLockObjectNamespace,
 			},
 		},
+		{
+			name: "emulated version out of range",
+			flags: []string{
+				"--kubeconfig", configKubeconfig,
+				"--emulated-version=1.28",
+			},
+			wantErr: true,
+		},
+		{
+			name: "default feature gates at binary version",
+			flags: []string{
+				"--kubeconfig", configKubeconfig,
+			},
+			wantFeaturesGates: map[string]bool{"kubeA": true, "kubeB": false},
+		},
+		{
+			name: "default feature gates at emulated version",
+			flags: []string{
+				"--kubeconfig", configKubeconfig,
+				"--emulated-version=1.31",
+			},
+			wantFeaturesGates: map[string]bool{"kubeA": false, "kubeB": false},
+		},
+		{
+			name: "set feature gates at emulated version",
+			flags: []string{
+				"--kubeconfig", configKubeconfig,
+				"--emulated-version=1.31",
+				"--feature-gates=kubeA=false,kubeB=true",
+			},
+			wantFeaturesGates: map[string]bool{"kubeA": false, "kubeB": true},
+		},
+		{
+			name: "cannot set locked feature gate",
+			flags: []string{
+				"--kubeconfig", configKubeconfig,
+				"--feature-gates=kubeA=false,kubeB=true",
+			},
+			wantErr: true,
+		},
 	}
 
 	makeListener := func(t *testing.T) net.Listener {
@@ -392,6 +437,23 @@ leaderElection:
 			for k, v := range tc.restoreFeatures {
 				featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)
 			}
+			componentGlobalsRegistry := utilversion.DefaultComponentGlobalsRegistry
+			t.Cleanup(func() {
+				componentGlobalsRegistry.Reset()
+			})
+			componentGlobalsRegistry.Reset()
+			verKube := utilversion.NewEffectiveVersion("1.32")
+			fg := feature.DefaultFeatureGate.DeepCopy()
+			utilruntime.Must(fg.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
+				"kubeA": {
+					{Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
+					{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta},
+				},
+				"kubeB": {
+					{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha},
+				},
+			}))
+			utilruntime.Must(componentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, verKube, fg))
 
 			fs := pflag.NewFlagSet("test", pflag.PanicOnError)
 			opts := options.NewOptions()
@@ -415,6 +477,12 @@ leaderElection:
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 			_, sched, err := Setup(ctx, opts, tc.registryOptions...)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatal("expected Setup error, got nil")
+				}
+				return
+			}
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -443,6 +511,12 @@ leaderElection:
 					t.Errorf("Unexpected clientConnection diff (-want, +got): %s", diff)
 				}
 			}
+			for f, v := range tc.wantFeaturesGates {
+				enabled := fg.Enabled(featuregate.Feature(f))
+				if enabled != v {
+					t.Errorf("expected featuregate.Enabled(%s)=%v, got %v", f, v, enabled)
+				}
+			}
 		})
 	}
 }

cmd/kube-scheduler/app/testing/testserver.go

@@ -103,6 +103,9 @@ func StartTestServer(ctx context.Context, customFlags []string) (result TestServ
 		fs.AddFlagSet(f)
 	}
 	fs.Parse(customFlags)
+	if err := opts.ComponentGlobalsRegistry.Set(); err != nil {
+		return result, err
+	}
 
 	if opts.SecureServing.BindPort != 0 {
 		opts.SecureServing.Listener, opts.SecureServing.BindPort, err = createListenerOnFreePort()

staging/src/k8s.io/apiserver/pkg/util/version/version.go

@@ -155,3 +155,15 @@ func DefaultKubeEffectiveVersion() MutableEffectiveVersion {
 	binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get())
 	return newEffectiveVersion(binaryVersion)
 }
+
+// ValidateKubeEffectiveVersion validates the EmulationVersion is equal to the binary version at 1.31 for kube components.
+// TODO: remove in 1.32
+// emulationVersion is introduced in 1.31, so it is only allowed to be equal to the binary version at 1.31.
+func ValidateKubeEffectiveVersion(effectiveVersion EffectiveVersion) error {
+	binaryVersion := version.MajorMinor(effectiveVersion.BinaryVersion().Major(), effectiveVersion.BinaryVersion().Minor())
+	if binaryVersion.EqualTo(version.MajorMinor(1, 31)) && !effectiveVersion.EmulationVersion().EqualTo(binaryVersion) {
+		return fmt.Errorf("emulation version needs to be equal to binary version(%s) in compatibility-version alpha, got %s",
+			binaryVersion.String(), effectiveVersion.EmulationVersion().String())
+	}
+	return nil
+}