scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-10-31 18:28:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor compatibility version code

Browse Source 
Replace DefaultComponentGlobalsRegistry with new instance of componentGlobalsRegistry in test api server.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

move kube effective version validation out of component base.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

move DefaultComponentGlobalsRegistry out of component base.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

move ComponentGlobalsRegistry out of featuregate pkg.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

remove usage of DefaultComponentGlobalsRegistry in test files.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

change non-test DefaultKubeEffectiveVersion to use DefaultBuildEffectiveVersion.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

Restore useDefaultBuildBinaryVersion in effective version.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

rename DefaultKubeEffectiveVersion to DefaultKubeEffectiveVersionForTest.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

pass options.ComponentGlobalsRegistry into config for controller manager and scheduler.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

Pass apiserver effective version to DefaultResourceEncodingConfig.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

change statusz registry to take effective version from the components.

Signed-off-by: Siyuan Zhang <sizhang@google.com>

Address review comments

Signed-off-by: Siyuan Zhang <sizhang@google.com>

update vendor

Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
Siyuan Zhang
2024-12-20 07:03:03 +00:00
parent 22f25efc2c
commit 8fc3a33454
66 changed files with 1008 additions and 805 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/kube-apiserver/app/options/options_test.go
View File

@@ -35,10 +35,10 @@ import (
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
utilversion "k8s.io/component-base/version"
kapi "k8s.io/kubernetes/pkg/apis/core"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
@@ -49,14 +49,12 @@ import (
)
func TestAddFlags(t *testing.T) {
componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry
t.Cleanup(func() {
componentGlobalsRegistry.Reset()
})
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
utilruntime.Must(componentGlobalsRegistry.Register("test", basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31"), featuregate.NewFeatureGate()))
s := NewServerRunOptions()
s.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry
for _, f := range s.Flags().FlagSets {
fs.AddFlagSet(f)
}
@@ -150,7 +148,7 @@ func TestAddFlags(t *testing.T) {
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
ComponentGlobalsRegistry: componentGlobalsRegistry,
ComponentName: featuregate.DefaultKubeComponent,
ComponentName: basecompatibility.DefaultKubeComponent,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{

6
cmd/kube-apiserver/app/options/validation.go
View File

@@ -24,7 +24,6 @@ import (
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilversion "k8s.io/component-base/version"
netutils "k8s.io/utils/net"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -142,10 +141,5 @@ func (s CompletedOptions) Validate() []error {
errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
}
effectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor(s.GenericServerRunOptions.ComponentName)
if err := utilversion.ValidateKubeEffectiveVersion(effectiveVersion); err != nil {
errs = append(errs, err)
}
return errs
}

8
cmd/kube-apiserver/app/server.go
View File

@@ -41,6 +41,7 @@ import (
"k8s.io/client-go/rest"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/cli/globalflag"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
@@ -64,10 +65,9 @@ func init() {
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
_, featureGate := featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
featuregate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
s := options.NewServerRunOptions()
ctx := genericapiserver.SetupSignalContext()
featureGate := s.GenericServerRunOptions.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)
cmd := &cobra.Command{
Use: "kube-apiserver",
@@ -79,7 +79,7 @@ cluster's shared state through which all other components interact.`,
// stop printing usage when the command errors
SilenceUsage: true,
PersistentPreRunE: func(*cobra.Command, []string) error {
if err := featuregate.DefaultComponentGlobalsRegistry.Set(); err != nil {
if err := s.GenericServerRunOptions.ComponentGlobalsRegistry.Set(); err != nil {
return err
}
// silence client-go warnings.
@@ -108,7 +108,7 @@ cluster's shared state through which all other components interact.`,
return utilerrors.NewAggregate(errs)
}
// add feature enablement metrics
featureGate.AddMetrics()
featureGate.(featuregate.MutableFeatureGate).AddMetrics()
return Run(ctx, completedOptions)
},
Args: func(cmd *cobra.Command, args []string) error {

67
cmd/kube-apiserver/app/testing/testserver.go
View File

@@ -43,22 +43,20 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
serveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storageversion"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
clientgotransport "k8s.io/client-go/transport"
"k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"
"k8s.io/component-base/featuregate"
basecompatibility "k8s.io/component-base/compatibility"
featuregatetesting "k8s.io/component-base/featuregate/testing"
logsapi "k8s.io/component-base/logs/api/v1"
utilversion "k8s.io/component-base/version"
"k8s.io/klog/v2"
"k8s.io/kube-aggregator/pkg/apiserver"
"k8s.io/kubernetes/pkg/features"
@@ -104,11 +102,8 @@ type TestServerInstanceOptions struct {
// an apiserver version skew scenario where all apiservers use the same proxyCA to verify client connections.
ProxyCA *ProxyCA
// Set the BinaryVersion of server effective version.
// If empty, effective version will default to version.DefaultKubeBinaryVersion.
// If empty, effective version will default to DefaultKubeEffectiveVersion.
BinaryVersion string
// Set the EmulationVersion of server effective version.
// If empty, emulation version will default to the effective version.
EmulationVersion string
// Set non-default request timeout in the server.
RequestTimeout time.Duration
}
@@ -194,21 +189,20 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
featureGate := utilfeature.DefaultMutableFeatureGate
featureGate.AddMetrics()
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
featureGate := utilfeature.DefaultMutableFeatureGate.DeepCopy()
effectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest()
if instanceOptions.BinaryVersion != "" {
effectiveVersion = utilversion.NewEffectiveVersion(instanceOptions.BinaryVersion)
effectiveVersion = basecompatibility.NewEffectiveVersionFromString(instanceOptions.BinaryVersion, "", "")
}
if instanceOptions.EmulationVersion != "" {
effectiveVersion.SetEmulationVersion(version.MustParse(instanceOptions.EmulationVersion))
effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion())
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
if err := componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate); err != nil {
return result, err
}
// need to call SetFeatureGateEmulationVersionDuringTest to reset the feature gate emulation version at the end of the test.
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion())
featuregate.DefaultComponentGlobalsRegistry.Reset()
utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
s := options.NewServerRunOptions()
// set up new instance of ComponentGlobalsRegistry instead of using the DefaultComponentGlobalsRegistry to avoid contention in parallel tests.
s.Options.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry
if instanceOptions.RequestTimeout > 0 {
s.GenericServerRunOptions.RequestTimeout = instanceOptions.RequestTimeout
}
@@ -330,15 +324,6 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
return result, err
}
s.Authentication.ClientCert.ClientCA = clientCACertFile
if utilfeature.DefaultFeatureGate.Enabled(features.UnknownVersionInteroperabilityProxy) {
// TODO: set up a general clean up for testserver
if clientgotransport.DialerStopCh == wait.NeverStop {
ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
t.Cleanup(cancel)
clientgotransport.DialerStopCh = ctx.Done()
}
s.PeerCAFile = filepath.Join(s.SecureServing.ServerCert.CertDirectory, s.SecureServing.ServerCert.PairName+".crt")
}
}
s.SecureServing.ExternalAddress = s.SecureServing.Listener.Addr().(*net.TCPAddr).IP // use listener addr although it is a loopback device
@@ -374,8 +359,32 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
s.Authentication.RequestHeader.ExtraHeaderPrefixes = extraHeaders
}
if err := featuregate.DefaultComponentGlobalsRegistry.Set(); err != nil {
return result, err
if err := componentGlobalsRegistry.Set(); err != nil {
return result, fmt.Errorf("%w\nIf you are using SetFeatureGate*DuringTest, try using --emulated-version and --feature-gates flags instead", err)
}
// If the local ComponentGlobalsRegistry is changed by the flags,
// we need to copy the new feature values back to the DefaultFeatureGate because most feature checks still use the DefaultFeatureGate.
// We cannot directly use DefaultFeatureGate in ComponentGlobalsRegistry because the changes done by ComponentGlobalsRegistry.Set() will not be undone at the end of the test.
if !featureGate.EmulationVersion().EqualTo(utilfeature.DefaultMutableFeatureGate.EmulationVersion()) {
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultMutableFeatureGate, effectiveVersion.EmulationVersion())
}
for f := range utilfeature.DefaultMutableFeatureGate.GetAll() {
if featureGate.Enabled(f) != utilfeature.DefaultFeatureGate.Enabled(f) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, f, featureGate.Enabled(f))
}
}
utilfeature.DefaultMutableFeatureGate.AddMetrics()
if instanceOptions.EnableCertAuth {
if featureGate.Enabled(features.UnknownVersionInteroperabilityProxy) {
// TODO: set up a general clean up for testserver
if clientgotransport.DialerStopCh == wait.NeverStop {
ctx, cancel := context.WithTimeout(context.Background(), time.Hour)
t.Cleanup(cancel)
clientgotransport.DialerStopCh = ctx.Done()
}
s.PeerCAFile = filepath.Join(s.SecureServing.ServerCert.CertDirectory, s.SecureServing.ServerCert.PairName+".crt")
}
}
saSigningKeyFile, err := os.CreateTemp("/tmp", "insecure_test_key")

4
cmd/kube-controller-manager/app/config/config.go
View File

@@ -21,6 +21,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
basecompatibility "k8s.io/component-base/compatibility"
kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
)
@@ -43,6 +44,9 @@ type Config struct {
EventBroadcaster record.EventBroadcaster
EventRecorder record.EventRecorder
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry
}
type completedConfig struct {

12
cmd/kube-controller-manager/app/controllermanager.go
View File

@@ -54,6 +54,7 @@ import (
"k8s.io/client-go/util/keyutil"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/cli/globalflag"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/configz"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
@@ -99,9 +100,6 @@ const (
// NewControllerManagerCommand creates a *cobra.Command object with default parameters
func NewControllerManagerCommand() *cobra.Command {
_, _ = featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
featuregate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
s, err := options.NewKubeControllerManagerOptions()
if err != nil {
klog.Background().Error(err, "Unable to initialize command options")
@@ -142,7 +140,7 @@ controller, and serviceaccounts controller.`,
}
// add feature enablement metrics
fg := s.ComponentGlobalsRegistry.FeatureGateFor(featuregate.DefaultKubeComponent)
fg := s.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)
fg.(featuregate.MutableFeatureGate).AddMetrics()
return Run(context.Background(), c.Complete())
},
@@ -218,7 +216,7 @@ func Run(ctx context.Context, c *config.CompletedConfig) error {
slis.SLIMetricsWithReset{}.Install(unsecuredMux)
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
statusz.Install(unsecuredMux, kubeControllerManager, statusz.NewRegistry())
statusz.Install(unsecuredMux, kubeControllerManager, statusz.NewRegistry(c.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent)))
}
handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
@@ -289,11 +287,11 @@ func Run(ctx context.Context, c *config.CompletedConfig) error {
}
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CoordinatedLeaderElection) {
binaryVersion, err := semver.ParseTolerant(featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent).BinaryVersion().String())
binaryVersion, err := semver.ParseTolerant(c.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent).BinaryVersion().String())
if err != nil {
return err
}
emulationVersion, err := semver.ParseTolerant(featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent).EmulationVersion().String())
emulationVersion, err := semver.ParseTolerant(c.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent).EmulationVersion().String())
if err != nil {
return err
}

26
cmd/kube-controller-manager/app/options/options.go
View File

@@ -25,6 +25,7 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientgofeaturegate "k8s.io/client-go/features"
clientset "k8s.io/client-go/kubernetes"
@@ -36,11 +37,11 @@ import (
cpnames "k8s.io/cloud-provider/names"
cpoptions "k8s.io/cloud-provider/options"
cliflag "k8s.io/component-base/cli/flag"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/metrics"
utilversion "k8s.io/component-base/version"
cmoptions "k8s.io/controller-manager/options"
"k8s.io/klog/v2"
kubectrlmgrconfigv1alpha1 "k8s.io/kube-controller-manager/config/v1alpha1"
@@ -106,7 +107,7 @@ type KubeControllerManagerOptions struct {
ShowHiddenMetricsForVersion string
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry featuregate.ComponentGlobalsRegistry
ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry
}
// NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
@@ -116,10 +117,12 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
return nil, err
}
if featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) == nil {
componentGlobalsRegistry := compatibility.DefaultComponentGlobalsRegistry
if componentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent) == nil {
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
effectiveVersion := compatibility.DefaultBuildEffectiveVersion()
utilruntime.Must(componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate))
}
s := KubeControllerManagerOptions{
@@ -211,7 +214,7 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(),
Metrics: metrics.NewOptions(),
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
}
s.Authentication.RemoteKubeConfigFileOptional = true
@@ -230,7 +233,6 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
s.GarbageCollectorController.GCIgnoredResources = gcIgnoredResources
s.Generic.LeaderElection.ResourceName = "kube-controller-manager"
s.Generic.LeaderElection.ResourceNamespace = "kube-system"
return &s, nil
}
@@ -452,7 +454,6 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable
errs = append(errs, s.Authentication.Validate()...)
errs = append(errs, s.Authorization.Validate()...)
errs = append(errs, s.Metrics.Validate()...)
errs = append(errs, utilversion.ValidateKubeEffectiveVersion(s.ComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent)))
// in-tree cloud providers are disabled since v1.31 (KEP-2395)
if len(s.KubeCloudShared.CloudProvider.Name) > 0 && !cloudprovider.IsExternal(s.KubeCloudShared.CloudProvider.Name) {
@@ -498,10 +499,11 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy
eventRecorder := eventBroadcaster.NewRecorder(clientgokubescheme.Scheme, v1.EventSource{Component: KubeControllerManagerUserAgent})
c := &kubecontrollerconfig.Config{
Client: client,
Kubeconfig: kubeconfig,
EventBroadcaster: eventBroadcaster,
EventRecorder: eventRecorder,
Client: client,
Kubeconfig: kubeconfig,
EventBroadcaster: eventBroadcaster,
EventRecorder: eventRecorder,
ComponentGlobalsRegistry: s.ComponentGlobalsRegistry,
}
if err := s.ApplyTo(c, allControllers, disabledByDefaultControllers, controllerAliases); err != nil {
return nil, err

51
cmd/kube-controller-manager/app/options/options_test.go
View File

@@ -34,7 +34,7 @@ import (
"k8s.io/apiserver/pkg/apis/apiserver"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
utilversion "k8s.io/component-base/version"
basecompatibility "k8s.io/component-base/compatibility"
componentbaseconfig "k8s.io/component-base/config"
"k8s.io/component-base/featuregate"
@@ -447,10 +447,11 @@ func TestAddFlags(t *testing.T) {
AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"}, // note: this does not match /healthz/ or /healthz/*
AlwaysAllowGroups: []string{"system:masters"},
},
Master: "192.168.4.20",
Metrics: &metrics.Options{},
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
Master: "192.168.4.20",
Metrics: &metrics.Options{},
Logs: logs.NewOptions(),
// ignores comparing ComponentGlobalsRegistry in this test.
ComponentGlobalsRegistry: s.ComponentGlobalsRegistry,
}
// Sort GCIgnoredResources because it's built from a map, which means the
@@ -736,27 +737,6 @@ func TestApplyTo(t *testing.T) {
}
func TestEmulatedVersion(t *testing.T) {
var cleanupAndSetupFunc = func() featuregate.FeatureGate {
componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry
componentGlobalsRegistry.Reset() // make sure this test have a clean state
t.Cleanup(func() {
componentGlobalsRegistry.Reset() // make sure this test doesn't leak a dirty state
})
verKube := utilversion.NewEffectiveVersion("1.32")
fg := featuregate.NewVersionedFeatureGate(version.MustParse("1.32"))
utilruntime.Must(fg.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"kubeA": {
{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
},
"kubeB": {
{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha},
},
}))
utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg))
return fg
}
testcases := []struct {
name string
@@ -808,9 +788,8 @@ func TestEmulatedVersion(t *testing.T) {
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
fg := cleanupAndSetupFunc()
fs, s := setupControllerManagerFlagSet(t)
fg := s.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)
err := fs.Parse(tc.flags)
checkTestError(t, err, false, "")
err = s.Validate([]string{""}, []string{""}, nil)
@@ -1558,6 +1537,22 @@ func setupControllerManagerFlagSet(t *testing.T) (*pflag.FlagSet, *KubeControlle
t.Fatal(fmt.Errorf("NewKubeControllerManagerOptions failed with %w", err))
}
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
verKube := basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31")
fg := featuregate.NewVersionedFeatureGate(version.MustParse("1.32"))
utilruntime.Must(fg.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"kubeA": {
{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.32"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
},
"kubeB": {
{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha},
},
}))
utilruntime.Must(componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, verKube, fg))
s.ComponentGlobalsRegistry = componentGlobalsRegistry
for _, f := range s.Flags([]string{""}, []string{""}, nil).FlagSets {
fs.AddFlagSet(f)
}

3
cmd/kube-proxy/app/server.go
View File

@@ -42,6 +42,7 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/mux"
"k8s.io/apiserver/pkg/server/routes"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
@@ -474,7 +475,7 @@ func serveMetrics(ctx context.Context, bindAddress string, proxyMode kubeproxyco
}
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
statusz.Install(proxyMux, kubeProxy, statusz.NewRegistry())
statusz.Install(proxyMux, kubeProxy, statusz.NewRegistry(compatibility.DefaultBuildEffectiveVersion()))
}
fn := func() {

4
cmd/kube-scheduler/app/config/config.go
View File

@@ -26,6 +26,7 @@ import (
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/events"
"k8s.io/client-go/tools/leaderelection"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/zpages/flagz"
kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
)
@@ -61,6 +62,9 @@ type Config struct {
// value, the pod will be moved from unschedulablePods to backoffQ or activeQ.
// If this value is empty, the default value (5min) will be used.
PodMaxInUnschedulablePodsDuration time.Duration
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry
}
type completedConfig struct {

25
cmd/kube-scheduler/app/options/options.go
View File

@@ -28,6 +28,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/uuid"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/dynamic/dynamicinformer"
@@ -39,13 +40,12 @@ import (
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/client-go/tools/record"
cliflag "k8s.io/component-base/cli/flag"
basecompatibility "k8s.io/component-base/compatibility"
componentbaseconfig "k8s.io/component-base/config"
"k8s.io/component-base/config/options"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/metrics"
utilversion "k8s.io/component-base/version"
zpagesfeatures "k8s.io/component-base/zpages/features"
"k8s.io/component-base/zpages/flagz"
"k8s.io/klog/v2"
@@ -78,7 +78,7 @@ type Options struct {
Master string
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry featuregate.ComponentGlobalsRegistry
ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry
// Flags hold the parsed CLI flags.
Flags *cliflag.NamedFlagSets
@@ -86,12 +86,17 @@ type Options struct {
// NewOptions returns default scheduler app options.
func NewOptions() *Options {
componentGlobalsRegistry := compatibility.DefaultComponentGlobalsRegistry
// make sure DefaultKubeComponent is registered in the DefaultComponentGlobalsRegistry.
if featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) == nil {
if componentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent) == nil {
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
effectiveVersion := compatibility.DefaultBuildEffectiveVersion()
utilruntime.Must(componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate))
}
return NewOptionsWithComponentGlobalsRegistry(componentGlobalsRegistry)
}
func NewOptionsWithComponentGlobalsRegistry(componentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry) *Options {
o := &Options{
SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(),
@@ -110,7 +115,7 @@ func NewOptions() *Options {
},
Metrics: metrics.NewOptions(),
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
}
o.Authentication.TolerateInClusterLookupFailure = true
@@ -287,11 +292,6 @@ func (o *Options) Validate() []error {
errs = append(errs, o.Authorization.Validate()...)
errs = append(errs, o.Metrics.Validate()...)
effectiveVersion := o.ComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent)
if err := utilversion.ValidateKubeEffectiveVersion(effectiveVersion); err != nil {
errs = append(errs, err)
}
return errs
}
@@ -337,6 +337,7 @@ func (o *Options) Config(ctx context.Context) (*schedulerappconfig.Config, error
dynClient := dynamic.NewForConfigOrDie(c.KubeConfig)
c.DynInformerFactory = dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynClient, 0, corev1.NamespaceAll, nil)
c.LeaderElection = leaderElectionConfig
c.ComponentGlobalsRegistry = o.ComponentGlobalsRegistry
return c, nil
}

28
cmd/kube-scheduler/app/options/options_test.go
View File

@@ -32,8 +32,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
basecompatibility "k8s.io/component-base/compatibility"
componentbaseconfig "k8s.io/component-base/config"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/klog/v2/ktesting"
v1 "k8s.io/kube-scheduler/config/v1"
@@ -282,6 +282,8 @@ profiles:
defaultPodMaxBackoffSeconds := int64(10)
defaultPercentageOfNodesToScore := ptr.To[int32](0)
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
testcases := []struct {
name string
options *Options
@@ -323,7 +325,7 @@ profiles:
AlwaysAllowGroups: []string{"system:masters"},
},
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedUsername: "config",
expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
@@ -375,7 +377,7 @@ profiles:
return cfg
}(),
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"componentconfig/v1alpha1\"",
},
@@ -384,7 +386,7 @@ profiles:
options: &Options{
ConfigFile: unknownVersionConfig,
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"kubescheduler.config.k8s.io/unknown\"",
},
@@ -393,7 +395,7 @@ profiles:
options: &Options{
ConfigFile: noVersionConfig,
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedError: "Object 'apiVersion' is missing",
},
@@ -430,7 +432,7 @@ profiles:
AlwaysAllowGroups: []string{"system:masters"},
},
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedUsername: "flag",
expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
@@ -503,7 +505,7 @@ profiles:
AlwaysAllowGroups: []string{"system:masters"},
},
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
TypeMeta: metav1.TypeMeta{
@@ -548,7 +550,7 @@ profiles:
options: &Options{
ConfigFile: pluginConfigFile,
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedUsername: "config",
expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
@@ -669,7 +671,7 @@ profiles:
options: &Options{
ConfigFile: multiProfilesConfig,
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedUsername: "config",
expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
@@ -784,7 +786,7 @@ profiles:
name: "no config",
options: &Options{
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedError: "no configuration has been provided",
},
@@ -793,7 +795,7 @@ profiles:
options: &Options{
ConfigFile: unknownFieldConfig,
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedError: `unknown field "foo"`,
checkErrFn: runtime.IsStrictDecodingError,
@@ -803,7 +805,7 @@ profiles:
options: &Options{
ConfigFile: duplicateFieldConfig,
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedError: `key "leaderElect" already set`,
checkErrFn: runtime.IsStrictDecodingError,
@@ -813,7 +815,7 @@ profiles:
options: &Options{
ConfigFile: highThroughputProfileConfig,
Logs: logs.NewOptions(),
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: componentGlobalsRegistry,
},
expectedUsername: "config",
expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{

11
cmd/kube-scheduler/app/server.go
View File

@@ -46,6 +46,7 @@ import (
"k8s.io/client-go/tools/leaderelection"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/cli/globalflag"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/configz"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
@@ -84,10 +85,6 @@ type Option func(runtime.Registry) error
// NewSchedulerCommand creates a *cobra.Command object with default parameters and registryOptions
func NewSchedulerCommand(registryOptions ...Option) *cobra.Command {
// explicitly register (if not already registered) the kube effective version and feature gate in DefaultComponentGlobalsRegistry,
// which will be used in NewOptions.
_, _ = featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
featuregate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
opts := options.NewOptions()
cmd := &cobra.Command{
@@ -138,7 +135,7 @@ for more information about scheduling and the kube-scheduler component.`,
// runCommand runs the scheduler.
func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Option) error {
verflag.PrintAndExitIfRequested()
fg := opts.ComponentGlobalsRegistry.FeatureGateFor(featuregate.DefaultKubeComponent)
fg := opts.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)
// Activate logging as soon as possible, after that
// show flags with the final logging configuration.
if err := logsapi.ValidateAndApply(opts.Logs, fg); err != nil {
@@ -216,11 +213,11 @@ func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *
readyzChecks = append(readyzChecks, handlerSyncCheck)
if cc.LeaderElection != nil && utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CoordinatedLeaderElection) {
binaryVersion, err := semver.ParseTolerant(featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent).BinaryVersion().String())
binaryVersion, err := semver.ParseTolerant(cc.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent).BinaryVersion().String())
if err != nil {
return err
}
emulationVersion, err := semver.ParseTolerant(featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent).EmulationVersion().String())
emulationVersion, err := semver.ParseTolerant(cc.ComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent).EmulationVersion().String())
if err != nil {
return err
}

14
cmd/kube-scheduler/app/server_test.go
View File

@@ -36,10 +36,10 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/util/feature"
basecompatibility "k8s.io/component-base/compatibility"
componentbaseconfig "k8s.io/component-base/config"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
utilversion "k8s.io/component-base/version"
configv1 "k8s.io/kube-scheduler/config/v1"
"k8s.io/kubernetes/cmd/kube-scheduler/app/options"
"k8s.io/kubernetes/pkg/features"
@@ -438,12 +438,8 @@ leaderElection:
for k, v := range tc.restoreFeatures {
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)
}
componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry
t.Cleanup(func() {
componentGlobalsRegistry.Reset()
})
componentGlobalsRegistry.Reset()
verKube := utilversion.NewEffectiveVersion("1.32")
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
verKube := basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31")
fg := feature.DefaultFeatureGate.DeepCopy()
utilruntime.Must(fg.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"kubeA": {
@@ -454,10 +450,10 @@ leaderElection:
{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Alpha},
},
}))
utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg))
utilruntime.Must(componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, verKube, fg))
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
opts := options.NewOptions()
opts := options.NewOptionsWithComponentGlobalsRegistry(componentGlobalsRegistry)
// use listeners instead of static ports so parallel test runs don't conflict
opts.SecureServing.Listener = makeListener(t)

3
pkg/controlplane/apiserver/config.go
View File

@@ -189,8 +189,7 @@ func BuildGenericConfig(
s.Etcd.StorageConfig.Transport.TracerProvider = noopoteltrace.NewTracerProvider()
}
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.CurrentVersion = genericConfig.EffectiveVersion
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfigEffectiveVersion(genericConfig.EffectiveVersion)
storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(genericConfig.EffectiveVersion)
storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New()

12
pkg/controlplane/apiserver/options/options_test.go
View File

@@ -40,10 +40,10 @@ import (
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
utilversion "k8s.io/component-base/version"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
"k8s.io/kubernetes/pkg/serviceaccount"
v1alpha1testing "k8s.io/kubernetes/pkg/serviceaccount/externaljwt/plugin/testing/v1alpha1"
@@ -51,13 +51,11 @@ import (
)
func TestAddFlags(t *testing.T) {
componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry
t.Cleanup(func() {
componentGlobalsRegistry.Reset()
})
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
utilruntime.Must(componentGlobalsRegistry.Register("test", basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31"), featuregate.NewFeatureGate()))
s := NewOptions()
s.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry
var fss cliflag.NamedFlagSets
s.AddFlags(&fss)
for _, f := range fss.FlagSets {
@@ -141,7 +139,7 @@ func TestAddFlags(t *testing.T) {
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
ComponentGlobalsRegistry: componentGlobalsRegistry,
ComponentName: featuregate.DefaultKubeComponent,
ComponentName: basecompatibility.DefaultKubeComponent,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{

5
pkg/controlplane/apiserver/options/validation_test.go
View File

@@ -25,6 +25,7 @@ import (
kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
basemetrics "k8s.io/component-base/metrics"
"k8s.io/kubernetes/pkg/features"
@@ -202,7 +203,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate master count equal 0",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry},
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: basecompatibility.NewComponentGlobalsRegistry()},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
@@ -229,7 +230,7 @@ func TestValidateOptions(t *testing.T) {
name: "validate token request enable not attempted",
expectErrors: true,
options: &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry},
GenericServerRunOptions: &genericoptions.ServerRunOptions{ComponentGlobalsRegistry: basecompatibility.NewComponentGlobalsRegistry()},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},

2
pkg/controlplane/apiserver/server.go
View File

@@ -161,7 +161,7 @@ func (c completedConfig) New(name string, delegationTarget genericapiserver.Dele
}
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
statusz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, statusz.NewRegistry())
statusz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, statusz.NewRegistry(c.Generic.EffectiveVersion))
}
if utilfeature.DefaultFeatureGate.Enabled(apiserverfeatures.CoordinatedLeaderElection) {

5
pkg/controlplane/instance_test.go
View File

@@ -55,6 +55,7 @@ import (
"k8s.io/apiserver/pkg/server/resourceconfig"
serverstorage "k8s.io/apiserver/pkg/server/storage"
etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
"k8s.io/apiserver/pkg/util/compatibility"
"k8s.io/apiserver/pkg/util/openapi"
"k8s.io/client-go/discovery"
"k8s.io/client-go/informers"
@@ -103,7 +104,7 @@ func setUp(t *testing.T) (*etcd3testing.EtcdTestServer, Config, *assert.Assertio
},
}
config.ControlPlane.Generic.EffectiveVersion = utilversion.DefaultKubeEffectiveVersion()
config.ControlPlane.Generic.EffectiveVersion = compatibility.DefaultKubeEffectiveVersionForTest()
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.DefaultResourceEncoding.SetEffectiveVersion(config.ControlPlane.Generic.EffectiveVersion)
storageConfig.StorageObjectCountTracker = config.ControlPlane.Generic.StorageObjectCountTracker
@@ -241,7 +242,7 @@ func TestVersion(t *testing.T) {
t.Errorf("unexpected error: %v", err)
}
expectedInfo := utilversion.Get()
kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion()
kubeVersion := compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion()
expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major())
expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor())

11
pkg/kubeapiserver/default_storage_factory_builder.go
View File

@@ -25,7 +25,8 @@ import (
"k8s.io/apiserver/pkg/server/resourceconfig"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
version "k8s.io/component-base/version"
"k8s.io/apiserver/pkg/util/compatibility"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/apis/admissionregistration"
"k8s.io/kubernetes/pkg/apis/apps"
@@ -61,6 +62,11 @@ func DefaultWatchCacheSizes() map[schema.GroupResource]int {
// NewStorageFactoryConfig returns a new StorageFactoryConfig set up with necessary resource overrides.
func NewStorageFactoryConfig() *StorageFactoryConfig {
return NewStorageFactoryConfigEffectiveVersion(compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent))
}
// NewStorageFactoryConfigEffectiveVersion returns a new StorageFactoryConfig set up with necessary resource overrides for a given EffectiveVersion.
func NewStorageFactoryConfigEffectiveVersion(effectiveVersion basecompatibility.EffectiveVersion) *StorageFactoryConfig {
resources := []schema.GroupVersionResource{
// If a resource has to be stored in a version that is not the
// latest, then it can be listed here. Usually this is the case
@@ -87,7 +93,7 @@ func NewStorageFactoryConfig() *StorageFactoryConfig {
return &StorageFactoryConfig{
Serializer: legacyscheme.Codecs,
DefaultResourceEncoding: serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme),
DefaultResourceEncoding: serverstorage.NewDefaultResourceEncodingConfigForEffectiveVersion(legacyscheme.Scheme, effectiveVersion),
ResourceEncodingOverrides: resources,
}
}
@@ -101,7 +107,6 @@ type StorageFactoryConfig struct {
Serializer runtime.StorageSerializer
ResourceEncodingOverrides []schema.GroupVersionResource
EtcdServersOverrides []string
CurrentVersion version.EffectiveVersion
}
// Complete completes the StorageFactoryConfig with provided etcdOptions returning completedStorageFactoryConfig.

3
pkg/kubelet/server/server.go
View File

@@ -59,6 +59,7 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/httplog"
"k8s.io/apiserver/pkg/server/routes"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/flushwriter"
"k8s.io/component-base/configz"
@@ -565,7 +566,7 @@ func (s *Server) InstallDebuggingHandlers() {
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
s.addMetricsBucketMatcher("statusz")
statusz.Install(s.restfulCont, ComponentKubelet, statusz.NewRegistry())
statusz.Install(s.restfulCont, ComponentKubelet, statusz.NewRegistry(compatibility.DefaultBuildEffectiveVersion()))
}
// The /runningpods endpoint is used for testing only.

3
staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/server.go
View File

@@ -24,7 +24,6 @@ import (
"k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/component-base/featuregate"
)
func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command {
@@ -34,7 +33,7 @@ func NewServerCommand(ctx context.Context, out, errOut io.Writer) *cobra.Command
Short: "Launch an API extensions API server",
Long: "Launch an API extensions API server",
PersistentPreRunE: func(*cobra.Command, []string) error {
return featuregate.DefaultComponentGlobalsRegistry.Set()
return o.ServerRunOptions.ComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {
if err := o.Complete(); err != nil {

38
staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/testing/testserver.go
View File

@@ -31,18 +31,18 @@ import (
extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
"k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/openapi"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
basecompatibility "k8s.io/component-base/compatibility"
featuregatetesting "k8s.io/component-base/featuregate/testing"
logsapi "k8s.io/component-base/logs/api/v1"
utilversion "k8s.io/component-base/version"
"k8s.io/klog/v2"
)
@@ -124,15 +124,17 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
featureGate := utilfeature.DefaultMutableFeatureGate
// Configure the effective version.
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion())
featuregate.DefaultComponentGlobalsRegistry.Reset()
utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
s := options.NewCustomResourceDefinitionsServerOptions(os.Stdout, os.Stderr)
// set up new instance of ComponentGlobalsRegistry instead of using the DefaultComponentGlobalsRegistry to avoid contention in parallel tests.
featureGate := utilfeature.DefaultMutableFeatureGate.DeepCopy()
effectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest()
effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion())
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
if err := componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate); err != nil {
return result, err
}
s.ServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry
s.AddFlags(fs)
s.RecommendedOptions.SecureServing.Listener, s.RecommendedOptions.SecureServing.BindPort, err = createLocalhostListenerOnFreePort()
@@ -155,8 +157,18 @@ func StartTestServer(t Logger, _ *TestServerInstanceOptions, customFlags []strin
fs.Parse(customFlags)
if err := featuregate.DefaultComponentGlobalsRegistry.Set(); err != nil {
return result, err
if err := componentGlobalsRegistry.Set(); err != nil {
return result, fmt.Errorf("%w\nIf you are using SetFeatureGate*DuringTest, try using --emulated-version and --feature-gates flags instead", err)
}
// If the local ComponentGlobalsRegistry is changed by the flags,
// we need to copy the new feature values back to the DefaultFeatureGate because most feature checks still use the DefaultFeatureGate.
if !featureGate.EmulationVersion().EqualTo(utilfeature.DefaultMutableFeatureGate.EmulationVersion()) {
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t.(featuregatetesting.TB), utilfeature.DefaultMutableFeatureGate, effectiveVersion.EmulationVersion())
}
for f := range utilfeature.DefaultMutableFeatureGate.GetAll() {
if featureGate.Enabled(f) != utilfeature.DefaultFeatureGate.Enabled(f) {
featuregatetesting.SetFeatureGateDuringTest(t.(featuregatetesting.TB), utilfeature.DefaultFeatureGate, f, featureGate.Enabled(f))
}
}
if err := s.Complete(); err != nil {

8
staging/src/k8s.io/apiextensions-apiserver/test/integration/fieldselector_test.go
View File

@@ -41,7 +41,6 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -593,9 +592,7 @@ func TestFieldSelectorOpenAPI(t *testing.T) {
func TestFieldSelectorDropFields(t *testing.T) {
_, ctx := ktesting.NewTestContext(t)
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"))
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceFieldSelectors, false)
tearDown, apiExtensionClient, _, err := fixtures.StartDefaultServerWithClients(t)
tearDown, apiExtensionClient, _, err := fixtures.StartDefaultServerWithClients(t, "--emulated-version=1.31", "--feature-gates=CustomResourceFieldSelectors=false")
if err != nil {
t.Fatal(err)
}
@@ -675,9 +672,8 @@ func TestFieldSelectorDropFields(t *testing.T) {
}
func TestFieldSelectorDisablement(t *testing.T) {
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"))
_, ctx := ktesting.NewTestContext(t)
tearDown, config, _, err := fixtures.StartDefaultServer(t)
tearDown, config, _, err := fixtures.StartDefaultServer(t, "--emulated-version=1.31")
if err != nil {
t.Fatal(err)
}

15
staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509_test.go
View File

@@ -34,7 +34,6 @@ import (
"github.com/stretchr/testify/assert"
asn1util "k8s.io/apimachinery/pkg/apis/asn1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/features"
@@ -799,9 +798,6 @@ func TestX509(t *testing.T) {
ExpectErr: false,
setupFunc: func(t *testing.T) {
t.Helper()
// This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33"))
},
},
"common name and empty UID with feature gate disabled": {
@@ -822,8 +818,6 @@ func TestX509(t *testing.T) {
ExpectErr: false,
setupFunc: func(t *testing.T) {
t.Helper()
// This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33"))
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.AllowParsingUserUIDFromCertAuth, false)
},
},
@@ -836,9 +830,6 @@ func TestX509(t *testing.T) {
ExpectErrMsg: regexp.MustCompile("UID cannot be an empty string"),
setupFunc: func(t *testing.T) {
t.Helper()
// This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33"))
},
},
"ca with non-string UID": {
@@ -850,9 +841,6 @@ func TestX509(t *testing.T) {
ExpectErrMsg: regexp.MustCompile("unable to parse UID into a string"),
setupFunc: func(t *testing.T) {
t.Helper()
// This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33"))
},
},
"ca with multiple UIDs": {
@@ -866,9 +854,6 @@ func TestX509(t *testing.T) {
ExpectErrMsg: regexp.MustCompile("expected 1 UID, but found multiple"),
setupFunc: func(t *testing.T) {
t.Helper()
// This statement can be removed once utilversion.DefaultKubeEffectiveVersion() is >= 1.33
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse("1.33"))
},
},
"ca with multiple organizations": {

8
staging/src/k8s.io/apiserver/pkg/cel/environment/base.go
View File

@@ -32,9 +32,9 @@ import (
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/cel/library"
genericfeatures "k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
utilversion "k8s.io/component-base/version"
basecompatibility "k8s.io/component-base/compatibility"
)
// DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet
@@ -50,9 +50,9 @@ import (
// A default version number equal to the current Kubernetes major.minor version
// indicates fast forward CEL features that can be used when rollback is no longer needed.
func DefaultCompatibilityVersion() *version.Version {
effectiveVer := featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent)
effectiveVer := compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent)
if effectiveVer == nil {
effectiveVer = utilversion.DefaultKubeEffectiveVersion()
effectiveVer = compatibility.DefaultBuildEffectiveVersion()
}
return effectiveVer.MinCompatibilityVersion()
}

4
staging/src/k8s.io/apiserver/pkg/server/config.go
View File

@@ -73,12 +73,12 @@ import (
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
"k8s.io/client-go/informers"
restclient "k8s.io/client-go/rest"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics/features"
"k8s.io/component-base/metrics/prometheus/slis"
"k8s.io/component-base/tracing"
utilversion "k8s.io/component-base/version"
"k8s.io/component-base/zpages/flagz"
"k8s.io/klog/v2"
openapicommon "k8s.io/kube-openapi/pkg/common"
@@ -153,7 +153,7 @@ type Config struct {
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
EffectiveVersion basecompatibility.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate
// AuditBackend is where audit events are sent to.

8
staging/src/k8s.io/apiserver/pkg/server/config_test.go
View File

@@ -47,9 +47,9 @@ import (
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/rest"
basecompatibility "k8s.io/component-base/compatibility"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/component-base/tracing"
utilversion "k8s.io/component-base/version"
"k8s.io/klog/v2/ktesting"
netutils "k8s.io/utils/net"
)
@@ -124,7 +124,7 @@ func TestNewWithDelegate(t *testing.T) {
delegateConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
delegateConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
delegateConfig.LoopbackClientConfig = &rest.Config{}
delegateConfig.EffectiveVersion = utilversion.NewEffectiveVersion("")
delegateConfig.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "")
clientset := fake.NewSimpleClientset()
if clientset == nil {
t.Fatal("unable to create fake client set")
@@ -157,7 +157,7 @@ func TestNewWithDelegate(t *testing.T) {
wrappingConfig.PublicAddress = netutils.ParseIPSloppy("192.168.10.4")
wrappingConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
wrappingConfig.LoopbackClientConfig = &rest.Config{}
wrappingConfig.EffectiveVersion = utilversion.NewEffectiveVersion("")
wrappingConfig.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "")
wrappingConfig.HealthzChecks = append(wrappingConfig.HealthzChecks, healthz.NamedCheck("wrapping-health", func(r *http.Request) error {
return fmt.Errorf("wrapping failed healthcheck")
@@ -434,7 +434,7 @@ func TestNewFeatureGatedSerializer(t *testing.T) {
}
})))
config.ExternalAddress = "192.168.10.4:443"
config.EffectiveVersion = utilversion.NewEffectiveVersion("")
config.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "")
config.LoopbackClientConfig = &rest.Config{}
if _, err := config.Complete(nil).New("test", NewEmptyDelegate()); err != nil {

4
staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go
View File

@@ -54,8 +54,8 @@ import (
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
restclient "k8s.io/client-go/rest"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
utilversion "k8s.io/component-base/version"
"k8s.io/klog/v2"
openapibuilder3 "k8s.io/kube-openapi/pkg/builder3"
openapicommon "k8s.io/kube-openapi/pkg/common"
@@ -244,7 +244,7 @@ type GenericAPIServer struct {
// EffectiveVersion determines which apis and features are available
// based on when the api/feature lifecyle.
EffectiveVersion utilversion.EffectiveVersion
EffectiveVersion basecompatibility.EffectiveVersion
// FeatureGate is a way to plumb feature gate through if you have them.
FeatureGate featuregate.FeatureGate

9
staging/src/k8s.io/apiserver/pkg/server/genericapiserver_test.go
View File

@@ -38,6 +38,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
utilversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/apis/example"
@@ -52,7 +53,7 @@ import (
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
restclient "k8s.io/client-go/rest"
utilversion "k8s.io/component-base/version"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/klog/v2/ktesting"
kubeopenapi "k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/validation/spec"
@@ -138,7 +139,7 @@ func setUp(t *testing.T) (Config, *assert.Assertions) {
if clientset == nil {
t.Fatal("unable to create fake client set")
}
config.EffectiveVersion = utilversion.NewEffectiveVersion("")
config.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString("", "", "")
config.OpenAPIConfig = DefaultOpenAPIConfig(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme()))
config.OpenAPIConfig.Info.Version = "unversioned"
config.OpenAPIV3Config = DefaultOpenAPIV3Config(testGetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(runtime.NewScheme()))
@@ -460,8 +461,8 @@ func TestNotRestRoutesHaveAuth(t *testing.T) {
config.EnableProfiling = true
kubeVersion := fakeVersion()
effectiveVersion := utilversion.NewEffectiveVersion(kubeVersion.String())
effectiveVersion.Set(effectiveVersion.BinaryVersion().WithInfo(kubeVersion), effectiveVersion.EmulationVersion(), effectiveVersion.MinCompatibilityVersion())
binaryVersion := utilversion.MustParse(kubeVersion.String())
effectiveVersion := basecompatibility.NewEffectiveVersion(binaryVersion, false, binaryVersion, binaryVersion.SubtractMinor(1))
config.EffectiveVersion = effectiveVersion
s, err := config.Complete(nil).New("test", NewEmptyDelegate())

16
staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go
View File

@@ -27,9 +27,9 @@ import (
"k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
utilversion "k8s.io/component-base/version"
basecompatibility "k8s.io/component-base/compatibility"
"github.com/spf13/pflag"
)
@@ -95,22 +95,22 @@ type ServerRunOptions struct {
ShutdownWatchTerminationGracePeriod time.Duration
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry featuregate.ComponentGlobalsRegistry
ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry
// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
ComponentName string
}
func NewServerRunOptions() *ServerRunOptions {
if featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) == nil {
if compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent) == nil {
featureGate := utilfeature.DefaultMutableFeatureGate
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
effectiveVersion := compatibility.DefaultBuildEffectiveVersion()
utilruntime.Must(compatibility.DefaultComponentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate))
}
return NewServerRunOptionsForComponent(featuregate.DefaultKubeComponent, featuregate.DefaultComponentGlobalsRegistry)
return NewServerRunOptionsForComponent(basecompatibility.DefaultKubeComponent, compatibility.DefaultComponentGlobalsRegistry)
}
func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry featuregate.ComponentGlobalsRegistry) *ServerRunOptions {
func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry) *ServerRunOptions {
defaults := server.NewConfig(serializer.CodecFactory{})
return &ServerRunOptions{
MaxRequestsInFlight: defaults.MaxRequestsInFlight,

28
staging/src/k8s.io/apiserver/pkg/server/options/server_run_options_test.go
View File

@@ -26,15 +26,15 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
utilversion "k8s.io/component-base/version"
basecompatibility "k8s.io/component-base/compatibility"
netutils "k8s.io/utils/net"
)
func TestServerRunOptionsValidate(t *testing.T) {
testRegistry := featuregate.NewComponentGlobalsRegistry()
defaultComponentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
testRegistry := basecompatibility.NewComponentGlobalsRegistry()
featureGate := utilfeature.DefaultFeatureGate.DeepCopy()
effectiveVersion := utilversion.NewEffectiveVersion("1.35")
effectiveVersion := basecompatibility.NewEffectiveVersionFromString("1.35", "1.32", "1.32")
effectiveVersion.SetEmulationVersion(version.MajorMinor(1, 31))
testComponent := "test"
utilruntime.Must(testRegistry.Register(testComponent, effectiveVersion, featureGate))
@@ -55,7 +55,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "--max-requests-inflight can not be negative value",
},
@@ -70,7 +70,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "--max-mutating-requests-inflight can not be negative value",
},
@@ -85,7 +85,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "--request-timeout can not be negative value",
},
@@ -100,7 +100,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: -1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "--min-request-timeout can not be negative value",
},
@@ -115,7 +115,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: -10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value",
},
@@ -130,7 +130,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: -10 * 1024 * 1024,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "ServerRunOptions.MaxRequestBodyBytes can not be negative value",
},
@@ -146,7 +146,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
LivezGracePeriod: -time.Second,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "--livez-grace-period can not be a negative value",
},
@@ -162,7 +162,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ShutdownDelayDuration: -time.Second,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "--shutdown-delay-duration can not be negative value",
},
@@ -178,7 +178,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
expectErr: "--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information",
},
@@ -211,7 +211,7 @@ func TestServerRunOptionsValidate(t *testing.T) {
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: 10 * 1024 * 1024,
MaxRequestBodyBytes: 10 * 1024 * 1024,
ComponentGlobalsRegistry: featuregate.DefaultComponentGlobalsRegistry,
ComponentGlobalsRegistry: defaultComponentGlobalsRegistry,
},
},
}

4
staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go
View File

@@ -46,7 +46,7 @@ import (
"k8s.io/client-go/discovery"
restclient "k8s.io/client-go/rest"
cliflag "k8s.io/component-base/cli/flag"
utilversion "k8s.io/component-base/version"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/klog/v2/ktesting"
netutils "k8s.io/utils/net"
)
@@ -278,7 +278,7 @@ func TestServerRunWithSNI(t *testing.T) {
// launch server
config := setUp(t)
v := fakeVersion()
config.EffectiveVersion = utilversion.NewEffectiveVersion(v.String())
config.EffectiveVersion = basecompatibility.NewEffectiveVersionFromString(v.String(), "", "")
config.EnableIndex = true
secureOptions := (&SecureServingOptions{

17
staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go
View File

@@ -22,7 +22,8 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
version "k8s.io/component-base/version"
"k8s.io/apiserver/pkg/util/compatibility"
basecompatibility "k8s.io/component-base/compatibility"
)
type ResourceEncodingConfig interface {
@@ -43,7 +44,7 @@ type DefaultResourceEncodingConfig struct {
// resources records the overriding encoding configs for individual resources.
resources map[schema.GroupResource]*OverridingResourceEncoding
scheme *runtime.Scheme
effectiveVersion version.EffectiveVersion
effectiveVersion basecompatibility.EffectiveVersion
}
type OverridingResourceEncoding struct {
@@ -54,7 +55,11 @@ type OverridingResourceEncoding struct {
var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{}
func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig {
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()}
return NewDefaultResourceEncodingConfigForEffectiveVersion(scheme, compatibility.DefaultComponentGlobalsRegistry.EffectiveVersionFor(basecompatibility.DefaultKubeComponent))
}
func NewDefaultResourceEncodingConfigForEffectiveVersion(scheme *runtime.Scheme, effectiveVersion basecompatibility.EffectiveVersion) *DefaultResourceEncodingConfig {
return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: effectiveVersion}
}
func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) {
@@ -64,7 +69,7 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored
}
}
func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) {
func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion basecompatibility.EffectiveVersion) {
o.effectiveVersion = effectiveVersion
}
@@ -121,7 +126,7 @@ type replacementInterface interface {
APILifecycleReplacement() schema.GroupVersionKind
}
func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) {
func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion basecompatibility.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) {
if example == nil || effectiveVersion == nil {
return binaryVersionOfResource, nil
}
@@ -172,7 +177,7 @@ func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example
}
// If it was introduced after current compatibility version, don't use it
// skip the introduced check for test when currentVersion is 0.0 to test all apis
// skip the introduced check for test when current compatibility version is 0.0 to test all apis
if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) {
// Skip versions that have a replacement.

8
staging/src/k8s.io/apiserver/pkg/server/storage/resource_encoding_config_test.go
View File

@@ -25,7 +25,7 @@ import (
"k8s.io/apimachinery/pkg/test"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
utilversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/version"
basecompatibility "k8s.io/component-base/compatibility"
)
func TestEmulatedStorageVersion(t *testing.T) {
@@ -33,21 +33,21 @@ func TestEmulatedStorageVersion(t *testing.T) {
name string
scheme *runtime.Scheme
binaryVersion schema.GroupVersion
effectiveVersion version.EffectiveVersion
effectiveVersion basecompatibility.EffectiveVersion
want schema.GroupVersion
}{
{
name: "pick compatible",
scheme: AlphaBetaScheme(utilversion.MustParse("1.31"), utilversion.MustParse("1.32")),
binaryVersion: v1beta1,
effectiveVersion: version.NewEffectiveVersion("1.32"),
effectiveVersion: basecompatibility.NewEffectiveVersionFromString("1.32", "", ""),
want: v1alpha1,
},
{
name: "alpha has been replaced, pick binary version",
scheme: AlphaReplacedBetaScheme(utilversion.MustParse("1.31"), utilversion.MustParse("1.32")),
binaryVersion: v1beta1,
effectiveVersion: version.NewEffectiveVersion("1.32"),
effectiveVersion: basecompatibility.NewEffectiveVersionFromString("1.32", "", ""),
want: v1beta1,
},
}

5
staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory_test.go
View File

@@ -33,7 +33,7 @@ import (
"k8s.io/apiserver/pkg/apis/example2"
example2install "k8s.io/apiserver/pkg/apis/example2/install"
"k8s.io/apiserver/pkg/storage/storagebackend"
version "k8s.io/component-base/version"
basecompatibility "k8s.io/component-base/compatibility"
)
var (
@@ -569,8 +569,7 @@ func TestStorageFactoryCompatibilityVersion(t *testing.T) {
gvk := gvks[0]
t.Run(gvk.GroupKind().String()+"@"+tc.effectiveVersion, func(t *testing.T) {
config := NewDefaultResourceEncodingConfig(sch)
config.SetEffectiveVersion(version.NewEffectiveVersion(tc.effectiveVersion))
config := NewDefaultResourceEncodingConfigForEffectiveVersion(sch, basecompatibility.NewEffectiveVersionFromString(tc.effectiveVersion, "", ""))
f := NewDefaultStorageFactory(
storagebackend.Config{},
"",

53
staging/src/k8s.io/apiserver/pkg/util/compatibility/registry.go Normal file
View File

@@ -0,0 +1,53 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package compatibility
import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
basecompatibility "k8s.io/component-base/compatibility"
)
// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access.
// Example usage:
// // register the component effective version and feature gate first
// wardleEffectiveVersion := basecompatibility.NewEffectiveVersion("1.2")
// wardleFeatureGate := featuregate.NewFeatureGate()
// utilruntime.Must(compatibility.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false))
//
// cmd := &cobra.Command{
// ...
// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE to ensure the feature gates are set based on emulation version right after parsing the flags.
// PersistentPreRunE: func(*cobra.Command, []string) error {
// if err := compatibility.DefaultComponentGlobalsRegistry.Set(); err != nil {
// return err
// }
// ...
// },
// RunE: func(c *cobra.Command, args []string) error {
// // call compatibility.DefaultComponentGlobalsRegistry.Validate() somewhere
// },
// }
//
// flags := cmd.Flags()
// // add flags
// compatibility.DefaultComponentGlobalsRegistry.AddFlags(flags)
var DefaultComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry = basecompatibility.NewComponentGlobalsRegistry()
func init() {
utilruntime.Must(DefaultComponentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate))
}

65
staging/src/k8s.io/apiserver/pkg/util/compatibility/version.go Normal file
View File

@@ -0,0 +1,65 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package compatibility
import (
"k8s.io/apimachinery/pkg/util/version"
basecompatibility "k8s.io/component-base/compatibility"
baseversion "k8s.io/component-base/version"
)
// minimumKubeEmulationVersion is the first release emulation version is introduced,
// so the emulation version cannot go lower than that.
var minimumKubeEmulationVersion *version.Version = version.MajorMinor(1, 31)
// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the
// current build information.
func DefaultBuildEffectiveVersion() basecompatibility.MutableEffectiveVersion {
binaryVersion := defaultBuildBinaryVersion()
useDefaultBuildBinaryVersion := true
// fall back to the hard coded kube version only when the git tag is not available for local unit tests.
if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 {
useDefaultBuildBinaryVersion = false
binaryVersion = version.MustParse(baseversion.DefaultKubeBinaryVersion)
}
versionFloor := kubeEffectiveVersionFloors(binaryVersion)
return basecompatibility.NewEffectiveVersion(binaryVersion, useDefaultBuildBinaryVersion, versionFloor, versionFloor)
}
func kubeEffectiveVersionFloors(binaryVersion *version.Version) *version.Version {
// both emulationVersion and minCompatibilityVersion can be set to binaryVersion - 3
versionFloor := binaryVersion.WithPatch(0).SubtractMinor(3)
if versionFloor.LessThan(minimumKubeEmulationVersion) {
versionFloor = minimumKubeEmulationVersion
}
return versionFloor
}
// DefaultKubeEffectiveVersionForTest returns the MutableEffectiveVersion based on the
// latest K8s release hardcoded in DefaultKubeBinaryVersion.
// DefaultKubeBinaryVersion is hard coded because defaultBuildBinaryVersion would return 0.0 when test is run without a git tag.
// We do not enforce the N-3..N emulation version range in tests so that the tests would not automatically fail when there is a version bump.
// Only used in tests.
func DefaultKubeEffectiveVersionForTest() basecompatibility.MutableEffectiveVersion {
binaryVersion := version.MustParse(baseversion.DefaultKubeBinaryVersion).WithInfo(baseversion.Get())
return basecompatibility.NewEffectiveVersion(binaryVersion, false, version.MustParse("0.0"), version.MustParse("0.0"))
}
func defaultBuildBinaryVersion() *version.Version {
verInfo := baseversion.Get()
return version.MustParse(verInfo.String()).WithInfo(verInfo)
}

76
staging/src/k8s.io/apiserver/pkg/util/compatibility/version_test.go Normal file
View File

@@ -0,0 +1,76 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package compatibility
import (
"testing"
"k8s.io/apimachinery/pkg/util/version"
basecompatibility "k8s.io/component-base/compatibility"
)
func TestValidateKubeEffectiveVersion(t *testing.T) {
tests := []struct {
name string
emulationVersion string
minCompatibilityVersion string
expectErr bool
}{
{
name: "valid versions",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.31.0",
expectErr: false,
},
{
name: "emulationVersion too low",
emulationVersion: "v1.30.0",
minCompatibilityVersion: "v1.31.0",
expectErr: true,
},
{
name: "minCompatibilityVersion too low",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.30.0",
expectErr: true,
},
{
name: "both versions too low",
emulationVersion: "v1.30.0",
minCompatibilityVersion: "v1.29.0",
expectErr: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
binaryVersion := version.MustParseGeneric("1.33")
versionFloor := kubeEffectiveVersionFloors(binaryVersion)
effective := basecompatibility.NewEffectiveVersion(binaryVersion, false, versionFloor, versionFloor)
effective.SetEmulationVersion(version.MustParseGeneric(test.emulationVersion))
effective.SetMinCompatibilityVersion(version.MustParseGeneric(test.minCompatibilityVersion))
err := effective.Validate()
if test.expectErr && err == nil {
t.Error("expected error, but got nil")
}
if !test.expectErr && err != nil {
t.Errorf("unexpected error: %v", err)
}
})
}
}

12
staging/src/k8s.io/component-base/compatibility/OWNERS Normal file
View File

@@ -0,0 +1,12 @@
# See the OWNERS docs at https://go.k8s.io/owners
# Currently assigned this directory to sig-api-machinery since this is
# an interface to the version definition in "k8s.io/apiserver/pkg/util/compatibility".
approvers:
- sig-api-machinery-api-approvers
reviewers:
- sig-api-machinery-api-reviewers
- siyuanfoundation
labels:
- sig/api-machinery

76
staging/src/k8s.io/component-base/featuregate/registry.go → staging/src/k8s.io/component-base/compatibility/registry.go
View File

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package featuregate
package compatibility
import (
"fmt"
@@ -26,38 +26,12 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
cliflag "k8s.io/component-base/cli/flag"
baseversion "k8s.io/component-base/version"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
)
// DefaultComponentGlobalsRegistry is the global var to store the effective versions and feature gates for all components for easy access.
// Example usage:
// // register the component effective version and feature gate first
// _, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(utilversion.DefaultKubeComponent, utilversion.DefaultKubeEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)
// wardleEffectiveVersion := utilversion.NewEffectiveVersion("1.2")
// wardleFeatureGate := featuregate.NewFeatureGate()
// utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(apiserver.WardleComponentName, wardleEffectiveVersion, wardleFeatureGate, false))
//
// cmd := &cobra.Command{
// ...
// // call DefaultComponentGlobalsRegistry.Set() in PersistentPreRunE
// PersistentPreRunE: func(*cobra.Command, []string) error {
// if err := utilversion.DefaultComponentGlobalsRegistry.Set(); err != nil {
// return err
// }
// ...
// },
// RunE: func(c *cobra.Command, args []string) error {
// // call utilversion.DefaultComponentGlobalsRegistry.Validate() somewhere
// },
// }
//
// flags := cmd.Flags()
// // add flags
// utilversion.DefaultComponentGlobalsRegistry.AddFlags(flags)
var DefaultComponentGlobalsRegistry ComponentGlobalsRegistry = NewComponentGlobalsRegistry()
const (
// DefaultKubeComponent is the component name for k8s control plane components.
DefaultKubeComponent = "kube"
klogLevel = 2
@@ -65,10 +39,10 @@ const (
type VersionMapping func(from *version.Version) *version.Version
// ComponentGlobals stores the global variables for a component for easy access.
// ComponentGlobals stores the global variables for a component for easy access, including feature gate and effective version.
type ComponentGlobals struct {
effectiveVersion baseversion.MutableEffectiveVersion
featureGate MutableVersionedFeatureGate
effectiveVersion MutableEffectiveVersion
featureGate featuregate.MutableVersionedFeatureGate
// emulationVersionMapping contains the mapping from the emulation version of this component
// to the emulation version of another component.
@@ -84,22 +58,24 @@ type ComponentGlobals struct {
dependentMinCompatibilityVersion bool
}
// ComponentGlobalsRegistry stores the global variables for different components for easy access, including feature gate and effective version of each component.
type ComponentGlobalsRegistry interface {
// EffectiveVersionFor returns the EffectiveVersion registered under the component.
// Returns nil if the component is not registered.
EffectiveVersionFor(component string) baseversion.EffectiveVersion
EffectiveVersionFor(component string) EffectiveVersion
// FeatureGateFor returns the FeatureGate registered under the component.
// Returns nil if the component is not registered.
FeatureGateFor(component string) FeatureGate
FeatureGateFor(component string) featuregate.FeatureGate
// Register registers the EffectiveVersion and FeatureGate for a component.
// returns error if the component is already registered.
Register(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) error
Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error
// ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry.
// Otherwise, the provided variables would be registered under the component, and the same variables would be returned.
ComponentGlobalsOrRegister(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) (baseversion.MutableEffectiveVersion, MutableVersionedFeatureGate)
ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate)
// AddFlags adds flags of "--emulated-version" and "--feature-gates"
AddFlags(fs *pflag.FlagSet)
// Set sets the flags for all global variables for all components registered.
// A component's feature gate and effective version would not be updated until Set() is called.
Set() error
// SetFallback calls Set() if it has never been called.
SetFallback() error
@@ -118,9 +94,13 @@ type ComponentGlobalsRegistry interface {
type componentGlobalsRegistry struct {
componentGlobals map[string]*ComponentGlobals
mutex sync.RWMutex
// list of component name to emulation version set from the flag.
// emulationVersionConfig stores the list of component name to emulation version set from the flag.
// When the `--emulated-version` flag is parsed, it would not take effect until Set() is called,
// because the emulation version needs to be set before the feature gate is set.
emulationVersionConfig []string
// map of component name to the list of feature gates set from the flag.
// featureGatesConfig stores the map of component name to the list of feature gates set from the flag.
// When the `--feature-gates` flag is parsed, it would not take effect until Set() is called,
// because the emulation version needs to be set before the feature gate is set.
featureGatesConfig map[string][]string
// set stores if the Set() function for the registry is already called.
set bool
@@ -143,7 +123,7 @@ func (r *componentGlobalsRegistry) Reset() {
r.set = false
}
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) baseversion.EffectiveVersion {
func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) EffectiveVersion {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
@@ -153,7 +133,7 @@ func (r *componentGlobalsRegistry) EffectiveVersionFor(component string) basever
return globals.effectiveVersion
}
func (r *componentGlobalsRegistry) FeatureGateFor(component string) FeatureGate {
func (r *componentGlobalsRegistry) FeatureGateFor(component string) featuregate.FeatureGate {
r.mutex.RLock()
defer r.mutex.RUnlock()
globals, ok := r.componentGlobals[component]
@@ -163,7 +143,7 @@ func (r *componentGlobalsRegistry) FeatureGateFor(component string) FeatureGate
return globals.featureGate
}
func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) error {
func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error {
if _, ok := r.componentGlobals[component]; ok {
return fmt.Errorf("component globals of %s already registered", component)
}
@@ -182,7 +162,7 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer
return nil
}
func (r *componentGlobalsRegistry) Register(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) error {
func (r *componentGlobalsRegistry) Register(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) error {
if effectiveVersion == nil {
return fmt.Errorf("cannot register nil effectiveVersion")
}
@@ -191,7 +171,7 @@ func (r *componentGlobalsRegistry) Register(component string, effectiveVersion b
return r.unsafeRegister(component, effectiveVersion, featureGate)
}
func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion baseversion.MutableEffectiveVersion, featureGate MutableVersionedFeatureGate) (baseversion.MutableEffectiveVersion, MutableVersionedFeatureGate) {
func (r *componentGlobalsRegistry) ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) {
r.mutex.Lock()
defer r.mutex.Unlock()
globals, ok := r.componentGlobals[component]
@@ -219,22 +199,16 @@ func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string {
func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string {
var vs []string
for component, globals := range r.componentGlobals {
binaryVer := globals.effectiveVersion.BinaryVersion()
if isEmulation {
if globals.dependentEmulationVersion {
continue
}
// emulated version could be between binaryMajor.{binaryMinor} and binaryMajor.{binaryMinor}
// TODO: change to binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor} in 1.32
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(0).String(), binaryVer.String(), globals.effectiveVersion.EmulationVersion().String()))
vs = append(vs, fmt.Sprintf("%s=%s", component, globals.effectiveVersion.AllowedEmulationVersionRange()))
} else {
if globals.dependentMinCompatibilityVersion {
continue
}
// min compatibility version could be between binaryMajor.{binaryMinor-1} and binaryMajor.{binaryMinor}
vs = append(vs, fmt.Sprintf("%s=%s..%s (default=%s)", component,
binaryVer.SubtractMinor(1).String(), binaryVer.String(), globals.effectiveVersion.MinCompatibilityVersion().String()))
vs = append(vs, fmt.Sprintf("%s=%s", component, globals.effectiveVersion.AllowedMinCompatibilityVersionRange()))
}
}
sort.Strings(vs)

76
staging/src/k8s.io/component-base/featuregate/registry_test.go → staging/src/k8s.io/component-base/compatibility/registry_test.go
View File

@@ -1,5 +1,5 @@
/*
Copyright 2024 The Kubernetes Authors.
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package featuregate
package compatibility
import (
"fmt"
@@ -25,7 +25,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
baseversion "k8s.io/component-base/version"
"k8s.io/component-base/featuregate"
)
const (
@@ -34,8 +34,8 @@ const (
func TestEffectiveVersionRegistry(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := baseversion.NewEffectiveVersion("1.31")
ver2 := baseversion.NewEffectiveVersion("1.28")
ver1 := NewEffectiveVersionFromString("1.31", "", "")
ver2 := NewEffectiveVersionFromString("1.28", "", "")
if r.EffectiveVersionFor(testComponent) != nil {
t.Fatalf("expected nil EffectiveVersion initially")
@@ -57,40 +57,40 @@ func TestEffectiveVersionRegistry(t *testing.T) {
func testRegistry(t *testing.T) *componentGlobalsRegistry {
r := NewComponentGlobalsRegistry()
verKube := baseversion.NewEffectiveVersion("1.31")
fgKube := NewVersionedFeatureGate(version.MustParse("0.0"))
err := fgKube.AddVersioned(map[Feature]VersionedSpecs{
verKube := NewEffectiveVersionFromString("1.31", "1.31", "1.30")
fgKube := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err := fgKube.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"kubeA": {
{Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha},
{Version: version.MustParse("1.28"), Default: false, PreRelease: Beta},
{Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: GA},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
{Version: version.MustParse("1.28"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("1.31"), Default: true, LockToDefault: true, PreRelease: featuregate.GA},
},
"kubeB": {
{Version: version.MustParse("1.30"), Default: false, PreRelease: Alpha},
{Version: version.MustParse("1.30"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("1.27"), Default: false, PreRelease: Alpha},
{Version: version.MustParse("1.29"), Default: true, PreRelease: Beta},
{Version: version.MustParse("1.27"), Default: false, PreRelease: featuregate.Alpha},
{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta},
},
})
if err != nil {
t.Fatal(err)
}
verTest := baseversion.NewEffectiveVersion("2.8")
fgTest := NewVersionedFeatureGate(version.MustParse("0.0"))
err = fgTest.AddVersioned(map[Feature]VersionedSpecs{
verTest := NewEffectiveVersionFromString("2.8", "2.8", "2.7")
fgTest := featuregate.NewVersionedFeatureGate(version.MustParse("0.0"))
err = fgTest.AddVersioned(map[featuregate.Feature]featuregate.VersionedSpecs{
"testA": {
{Version: version.MustParse("2.7"), Default: false, PreRelease: Alpha},
{Version: version.MustParse("2.8"), Default: false, PreRelease: Beta},
{Version: version.MustParse("2.10"), Default: true, PreRelease: GA},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
{Version: version.MustParse("2.8"), Default: false, PreRelease: featuregate.Beta},
{Version: version.MustParse("2.10"), Default: true, PreRelease: featuregate.GA},
},
"testB": {
{Version: version.MustParse("2.9"), Default: false, PreRelease: Alpha},
{Version: version.MustParse("2.9"), Default: false, PreRelease: featuregate.Alpha},
},
"commonC": {
{Version: version.MustParse("2.7"), Default: false, PreRelease: Alpha},
{Version: version.MustParse("2.9"), Default: true, PreRelease: Beta},
{Version: version.MustParse("2.7"), Default: false, PreRelease: featuregate.Alpha},
{Version: version.MustParse("2.9"), Default: true, PreRelease: featuregate.Beta},
},
})
if err != nil {
@@ -154,8 +154,8 @@ func TestFlags(t *testing.T) {
parseError string
expectedKubeEmulationVersion string
expectedTestEmulationVersion string
expectedKubeFeatureValues map[Feature]bool
expectedTestFeatureValues map[Feature]bool
expectedKubeFeatureValues map[featuregate.Feature]bool
expectedTestFeatureValues map[featuregate.Feature]bool
}{
{
name: "setting kube emulation version",
@@ -214,8 +214,8 @@ func TestFlags(t *testing.T) {
},
expectedKubeEmulationVersion: "1.31",
expectedTestEmulationVersion: "2.7",
expectedKubeFeatureValues: map[Feature]bool{"kubeA": true, "kubeB": false, "commonC": true},
expectedTestFeatureValues: map[Feature]bool{"testA": true, "testB": false, "commonC": false},
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": true, "kubeB": false, "commonC": true},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": true, "testB": false, "commonC": false},
},
{
name: "setting future test feature flag",
@@ -235,8 +235,8 @@ func TestFlags(t *testing.T) {
},
expectedKubeEmulationVersion: "1.30",
expectedTestEmulationVersion: "2.7",
expectedKubeFeatureValues: map[Feature]bool{"kubeA": false, "kubeB": true, "commonC": false},
expectedTestFeatureValues: map[Feature]bool{"testA": false, "testB": false, "commonC": true},
expectedKubeFeatureValues: map[featuregate.Feature]bool{"kubeA": false, "kubeB": true, "commonC": false},
expectedTestFeatureValues: map[featuregate.Feature]bool{"testA": false, "testB": false, "commonC": true},
},
{
name: "setting kube feature flag with different prefix",
@@ -313,9 +313,9 @@ func TestFlags(t *testing.T) {
func TestVersionMapping(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := baseversion.NewEffectiveVersion("0.58")
ver2 := baseversion.NewEffectiveVersion("1.28")
ver3 := baseversion.NewEffectiveVersion("2.10")
ver1 := NewEffectiveVersionFromString("0.58", "", "")
ver2 := NewEffectiveVersionFromString("1.28", "", "")
ver3 := NewEffectiveVersionFromString("2.10", "", "")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))
@@ -355,9 +355,9 @@ func TestVersionMapping(t *testing.T) {
func TestVersionMappingWithMultipleDependency(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := baseversion.NewEffectiveVersion("0.58")
ver2 := baseversion.NewEffectiveVersion("1.28")
ver3 := baseversion.NewEffectiveVersion("2.10")
ver1 := NewEffectiveVersionFromString("0.58", "", "")
ver2 := NewEffectiveVersionFromString("1.28", "", "")
ver3 := NewEffectiveVersionFromString("2.10", "", "")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))
@@ -382,9 +382,9 @@ func TestVersionMappingWithMultipleDependency(t *testing.T) {
func TestVersionMappingWithCyclicDependency(t *testing.T) {
r := NewComponentGlobalsRegistry()
ver1 := baseversion.NewEffectiveVersion("0.58")
ver2 := baseversion.NewEffectiveVersion("1.28")
ver3 := baseversion.NewEffectiveVersion("2.10")
ver1 := NewEffectiveVersionFromString("0.58", "", "")
ver2 := NewEffectiveVersionFromString("1.28", "", "")
ver3 := NewEffectiveVersionFromString("2.10", "", "")
utilruntime.Must(r.Register("test1", ver1, nil))
utilruntime.Must(r.Register("test2", ver2, nil))

213
staging/src/k8s.io/component-base/compatibility/version.go Normal file
View File

@@ -0,0 +1,213 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package compatibility
import (
"fmt"
"sync/atomic"
"k8s.io/apimachinery/pkg/util/version"
baseversion "k8s.io/component-base/version"
)
// EffectiveVersion stores all the version information of a component.
type EffectiveVersion interface {
// BinaryVersion is the binary version of a component. Tied to a particular binary release.
BinaryVersion() *version.Version
// EmulationVersion is the version a component emulate its capabilities (APIs, features, ...) of.
// If EmulationVersion is set to be different from BinaryVersion, the component will emulate the behavior of this version instead of the underlying binary version.
EmulationVersion() *version.Version
// MinCompatibilityVersion is the minimum version a component is compatible with (in terms of storage versions, validation rules, ...).
MinCompatibilityVersion() *version.Version
EqualTo(other EffectiveVersion) bool
String() string
Validate() []error
// AllowedEmulationVersionRange returns the string of the allowed range of emulation version.
// Used only for docs/help.
AllowedEmulationVersionRange() string
// AllowedMinCompatibilityVersionRange returns the string of the allowed range of min compatibility version.
// Used only for docs/help.
AllowedMinCompatibilityVersionRange() string
}
type MutableEffectiveVersion interface {
EffectiveVersion
SetEmulationVersion(emulationVersion *version.Version)
SetMinCompatibilityVersion(minCompatibilityVersion *version.Version)
}
type effectiveVersion struct {
// When true, BinaryVersion() returns the current binary version
useDefaultBuildBinaryVersion atomic.Bool
// Holds the last binary version stored in Set()
binaryVersion atomic.Pointer[version.Version]
// If the emulationVersion is set by the users, it could only contain major and minor versions.
// In tests, emulationVersion could be the same as the binary version, or set directly,
// which can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
emulationVersion atomic.Pointer[version.Version]
// minCompatibilityVersion could only contain major and minor versions.
minCompatibilityVersion atomic.Pointer[version.Version]
// emulationVersionFloor is the minimum emulationVersion allowed. No limit if nil.
emulationVersionFloor *version.Version
// minCompatibilityVersionFloor is the minimum minCompatibilityVersionFloor allowed. No limit if nil.
minCompatibilityVersionFloor *version.Version
}
func (m *effectiveVersion) BinaryVersion() *version.Version {
if m.useDefaultBuildBinaryVersion.Load() {
return defaultBuildBinaryVersion()
}
return m.binaryVersion.Load()
}
func (m *effectiveVersion) EmulationVersion() *version.Version {
ver := m.emulationVersion.Load()
if ver != nil {
// Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
// The pre-release should not be accessible to the users.
return ver.WithPreRelease(m.BinaryVersion().PreRelease())
}
return ver
}
func (m *effectiveVersion) MinCompatibilityVersion() *version.Version {
return m.minCompatibilityVersion.Load()
}
func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool {
return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion())
}
func (m *effectiveVersion) String() string {
if m == nil {
return "<nil>"
}
return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}",
m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String())
}
func majorMinor(ver *version.Version) *version.Version {
if ver == nil {
return ver
}
return version.MajorMinor(ver.Major(), ver.Minor())
}
func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) {
m.emulationVersion.Store(majorMinor(emulationVersion))
// set the default minCompatibilityVersion to be emulationVersion - 1 if possible
minCompatibilityVersion := majorMinor(emulationVersion.SubtractMinor(1))
if minCompatibilityVersion.LessThan(m.minCompatibilityVersionFloor) {
minCompatibilityVersion = m.minCompatibilityVersionFloor
}
m.minCompatibilityVersion.Store(minCompatibilityVersion)
}
// SetMinCompatibilityVersion should be called after SetEmulationVersion
func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) {
m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion))
}
func (m *effectiveVersion) AllowedEmulationVersionRange() string {
binaryVersion := m.BinaryVersion()
if binaryVersion == nil {
return ""
}
// Consider patch version to be 0.
binaryVersion = version.MajorMinor(binaryVersion.Major(), binaryVersion.Minor())
floor := m.emulationVersionFloor
if floor == nil {
floor = version.MajorMinor(0, 0)
}
return fmt.Sprintf("%s..%s (default=%s)", floor.String(), binaryVersion.String(), m.EmulationVersion().String())
}
func (m *effectiveVersion) AllowedMinCompatibilityVersionRange() string {
binaryVersion := m.BinaryVersion()
if binaryVersion == nil {
return ""
}
// Consider patch version to be 0.
binaryVersion = version.MajorMinor(binaryVersion.Major(), binaryVersion.Minor())
floor := m.minCompatibilityVersionFloor
if floor == nil {
floor = version.MajorMinor(0, 0)
}
return fmt.Sprintf("%s..%s (default=%s)", floor.String(), binaryVersion.String(), m.MinCompatibilityVersion().String())
}
func (m *effectiveVersion) Validate() []error {
var errs []error
// Validate only checks the major and minor versions.
binaryVersion := m.BinaryVersion().WithPatch(0)
emulationVersion := m.emulationVersion.Load()
minCompatibilityVersion := m.minCompatibilityVersion.Load()
// emulationVersion can only be between emulationVersionFloor and binaryVersion
if emulationVersion.GreaterThan(binaryVersion) || emulationVersion.LessThan(m.emulationVersionFloor) {
errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), m.emulationVersionFloor.String(), binaryVersion.String()))
}
// minCompatibilityVersion can only be between minCompatibilityVersionFloor and emulationVersion
if minCompatibilityVersion.GreaterThan(emulationVersion) || minCompatibilityVersion.LessThan(m.minCompatibilityVersionFloor) {
errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), m.minCompatibilityVersionFloor.String(), emulationVersion.String()))
}
return errs
}
// NewEffectiveVersion creates a MutableEffectiveVersion from the binaryVersion.
// If useDefaultBuildBinaryVersion is true, the call of BinaryVersion() will always return the current binary version.
// NewEffectiveVersion(binaryVersion, true) should only be used if the binary version is dynamic.
// Otherwise, use NewEffectiveVersion(binaryVersion, false) or NewEffectiveVersionFromString.
func NewEffectiveVersion(binaryVersion *version.Version, useDefaultBuildBinaryVersion bool, emulationVersionFloor, minCompatibilityVersionFloor *version.Version) MutableEffectiveVersion {
effective := &effectiveVersion{
emulationVersionFloor: emulationVersionFloor,
minCompatibilityVersionFloor: minCompatibilityVersionFloor,
}
compatVersion := binaryVersion.SubtractMinor(1)
effective.binaryVersion.Store(binaryVersion)
effective.useDefaultBuildBinaryVersion.Store(useDefaultBuildBinaryVersion)
effective.SetEmulationVersion(binaryVersion)
effective.SetMinCompatibilityVersion(compatVersion)
return effective
}
// NewEffectiveVersionFromString creates a MutableEffectiveVersion from the binaryVersion string.
func NewEffectiveVersionFromString(binaryVer, emulationVerFloor, minCompatibilityVerFloor string) MutableEffectiveVersion {
if binaryVer == "" {
return &effectiveVersion{}
}
binaryVersion := version.MustParse(binaryVer)
emulationVersionFloor := version.MajorMinor(0, 0)
if emulationVerFloor != "" {
emulationVersionFloor = version.MustParse(emulationVerFloor)
}
minCompatibilityVersionFloor := version.MajorMinor(0, 0)
if minCompatibilityVerFloor != "" {
minCompatibilityVersionFloor = version.MustParse(minCompatibilityVerFloor)
}
return NewEffectiveVersion(binaryVersion, false, emulationVersionFloor, minCompatibilityVersionFloor)
}
func defaultBuildBinaryVersion() *version.Version {
verInfo := baseversion.Get()
return version.MustParse(verInfo.String()).WithInfo(verInfo)
}

148
staging/src/k8s.io/component-base/compatibility/version_test.go Normal file
View File

@@ -0,0 +1,148 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package compatibility
import (
"testing"
"k8s.io/apimachinery/pkg/util/version"
)
func TestValidate(t *testing.T) {
tests := []struct {
name string
binaryVersion string
emulationVersion string
minCompatibilityVersion string
emulationVersionFloor string
minCompatibilityVersionFloor string
expectErrors bool
}{
{
name: "patch version diff ok",
binaryVersion: "v1.32.1",
emulationVersion: "v1.32.2",
minCompatibilityVersion: "v1.32.5",
},
{
name: "emulation version greater than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.33.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "min compatibility version greater than emulation version not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "between floor and binary ok",
binaryVersion: "v1.32.1",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.30.0",
emulationVersionFloor: "v1.31.0",
minCompatibilityVersionFloor: "v1.30.0",
},
{
name: "emulation version less than floor not ok",
binaryVersion: "v1.32.1",
emulationVersion: "v1.30.0",
minCompatibilityVersion: "v1.30.0",
emulationVersionFloor: "v1.31.0",
minCompatibilityVersionFloor: "v1.30.0",
expectErrors: true,
},
{
name: "min compatibility version less than floor not ok",
binaryVersion: "v1.32.1",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.29.0",
emulationVersionFloor: "v1.31.0",
minCompatibilityVersionFloor: "v1.30.0",
expectErrors: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
effective := NewEffectiveVersionFromString(test.binaryVersion, test.emulationVersionFloor, test.minCompatibilityVersionFloor)
emulationVersion := version.MustParseGeneric(test.emulationVersion)
minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion)
effective.SetEmulationVersion(emulationVersion)
effective.SetMinCompatibilityVersion(minCompatibilityVersion)
errs := effective.Validate()
if len(errs) > 0 && !test.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}
if len(errs) == 0 && test.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}
func TestSetEmulationVersion(t *testing.T) {
tests := []struct {
name string
binaryVersion string
emulationVersion string
expectMinCompatibilityVersion string
emulationVersionFloor string
minCompatibilityVersionFloor string
}{
{
name: "minCompatibilityVersion default to 1 minor less than emulationVersion",
binaryVersion: "v1.34",
emulationVersion: "v1.32",
expectMinCompatibilityVersion: "v1.31",
emulationVersionFloor: "v1.31",
minCompatibilityVersionFloor: "v1.31",
},
{
name: "minCompatibilityVersion default to emulationVersion when hitting the floor",
binaryVersion: "v1.34",
emulationVersion: "v1.31",
expectMinCompatibilityVersion: "v1.31",
emulationVersionFloor: "v1.31",
minCompatibilityVersionFloor: "v1.31",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
effective := NewEffectiveVersionFromString(test.binaryVersion, test.emulationVersionFloor, test.minCompatibilityVersionFloor)
emulationVersion := version.MustParseGeneric(test.emulationVersion)
effective.SetEmulationVersion(emulationVersion)
errs := effective.Validate()
if len(errs) > 0 {
t.Fatalf("expected no Validate errors, errors found %+v", errs)
return
}
expectMinCompatibilityVersion := version.MustParseGeneric(test.expectMinCompatibilityVersion)
if !effective.MinCompatibilityVersion().EqualTo(expectMinCompatibilityVersion) {
t.Errorf("expected minCompatibilityVersion %s, got %s", expectMinCompatibilityVersion.String(), effective.MinCompatibilityVersion().String())
}
})
}
}

164
staging/src/k8s.io/component-base/version/version.go
View File

@@ -19,43 +19,10 @@ package version
import (
"fmt"
"runtime"
"sync/atomic"
"k8s.io/apimachinery/pkg/util/version"
apimachineryversion "k8s.io/apimachinery/pkg/version"
)
var minimumKubeEmulationVersion *version.Version = version.MajorMinor(1, 31)
type EffectiveVersion interface {
BinaryVersion() *version.Version
EmulationVersion() *version.Version
MinCompatibilityVersion() *version.Version
EqualTo(other EffectiveVersion) bool
String() string
Validate() []error
}
type MutableEffectiveVersion interface {
EffectiveVersion
Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version)
SetEmulationVersion(emulationVersion *version.Version)
SetMinCompatibilityVersion(minCompatibilityVersion *version.Version)
}
type effectiveVersion struct {
// When true, BinaryVersion() returns the current binary version
useDefaultBuildBinaryVersion atomic.Bool
// Holds the last binary version stored in Set()
binaryVersion atomic.Pointer[version.Version]
// If the emulationVersion is set by the users, it could only contain major and minor versions.
// In tests, emulationVersion could be the same as the binary version, or set directly,
// which can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
emulationVersion atomic.Pointer[version.Version]
// minCompatibilityVersion could only contain major and minor versions.
minCompatibilityVersion atomic.Pointer[version.Version]
}
// Get returns the overall codebase version. It's for detecting
// what code a binary was built from.
func Get() apimachineryversion.Info {
@@ -73,134 +40,3 @@ func Get() apimachineryversion.Info {
Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH),
}
}
func (m *effectiveVersion) BinaryVersion() *version.Version {
if m.useDefaultBuildBinaryVersion.Load() {
return defaultBuildBinaryVersion()
}
return m.binaryVersion.Load()
}
func (m *effectiveVersion) EmulationVersion() *version.Version {
ver := m.emulationVersion.Load()
if ver != nil {
// Emulation version can have "alpha" as pre-release to continue serving expired apis while we clean up the test.
// The pre-release should not be accessible to the users.
return ver.WithPreRelease(m.BinaryVersion().PreRelease())
}
return ver
}
func (m *effectiveVersion) MinCompatibilityVersion() *version.Version {
return m.minCompatibilityVersion.Load()
}
func (m *effectiveVersion) EqualTo(other EffectiveVersion) bool {
return m.BinaryVersion().EqualTo(other.BinaryVersion()) && m.EmulationVersion().EqualTo(other.EmulationVersion()) && m.MinCompatibilityVersion().EqualTo(other.MinCompatibilityVersion())
}
func (m *effectiveVersion) String() string {
if m == nil {
return "<nil>"
}
return fmt.Sprintf("{BinaryVersion: %s, EmulationVersion: %s, MinCompatibilityVersion: %s}",
m.BinaryVersion().String(), m.EmulationVersion().String(), m.MinCompatibilityVersion().String())
}
func majorMinor(ver *version.Version) *version.Version {
if ver == nil {
return ver
}
return version.MajorMinor(ver.Major(), ver.Minor())
}
func (m *effectiveVersion) Set(binaryVersion, emulationVersion, minCompatibilityVersion *version.Version) {
m.binaryVersion.Store(binaryVersion)
m.useDefaultBuildBinaryVersion.Store(false)
m.emulationVersion.Store(majorMinor(emulationVersion))
m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion))
}
func (m *effectiveVersion) SetEmulationVersion(emulationVersion *version.Version) {
m.emulationVersion.Store(majorMinor(emulationVersion))
// set the default minCompatibilityVersion to be emulationVersion - 1
m.minCompatibilityVersion.Store(majorMinor(emulationVersion.SubtractMinor(1)))
}
// SetMinCompatibilityVersion should be called after SetEmulationVersion
func (m *effectiveVersion) SetMinCompatibilityVersion(minCompatibilityVersion *version.Version) {
m.minCompatibilityVersion.Store(majorMinor(minCompatibilityVersion))
}
func (m *effectiveVersion) Validate() []error {
var errs []error
// Validate only checks the major and minor versions.
binaryVersion := m.BinaryVersion().WithPatch(0)
emulationVersion := m.emulationVersion.Load()
minCompatibilityVersion := m.minCompatibilityVersion.Load()
// emulationVersion can only be 1.{binaryMinor-3}...1.{binaryMinor}
maxEmuVer := binaryVersion
minEmuVer := binaryVersion.SubtractMinor(3)
if emulationVersion.GreaterThan(maxEmuVer) || emulationVersion.LessThan(minEmuVer) {
errs = append(errs, fmt.Errorf("emulation version %s is not between [%s, %s]", emulationVersion.String(), minEmuVer.String(), maxEmuVer.String()))
}
// minCompatibilityVersion can only be 1.{binaryMinor-3} to 1.{binaryMinor}
maxCompVer := emulationVersion
minCompVer := binaryVersion.SubtractMinor(4)
if minCompatibilityVersion.GreaterThan(maxCompVer) || minCompatibilityVersion.LessThan(minCompVer) {
errs = append(errs, fmt.Errorf("minCompatibilityVersion version %s is not between [%s, %s]", minCompatibilityVersion.String(), minCompVer.String(), maxCompVer.String()))
}
return errs
}
func newEffectiveVersion(binaryVersion *version.Version, useDefaultBuildBinaryVersion bool) MutableEffectiveVersion {
effective := &effectiveVersion{}
compatVersion := binaryVersion.SubtractMinor(1)
effective.Set(binaryVersion, binaryVersion, compatVersion)
effective.useDefaultBuildBinaryVersion.Store(useDefaultBuildBinaryVersion)
return effective
}
func NewEffectiveVersion(binaryVer string) MutableEffectiveVersion {
if binaryVer == "" {
return &effectiveVersion{}
}
binaryVersion := version.MustParse(binaryVer)
return newEffectiveVersion(binaryVersion, false)
}
func defaultBuildBinaryVersion() *version.Version {
verInfo := Get()
return version.MustParse(verInfo.String()).WithInfo(verInfo)
}
// DefaultBuildEffectiveVersion returns the MutableEffectiveVersion based on the
// current build information.
func DefaultBuildEffectiveVersion() MutableEffectiveVersion {
binaryVersion := defaultBuildBinaryVersion()
if binaryVersion.Major() == 0 && binaryVersion.Minor() == 0 {
return DefaultKubeEffectiveVersion()
}
return newEffectiveVersion(binaryVersion, true)
}
// DefaultKubeEffectiveVersion returns the MutableEffectiveVersion based on the
// latest K8s release.
func DefaultKubeEffectiveVersion() MutableEffectiveVersion {
binaryVersion := version.MustParse(DefaultKubeBinaryVersion).WithInfo(Get())
return newEffectiveVersion(binaryVersion, false)
}
// ValidateKubeEffectiveVersion validates the EmulationVersion is at least 1.31 and MinCompatibilityVersion
// is at least 1.30 for kube components.
func ValidateKubeEffectiveVersion(effectiveVersion EffectiveVersion) error {
if !effectiveVersion.EmulationVersion().AtLeast(minimumKubeEmulationVersion) {
return fmt.Errorf("emulation version needs to be greater or equal to 1.31, got %s", effectiveVersion.EmulationVersion().String())
}
if !effectiveVersion.MinCompatibilityVersion().AtLeast(minimumKubeEmulationVersion.SubtractMinor(1)) {
return fmt.Errorf("minCompatibilityVersion version needs to be greater or equal to 1.30, got %s", effectiveVersion.MinCompatibilityVersion().String())
}
return nil
}

185
staging/src/k8s.io/component-base/version/version_test.go
View File

@@ -1,185 +0,0 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package version
import (
"testing"
"k8s.io/apimachinery/pkg/util/version"
)
func TestValidate(t *testing.T) {
tests := []struct {
name string
binaryVersion string
emulationVersion string
minCompatibilityVersion string
expectErrors bool
}{
{
name: "patch version diff ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.1",
minCompatibilityVersion: "v1.31.5",
},
{
name: "emulation version one minor lower than binary ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.31.0",
},
{
name: "emulation version two minor lower than binary ok",
binaryVersion: "v1.33.2",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: false,
},
{
name: "emulation version three minor lower than binary ok",
binaryVersion: "v1.35.0",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.32.0",
},
{
name: "emulation version four minor lower than binary not ok",
binaryVersion: "v1.36.0",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: true,
},
{
name: "emulation version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.33.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "emulation version two minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.34.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: true,
},
{
name: "compatibility version same as binary ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.32.0",
expectErrors: false,
},
{
name: "compatibility version two minor lower than binary ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.30.0",
expectErrors: false,
},
{
name: "compatibility version three minor lower than binary ok",
binaryVersion: "v1.34.2",
emulationVersion: "v1.33.0",
minCompatibilityVersion: "v1.31.0",
expectErrors: false,
},
{
name: "compatibility version one minor higher than binary not ok",
binaryVersion: "v1.32.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.33.0",
expectErrors: true,
},
{
name: "emulation version lower than compatibility version not ok",
binaryVersion: "v1.34.2",
emulationVersion: "v1.32.0",
minCompatibilityVersion: "v1.33.0",
expectErrors: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
binaryVersion := version.MustParseGeneric(test.binaryVersion)
effective := &effectiveVersion{}
emulationVersion := version.MustParseGeneric(test.emulationVersion)
minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion)
effective.Set(binaryVersion, emulationVersion, minCompatibilityVersion)
errs := effective.Validate()
if len(errs) > 0 && !test.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}
if len(errs) == 0 && test.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}
func TestValidateKubeEffectiveVersion(t *testing.T) {
tests := []struct {
name string
emulationVersion string
minCompatibilityVersion string
expectErr bool
}{
{
name: "valid versions",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.31.0",
expectErr: false,
},
{
name: "emulationVersion too low",
emulationVersion: "v1.30.0",
minCompatibilityVersion: "v1.31.0",
expectErr: true,
},
{
name: "minCompatibilityVersion too low",
emulationVersion: "v1.31.0",
minCompatibilityVersion: "v1.29.0",
expectErr: true,
},
{
name: "both versions too low",
emulationVersion: "v1.30.0",
minCompatibilityVersion: "v1.30.0",
expectErr: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
effective := NewEffectiveVersion("1.32")
effective.SetEmulationVersion(version.MustParseGeneric(test.emulationVersion))
effective.SetMinCompatibilityVersion(version.MustParseGeneric(test.minCompatibilityVersion))
err := ValidateKubeEffectiveVersion(effective)
if test.expectErr && err == nil {
t.Error("expected error, but got nil")
}
if !test.expectErr && err != nil {
t.Errorf("unexpected error: %v", err)
}
})
}
}

28
staging/src/k8s.io/component-base/zpages/statusz/registry.go
View File

@@ -20,9 +20,9 @@ import (
"time"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
"k8s.io/component-base/compatibility"
compbasemetrics "k8s.io/component-base/metrics"
utilversion "k8s.io/component-base/version"
)
@@ -34,9 +34,12 @@ type statuszRegistry interface {
emulationVersion() *version.Version
}
type registry struct{}
type registry struct {
// componentGlobalsRegistry compatibility.ComponentGlobalsRegistry
effectiveVersion compatibility.EffectiveVersion
}
func (registry) processStartTime() time.Time {
func (*registry) processStartTime() time.Time {
start, err := compbasemetrics.GetProcessStart()
if err != nil {
klog.Errorf("Could not get process start time, %v", err)
@@ -45,23 +48,20 @@ func (registry) processStartTime() time.Time {
return time.Unix(int64(start), 0)
}
func (registry) goVersion() string {
func (*registry) goVersion() string {
return utilversion.Get().GoVersion
}
func (registry) binaryVersion() *version.Version {
effectiveVer := featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent)
if effectiveVer != nil {
return effectiveVer.BinaryVersion()
func (r *registry) binaryVersion() *version.Version {
if r.effectiveVersion != nil {
return r.effectiveVersion.BinaryVersion()
}
return utilversion.DefaultKubeEffectiveVersion().BinaryVersion()
return version.MustParse(utilversion.Get().String())
}
func (registry) emulationVersion() *version.Version {
effectiveVer := featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent)
if effectiveVer != nil {
return effectiveVer.EmulationVersion()
func (r *registry) emulationVersion() *version.Version {
if r.effectiveVersion != nil {
return r.effectiveVersion.EmulationVersion()
}
return nil

23
staging/src/k8s.io/component-base/zpages/statusz/registry_test.go
View File

@@ -20,14 +20,12 @@ import (
"testing"
"github.com/stretchr/testify/assert"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/compatibility"
utilversion "k8s.io/component-base/version"
)
func TestBinaryVersion(t *testing.T) {
componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry
tests := []struct {
name string
setFakeEffectiveVersion bool
@@ -42,20 +40,18 @@ func TestBinaryVersion(t *testing.T) {
},
{
name: "binaryVersion without effective version",
wantBinaryVersion: utilversion.DefaultKubeEffectiveVersion().BinaryVersion(),
wantBinaryVersion: version.MustParse(utilversion.Get().String()),
},
}
for _, tt := range tests {
componentGlobalsRegistry.Reset()
t.Run(tt.name, func(t *testing.T) {
registry := &registry{}
if tt.setFakeEffectiveVersion {
verKube := utilversion.NewEffectiveVersion(tt.fakeVersion)
fg := featuregate.NewVersionedFeatureGate(version.MustParse(tt.fakeVersion))
utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg))
verKube := compatibility.NewEffectiveVersionFromString(tt.fakeVersion, "", "")
registry.effectiveVersion = verKube
}
registry := &registry{}
got := registry.binaryVersion()
assert.Equal(t, tt.wantBinaryVersion, got)
})
@@ -63,7 +59,6 @@ func TestBinaryVersion(t *testing.T) {
}
func TestEmulationVersion(t *testing.T) {
componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry
tests := []struct {
name string
setFakeEffectiveVersion bool
@@ -83,16 +78,14 @@ func TestEmulationVersion(t *testing.T) {
}
for _, tt := range tests {
componentGlobalsRegistry.Reset()
t.Run(tt.name, func(t *testing.T) {
registry := &registry{}
if tt.setFakeEffectiveVersion {
verKube := utilversion.NewEffectiveVersion("0.0.0")
verKube := compatibility.NewEffectiveVersionFromString("0.0.0", "", "")
verKube.SetEmulationVersion(version.MustParse(tt.fakeEmulVer))
fg := featuregate.NewVersionedFeatureGate(version.MustParse(tt.fakeEmulVer))
utilruntime.Must(componentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, verKube, fg))
registry.effectiveVersion = verKube
}
registry := &registry{}
got := registry.emulationVersion()
if tt.wantEmul != nil && got != nil {
assert.Equal(t, tt.wantEmul.Major(), got.Major())

5
staging/src/k8s.io/component-base/zpages/statusz/statusz.go
View File

@@ -24,6 +24,7 @@ import (
"net/http"
"time"
"k8s.io/component-base/compatibility"
"k8s.io/component-base/zpages/httputil"
"k8s.io/klog/v2"
)
@@ -62,8 +63,8 @@ type mux interface {
Handle(path string, handler http.Handler)
}
func NewRegistry() statuszRegistry {
return registry{}
func NewRegistry(effectiveVersion compatibility.EffectiveVersion) statuszRegistry {
return &registry{effectiveVersion: effectiveVersion}
}
func Install(m mux, componentName string, reg statuszRegistry) {

2
staging/src/k8s.io/cri-client/go.mod
View File

@@ -45,7 +45,6 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
@@ -57,7 +56,6 @@ require (
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect

4
staging/src/k8s.io/cri-client/go.sum generated
View File

@@ -18,7 +18,6 @@ github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -70,7 +69,6 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -117,9 +115,7 @@ github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoG
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=

3
staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go
View File

@@ -31,7 +31,6 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/filters"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/component-base/featuregate"
"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
"k8s.io/kube-aggregator/pkg/apiserver"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
@@ -63,7 +62,7 @@ func NewCommandStartAggregator(ctx context.Context, defaults *AggregatorOptions)
Short: "Launch a API aggregator and proxy server",
Long: "Launch a API aggregator and proxy server",
PersistentPreRunE: func(*cobra.Command, []string) error {
return featuregate.DefaultComponentGlobalsRegistry.Set()
return o.ServerRunOptions.ComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {
if err := o.Complete(); err != nil {

31
staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go
View File

@@ -33,9 +33,11 @@ import (
"k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
utilversion "k8s.io/component-base/version"
baseversion "k8s.io/component-base/version"
"k8s.io/sample-apiserver/pkg/admission/plugin/banflunder"
"k8s.io/sample-apiserver/pkg/admission/wardleinitializer"
"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
@@ -51,6 +53,8 @@ const defaultEtcdPathPrefix = "/registry/wardle.example.com"
// WardleServerOptions contains state for master/api server
type WardleServerOptions struct {
RecommendedOptions *genericoptions.RecommendedOptions
// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
ComponentGlobalsRegistry basecompatibility.ComponentGlobalsRegistry
SharedInformerFactory informers.SharedInformerFactory
StdOut io.Writer
@@ -63,7 +67,7 @@ func WardleVersionToKubeVersion(ver *version.Version) *version.Version {
if ver.Major() != 1 {
return nil
}
kubeVer := utilversion.DefaultKubeEffectiveVersion().BinaryVersion()
kubeVer := version.MustParse(baseversion.DefaultKubeBinaryVersion)
// "1.2" maps to kubeVer
offset := int(ver.Minor()) - 2
mappedVer := kubeVer.OffsetMinor(offset)
@@ -80,6 +84,7 @@ func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions {
defaultEtcdPathPrefix,
apiserver.Codecs.LegacyCodec(v1alpha1.SchemeGroupVersion),
),
ComponentGlobalsRegistry: compatibility.DefaultComponentGlobalsRegistry,
StdOut: out,
StdErr: errOut,
@@ -99,7 +104,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
if skipDefaultComponentGlobalsRegistrySet {
return nil
}
return featuregate.DefaultComponentGlobalsRegistry.Set()
return defaults.ComponentGlobalsRegistry.Set()
},
RunE: func(c *cobra.Command, args []string) error {
if err := o.Complete(); err != nil {
@@ -135,8 +140,8 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
// Register the "Wardle" component with the global component registry,
// associating it with its effective version and feature gate configuration.
// Will skip if the component has been registered, like in the integration test.
_, wardleFeatureGate := featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
apiserver.WardleComponentName, utilversion.NewEffectiveVersion(defaultWardleVersion),
_, wardleFeatureGate := defaults.ComponentGlobalsRegistry.ComponentGlobalsOrRegister(
apiserver.WardleComponentName, basecompatibility.NewEffectiveVersionFromString(defaultWardleVersion, "", ""),
featuregate.NewVersionedFeatureGate(version.MustParse(defaultWardleVersion)))
// Add versioned feature specifications for the "BanFlunder" feature.
@@ -150,14 +155,14 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
}))
// Register the default kube component if not already present in the global registry.
_, _ = featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(featuregate.DefaultKubeComponent,
utilversion.NewEffectiveVersion(utilversion.DefaultKubeBinaryVersion), utilfeature.DefaultMutableFeatureGate)
_, _ = defaults.ComponentGlobalsRegistry.ComponentGlobalsOrRegister(basecompatibility.DefaultKubeComponent,
basecompatibility.NewEffectiveVersionFromString(baseversion.DefaultKubeBinaryVersion, "", ""), utilfeature.DefaultMutableFeatureGate)
// Set the emulation version mapping from the "Wardle" component to the kube component.
// This ensures that the emulation version of the latter is determined by the emulation version of the former.
utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.SetEmulationVersionMapping(apiserver.WardleComponentName, featuregate.DefaultKubeComponent, WardleVersionToKubeVersion))
utilruntime.Must(defaults.ComponentGlobalsRegistry.SetEmulationVersionMapping(apiserver.WardleComponentName, basecompatibility.DefaultKubeComponent, WardleVersionToKubeVersion))
featuregate.DefaultComponentGlobalsRegistry.AddFlags(flags)
defaults.ComponentGlobalsRegistry.AddFlags(flags)
return cmd
}
@@ -166,13 +171,13 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
func (o WardleServerOptions) Validate(args []string) error {
errors := []error{}
errors = append(errors, o.RecommendedOptions.Validate()...)
errors = append(errors, featuregate.DefaultComponentGlobalsRegistry.Validate()...)
errors = append(errors, o.ComponentGlobalsRegistry.Validate()...)
return utilerrors.NewAggregate(errors)
}
// Complete fills in fields required to have valid data
func (o *WardleServerOptions) Complete() error {
if featuregate.DefaultComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName).Enabled("BanFlunder") {
if o.ComponentGlobalsRegistry.FeatureGateFor(apiserver.WardleComponentName).Enabled("BanFlunder") {
// register admission plugins
banflunder.Register(o.RecommendedOptions.Admission.Plugins)
@@ -209,8 +214,8 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) {
serverConfig.OpenAPIV3Config.Info.Title = "Wardle"
serverConfig.OpenAPIV3Config.Info.Version = "0.1"
serverConfig.FeatureGate = featuregate.DefaultComponentGlobalsRegistry.FeatureGateFor(featuregate.DefaultKubeComponent)
serverConfig.EffectiveVersion = featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(apiserver.WardleComponentName)
serverConfig.FeatureGate = o.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)
serverConfig.EffectiveVersion = o.ComponentGlobalsRegistry.EffectiveVersionFor(apiserver.WardleComponentName)
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return nil, err

4
staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start_test.go
View File

@@ -20,13 +20,13 @@ import (
"testing"
"k8s.io/apimachinery/pkg/util/version"
utilversion "k8s.io/component-base/version"
"k8s.io/apiserver/pkg/util/compatibility"
"github.com/stretchr/testify/assert"
)
func TestWardleEmulationVersionToKubeEmulationVersion(t *testing.T) {
defaultKubeEffectiveVersion := utilversion.DefaultKubeEffectiveVersion()
defaultKubeEffectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest()
testCases := []struct {
desc string

7
test/integration/apiserver/admissionwebhook/match_conditions_test.go
View File

@@ -22,7 +22,6 @@ import (
"crypto/x509"
"encoding/json"
"io"
"k8s.io/apimachinery/pkg/util/version"
"net/http"
"net/http/httptest"
"strconv"
@@ -634,9 +633,9 @@ func TestMatchConditionsWithoutStrictCostEnforcement(t *testing.T) {
for _, testcase := range testcases {
t.Run(testcase.name, func(t *testing.T) {
upCh := recorder.Reset()
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"))
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.StrictCostEnforcementForWebhooks, false)
server, err := apiservertesting.StartTestServer(t, &apiservertesting.TestServerInstanceOptions{EmulationVersion: "1.31"}, []string{
server, err := apiservertesting.StartTestServer(t, nil, []string{
"--emulated-version", "1.31",
"--feature-gates", "StrictCostEnforcementForWebhooks=false",
"--disable-admission-plugins=ServiceAccount",
}, framework.SharedEtcd())
if err != nil {

10
test/integration/apiserver/apiserver_test.go
View File

@@ -56,10 +56,12 @@ import (
"k8s.io/apimachinery/pkg/types"
utiljson "k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/endpoints/handlers"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
@@ -70,7 +72,6 @@ import (
"k8s.io/client-go/restmapper"
"k8s.io/client-go/tools/pager"
featuregatetesting "k8s.io/component-base/featuregate/testing"
utilversion "k8s.io/component-base/version"
"k8s.io/klog/v2"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
@@ -3198,8 +3199,7 @@ func TestEmulatedStorageVersion(t *testing.T) {
for emulatedVersion, cases := range groupedCases {
t.Run(emulatedVersion, func(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(
t, &kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: emulatedVersion},
[]string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd())
t, nil, []string{"--emulated-version=kube=" + emulatedVersion, `--storage-media-type=application/json`}, framework.SharedEtcd())
defer server.TearDownFn()
client := clientset.NewForConfigOrDie(server.ClientConfig)
@@ -3302,7 +3302,7 @@ func TestAllowedEmulationVersions(t *testing.T) {
}{
{
name: "default",
emulationVersion: utilversion.DefaultKubeEffectiveVersion().EmulationVersion().String(),
emulationVersion: compatibility.DefaultKubeEffectiveVersionForTest().EmulationVersion().String(),
},
}
@@ -3337,6 +3337,7 @@ func TestAllowedEmulationVersions(t *testing.T) {
}
func TestEnableEmulationVersion(t *testing.T) {
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32"))
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
[]string{"--emulated-version=kube=1.31"}, framework.SharedEtcd())
@@ -3398,6 +3399,7 @@ func TestEnableEmulationVersion(t *testing.T) {
}
func TestDisableEmulationVersion(t *testing.T) {
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.32"))
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
[]string{}, framework.SharedEtcd())

7
test/integration/apiserver/cel/validatingadmissionpolicy_test.go
View File

@@ -20,7 +20,6 @@ import (
"context"
"encoding/json"
"fmt"
"k8s.io/apimachinery/pkg/util/version"
"net/http"
"net/http/httptest"
"os"
@@ -2235,9 +2234,9 @@ func Test_CostLimitForValidation(t *testing.T) {
func Test_CostLimitForValidationWithFeatureDisabled(t *testing.T) {
resetPolicyRefreshInterval := generic.SetPolicyRefreshIntervalForTests(policyRefreshInterval)
defer resetPolicyRefreshInterval()
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"))
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.StrictCostEnforcementForVAP, false)
server, err := apiservertesting.StartTestServer(t, &apiservertesting.TestServerInstanceOptions{EmulationVersion: "1.31"}, []string{
server, err := apiservertesting.StartTestServer(t, nil, []string{
"--emulated-version", "1.31",
"--feature-gates", "StrictCostEnforcementForVAP=false",
"--enable-admission-plugins", "ValidatingAdmissionPolicy",
}, framework.SharedEtcd())
if err != nil {

3
test/integration/client/client_test.go
View File

@@ -47,6 +47,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1"
autoscalingv1ac "k8s.io/client-go/applyconfigurations/autoscaling/v1"
@@ -83,7 +84,7 @@ func TestClient(t *testing.T) {
t.Fatalf("unexpected error: %v", err)
}
expectedInfo := utilversion.Get()
kubeVersion := utilversion.DefaultKubeEffectiveVersion().BinaryVersion()
kubeVersion := compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion()
expectedInfo.Major = fmt.Sprintf("%d", kubeVersion.Major())
expectedInfo.Minor = fmt.Sprintf("%d", kubeVersion.Minor())

12
test/integration/controlplane/transformation/transformation_test.go
View File

@@ -25,7 +25,6 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
@@ -117,7 +116,7 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool,
return nil, fmt.Errorf("failed to read config file: %w", err)
}
if e.kubeAPIServer, err = startTestServerLocked(
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(
tb, nil,
e.getEncryptionOptions(reload), e.storageConfig); err != nil {
e.cleanUp()
@@ -148,15 +147,6 @@ func newTransformTest(tb testing.TB, transformerConfigYAML string, reload bool,
return &e, nil
}
var startTestServerLock sync.Mutex
// startTestServerLocked prevents parallel calls to kubeapiservertesting.StartTestServer because it messes with global state.
func startTestServerLocked(t ktesting.TB, instanceOptions *kubeapiservertesting.TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result kubeapiservertesting.TestServer, err error) {
startTestServerLock.Lock()
defer startTestServerLock.Unlock()
return kubeapiservertesting.StartTestServer(t, instanceOptions, customFlags, storageConfig)
}
func (e *transformTest) cleanUp() {
if e.configDir != "" {
os.RemoveAll(e.configDir)

7
test/integration/etcd/data.go
View File

@@ -25,6 +25,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/util/compatibility"
utilversion "k8s.io/component-base/version"
"k8s.io/kubernetes/test/utils/image"
@@ -34,9 +35,9 @@ import (
// Tests aiming for full coverage of versions should test fixtures of all supported versions.
func GetSupportedEmulatedVersions() []string {
return []string{
utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(2).String(),
utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(1).String(),
utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String(),
compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion().SubtractMinor(2).String(),
compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion().SubtractMinor(1).String(),
compatibility.DefaultKubeEffectiveVersionForTest().BinaryVersion().String(),
}
}

1
test/integration/etcd/etcd_storage_path_test.go
View File

@@ -104,7 +104,6 @@ func testEtcdStoragePathWithVersion(t *testing.T, v string) {
// only understand v1beta1.
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, false)
}
registerEffectiveEmulationVersion(t)
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
// Disable alphas when emulating previous versions.

40
test/integration/etcd/server.go
View File

@@ -36,9 +36,9 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/util/compatibility"
"k8s.io/apiserver/pkg/util/feature"
cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
@@ -46,9 +46,8 @@ import (
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
utiltesting "k8s.io/client-go/util/testing"
"k8s.io/component-base/featuregate"
basecompatibility "k8s.io/component-base/compatibility"
featuregatetesting "k8s.io/component-base/featuregate/testing"
utilversion "k8s.io/component-base/version"
"k8s.io/kubernetes/cmd/kube-apiserver/app"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/integration"
@@ -67,17 +66,6 @@ AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0
/IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg==
-----END EC PRIVATE KEY-----`
func registerEffectiveEmulationVersion(t *testing.T) {
featureGate := feature.DefaultMutableFeatureGate
featureGate.AddMetrics()
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion())
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion())
featuregate.DefaultComponentGlobalsRegistry.Reset()
utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
}
// StartRealAPIServerOrDie starts an API server that is appropriate for use in tests that require one of every resource
func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOptions)) *APIServer {
tCtx := ktesting.Init(t)
@@ -108,6 +96,17 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu
}
opts := options.NewServerRunOptions()
// If EmulationVersion of DefaultFeatureGate is set during test, we need to propagate it to the apiserver ComponentGlobalsRegistry.
featureGate := feature.DefaultMutableFeatureGate.DeepCopy()
effectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest()
effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion())
// set up new instance of ComponentGlobalsRegistry instead of using the DefaultComponentGlobalsRegistry to avoid contention in parallel tests.
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
if err := componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate); err != nil {
t.Fatal(err)
}
opts.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry
opts.Options.SecureServing.Listener = listener
opts.Options.SecureServing.ServerCert.CertDirectory = certDir
opts.Options.ServiceAccountSigningKeyFile = saSigningKeyFile.Name()
@@ -123,6 +122,19 @@ func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRu
for _, f := range configFuncs {
f(opts)
}
// If the local ComponentGlobalsRegistry is changed by configFuncs,
// we need to copy the new feature values back to the DefaultFeatureGate because most feature checks still use the DefaultFeatureGate.
if !featureGate.EmulationVersion().EqualTo(feature.DefaultMutableFeatureGate.EmulationVersion()) {
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultMutableFeatureGate, effectiveVersion.EmulationVersion())
}
for f := range feature.DefaultMutableFeatureGate.GetAll() {
if featureGate.Enabled(f) != feature.DefaultFeatureGate.Enabled(f) {
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, f, featureGate.Enabled(f))
}
}
feature.DefaultMutableFeatureGate.AddMetrics()
completedOptions, err := opts.Complete(tCtx)
if err != nil {
t.Fatal(err)

26
test/integration/examples/apiserver_test.go
View File

@@ -55,9 +55,9 @@ import (
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/client-go/transport"
"k8s.io/client-go/util/cert"
basecompatibility "k8s.io/component-base/compatibility"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
utilversion "k8s.io/component-base/version"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/cmd/kube-apiserver/app"
@@ -282,11 +282,8 @@ func testFrontProxyConfig(t *testing.T, withUID bool) {
extraKASFlags = []string{"--requestheader-uid-headers=x-remote-uid"}
}
// each wardle binary is bundled with a specific kube binary.
kubeBinaryVersion := sampleserver.WardleVersionToKubeVersion(version.MustParse(wardleBinaryVersion)).String()
// start up the KAS and prepare the options for the wardle API server
testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, kubeBinaryVersion, wardleBinaryVersion, extraKASFlags, withUID)
testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, wardleBinaryVersion, extraKASFlags, withUID)
kubeConfig := getKubeConfig(testKAS)
// create the SA that we will use to query the aggregated API
@@ -402,10 +399,7 @@ func testAggregatedAPIServer(t *testing.T, setWardleFeatureGate, banFlunder bool
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
t.Cleanup(cancel)
// each wardle binary is bundled with a specific kube binary.
kubeBinaryVersion := sampleserver.WardleVersionToKubeVersion(version.MustParse(wardleBinaryVersion)).String()
testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, kubeBinaryVersion, wardleBinaryVersion, nil, false)
testKAS, wardleOptions, wardlePort := prepareAggregatedWardleAPIServer(ctx, t, testNamespace, wardleBinaryVersion, nil, false)
kubeClientConfig := getKubeConfig(testKAS)
wardleCertDir, _ := os.MkdirTemp("", "test-integration-wardle-server")
@@ -685,7 +679,7 @@ func TestAggregatedAPIServerRejectRedirectResponse(t *testing.T) {
}
}
func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespace, kubebinaryVersion, wardleBinaryVersion string, kubeAPIServerFlags []string, withUID bool) (*kastesting.TestServer, *sampleserver.WardleServerOptions, int) {
func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespace, wardleBinaryVersion string, kubeAPIServerFlags []string, withUID bool) (*kastesting.TestServer, *sampleserver.WardleServerOptions, int) {
// makes the kube-apiserver very responsive. it's normally a minute
dynamiccertificates.FileRefreshDuration = 1 * time.Second
@@ -697,22 +691,17 @@ func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespa
// endpoints cannot have loopback IPs so we need to override the resolver itself
t.Cleanup(app.SetServiceResolverForTests(staticURLServiceResolver(fmt.Sprintf("https://127.0.0.1:%d", wardlePort))))
// TODO figure out how to actually make BinaryVersion/EmulationVersion work with Wardle and KAS at the same time when Alpha FG are being set
if withUID {
kubebinaryVersion = ""
}
testServer := kastesting.StartTestServerOrDie(t,
&kastesting.TestServerInstanceOptions{
EnableCertAuth: true,
BinaryVersion: kubebinaryVersion,
},
kubeAPIServerFlags,
framework.SharedEtcd())
t.Cleanup(func() { testServer.TearDownFn() })
_, _ = featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
apiserver.WardleComponentName, utilversion.NewEffectiveVersion(wardleBinaryVersion),
componentGlobalsRegistry := testServer.ServerOpts.Options.GenericServerRunOptions.ComponentGlobalsRegistry
_, _ = componentGlobalsRegistry.ComponentGlobalsOrRegister(
apiserver.WardleComponentName, basecompatibility.NewEffectiveVersionFromString(wardleBinaryVersion, "", ""),
featuregate.NewVersionedFeatureGate(version.MustParse(wardleBinaryVersion)))
kubeClient := client.NewForConfigOrDie(getKubeConfig(testServer))
@@ -740,6 +729,7 @@ func prepareAggregatedWardleAPIServer(ctx context.Context, t *testing.T, namespa
}
wardleOptions := sampleserver.NewWardleServerOptions(os.Stdout, os.Stderr)
wardleOptions.ComponentGlobalsRegistry = componentGlobalsRegistry
// ensure this is a SAN on the generated cert for service FQDN
wardleOptions.AlternateDNS = []string{
fmt.Sprintf("api.%s.svc", namespace),

27
test/integration/framework/test_server.go
View File

@@ -35,9 +35,13 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
genericapiserver "k8s.io/apiserver/pkg/server"
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/util/compatibility"
utilfeature "k8s.io/apiserver/pkg/util/feature"
client "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/cert"
basecompatibility "k8s.io/component-base/compatibility"
featuregatetesting "k8s.io/component-base/featuregate/testing"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
netutils "k8s.io/utils/net"
@@ -136,6 +140,17 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
}
opts := options.NewServerRunOptions()
// If EmulationVersion of DefaultFeatureGate is set during test, we need to propagate it to the apiserver ComponentGlobalsRegistry.
featureGate := utilfeature.DefaultMutableFeatureGate.DeepCopy()
effectiveVersion := compatibility.DefaultKubeEffectiveVersionForTest()
effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion())
// set up new instance of ComponentGlobalsRegistry instead of using the DefaultComponentGlobalsRegistry to avoid contention in parallel tests.
componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry()
if err := componentGlobalsRegistry.Register(basecompatibility.DefaultKubeComponent, effectiveVersion, featureGate); err != nil {
t.Fatal(err)
}
opts.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry
opts.SecureServing.Listener = listener
opts.SecureServing.BindAddress = netutils.ParseIPSloppy("127.0.0.1")
opts.SecureServing.ServerCert.CertDirectory = certDir
@@ -158,6 +173,18 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) (
setup.ModifyServerRunOptions(opts)
}
// If the local ComponentGlobalsRegistry is changed by ModifyServerRunOptions,
// we need to copy the new feature values back to the DefaultFeatureGate because most feature checks still use the DefaultFeatureGate.
if !featureGate.EmulationVersion().EqualTo(utilfeature.DefaultMutableFeatureGate.EmulationVersion()) {
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultMutableFeatureGate, effectiveVersion.EmulationVersion())
}
for f := range utilfeature.DefaultMutableFeatureGate.GetAll() {
if featureGate.Enabled(f) != utilfeature.DefaultFeatureGate.Enabled(f) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, f, featureGate.Enabled(f))
}
}
utilfeature.DefaultMutableFeatureGate.AddMetrics()
completedOptions, err := opts.Complete(ctx)
if err != nil {
t.Fatal(err)

5
test/integration/pods/pods_test.go
View File

@@ -1230,9 +1230,8 @@ func TestMutablePodSchedulingDirectives(t *testing.T) {
// Test disabling of RelaxedDNSSearchValidation after a Pod has been created
func TestRelaxedDNSSearchValidation(t *testing.T) {
// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
server := kubeapiservertesting.StartTestServerOrDie(t,
&kubeapiservertesting.TestServerInstanceOptions{BinaryVersion: "1.32"},
framework.DefaultTestServerFlags(), framework.SharedEtcd())
server := kubeapiservertesting.StartTestServerOrDie(t, nil,
append(framework.DefaultTestServerFlags(), "--emulated-version=1.32"), framework.SharedEtcd())
defer server.TearDownFn()
client := clientset.NewForConfigOrDie(server.ClientConfig)

13
test/integration/service/loadbalancer_test.go
View File

@@ -19,7 +19,7 @@ package service
import (
"context"
"encoding/json"
"k8s.io/apimachinery/pkg/util/version"
"fmt"
"testing"
"time"
@@ -34,10 +34,8 @@ import (
clientset "k8s.io/client-go/kubernetes"
servicecontroller "k8s.io/cloud-provider/controllers/service"
fakecloud "k8s.io/cloud-provider/fake"
featuregatetesting "k8s.io/component-base/featuregate/testing"
controllersmetrics "k8s.io/component-base/metrics/prometheus/controllers"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/test/integration/framework"
"k8s.io/utils/net"
utilpointer "k8s.io/utils/pointer"
@@ -708,13 +706,12 @@ func Test_ServiceLoadBalancerIPMode(t *testing.T) {
for _, tc := range testCases {
t.Run("", func(t *testing.T) {
var testServerOptions *kubeapiservertesting.TestServerInstanceOptions
serverFlags := framework.DefaultTestServerFlags()
if !tc.ipModeEnabled {
testServerOptions = &kubeapiservertesting.TestServerInstanceOptions{EmulationVersion: "1.31"}
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"))
serverFlags = append(serverFlags, "--emulated-version=1.31")
}
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LoadBalancerIPMode, tc.ipModeEnabled)
server := kubeapiservertesting.StartTestServerOrDie(t, testServerOptions, framework.DefaultTestServerFlags(), framework.SharedEtcd())
serverFlags = append(serverFlags, fmt.Sprintf("--feature-gates=LoadBalancerIPMode=%v", tc.ipModeEnabled))
server := kubeapiservertesting.StartTestServerOrDie(t, nil, serverFlags, framework.SharedEtcd())
defer server.TearDownFn()
client, err := clientset.NewForConfig(server.ClientConfig)