Merge pull request #51372 from mtaufen/feature-gate-file

Automatic merge from submit-queue (batch tested with PRs 49971, 51357, 51616, 51649, 51372)

Separate feature gates for dynamic kubelet config vs loading from a file

This makes it so these two features can be turned on independently, rather than bundling both under dynamic kubelet config.

fixes: #51664

```release-note
NONE
```

cmd/kubelet/BUILD

@@ -20,9 +20,6 @@ go_library(
         "//cmd/kubelet/app:go_default_library",
         "//cmd/kubelet/app/options:go_default_library",
         "//pkg/client/metrics/prometheus:go_default_library",
-        "//pkg/features:go_default_library",
-        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
-        "//pkg/kubelet/kubeletconfig:go_default_library",
         "//pkg/version/prometheus:go_default_library",
         "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

cmd/kubelet/app/BUILD

@@ -112,6 +112,7 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/authentication/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library",

cmd/kubelet/app/options/options.go

@@ -138,8 +138,8 @@ func ValidateKubeletFlags(f *KubeletFlags) error {
 		return fmt.Errorf("the DynamicKubeletConfig feature gate must be enabled in order to use the --dynamic-config-dir flag")
 	}
 	// ensure that nobody sets InitConfigDir if the dynamic config feature gate is turned off
-	if f.InitConfigDir.Provided() && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
-		return fmt.Errorf("the DynamicKubeletConfig feature gate must be enabled in order to use the --init-config-dir flag")
+	if f.InitConfigDir.Provided() && !utilfeature.DefaultFeatureGate.Enabled(features.KubeletConfigFile) {
+		return fmt.Errorf("the KubeletConfigFile feature gate must be enabled in order to use the --init-config-dir flag")
 	}
 	return nil
 }

cmd/kubelet/app/server.go

@@ -44,6 +44,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/apiserver/pkg/util/flag"
 	clientgoclientset "k8s.io/client-go/kubernetes"
 	clientset "k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -768,29 +769,29 @@ func parseResourceList(m kubeletconfiginternal.ConfigurationMap) (v1.ResourceLis
 }
 
 // BootstrapKubeletConfigController constructs and bootstrap a configuration controller
-func BootstrapKubeletConfigController(
-	flags *options.KubeletFlags,
-	defaultConfig *kubeletconfiginternal.KubeletConfiguration) (*kubeletconfiginternal.KubeletConfiguration, *kubeletconfig.Controller, error) {
+func BootstrapKubeletConfigController(defaultConfig *kubeletconfiginternal.KubeletConfiguration,
+	initConfigDirFlag flag.StringFlag,
+	dynamicConfigDirFlag flag.StringFlag) (*kubeletconfiginternal.KubeletConfiguration, *kubeletconfig.Controller, error) {
 	var err error
 	// Alpha Dynamic Configuration Implementation; this section only loads config from disk, it does not contact the API server
 	// compute absolute paths based on current working dir
 	initConfigDir := ""
-	if flags.InitConfigDir.Provided() {
-		initConfigDir, err = filepath.Abs(flags.InitConfigDir.Value())
+	if utilfeature.DefaultFeatureGate.Enabled(features.KubeletConfigFile) && initConfigDirFlag.Provided() {
+		initConfigDir, err = filepath.Abs(initConfigDirFlag.Value())
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to get absolute path for --init-config-dir")
 		}
 	}
 	dynamicConfigDir := ""
-	if flags.DynamicConfigDir.Provided() {
-		dynamicConfigDir, err = filepath.Abs(flags.DynamicConfigDir.Value())
+	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) && dynamicConfigDirFlag.Provided() {
+		dynamicConfigDir, err = filepath.Abs(dynamicConfigDirFlag.Value())
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to get absolute path for --dynamic-config-dir")
 		}
 	}
 
 	// get the latest KubeletConfiguration checkpoint from disk, or load the init or default config if no valid checkpoints exist
-	kubeletConfigController, err := kubeletconfig.NewController(initConfigDir, dynamicConfigDir, defaultConfig)
+	kubeletConfigController, err := kubeletconfig.NewController(defaultConfig, initConfigDir, dynamicConfigDir)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to construct controller, error: %v", err)
 	}

cmd/kubelet/kubelet.go

@@ -32,10 +32,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubelet/app"
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
-	"k8s.io/kubernetes/pkg/features"
-	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
-	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
-	_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
 	"k8s.io/kubernetes/pkg/version/verflag"
 )
 
@@ -75,17 +72,12 @@ func main() {
 	if err := options.ValidateKubeletFlags(kubeletFlags); err != nil {
 		die(err)
 	}
-	// if dynamic kubelet config is enabled, bootstrap the kubelet config controller
-	var kubeletConfig *kubeletconfiginternal.KubeletConfiguration
-	var kubeletConfigController *kubeletconfig.Controller
-	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
-		var err error
-		kubeletConfig, kubeletConfigController, err = app.BootstrapKubeletConfigController(kubeletFlags, defaultConfig)
-		if err != nil {
-			die(err)
-		}
-	} else if kubeletConfig == nil {
-		kubeletConfig = defaultConfig
+	// bootstrap the kubelet config controller, app.BootstrapKubeletConfigController will check
+	// feature gates and only turn on relevant parts of the controller
+	kubeletConfig, kubeletConfigController, err := app.BootstrapKubeletConfigController(
+		defaultConfig, kubeletFlags.InitConfigDir, kubeletFlags.DynamicConfigDir)
+	if err != nil {
+		die(err)
 	}
 
 	// construct a KubeletServer from kubeletFlags and kubeletConfig

pkg/features/kube_features.go

@@ -45,6 +45,10 @@ const (
 	// alpha: v1.4
 	DynamicKubeletConfig utilfeature.Feature = "DynamicKubeletConfig"
 
+	// owner: @mtaufen
+	// alpha: v1.8
+	KubeletConfigFile utilfeature.Feature = "KubeletConfigFile"
+
 	// owner: @pweil-
 	// alpha: v1.5
 	//
@@ -152,6 +156,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	ExternalTrafficLocalOnly:                    {Default: true, PreRelease: utilfeature.GA},
 	AppArmor:                                    {Default: true, PreRelease: utilfeature.Beta},
 	DynamicKubeletConfig:                        {Default: false, PreRelease: utilfeature.Alpha},
+	KubeletConfigFile:                           {Default: false, PreRelease: utilfeature.Alpha},
 	DynamicVolumeProvisioning:                   {Default: true, PreRelease: utilfeature.Alpha},
 	ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: utilfeature.Beta},
 	ExperimentalCriticalPodAnnotation:           {Default: false, PreRelease: utilfeature.Alpha},

pkg/kubelet/kubeletconfig/controller.go

@@ -78,9 +78,9 @@ type Controller struct {
 // If the `initConfigDir` is an empty string, skips trying to load the init config.
 // If the `dynamicConfigDir` is an empty string, skips trying to load checkpoints or download new config,
 // but will still sync the ConfigOK condition if you call StartSync with a non-nil client.
-func NewController(initConfigDir string,
-	dynamicConfigDir string,
-	defaultConfig *kubeletconfig.KubeletConfiguration) (*Controller, error) {
+func NewController(defaultConfig *kubeletconfig.KubeletConfiguration,
+	initConfigDir string,
+	dynamicConfigDir string) (*Controller, error) {
 	var err error
 
 	fs := utilfs.DefaultFs{}