/* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package options import ( "context" "crypto/rand" "crypto/rsa" "crypto/x509" "encoding/pem" "fmt" "os" "reflect" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/spf13/pflag" noopoteltrace "go.opentelemetry.io/otel/trace/noop" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" apiserveroptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" cliflag "k8s.io/component-base/cli/flag" basecompatibility "k8s.io/component-base/compatibility" "k8s.io/component-base/featuregate" "k8s.io/component-base/logs" "k8s.io/component-base/metrics" kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" "k8s.io/kubernetes/pkg/serviceaccount" v1alpha1testing "k8s.io/kubernetes/pkg/serviceaccount/externaljwt/plugin/testing/v1alpha1" netutils "k8s.io/utils/net" ) func TestAddFlags(t *testing.T) { componentGlobalsRegistry := basecompatibility.NewComponentGlobalsRegistry() fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) utilruntime.Must(componentGlobalsRegistry.Register("test", basecompatibility.NewEffectiveVersionFromString("1.32", "1.31", "1.31"), featuregate.NewFeatureGate())) s := NewOptions() s.GenericServerRunOptions.ComponentGlobalsRegistry = componentGlobalsRegistry var fss cliflag.NamedFlagSets s.AddFlags(&fss) for _, f := range fss.FlagSets { fs.AddFlagSet(f) } args := []string{ "--enable-admission-plugins=AlwaysDeny", "--admission-control-config-file=/admission-control-config", "--advertise-address=192.168.10.10", "--anonymous-auth=false", "--audit-log-maxage=11", "--audit-log-maxbackup=12", "--audit-log-maxsize=13", "--audit-log-path=/var/log", "--audit-log-mode=blocking", "--audit-log-batch-buffer-size=46", "--audit-log-batch-max-size=47", "--audit-log-batch-max-wait=48s", "--audit-log-batch-throttle-enable=true", "--audit-log-batch-throttle-qps=49.5", "--audit-log-batch-throttle-burst=50", "--audit-log-truncate-enabled=true", "--audit-log-truncate-max-batch-size=45", "--audit-log-truncate-max-event-size=44", "--audit-log-version=audit.k8s.io/v1", "--audit-policy-file=/policy", "--audit-webhook-config-file=/webhook-config", "--audit-webhook-mode=blocking", "--audit-webhook-batch-buffer-size=42", "--audit-webhook-batch-max-size=43", "--audit-webhook-batch-max-wait=1s", "--audit-webhook-batch-throttle-enable=false", "--audit-webhook-batch-throttle-qps=43.5", "--audit-webhook-batch-throttle-burst=44", "--audit-webhook-truncate-enabled=true", "--audit-webhook-truncate-max-batch-size=43", "--audit-webhook-truncate-max-event-size=42", "--audit-webhook-initial-backoff=2s", "--audit-webhook-version=audit.k8s.io/v1", "--authentication-token-webhook-cache-ttl=3m", "--authentication-token-webhook-config-file=/token-webhook-config", "--authorization-mode=AlwaysDeny,RBAC", "--authorization-policy-file=/policy", "--authorization-webhook-cache-authorized-ttl=3m", "--authorization-webhook-cache-unauthorized-ttl=1m", "--authorization-webhook-config-file=/webhook-config", "--bind-address=192.168.10.20", "--client-ca-file=/client-ca", "--cors-allowed-origins=10.10.10.100,10.10.10.200", "--contention-profiling=true", "--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml", "--enable-aggregator-routing=true", "--enable-priority-and-fairness=false", "--enable-logs-handler=false", "--etcd-keyfile=/var/run/kubernetes/etcd.key", "--etcd-certfile=/var/run/kubernetes/etcdce.crt", "--etcd-cafile=/var/run/kubernetes/etcdca.crt", "--http2-max-streams-per-connection=42", "--tracing-config-file=/var/run/kubernetes/tracing_config.yaml", "--proxy-client-cert-file=/var/run/kubernetes/proxy.crt", "--proxy-client-key-file=/var/run/kubernetes/proxy.key", "--request-timeout=2m", "--storage-backend=etcd3", "--lease-reuse-duration-seconds=100", "--emulated-version=test=1.31", } fs.Parse(args) utilruntime.Must(componentGlobalsRegistry.Set()) // This is a snapshot of expected options parsed by args. expected := &Options{ GenericServerRunOptions: &apiserveroptions.ServerRunOptions{ AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"), CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"}, MaxRequestsInFlight: 400, MaxMutatingRequestsInFlight: 200, RequestTimeout: time.Duration(2) * time.Minute, MinRequestTimeout: 1800, StorageInitializationTimeout: time.Minute, JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), MaxRequestBodyBytes: int64(3 * 1024 * 1024), ComponentGlobalsRegistry: componentGlobalsRegistry, ComponentName: basecompatibility.DefaultKubeComponent, }, Admission: &kubeoptions.AdmissionOptions{ GenericAdmission: &apiserveroptions.AdmissionOptions{ RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder, DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins, EnablePlugins: []string{"AlwaysDeny"}, ConfigFile: "/admission-control-config", Plugins: s.Admission.GenericAdmission.Plugins, Decorators: s.Admission.GenericAdmission.Decorators, }, }, Etcd: &apiserveroptions.EtcdOptions{ StorageConfig: storagebackend.Config{ Type: "etcd3", Transport: storagebackend.TransportConfig{ ServerList: nil, KeyFile: "/var/run/kubernetes/etcd.key", TrustedCAFile: "/var/run/kubernetes/etcdca.crt", CertFile: "/var/run/kubernetes/etcdce.crt", TracerProvider: noopoteltrace.NewTracerProvider(), }, Prefix: "/registry", CompactionInterval: storagebackend.DefaultCompactInterval, CountMetricPollPeriod: time.Minute, DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval, EventsHistoryWindow: storagebackend.DefaultEventsHistoryWindow, HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout, ReadycheckTimeout: storagebackend.DefaultReadinessTimeout, LeaseManagerConfig: etcd3.LeaseManagerConfig{ ReuseDurationSeconds: 100, MaxObjectCount: 1000, }, }, DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", DeleteCollectionWorkers: 1, EnableGarbageCollection: true, EnableWatchCache: true, DefaultWatchCacheSize: 100, }, SecureServing: (&apiserveroptions.SecureServingOptions{ BindAddress: netutils.ParseIPSloppy("192.168.10.20"), BindPort: 6443, ServerCert: apiserveroptions.GeneratableKeyCert{ CertDirectory: "/var/run/kubernetes", PairName: "apiserver", }, HTTP2MaxStreamsPerConnection: 42, Required: true, }).WithLoopback(), EventTTL: 1 * time.Hour, Audit: &apiserveroptions.AuditOptions{ LogOptions: apiserveroptions.AuditLogOptions{ Path: "/var/log", MaxAge: 11, MaxBackups: 12, MaxSize: 13, Format: "json", BatchOptions: apiserveroptions.AuditBatchOptions{ Mode: "blocking", BatchConfig: auditbuffered.BatchConfig{ BufferSize: 46, MaxBatchSize: 47, MaxBatchWait: 48 * time.Second, ThrottleEnable: true, ThrottleQPS: 49.5, ThrottleBurst: 50, }, }, TruncateOptions: apiserveroptions.AuditTruncateOptions{ Enabled: true, TruncateConfig: audittruncate.Config{ MaxBatchSize: 45, MaxEventSize: 44, }, }, GroupVersionString: "audit.k8s.io/v1", }, WebhookOptions: apiserveroptions.AuditWebhookOptions{ ConfigFile: "/webhook-config", BatchOptions: apiserveroptions.AuditBatchOptions{ Mode: "blocking", BatchConfig: auditbuffered.BatchConfig{ BufferSize: 42, MaxBatchSize: 43, MaxBatchWait: 1 * time.Second, ThrottleEnable: false, ThrottleQPS: 43.5, ThrottleBurst: 44, AsyncDelegate: true, }, }, TruncateOptions: apiserveroptions.AuditTruncateOptions{ Enabled: true, TruncateConfig: audittruncate.Config{ MaxBatchSize: 43, MaxEventSize: 42, }, }, InitialBackoff: 2 * time.Second, GroupVersionString: "audit.k8s.io/v1", }, PolicyFile: "/policy", }, Features: &apiserveroptions.FeatureOptions{ EnableProfiling: true, EnableContentionProfiling: true, }, Authentication: &kubeoptions.BuiltInAuthenticationOptions{ Anonymous: s.Authentication.Anonymous, ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{ ClientCA: "/client-ca", }, WebHook: &kubeoptions.WebHookAuthenticationOptions{ CacheTTL: 180000000000, ConfigFile: "/token-webhook-config", Version: "v1beta1", RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), }, BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{}, OIDC: s.Authentication.OIDC, RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{}, ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ Lookup: true, ExtendExpiration: true, MaxExtendedExpiration: serviceaccount.ExpirationExtensionSeconds * time.Second, }, TokenFile: &kubeoptions.TokenFileAuthenticationOptions{}, TokenSuccessCacheTTL: 10 * time.Second, TokenFailureCacheTTL: 0, }, Authorization: &kubeoptions.BuiltInAuthorizationOptions{ Modes: []string{"AlwaysDeny", "RBAC"}, PolicyFile: "/policy", WebhookConfigFile: "/webhook-config", WebhookCacheAuthorizedTTL: 180000000000, WebhookCacheUnauthorizedTTL: 60000000000, WebhookVersion: "v1beta1", WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), }, APIEnablement: &apiserveroptions.APIEnablementOptions{ RuntimeConfig: cliflag.ConfigurationMap{}, }, EgressSelector: &apiserveroptions.EgressSelectorOptions{ ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml", }, EnableLogsHandler: false, EnableAggregatorRouting: true, ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", ProxyClientCertFile: "/var/run/kubernetes/proxy.crt", Metrics: &metrics.Options{}, Logs: logs.NewOptions(), Traces: &apiserveroptions.TracingOptions{ ConfigFile: "/var/run/kubernetes/tracing_config.yaml", }, AggregatorRejectForwardingRedirects: true, SystemNamespaces: []string{"kube-system", "kube-public", "default"}, } expected.Authentication.OIDC.UsernameClaim = "sub" expected.Authentication.OIDC.SigningAlgs = []string{"RS256"} if !s.Authorization.AreLegacyFlagsSet() { t.Errorf("expected legacy authorization flags to be set") } // setting the method to nil since methods can't be compared with reflect.DeepEqual s.Authorization.AreLegacyFlagsSet = nil if !reflect.DeepEqual(expected, s) { t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreFields(apiserveroptions.ServerRunOptions{}, "ComponentGlobalsRegistry"), cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}, kubeoptions.AnonymousAuthenticationOptions{}))) } testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test") if testEffectiveVersion.EmulationVersion().String() != "1.31" { t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31") } } func TestCompleteForServiceAccount(t *testing.T) { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { panic("Error while generating first RSA key") } // Marshal the private key into PEM format privateKeyPEM := &pem.Block{ Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key), } // Open a file to write the private key privateKeyFile, err := os.Create("private_key.pem") if err != nil { t.Fatalf("Failed to create private key file: %v", err) } t.Cleanup(func() { _ = privateKeyFile.Close() _ = os.Remove("private_key.pem") }) // Write the PEM-encoded private key to the file if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { t.Fatalf("Failed to encode private key: %v", err) } testCases := []struct { desc string issuers []string externalSigner bool signingKeyFiles string maxExpiration time.Duration maxExtendedExpiration time.Duration externalMaxExpirationSec int64 fetchError error metadataError error wantError error expectedMaxtokenExp time.Duration expectedExtendedMaxTokenExp time.Duration externalPublicKeyGetterPresent bool }{ { desc: "endpoint and key file", issuers: []string{ "iss", }, externalSigner: true, signingKeyFiles: "private_key.pem", maxExpiration: time.Second * 3600, wantError: fmt.Errorf("service-account-signing-key-file and service-account-signing-endpoint are mutually exclusive and cannot be set at the same time"), }, { desc: "max token expiration breaching acceptable values", issuers: []string{ "iss", }, externalSigner: true, signingKeyFiles: "private_key.pem", maxExpiration: time.Second * 10, wantError: fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds"), }, { desc: "path to a signing key provided", issuers: []string{ "iss", }, externalSigner: false, signingKeyFiles: "private_key.pem", maxExpiration: time.Second * 3600, externalPublicKeyGetterPresent: false, expectedMaxtokenExp: time.Second * 3600, }, { desc: "signing endpoint provided, use endpoint expiration", issuers: []string{ "iss", }, externalSigner: true, signingKeyFiles: "", maxExpiration: 0, maxExtendedExpiration: 365 * 24 * time.Hour, externalMaxExpirationSec: 600, // 10m expectedMaxtokenExp: 10 * time.Minute, expectedExtendedMaxTokenExp: 10 * time.Minute, externalPublicKeyGetterPresent: true, }, { desc: "signing endpoint provided, use local smaller expirations", issuers: []string{ "iss", }, externalSigner: true, signingKeyFiles: "", maxExpiration: 1 * time.Hour, maxExtendedExpiration: 24 * time.Hour, externalMaxExpirationSec: 31556952, // 1 year expectedMaxtokenExp: 1 * time.Hour, expectedExtendedMaxTokenExp: 24 * time.Hour, externalPublicKeyGetterPresent: true, }, { desc: "signing endpoint provided and want larger than signer can provide", issuers: []string{ "iss", }, externalSigner: true, signingKeyFiles: "", maxExpiration: 1 * time.Hour, // want 1hr externalMaxExpirationSec: 600, // signer can only sign 10m wantError: fmt.Errorf("service-account-max-token-expiration cannot be set longer than the token expiration supported by service-account-signing-endpoint: 1h0m0s > 10m0s"), }, { desc: "signing endpoint provided but return smaller than accaptable max token exp", issuers: []string{ "iss", }, externalSigner: true, signingKeyFiles: "", maxExpiration: 0, externalMaxExpirationSec: 300, // 5m wantError: fmt.Errorf("max token life supported by external-jwt-signer (300s) is less than acceptable (min 600s)"), }, { desc: "signing endpoint provided and error when getting metadata", issuers: []string{ "iss", }, externalSigner: true, signingKeyFiles: "", maxExpiration: 0, externalMaxExpirationSec: 900, // 15m metadataError: fmt.Errorf("metadata error"), wantError: fmt.Errorf("while setting up external-jwt-signer: rpc error: code = Unknown desc = metadata error"), }, { desc: "signing endpoint provided and error when creating plugin (during initial fetch)", issuers: []string{ "iss", }, externalSigner: true, signingKeyFiles: "", maxExpiration: 0, externalMaxExpirationSec: 900, // 15m fetchError: fmt.Errorf("keys fetch error"), wantError: fmt.Errorf("while setting up external-jwt-signer: while initially filling key cache: while performing initial cache fill: while fetching token verification keys: while getting externally supported jwt signing keys: rpc error: code = Unknown desc = keys fetch error"), }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { options := NewOptions() if tc.externalSigner { // create and start mock signer. socketPath := fmt.Sprintf("@mock-external-jwt-signer-%d.sock", time.Now().Nanosecond()) mockSigner := v1alpha1testing.NewMockSigner(t, socketPath) defer mockSigner.CleanUp() mockSigner.MaxTokenExpirationSeconds = tc.externalMaxExpirationSec mockSigner.MetadataError = tc.metadataError mockSigner.FetchError = tc.fetchError options.ServiceAccountSigningEndpoint = socketPath } options.ServiceAccountSigningKeyFile = tc.signingKeyFiles options.Authentication = &kubeoptions.BuiltInAuthenticationOptions{ ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ Issuers: tc.issuers, MaxExpiration: tc.maxExpiration, MaxExtendedExpiration: tc.maxExtendedExpiration, }, } co := completedOptions{ Options: *options, } ctx, cancel := context.WithCancel(context.Background()) defer cancel() err := options.completeServiceAccountOptions(ctx, &co) if tc.wantError != nil { if err == nil || tc.wantError.Error() != err.Error() { t.Errorf("Expected error: %v, got: %v", tc.wantError, err) } return } if err != nil { t.Errorf("Didn't expect any error but got: %v", err) } if tc.externalPublicKeyGetterPresent != (co.Authentication.ServiceAccounts.ExternalPublicKeysGetter != nil) { t.Errorf("Unexpected value of ExternalPublicKeysGetter: %v", co.Authentication.ServiceAccounts.ExternalPublicKeysGetter) } if tc.expectedExtendedMaxTokenExp != co.Authentication.ServiceAccounts.MaxExtendedExpiration { t.Errorf("Expected MaxExtendedExpiration %v, found %v", tc.expectedExtendedMaxTokenExp, co.Authentication.ServiceAccounts.MaxExtendedExpiration) } if tc.expectedMaxtokenExp.Seconds() != co.Authentication.ServiceAccounts.MaxExpiration.Seconds() { t.Errorf("Expected MaxExpiration to be %v, found %v", tc.expectedMaxtokenExp, co.Authentication.ServiceAccounts.MaxExpiration) } }) } }