Files
kubernetes/pkg/controlplane/apiserver/options/options_test.go
2024-11-07 03:16:23 +00:00

514 lines
18 KiB
Go

/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"fmt"
"os"
"reflect"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/spf13/pflag"
noopoteltrace "go.opentelemetry.io/otel/trace/noop"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/featuregate"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
utilversion "k8s.io/component-base/version"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
v1alpha1testing "k8s.io/kubernetes/pkg/serviceaccount/externaljwt/plugin/testing/v1alpha1"
netutils "k8s.io/utils/net"
)
func TestAddFlags(t *testing.T) {
componentGlobalsRegistry := featuregate.DefaultComponentGlobalsRegistry
t.Cleanup(func() {
componentGlobalsRegistry.Reset()
})
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
utilruntime.Must(componentGlobalsRegistry.Register("test", utilversion.NewEffectiveVersion("1.32"), featuregate.NewFeatureGate()))
s := NewOptions()
var fss cliflag.NamedFlagSets
s.AddFlags(&fss)
for _, f := range fss.FlagSets {
fs.AddFlagSet(f)
}
args := []string{
"--enable-admission-plugins=AlwaysDeny",
"--admission-control-config-file=/admission-control-config",
"--advertise-address=192.168.10.10",
"--anonymous-auth=false",
"--audit-log-maxage=11",
"--audit-log-maxbackup=12",
"--audit-log-maxsize=13",
"--audit-log-path=/var/log",
"--audit-log-mode=blocking",
"--audit-log-batch-buffer-size=46",
"--audit-log-batch-max-size=47",
"--audit-log-batch-max-wait=48s",
"--audit-log-batch-throttle-enable=true",
"--audit-log-batch-throttle-qps=49.5",
"--audit-log-batch-throttle-burst=50",
"--audit-log-truncate-enabled=true",
"--audit-log-truncate-max-batch-size=45",
"--audit-log-truncate-max-event-size=44",
"--audit-log-version=audit.k8s.io/v1",
"--audit-policy-file=/policy",
"--audit-webhook-config-file=/webhook-config",
"--audit-webhook-mode=blocking",
"--audit-webhook-batch-buffer-size=42",
"--audit-webhook-batch-max-size=43",
"--audit-webhook-batch-max-wait=1s",
"--audit-webhook-batch-throttle-enable=false",
"--audit-webhook-batch-throttle-qps=43.5",
"--audit-webhook-batch-throttle-burst=44",
"--audit-webhook-truncate-enabled=true",
"--audit-webhook-truncate-max-batch-size=43",
"--audit-webhook-truncate-max-event-size=42",
"--audit-webhook-initial-backoff=2s",
"--audit-webhook-version=audit.k8s.io/v1",
"--authentication-token-webhook-cache-ttl=3m",
"--authentication-token-webhook-config-file=/token-webhook-config",
"--authorization-mode=AlwaysDeny,RBAC",
"--authorization-policy-file=/policy",
"--authorization-webhook-cache-authorized-ttl=3m",
"--authorization-webhook-cache-unauthorized-ttl=1m",
"--authorization-webhook-config-file=/webhook-config",
"--bind-address=192.168.10.20",
"--client-ca-file=/client-ca",
"--cors-allowed-origins=10.10.10.100,10.10.10.200",
"--contention-profiling=true",
"--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml",
"--enable-aggregator-routing=true",
"--enable-priority-and-fairness=false",
"--enable-logs-handler=false",
"--etcd-keyfile=/var/run/kubernetes/etcd.key",
"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
"--etcd-cafile=/var/run/kubernetes/etcdca.crt",
"--http2-max-streams-per-connection=42",
"--tracing-config-file=/var/run/kubernetes/tracing_config.yaml",
"--proxy-client-cert-file=/var/run/kubernetes/proxy.crt",
"--proxy-client-key-file=/var/run/kubernetes/proxy.key",
"--request-timeout=2m",
"--storage-backend=etcd3",
"--lease-reuse-duration-seconds=100",
"--emulated-version=test=1.31",
}
fs.Parse(args)
utilruntime.Must(componentGlobalsRegistry.Set())
// This is a snapshot of expected options parsed by args.
expected := &Options{
GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"),
CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"},
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
StorageInitializationTimeout: time.Minute,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
ComponentGlobalsRegistry: componentGlobalsRegistry,
ComponentName: featuregate.DefaultKubeComponent,
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins,
EnablePlugins: []string{"AlwaysDeny"},
ConfigFile: "/admission-control-config",
Plugins: s.Admission.GenericAdmission.Plugins,
Decorators: s.Admission.GenericAdmission.Decorators,
},
},
Etcd: &apiserveroptions.EtcdOptions{
StorageConfig: storagebackend.Config{
Type: "etcd3",
Transport: storagebackend.TransportConfig{
ServerList: nil,
KeyFile: "/var/run/kubernetes/etcd.key",
TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
CertFile: "/var/run/kubernetes/etcdce.crt",
TracerProvider: noopoteltrace.NewTracerProvider(),
},
Prefix: "/registry",
CompactionInterval: storagebackend.DefaultCompactInterval,
CountMetricPollPeriod: time.Minute,
DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval,
HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout,
ReadycheckTimeout: storagebackend.DefaultReadinessTimeout,
LeaseManagerConfig: etcd3.LeaseManagerConfig{
ReuseDurationSeconds: 100,
MaxObjectCount: 1000,
},
},
DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
DeleteCollectionWorkers: 1,
EnableGarbageCollection: true,
EnableWatchCache: true,
DefaultWatchCacheSize: 100,
},
SecureServing: (&apiserveroptions.SecureServingOptions{
BindAddress: netutils.ParseIPSloppy("192.168.10.20"),
BindPort: 6443,
ServerCert: apiserveroptions.GeneratableKeyCert{
CertDirectory: "/var/run/kubernetes",
PairName: "apiserver",
},
HTTP2MaxStreamsPerConnection: 42,
Required: true,
}).WithLoopback(),
EventTTL: 1 * time.Hour,
Audit: &apiserveroptions.AuditOptions{
LogOptions: apiserveroptions.AuditLogOptions{
Path: "/var/log",
MaxAge: 11,
MaxBackups: 12,
MaxSize: 13,
Format: "json",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 46,
MaxBatchSize: 47,
MaxBatchWait: 48 * time.Second,
ThrottleEnable: true,
ThrottleQPS: 49.5,
ThrottleBurst: 50,
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 45,
MaxEventSize: 44,
},
},
GroupVersionString: "audit.k8s.io/v1",
},
WebhookOptions: apiserveroptions.AuditWebhookOptions{
ConfigFile: "/webhook-config",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 42,
MaxBatchSize: 43,
MaxBatchWait: 1 * time.Second,
ThrottleEnable: false,
ThrottleQPS: 43.5,
ThrottleBurst: 44,
AsyncDelegate: true,
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 43,
MaxEventSize: 42,
},
},
InitialBackoff: 2 * time.Second,
GroupVersionString: "audit.k8s.io/v1",
},
PolicyFile: "/policy",
},
Features: &apiserveroptions.FeatureOptions{
EnableProfiling: true,
EnableContentionProfiling: true,
},
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
Anonymous: s.Authentication.Anonymous,
ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
ClientCA: "/client-ca",
},
WebHook: &kubeoptions.WebHookAuthenticationOptions{
CacheTTL: 180000000000,
ConfigFile: "/token-webhook-config",
Version: "v1beta1",
RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
},
BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
OIDC: s.Authentication.OIDC,
RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
Lookup: true,
ExtendExpiration: true,
},
TokenFile: &kubeoptions.TokenFileAuthenticationOptions{},
TokenSuccessCacheTTL: 10 * time.Second,
TokenFailureCacheTTL: 0,
},
Authorization: &kubeoptions.BuiltInAuthorizationOptions{
Modes: []string{"AlwaysDeny", "RBAC"},
PolicyFile: "/policy",
WebhookConfigFile: "/webhook-config",
WebhookCacheAuthorizedTTL: 180000000000,
WebhookCacheUnauthorizedTTL: 60000000000,
WebhookVersion: "v1beta1",
WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
},
APIEnablement: &apiserveroptions.APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{},
},
EgressSelector: &apiserveroptions.EgressSelectorOptions{
ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
},
EnableLogsHandler: false,
EnableAggregatorRouting: true,
ProxyClientKeyFile: "/var/run/kubernetes/proxy.key",
ProxyClientCertFile: "/var/run/kubernetes/proxy.crt",
Metrics: &metrics.Options{},
Logs: logs.NewOptions(),
Traces: &apiserveroptions.TracingOptions{
ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
},
AggregatorRejectForwardingRedirects: true,
SystemNamespaces: []string{"kube-system", "kube-public", "default"},
}
expected.Authentication.OIDC.UsernameClaim = "sub"
expected.Authentication.OIDC.SigningAlgs = []string{"RS256"}
if !s.Authorization.AreLegacyFlagsSet() {
t.Errorf("expected legacy authorization flags to be set")
}
// setting the method to nil since methods can't be compared with reflect.DeepEqual
s.Authorization.AreLegacyFlagsSet = nil
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}, kubeoptions.OIDCAuthenticationOptions{}, kubeoptions.AnonymousAuthenticationOptions{})))
}
testEffectiveVersion := s.GenericServerRunOptions.ComponentGlobalsRegistry.EffectiveVersionFor("test")
if testEffectiveVersion.EmulationVersion().String() != "1.31" {
t.Errorf("Got emulation version %s, wanted %s", testEffectiveVersion.EmulationVersion().String(), "1.31")
}
}
func TestCompleteForServiceAccount(t *testing.T) {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic("Error while generating first RSA key")
}
// Marshal the private key into PEM format
privateKeyPEM := &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(key),
}
// Open a file to write the private key
privateKeyFile, err := os.Create("private_key.pem")
if err != nil {
t.Fatalf("Failed to create private key file: %v", err)
}
t.Cleanup(func() {
_ = privateKeyFile.Close()
_ = os.Remove("private_key.pem")
})
// Write the PEM-encoded private key to the file
if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil {
t.Fatalf("Failed to encode private key: %v", err)
}
// create and start mock signer.
socketPath := "@mock-external-jwt-signer.sock"
mockSigner := v1alpha1testing.NewMockSigner(t, socketPath)
defer mockSigner.CleanUp()
testCases := []struct {
desc string
issuers []string
signingEndpoint string
signingKeyFiles string
maxExpiration time.Duration
externalMaxExpirationSec int64
fetchError error
metadataError error
wantError error
expectedMaxtokenExp time.Duration
expectedIsExternalSigner bool
externalPublicKeyGetterPresent bool
}{
{
desc: "no endpoint or key file",
issuers: []string{
"iss",
},
signingEndpoint: socketPath,
signingKeyFiles: "private_key.pem",
maxExpiration: time.Second * 3600,
wantError: fmt.Errorf("service-account-signing-key-file and service-account-signing-endpoint are mutually exclusive and cannot be set at the same time"),
},
{
desc: "max token expiration breaching accepteable values",
issuers: []string{
"iss",
},
signingEndpoint: socketPath,
signingKeyFiles: "private_key.pem",
maxExpiration: time.Second * 10,
wantError: fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds"),
},
{
desc: "path to a signing key provided",
issuers: []string{
"iss",
},
signingEndpoint: "",
signingKeyFiles: "private_key.pem",
maxExpiration: time.Second * 3600,
expectedIsExternalSigner: false,
externalPublicKeyGetterPresent: false,
expectedMaxtokenExp: time.Second * 3600,
},
{
desc: "signing endpoint provided",
issuers: []string{
"iss",
},
signingEndpoint: socketPath,
signingKeyFiles: "",
maxExpiration: 0,
externalMaxExpirationSec: 600, // 10m
expectedIsExternalSigner: true,
externalPublicKeyGetterPresent: true,
expectedMaxtokenExp: time.Second * 600, // 10m
},
{
desc: "signing endpoint provided and max token expiration set",
issuers: []string{
"iss",
},
signingEndpoint: socketPath,
signingKeyFiles: "",
maxExpiration: time.Second * 3600,
externalMaxExpirationSec: 600, // 10m
wantError: fmt.Errorf("service-account-max-token-expiration and service-account-signing-endpoint are mutually exclusive and cannot be set at the same time"),
},
{
desc: "signing endpoint provided but return smaller than accaptable max token exp",
issuers: []string{
"iss",
},
signingEndpoint: socketPath,
signingKeyFiles: "",
maxExpiration: 0,
externalMaxExpirationSec: 300, // 5m
wantError: fmt.Errorf("max token life supported by external-jwt-signer (300s) is less than acceptable (min 600s)"),
},
{
desc: "signing endpoint provided and error when getting metadata",
issuers: []string{
"iss",
},
signingEndpoint: socketPath,
signingKeyFiles: "",
maxExpiration: 0,
externalMaxExpirationSec: 900, // 15m
metadataError: fmt.Errorf("metadata error"),
wantError: fmt.Errorf("while setting up external-jwt-signer: rpc error: code = Unknown desc = metadata error"),
},
{
desc: "signing endpoint provided and error when creating plugin (during initial fetch)",
issuers: []string{
"iss",
},
signingEndpoint: socketPath,
signingKeyFiles: "",
maxExpiration: 0,
externalMaxExpirationSec: 900, // 15m
fetchError: fmt.Errorf("keys fetch error"),
wantError: fmt.Errorf("while setting up external-jwt-signer: while initially filling key cache: while performing initial cache fill: while fetching token verification keys: while getting externally supported jwt signing keys: rpc error: code = Unknown desc = keys fetch error"),
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
options := NewOptions()
options.ServiceAccountSigningEndpoint = tc.signingEndpoint
options.ServiceAccountSigningKeyFile = tc.signingKeyFiles
options.Authentication = &kubeoptions.BuiltInAuthenticationOptions{
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
Issuers: tc.issuers,
MaxExpiration: tc.maxExpiration,
},
}
_ = mockSigner.Reset()
mockSigner.MaxTokenExpirationSeconds = tc.externalMaxExpirationSec
mockSigner.MetadataError = tc.metadataError
mockSigner.FetchError = tc.fetchError
co := completedOptions{
Options: *options,
}
err := options.completeServiceAccountOptions(context.Background(), &co)
if tc.wantError != nil {
if err == nil || tc.wantError.Error() != err.Error() {
t.Errorf("Expected error: %v, got: %v", tc.wantError, err)
}
return
}
if err != nil {
t.Errorf("Didn't expect any error but got: %v", err)
}
if tc.externalPublicKeyGetterPresent != (co.Authentication.ServiceAccounts.ExternalPublicKeysGetter != nil) {
t.Errorf("Unexpected value of ExternalPublicKeysGetter: %v", co.Authentication.ServiceAccounts.ExternalPublicKeysGetter)
}
if tc.expectedIsExternalSigner != co.Authentication.ServiceAccounts.IsTokenSignerExternal {
t.Errorf("Expected IsTokenSignerExternal %v, found %v", tc.expectedIsExternalSigner, co.Authentication.ServiceAccounts.IsTokenSignerExternal)
}
if tc.expectedMaxtokenExp.Seconds() != co.Authentication.ServiceAccounts.MaxExpiration.Seconds() {
t.Errorf("Expected MaxExpiration to be %v, found %v", tc.expectedMaxtokenExp, co.Authentication.ServiceAccounts.MaxExpiration)
}
})
}
}