Merge pull request #42059 from deads2k/agg-19-integration-test

Automatic merge from submit-queue (batch tested with PRs 35094, 42095, 42059, 42143, 41944)

add aggregation integration test

Wires up an integration test which runs a full kube-apiserver, the wardle server, and the kube-aggregator and creates the APIservice object for the wardle server.  Without services and DNS the aggregator doesn't proxy, but it does ensure we don't have an obvious panic or bring up failure.

@sttts @ncdc

staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -189,16 +189,16 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 
 // Run spawns the http servers (secure and insecure). It only returns if stopCh is closed
 // or one of the ports cannot be listened on initially.
-func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) {
+func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 	if s.SecureServingInfo != nil && s.Handler != nil {
 		if err := s.serveSecurely(stopCh); err != nil {
-			glog.Fatal(err)
+			return err
 		}
 	}
 
 	if s.InsecureServingInfo != nil && s.InsecureHandler != nil {
 		if err := s.serveInsecurely(stopCh); err != nil {
-			glog.Fatal(err)
+			return err
 		}
 	}
 
@@ -210,6 +210,7 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) {
 	}
 
 	<-stopCh
+	return nil
 }
 
 // EffectiveSecurePort returns the secure port we bound to.

staging/src/k8s.io/apiserver/pkg/server/options/serving_test.go

@@ -448,114 +448,121 @@ NextTest:
 		}
 
 		stopCh := make(chan struct{})
+		func() {
+			defer close(stopCh)
 
-		// launch server
-		config := setUp(t)
+			// launch server
+			config := setUp(t)
 
-		v := fakeVersion()
-		config.Version = &v
+			v := fakeVersion()
+			config.Version = &v
 
-		config.EnableIndex = true
-		secureOptions := &SecureServingOptions{
-			ServingOptions: ServingOptions{
-				BindAddress: net.ParseIP("127.0.0.1"),
-				BindPort:    6443,
-			},
-			ServerCert: GeneratableKeyCert{
-				CertKey: CertKey{
-					CertFile: serverCertBundleFile,
-					KeyFile:  serverKeyFile,
+			config.EnableIndex = true
+			secureOptions := &SecureServingOptions{
+				ServingOptions: ServingOptions{
+					BindAddress: net.ParseIP("127.0.0.1"),
+					BindPort:    6443,
 				},
-			},
-			SNICertKeys: namedCertKeys,
-		}
-		config.LoopbackClientConfig = &restclient.Config{}
-		if err := secureOptions.ApplyTo(&config); err != nil {
-			t.Errorf("%q - failed applying the SecureServingOptions: %v", title, err)
-			continue NextTest
-		}
-		config.InsecureServingInfo = nil
-
-		s, err := config.Complete().New()
-		if err != nil {
-			t.Errorf("%q - failed creating the server: %v", title, err)
-			continue NextTest
-		}
-
-		// patch in a 0-port to enable auto port allocation
-		s.SecureServingInfo.BindAddress = "127.0.0.1:0"
-
-		// add poststart hook to know when the server is up.
-		startedCh := make(chan struct{})
-		s.AddPostStartHook("test-notifier", func(context PostStartHookContext) error {
-			close(startedCh)
-			return nil
-		})
-		preparedServer := s.PrepareRun()
-		go preparedServer.Run(stopCh)
-
-		// load ca certificates into a pool
-		roots := x509.NewCertPool()
-		for _, caCert := range caCerts {
-			roots.AddCert(caCert)
-		}
-
-		<-startedCh
-
-		effectiveSecurePort := fmt.Sprintf("%d", preparedServer.EffectiveSecurePort())
-		// try to dial
-		addr := fmt.Sprintf("localhost:%s", effectiveSecurePort)
-		t.Logf("Dialing %s as %q", addr, test.ServerName)
-		conn, err := tls.Dial("tcp", addr, &tls.Config{
-			RootCAs:    roots,
-			ServerName: test.ServerName, // used for SNI in the client HELLO packet
-		})
-		if err != nil {
-			t.Errorf("%q - failed to connect: %v", title, err)
-			continue NextTest
-		}
-
-		// check returned server certificate
-		sig := x509CertSignature(conn.ConnectionState().PeerCertificates[0])
-		gotCertIndex, found := signatures[sig]
-		if !found {
-			t.Errorf("%q - unknown signature returned from server: %s", title, sig)
-		}
-		if gotCertIndex != test.ExpectedCertIndex {
-			t.Errorf("%q - expected cert index %d, got cert index %d", title, test.ExpectedCertIndex, gotCertIndex)
-		}
-
-		conn.Close()
-
-		// check that the loopback client can connect
-		host := "127.0.0.1"
-		if len(test.LoopbackClientBindAddressOverride) != 0 {
-			host = test.LoopbackClientBindAddressOverride
-		}
-		s.LoopbackClientConfig.Host = net.JoinHostPort(host, effectiveSecurePort)
-		if test.ExpectLoopbackClientError {
-			if err == nil {
-				t.Errorf("%q - expected error creating loopback client config", title)
+				ServerCert: GeneratableKeyCert{
+					CertKey: CertKey{
+						CertFile: serverCertBundleFile,
+						KeyFile:  serverKeyFile,
+					},
+				},
+				SNICertKeys: namedCertKeys,
 			}
-			continue NextTest
-		}
-		if err != nil {
-			t.Errorf("%q - failed creating loopback client config: %v", title, err)
-			continue NextTest
-		}
-		client, err := discovery.NewDiscoveryClientForConfig(s.LoopbackClientConfig)
-		if err != nil {
-			t.Errorf("%q - failed to create loopback client: %v", title, err)
-			continue NextTest
-		}
-		got, err := client.ServerVersion()
-		if err != nil {
-			t.Errorf("%q - failed to connect with loopback client: %v", title, err)
-			continue NextTest
-		}
-		if expected := &v; !reflect.DeepEqual(got, expected) {
-			t.Errorf("%q - loopback client didn't get correct version info: expected=%v got=%v", title, expected, got)
-		}
+			config.LoopbackClientConfig = &restclient.Config{}
+			if err := secureOptions.ApplyTo(&config); err != nil {
+				t.Errorf("%q - failed applying the SecureServingOptions: %v", title, err)
+				return
+			}
+			config.InsecureServingInfo = nil
+
+			s, err := config.Complete().New()
+			if err != nil {
+				t.Errorf("%q - failed creating the server: %v", title, err)
+				return
+			}
+
+			// patch in a 0-port to enable auto port allocation
+			s.SecureServingInfo.BindAddress = "127.0.0.1:0"
+
+			// add poststart hook to know when the server is up.
+			startedCh := make(chan struct{})
+			s.AddPostStartHook("test-notifier", func(context PostStartHookContext) error {
+				close(startedCh)
+				return nil
+			})
+			preparedServer := s.PrepareRun()
+			go func() {
+				if err := preparedServer.Run(stopCh); err != nil {
+					t.Fatal(err)
+				}
+			}()
+
+			// load ca certificates into a pool
+			roots := x509.NewCertPool()
+			for _, caCert := range caCerts {
+				roots.AddCert(caCert)
+			}
+
+			<-startedCh
+
+			effectiveSecurePort := fmt.Sprintf("%d", preparedServer.EffectiveSecurePort())
+			// try to dial
+			addr := fmt.Sprintf("localhost:%s", effectiveSecurePort)
+			t.Logf("Dialing %s as %q", addr, test.ServerName)
+			conn, err := tls.Dial("tcp", addr, &tls.Config{
+				RootCAs:    roots,
+				ServerName: test.ServerName, // used for SNI in the client HELLO packet
+			})
+			if err != nil {
+				t.Errorf("%q - failed to connect: %v", title, err)
+				return
+			}
+
+			// check returned server certificate
+			sig := x509CertSignature(conn.ConnectionState().PeerCertificates[0])
+			gotCertIndex, found := signatures[sig]
+			if !found {
+				t.Errorf("%q - unknown signature returned from server: %s", title, sig)
+			}
+			if gotCertIndex != test.ExpectedCertIndex {
+				t.Errorf("%q - expected cert index %d, got cert index %d", title, test.ExpectedCertIndex, gotCertIndex)
+			}
+
+			conn.Close()
+
+			// check that the loopback client can connect
+			host := "127.0.0.1"
+			if len(test.LoopbackClientBindAddressOverride) != 0 {
+				host = test.LoopbackClientBindAddressOverride
+			}
+			s.LoopbackClientConfig.Host = net.JoinHostPort(host, effectiveSecurePort)
+			if test.ExpectLoopbackClientError {
+				if err == nil {
+					t.Errorf("%q - expected error creating loopback client config", title)
+				}
+				return
+			}
+			if err != nil {
+				t.Errorf("%q - failed creating loopback client config: %v", title, err)
+				return
+			}
+			client, err := discovery.NewDiscoveryClientForConfig(s.LoopbackClientConfig)
+			if err != nil {
+				t.Errorf("%q - failed to create loopback client: %v", title, err)
+				return
+			}
+			got, err := client.ServerVersion()
+			if err != nil {
+				t.Errorf("%q - failed to connect with loopback client: %v", title, err)
+				return
+			}
+			if expected := &v; !reflect.DeepEqual(got, expected) {
+				t.Errorf("%q - loopback client didn't get correct version info: expected=%v got=%v", title, expected, got)
+			}
+		}()
 	}
 }
 

staging/src/k8s.io/kube-aggregator/main.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"runtime"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kube-aggregator/pkg/cmd/server"
 
@@ -40,7 +41,7 @@ func main() {
 		runtime.GOMAXPROCS(runtime.NumCPU())
 	}
 
-	cmd := server.NewCommandStartAggregator(os.Stdout, os.Stderr)
+	cmd := server.NewCommandStartAggregator(os.Stdout, os.Stderr, wait.NeverStop)
 	cmd.Flags().AddGoFlagSet(flag.CommandLine)
 	if err := cmd.Execute(); err != nil {
 		panic(err)

staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apimachinery/pkg/util/wait"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -106,7 +105,7 @@ func (c *Config) SkipComplete() completedConfig {
 }
 
 // New returns a new instance of APIAggregator from the given config.
-func (c completedConfig) New() (*APIAggregator, error) {
+func (c completedConfig) New(stopCh <-chan struct{}) (*APIAggregator, error) {
 	informerFactory := informers.NewSharedInformerFactory(
 		internalclientset.NewForConfigOrDie(c.Config.GenericConfig.LoopbackClientConfig),
 		5*time.Minute, // this is effectively used as a refresh interval right now.  Might want to do something nicer later on.
@@ -154,12 +153,12 @@ func (c completedConfig) New() (*APIAggregator, error) {
 	apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().InternalVersion().APIServices(), s)
 
 	s.GenericAPIServer.AddPostStartHook("start-informers", func(context genericapiserver.PostStartHookContext) error {
-		informerFactory.Start(wait.NeverStop)
-		kubeInformers.Start(wait.NeverStop)
+		informerFactory.Start(stopCh)
+		kubeInformers.Start(stopCh)
 		return nil
 	})
 	s.GenericAPIServer.AddPostStartHook("apiservice-registration-controller", func(context genericapiserver.PostStartHookContext) error {
-		apiserviceRegistrationController.Run(wait.NeverStop)
+		apiserviceRegistrationController.Run(stopCh)
 		return nil
 	})
 

staging/src/k8s.io/kube-aggregator/pkg/cmd/server/start.go

@@ -25,7 +25,6 @@ import (
 	"github.com/spf13/pflag"
 
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apimachinery/pkg/util/wait"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/filters"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
@@ -56,7 +55,7 @@ type AggregatorOptions struct {
 }
 
 // NewCommandStartAggregator provides a CLI handler for 'start master' command
-func NewCommandStartAggregator(out, err io.Writer) *cobra.Command {
+func NewCommandStartAggregator(out, err io.Writer, stopCh <-chan struct{}) *cobra.Command {
 	o := NewDefaultOptions(out, err)
 
 	cmd := &cobra.Command{
@@ -69,7 +68,7 @@ func NewCommandStartAggregator(out, err io.Writer) *cobra.Command {
 			if err := o.Validate(args); err != nil {
 				return err
 			}
-			if err := o.RunAggregator(); err != nil {
+			if err := o.RunAggregator(stopCh); err != nil {
 				return err
 			}
 			return nil
@@ -110,7 +109,7 @@ func (o *AggregatorOptions) Complete() error {
 	return nil
 }
 
-func (o AggregatorOptions) RunAggregator() error {
+func (o AggregatorOptions) RunAggregator(stopCh <-chan struct{}) error {
 	// TODO have a "real" external address
 	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost"); err != nil {
 		return fmt.Errorf("error creating self-signed certificates: %v", err)
@@ -161,11 +160,9 @@ func (o AggregatorOptions) RunAggregator() error {
 		return err
 	}
 
-	server, err := config.Complete().New()
+	server, err := config.Complete().New(stopCh)
 	if err != nil {
 		return err
 	}
-	server.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
-
-	return nil
+	return server.GenericAPIServer.PrepareRun().Run(stopCh)
 }

staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go

@@ -19,6 +19,7 @@ package server
 import (
 	"fmt"
 	"io"
+	"net"
 
 	"github.com/spf13/cobra"
 
@@ -85,7 +86,7 @@ func (o *WardleServerOptions) Complete() error {
 
 func (o WardleServerOptions) Config() (*apiserver.Config, error) {
 	// TODO have a "real" external address
-	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost"); err != nil {
+	if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", net.ParseIP("127.0.0.1")); err != nil {
 		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
 
@@ -110,7 +111,5 @@ func (o WardleServerOptions) RunWardleServer(stopCh <-chan struct{}) error {
 	if err != nil {
 		return err
 	}
-	server.GenericAPIServer.PrepareRun().Run(stopCh)
-
-	return nil
+	return server.GenericAPIServer.PrepareRun().Run(stopCh)
 }