Merge pull request #41852 from mml/etcd-upgrade-test

Automatic merge from submit-queue (batch tested with PRs 42106, 42094, 42069, 42098, 41852)

Write etcd_upgrade test.

Part of the fix for #40636

hack/verify-flags/known-flags.txt

@@ -165,6 +165,10 @@ dump-logs-on-failure
 duration-sec
 e2e-output-dir
 e2e-verify-service-account
+etcd-metrics-scrape-uri
+etcd-upgrade-storage
+etcd-upgrade-version
+etcd-version-scrape-uri
 enable-controller-attach-detach
 enable-cri
 enable-custom-metrics

test/e2e/cluster_upgrade.go

@@ -102,6 +102,28 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 	})
 })
 
+var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
+	f := framework.NewDefaultFramework("etcd-upgrade")
+
+	framework.KubeDescribe("etcd upgrade", func() {
+		It("should maintain a functioning cluster", func() {
+			cm := chaosmonkey.New(func() {
+				framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion))
+				// TODO(mml): verify the etcd version
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.EtcdUpgrade,
+				})
+			}
+
+			cm.Do()
+		})
+	})
+})
+
 type chaosMonkeyAdapter struct {
 	test        upgrades.Test
 	framework   *framework.Framework

test/e2e/framework/nodes_util.go

@@ -30,6 +30,15 @@ import (
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 )
 
+func EtcdUpgrade(target_storage, target_version string) error {
+	switch TestContext.Provider {
+	case "gce":
+		return etcdUpgradeGCE(target_storage, target_version)
+	default:
+		return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider)
+	}
+}
+
 func MasterUpgrade(v string) error {
 	switch TestContext.Provider {
 	case "gce":
@@ -41,6 +50,17 @@ func MasterUpgrade(v string) error {
 	}
 }
 
+func etcdUpgradeGCE(target_storage, target_version string) error {
+	env := append(
+		os.Environ(),
+		"TEST_ETCD_VERSION="+target_version,
+		"STORAGE_BACKEND="+target_storage,
+		"TEST_ETCD_IMAGE=3.0.14")
+
+	_, _, err := RunCmdEnv(env, path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-l", "-M")
+	return err
+}
+
 func masterUpgradeGCE(rawV string) error {
 	v := "v" + rawV
 	_, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", v)

test/e2e/framework/test_context.go

@@ -49,6 +49,8 @@ type TestContextType struct {
 	// Timeout for waiting for system pods to be running
 	SystemPodsStartupTimeout time.Duration
 	UpgradeTarget            string
+	EtcdUpgradeStorage       string
+	EtcdUpgradeVersion       string
 	UpgradeImage             string
 	PrometheusPushGateway    string
 	ContainerRuntime         string
@@ -195,6 +197,8 @@ func RegisterClusterFlags() {
 	flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.")
 	flag.DurationVar(&TestContext.NodeSchedulableTimeout, "node-schedulable-timeout", 4*time.Hour, "Timeout for waiting for all nodes to be schedulable.")
 	flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
+	flag.StringVar(&TestContext.EtcdUpgradeStorage, "etcd-upgrade-storage", "", "The storage version to upgrade to (either 'etcdv2' or 'etcdv3') if doing an etcd upgrade test.")
+	flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.")
 	flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")

test/e2e/upgrades/upgrade.go

@@ -33,6 +33,10 @@ const (
 	// ClusterUpgrade indicates that both master and nodes are
 	// being upgraded.
 	ClusterUpgrade
+
+	// EtcdUpgrade indicates that only etcd is being upgraded (or migrated
+	// between storage versions).
+	EtcdUpgrade
 )
 
 // Test is an interface for upgrade tests.