Merge pull request #33240 from spxtr/xmloutput

Automatic merge from submit-queue

Make e2e.go give us JUnit results.

Will assign to @fejta when it's ready to go. https://github.com/kubernetes/test-infra/issues/76

docs/devel/e2e-tests.md

@@ -125,12 +125,6 @@ go run hack/e2e.go -v --build
 # Create a fresh cluster.  Deletes a cluster first, if it exists
 go run hack/e2e.go -v --up
 
-# Push code to an existing cluster
-go run hack/e2e.go -v --push
-
-# Push to an existing cluster, or bring up a cluster if it's down.
-go run hack/e2e.go -v --pushup
-
 # Run all tests
 go run hack/e2e.go -v --test
 
@@ -144,12 +138,12 @@ go run hack/e2e.go -v --test --test_args="--ginkgo.skip=Pods.*env"
 GINKGO_PARALLEL=y go run hack/e2e.go --v --test --test_args="--ginkgo.skip=\[Serial\]"
 
 # Flags can be combined, and their actions will take place in this order:
-# --build, --push|--up|--pushup, --test, --down
+# --build, --up, --test, --down
 #
 # You can also specify an alternative provider, such as 'aws'
 #
 # e.g.:
-KUBERNETES_PROVIDER=aws go run hack/e2e.go -v --build --pushup --test --down
+KUBERNETES_PROVIDER=aws go run hack/e2e.go -v --build --up --test --down
 
 # -ctl can be used to quickly call kubectl against your e2e cluster. Useful for
 # cleaning up after a failed test or viewing logs. Use -v to avoid suppressing

hack/e2e-internal/e2e-push.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
-
-: ${KUBECTL:=${KUBE_ROOT}/cluster/kubectl.sh}
-: ${KUBE_CONFIG_FILE:="config-test.sh"}
-
-export KUBECTL KUBE_CONFIG_FILE
-
-source "${KUBE_ROOT}/cluster/kube-util.sh"
-
-prepare-e2e
-
-"${KUBE_ROOT}/cluster/kube-push.sh" $@

hack/e2e.go

@@ -18,6 +18,7 @@ limitations under the License.
 package main
 
 import (
+	"encoding/xml"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -25,7 +26,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 )
@@ -33,31 +33,24 @@ import (
 var (
 	// TODO(fejta): change all these _ flags to -
 	build            = flag.Bool("build", false, "If true, build a new release. Otherwise, use whatever is there.")
-	checkNodeCount = flag.Bool("check_node_count", true, ""+
-		"By default, verify that the cluster has at least two nodes."+
-		"You can explicitly set to false if you're, e.g., testing single-node clusters "+
-		"for which the node count is supposed to be one.")
 	checkVersionSkew = flag.Bool("check_version_skew", true, ""+
 		"By default, verify that client and server have exact version match. "+
 		"You can explicitly set to false if you're, e.g., testing client changes "+
 		"for which the server version doesn't make a difference.")
 	checkLeakedResources = flag.Bool("check_leaked_resources", false, "Ensure project ends with the same resources")
-	ctlCmd               = flag.String("ctl", "", "If nonempty, pass this as an argument, and call kubectl. Implies -v.")
 	down                 = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
 	dump                 = flag.String("dump", "", "If set, dump cluster logs to this location on test or cluster-up failure")
 	kubemark             = flag.Bool("kubemark", false, "If true, run kubemark tests.")
-	push                 = flag.Bool("push", false, "If true, push to e2e cluster. Has no effect if -up is true.")
-	pushup               = flag.Bool("pushup", false, "If true, push to e2e cluster if it's up, otherwise start the e2e cluster.")
 	skewTests            = flag.Bool("skew", false, "If true, run tests in another version at ../kubernetes/hack/e2e.go")
 	testArgs             = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
 	test                 = flag.Bool("test", false, "Run Ginkgo tests.")
 	up                   = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
 	upgradeArgs          = flag.String("upgrade_args", "", "If set, run upgrade tests before other tests")
 	verbose              = flag.Bool("v", false, "If true, print all command output.")
-)
 
-const (
+	deprecatedPush   = flag.Bool("push", false, "Deprecated. Does nothing.")
-	minNodeCount = 2
+	deprecatedPushup = flag.Bool("pushup", false, "Deprecated. Does nothing.")
+	deprecatedCtlCmd = flag.String("ctl", "", "Deprecated. Does nothing.")
 )
 
 func appendError(errs []error, err error) []error {
@@ -67,125 +60,198 @@ func appendError(errs []error, err error) []error {
 	return errs
 }
 
+func validWorkingDirectory() error {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("could not get pwd: %v", err)
+	}
+	acwd, err := filepath.Abs(cwd)
+	if err != nil {
+		return fmt.Errorf("failed to convert %s to an absolute path: %v", cwd, err)
+	}
+	// This also matches "kubernetes_skew" for upgrades.
+	if !strings.Contains(filepath.Base(acwd), "kubernetes") {
+		return fmt.Errorf("must run from kubernetes directory root: %v", acwd)
+	}
+	return nil
+}
+
+type TestCase struct {
+	XMLName   xml.Name `xml:"testcase"`
+	ClassName string   `xml:"classname,attr"`
+	Name      string   `xml:"name,attr"`
+	Time      float64  `xml:"time,attr"`
+	Failure   string   `xml:"failure,omitempty"`
+}
+
+type TestSuite struct {
+	XMLName  xml.Name `xml:"testsuite"`
+	Failures int      `xml:"failures,attr"`
+	Tests    int      `xml:"tests,attr"`
+	Time     float64  `xml:"time,attr"`
+	Cases    []TestCase
+}
+
+var suite TestSuite
+
+func xmlWrap(name string, f func() error) error {
+	start := time.Now()
+	err := f()
+	duration := time.Since(start)
+	c := TestCase{
+		Name:      name,
+		ClassName: "e2e.go",
+		Time:      duration.Seconds(),
+	}
+	if err != nil {
+		c.Failure = err.Error()
+		suite.Failures++
+	}
+	suite.Cases = append(suite.Cases, c)
+	suite.Tests++
+	return err
+}
+
+func writeXML(start time.Time) {
+	suite.Time = time.Since(start).Seconds()
+	out, err := xml.MarshalIndent(&suite, "", "    ")
+	if err != nil {
+		log.Fatalf("Could not marshal XML: %s", err)
+	}
+	path := filepath.Join(*dump, "junit_runner.xml")
+	f, err := os.Create(path)
+	if err != nil {
+		log.Fatalf("Could not create file: %s", err)
+	}
+	defer f.Close()
+	if _, err := f.WriteString(xml.Header); err != nil {
+		log.Fatalf("Error writing XML header: %s", err)
+	}
+	if _, err := f.Write(out); err != nil {
+		log.Fatalf("Error writing XML data: %s", err)
+	}
+	log.Printf("Saved XML output to %s.", path)
+}
+
 func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 	flag.Parse()
 
-	cwd, err := os.Getwd()
+	if err := validWorkingDirectory(); err != nil {
-	if err != nil {
+		log.Fatalf("Called from invalid working directory: %v", err)
-		log.Fatalf("Could not get pwd: %v", err)
 	}
-	acwd, err := filepath.Abs(cwd)
+
-	if err != nil {
+	if err := run(); err != nil {
-		log.Fatalf("Failed to convert to an absolute path: %v", err)
+		log.Fatalf("Something went wrong: %s", err)
 	}
-	if !strings.Contains(filepath.Base(acwd), "kubernetes") {
+}
-		// TODO(fejta): cd up into  the kubernetes directory
+
-		log.Fatalf("Must run from kubernetes directory: %v", cwd)
+func run() error {
+	if *dump != "" {
+		defer writeXML(time.Now())
 	}
 
 	if *build {
-		// The build-release script needs stdin to ask the user whether
+		if err := xmlWrap("Build", Build); err != nil {
-		// it's OK to download the docker image.
+			return fmt.Errorf("error building: %s", err)
-		cmd := exec.Command("make", "quick-release")
-		cmd.Stdin = os.Stdin
-		if err := finishRunning("build-release", cmd); err != nil {
-			log.Fatalf("error building kubernetes: %v", err)
 		}
 	}
 
+	if *checkVersionSkew {
+		os.Setenv("KUBECTL", "./cluster/kubectl.sh --match-server-version")
+	} else {
+		os.Setenv("KUBECTL", "./cluster/kubectl.sh")
+	}
+	os.Setenv("KUBE_CONFIG_FILE", "config-test.sh")
+	// force having batch/v2alpha1 always on for e2e tests
+	os.Setenv("KUBE_RUNTIME_CONFIG", "batch/v2alpha1=true")
+
 	if *up {
-		if err := TearDown(); err != nil {
+		if err := xmlWrap("TearDown", TearDown); err != nil {
-			log.Fatalf("error tearing down previous cluster: %v", err)
+			return fmt.Errorf("error tearing down previous cluster: %s", err)
 		}
 	}
 
+	var err error
 	var errs []error
 
-	var beforeResources []byte
+	var (
+		beforeResources []byte
+		upResources     []byte
+		afterResources  []byte
+	)
+
 	if *checkLeakedResources {
+		errs = appendError(errs, xmlWrap("ListResources Before", func() error {
 			beforeResources, err = ListResources()
-		errs = appendError(errs, err)
+			return err
+		}))
 	}
 
-	os.Setenv("KUBECTL", strings.Join(append([]string{"./cluster/kubectl.sh"}, kubectlArgs()...), " "))
-
-	if *upgradeArgs != "" { // Start the cluster using a previous version.
-		if err := UpgradeUp(); err != nil {
-			log.Fatalf("error starting cluster to upgrade: %v", err)
-		}
-	} else { // Start the cluster using this version.
-		if *pushup {
-			if IsUp() {
-				log.Printf("e2e cluster is up, pushing.")
-				*up = false
-				*push = true
-			} else {
-				log.Printf("e2e cluster is down, creating.")
-				*up = true
-				*push = false
-			}
-		}
 	if *up {
-			if err := Up(); err != nil {
+		// Start the cluster using this version.
-				log.Fatalf("starting e2e cluster: %v", err)
+		if err := xmlWrap("Up", Up); err != nil {
-			}
+			return fmt.Errorf("starting e2e cluster: %s", err)
-		} else if *push {
-			if err := finishRunning("push", exec.Command("./hack/e2e-internal/e2e-push.sh")); err != nil {
-				log.Fatalf("error pushing e2e clsuter: %v", err)
-			}
 		}
 	}
 
-	var upResources []byte
 	if *checkLeakedResources {
+		errs = appendError(errs, xmlWrap("ListResources Up", func() error {
 			upResources, err = ListResources()
-		errs = appendError(errs, err)
+			return err
-	}
+		}))
-
-	if *ctlCmd != "" {
-		ctlArgs := strings.Fields(*ctlCmd)
-		os.Setenv("KUBE_CONFIG_FILE", "config-test.sh")
-		errs = appendError(errs, finishRunning("'kubectl "+*ctlCmd+"'", exec.Command("./cluster/kubectl.sh", ctlArgs...)))
 	}
 
 	if *upgradeArgs != "" {
-		errs = appendError(errs, UpgradeTest(*upgradeArgs))
+		errs = appendError(errs, xmlWrap("UpgradeTest", func() error {
+			return UpgradeTest(*upgradeArgs)
+		}))
 	}
 
 	if *test {
+		errs = appendError(errs, xmlWrap("kubectl version", func() error {
+			return finishRunning("kubectl version", exec.Command("./cluster/kubectl.sh", "version", "--match-server-version=false"))
+		}))
 		if *skewTests {
-			errs = appendError(errs, SkewTest())
+			errs = appendError(errs, xmlWrap("SkewTest", SkewTest))
 		} else {
-			errs = appendError(errs, Test())
+			errs = appendError(errs, xmlWrap("Test", Test))
 		}
 	}
 
 	if *kubemark {
-		errs = appendError(errs, KubemarkTest())
+		errs = appendError(errs, xmlWrap("KubemarkTest", KubemarkTest))
+	}
+
+	if len(errs) > 0 && *dump != "" {
+		errs = appendError(errs, xmlWrap("DumpClusterLogs", func() error {
+			return DumpClusterLogs(*dump)
+		}))
 	}
 
 	if *down {
-		if errs != nil && *dump != "" {
+		errs = appendError(errs, xmlWrap("TearDown", TearDown))
-			DumpClusterLogs(*dump)
-		}
-		errs = appendError(errs, TearDown())
 	}
 
 	if *checkLeakedResources {
 		log.Print("Sleeping for 30 seconds...") // Wait for eventually consistent listing
 		time.Sleep(30 * time.Second)
-		afterResources, err := ListResources()
+		if err := xmlWrap("ListResources After", func() error {
-		if err != nil {
+			afterResources, err = ListResources()
+			return err
+		}); err != nil {
 			errs = append(errs, err)
 		} else {
-			errs = appendError(errs, DiffResources(beforeResources, upResources, afterResources, *dump))
+			errs = appendError(errs, xmlWrap("DiffResources", func() error {
+				return DiffResources(beforeResources, upResources, afterResources, *dump)
+			}))
 		}
 	}
 
 	if len(errs) != 0 {
-		log.Fatalf("Encountered %d errors: %v", len(errs), errs)
+		return fmt.Errorf("encountered %d errors: %v", len(errs), errs)
 	}
+	return nil
 }
 
 func DiffResources(before, clusterUp, after []byte, location string) error {
@@ -255,40 +321,25 @@ func ListResources() ([]byte, error) {
 	return stdout, nil
 }
 
+func Build() error {
+	// The build-release script needs stdin to ask the user whether
+	// it's OK to download the docker image.
+	cmd := exec.Command("make", "quick-release")
+	cmd.Stdin = os.Stdin
+	if err := finishRunning("build-release", cmd); err != nil {
+		return fmt.Errorf("error building kubernetes: %v", err)
+	}
+	return nil
+}
+
 func TearDown() error {
 	return finishRunning("teardown", exec.Command("./hack/e2e-internal/e2e-down.sh"))
 }
 
-// Up brings an e2e cluster up, recreating it if one is already running.
 func Up() error {
-	// force having batch/v2alpha1 always on for e2e tests
-	os.Setenv("KUBE_RUNTIME_CONFIG", "batch/v2alpha1=true")
 	return finishRunning("up", exec.Command("./hack/e2e-internal/e2e-up.sh"))
 }
 
-// Ensure that the cluster is large engough to run the e2e tests.
-func ValidateClusterSize() error {
-	// Check that there are at least minNodeCount nodes running
-	cmd := exec.Command("./hack/e2e-internal/e2e-cluster-size.sh")
-	if *verbose {
-		cmd.Stderr = os.Stderr
-	}
-	stdout, err := cmd.Output()
-	if err != nil {
-		return fmt.Errorf("Could not get nodes to validate cluster size (%s)", err)
-	}
-
-	numNodes, err := strconv.Atoi(strings.TrimSpace(string(stdout)))
-	if err != nil {
-		return fmt.Errorf("Could not count number of nodes to validate cluster size (%s)", err)
-	}
-
-	if numNodes < minNodeCount {
-		return fmt.Errorf("Cluster size (%d) is too small to run e2e tests.  %d Nodes are required.", numNodes, minNodeCount)
-	}
-	return nil
-}
-
 // Is the e2e cluster up?
 func IsUp() bool {
 	return finishRunning("get status", exec.Command("./hack/e2e-internal/e2e-status.sh")) == nil
@@ -345,32 +396,15 @@ func KubemarkTest() error {
 func chdirSkew() (string, error) {
 	old, err := os.Getwd()
 	if err != nil {
-		return "", fmt.Errorf("Failed to os.Getwd(): %v", err)
+		return "", fmt.Errorf("failed to os.Getwd(): %v", err)
 	}
 	err = os.Chdir("../kubernetes_skew")
 	if err != nil {
-		return "", fmt.Errorf("Failed to cd ../kubernetes_skew: %v", err)
+		return "", fmt.Errorf("failed to cd ../kubernetes_skew: %v", err)
 	}
 	return old, nil
 }
 
-func UpgradeUp() error {
-	old, err := chdirSkew()
-	if err != nil {
-		return err
-	}
-	defer os.Chdir(old)
-	return finishRunning("UpgradeUp",
-		exec.Command(
-			"go", "run", "./hack/e2e.go",
-			fmt.Sprintf("--check_version_skew=%t", *checkVersionSkew),
-			fmt.Sprintf("--push=%t", *push),
-			fmt.Sprintf("--pushup=%t", *pushup),
-			fmt.Sprintf("--up=%t", *up),
-			fmt.Sprintf("--v=%t", *verbose),
-		))
-}
-
 func UpgradeTest(args string) error {
 	old, err := chdirSkew()
 	if err != nil {
@@ -410,16 +444,11 @@ func SkewTest() error {
 
 func Test() error {
 	if !IsUp() {
-		log.Fatal("Testing requested, but e2e cluster not up!")
+		return fmt.Errorf("testing requested, but e2e cluster not up!")
 	}
 
 	// TODO(fejta): add a --federated or something similar
 	if os.Getenv("FEDERATION") != "true" {
-		if *checkNodeCount {
-			if err := ValidateClusterSize(); err != nil {
-				return err
-			}
-		}
 		return finishRunning("Ginkgo tests", exec.Command("./hack/ginkgo-e2e.sh", strings.Fields(*testArgs)...))
 	}
 
@@ -444,12 +473,3 @@ func finishRunning(stepName string, cmd *exec.Cmd) error {
 	}
 	return nil
 }
-
-// returns either "", or a list of args intended for appending with the
-// kubectl command (beginning with a space).
-func kubectlArgs() []string {
-	if !*checkVersionSkew {
-		return []string{}
-	}
-	return []string{"--match-server-version"}
-}