Merge pull request #79284 from johnSchnake/conformanceGoRunner

Adds an optional golang runner to the conformance test image

build/BUILD

@@ -172,6 +172,7 @@ filegroup(
             "//cmd/linkcheck",
             "//test/e2e:e2e.test_binary",
             "//vendor/github.com/onsi/ginkgo/ginkgo",
+            "//cluster/images/conformance/go-runner",
         ],
     )),
 )

cluster/images/conformance/BUILD

@@ -13,6 +13,7 @@ container_layer(
     name = "bins",
     directory = "/usr/local/bin",
     files = [
+        "//cluster/images/conformance/go-runner",
         "//cmd/kubectl",
         "//test/e2e:e2e.test_binary",
         "//vendor/github.com/onsi/ginkgo/ginkgo",
@@ -62,7 +63,10 @@ filegroup(
 
 filegroup(
     name = "all-srcs",
-    srcs = [":package-srcs"],
+    srcs = [
+        ":package-srcs",
+        "//cluster/images/conformance/go-runner:all-srcs",
+    ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )

cluster/images/conformance/Dockerfile

@@ -18,6 +18,7 @@ COPY ginkgo /usr/local/bin/
 COPY e2e.test /usr/local/bin/
 COPY kubectl /usr/local/bin/
 COPY run_e2e.sh /run_e2e.sh
+COPY gorunner /gorunner
 COPY cluster /kubernetes/cluster
 WORKDIR /usr/local/bin
 

cluster/images/conformance/Makefile

@@ -27,6 +27,7 @@ DOCKERIZED_OUTPUT_PATH=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(A
 GINKGO_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/ginkgo && echo $(LOCAL_OUTPUT_PATH)/ginkgo || echo $(DOCKERIZED_OUTPUT_PATH)/ginkgo)
 KUBECTL_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/kubectl && echo $(LOCAL_OUTPUT_PATH)/kubectl || echo $(DOCKERIZED_OUTPUT_PATH)/kubectl)
 E2E_TEST_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/e2e.test && echo $(LOCAL_OUTPUT_PATH)/e2e.test || echo $(DOCKERIZED_OUTPUT_PATH)/e2e.test)
+E2E_GO_RUNNER_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/go-runner && echo $(LOCAL_OUTPUT_PATH)/go-runner || echo $(DOCKERIZED_OUTPUT_PATH)/go-runner)
 
 CLUSTER_DIR?=$(shell pwd)/../../../cluster/
 
@@ -45,11 +46,13 @@ endif
 	cp ${GINKGO_BIN} ${TEMP_DIR}
 	cp ${KUBECTL_BIN} ${TEMP_DIR}
 	cp ${E2E_TEST_BIN} ${TEMP_DIR}
+	cp ${E2E_GO_RUNNER_BIN} ${TEMP_DIR}/gorunner
 	cp -r ${CLUSTER_DIR} ${TEMP_DIR}/cluster
 
 	chmod a+rx ${TEMP_DIR}/ginkgo
 	chmod a+rx ${TEMP_DIR}/kubectl
 	chmod a+rx ${TEMP_DIR}/e2e.test
+	chmod a+rx ${TEMP_DIR}/gorunner
 
 	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
 

cluster/images/conformance/README.md

@@ -7,7 +7,7 @@
 
 ```console
 # First, build the binaries by running make from the root directory
-$ make WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo cmd/kubectl"
+$ make WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo cmd/kubectl cluster/images/conformance/go-runner"
 
 # Build for linux/amd64 (default)
 # export REGISTRY=$HOST/$ORG to switch from k8s.gcr.io

cluster/images/conformance/go-runner/BUILD

@@ -0,0 +1,47 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cmd.go",
+        "const.go",
+        "e2erunner.go",
+        "env.go",
+        "tar.go",
+    ],
+    importpath = "k8s.io/kubernetes/cluster/images/conformance/go-runner",
+    visibility = ["//visibility:private"],
+    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+)
+
+go_binary(
+    name = "go-runner",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cmd_test.go",
+        "env_test.go",
+        "tar_test.go",
+    ],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+)

cluster/images/conformance/go-runner/Makefile

@@ -0,0 +1,27 @@
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOST_GOOS ?= $(shell go env GOOS)
+HOST_GOARCH ?= $(shell go env GOARCH)
+GO_BUILD ?= go build
+
+.PHONY: all build clean
+
+all: build
+
+build:
+	$(GO_BUILD)
+
+clean:
+	rm done e2e.log e2e.tar.gz go-runner

cluster/images/conformance/go-runner/cmd.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+)
+
+// getCmd uses the given environment to form the ginkgo command to run tests. It will
+// set the stdout/stderr to the given writer.
+func getCmd(env Getenver, w io.Writer) *exec.Cmd {
+	ginkgoArgs := []string{}
+
+	// The logic of the parallel env var impacting the skip value necessitates it
+	// being placed before the rest of the flag resolution.
+	skip := env.Getenv(skipEnvKey)
+	switch env.Getenv(parallelEnvKey) {
+	case "y", "Y", "true":
+		ginkgoArgs = append(ginkgoArgs, "--p")
+		if len(skip) == 0 {
+			skip = serialTestsRegexp
+		}
+	}
+
+	ginkgoArgs = append(ginkgoArgs, []string{
+		"--focus=" + env.Getenv(focusEnvKey),
+		"--skip=" + skip,
+		"--noColor=true",
+	}...)
+
+	extraArgs := []string{
+		"--disable-log-dump",
+		"--repo-root=/kubernetes",
+		"--provider=" + env.Getenv(providerEnvKey),
+		"--report-dir=" + env.Getenv(resultsDirEnvKey),
+		"--kubeconfig=" + env.Getenv(kubeconfigEnvKey),
+	}
+
+	// Extra args handling
+	sep := " "
+	if len(env.Getenv(extraArgsSeparaterEnvKey)) > 0 {
+		sep = env.Getenv(extraArgsSeparaterEnvKey)
+	}
+
+	if len(env.Getenv(extraGinkgoArgsEnvKey)) > 0 {
+		ginkgoArgs = append(ginkgoArgs, strings.Split(env.Getenv(extraGinkgoArgsEnvKey), sep)...)
+	}
+
+	if len(env.Getenv(extraArgsEnvKey)) > 0 {
+		fmt.Printf("sep is %q args are %q", sep, env.Getenv(extraArgsEnvKey))
+		fmt.Println("split", strings.Split(env.Getenv(extraArgsEnvKey), sep))
+		extraArgs = append(extraArgs, strings.Split(env.Getenv(extraArgsEnvKey), sep)...)
+	}
+
+	if len(env.Getenv(dryRunEnvKey)) > 0 {
+		ginkgoArgs = append(ginkgoArgs, "--dryRun=true")
+	}
+
+	args := []string{}
+	args = append(args, ginkgoArgs...)
+	args = append(args, env.Getenv(testBinEnvKey))
+	args = append(args, "--")
+	args = append(args, extraArgs...)
+
+	cmd := exec.Command(env.Getenv(ginkgoEnvKey), args...)
+	cmd.Stdout = w
+	cmd.Stderr = w
+	return cmd
+}
+
+// cmdInfo generates a useful look at what the command is for printing/debug.
+func cmdInfo(cmd *exec.Cmd) string {
+	return fmt.Sprintf(
+		`Command env: %v
+Run from directory: %v
+Executable path: %v
+Args (comma-delimited): %v`, cmd.Env, cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","),
+	)
+}

cluster/images/conformance/go-runner/cmd_test.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+	"reflect"
+	"testing"
+)
+
+func TestGetCmd(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		env        Getenver
+		expectArgs []string
+	}{
+		{
+			desc: "Default",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:  "ginkgobin",
+					testBinEnvKey: "testbin",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin",
+				"--focus=", "--skip=",
+				"--noColor=true", "testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=", "--report-dir=", "--kubeconfig=",
+			},
+		}, {
+			desc: "Filling in defaults",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:     "ginkgobin",
+					testBinEnvKey:    "testbin",
+					focusEnvKey:      "focus",
+					skipEnvKey:       "skip",
+					providerEnvKey:   "provider",
+					resultsDirEnvKey: "results",
+					kubeconfigEnvKey: "kubeconfig",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin",
+				"--focus=focus", "--skip=skip",
+				"--noColor=true", "testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=provider", "--report-dir=results", "--kubeconfig=kubeconfig",
+			},
+		}, {
+			desc: "Parallel gets set and skips serial",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:   "ginkgobin",
+					testBinEnvKey:  "testbin",
+					parallelEnvKey: "true",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin", "--p",
+				"--focus=", "--skip=[Serial]",
+				"--noColor=true", "testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=", "--report-dir=", "--kubeconfig=",
+			},
+		}, {
+			desc: "Arbitrary options before and after double dash split by space",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:          "ginkgobin",
+					testBinEnvKey:         "testbin",
+					extraArgsEnvKey:       "--extra=1 --extra=2",
+					extraGinkgoArgsEnvKey: "--ginkgo1 --ginkgo2",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin", "--focus=", "--skip=",
+				"--noColor=true", "--ginkgo1", "--ginkgo2",
+				"testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=", "--report-dir=", "--kubeconfig=",
+				"--extra=1", "--extra=2",
+			},
+		}, {
+			desc: "Arbitrary options can be split by other tokens",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:             "ginkgobin",
+					testBinEnvKey:            "testbin",
+					extraArgsEnvKey:          "--extra=value with spaces:--extra=value with % anything!$$",
+					extraGinkgoArgsEnvKey:    `--ginkgo='with "quotes" and ':--ginkgo2=true$(foo)`,
+					extraArgsSeparaterEnvKey: ":",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin", "--focus=", "--skip=",
+				"--noColor=true", `--ginkgo='with "quotes" and '`, "--ginkgo2=true$(foo)",
+				"testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=", "--report-dir=", "--kubeconfig=",
+				"--extra=value with spaces", "--extra=value with % anything!$$",
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			c := getCmd(tc.env, os.Stdout)
+			if !reflect.DeepEqual(c.Args, tc.expectArgs) {
+				t.Errorf("Expected args %q but got %q", tc.expectArgs, c.Args)
+			}
+		})
+	}
+}

cluster/images/conformance/go-runner/const.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+const (
+	// resultsTarballName is the name of the tarball we create with all the results.
+	resultsTarballName = "e2e.tar.gz"
+
+	// doneFileName is the name of the file that signals to the Sonobuoy worker we are
+	// done. The file should contain the path to the results file.
+	doneFileName = "done"
+
+	// resultsDirEnvKey is the env var which stores which directory to put the donefile
+	// and results into. It is a shared, mounted volume between the plugin and Sonobuoy.
+	resultsDirEnvKey = "RESULTS_DIR"
+
+	// logFileName is the name of the file which stdout is tee'd to.
+	logFileName = "e2e.log"
+
+	// Misc env vars which were explicitly supported prior to the go runner.
+	dryRunEnvKey     = "E2E_DRYRUN"
+	parallelEnvKey   = "E2E_PARALLEL"
+	focusEnvKey      = "E2E_FOCUS"
+	skipEnvKey       = "E2E_SKIP"
+	providerEnvKey   = "E2E_PROVIDER"
+	kubeconfigEnvKey = "KUBECONFIG"
+	ginkgoEnvKey     = "GINKGO_BIN"
+	testBinEnvKey    = "TEST_BIN"
+
+	// extraGinkgoArgsEnvKey, if set, will is a list of other arguments to pass to ginkgo.
+	// These are passed before the test binary and include things like `--afterSuiteHook`.
+	extraGinkgoArgsEnvKey = "E2E_EXTRA_GINKGO_ARGS"
+
+	// extraArgsEnvKey, if set, will is a list of other arguments to pass to the tests.
+	// These are passed after the `--` and include things like `--provider`.
+	extraArgsEnvKey = "E2E_EXTRA_ARGS"
+
+	// extraArgsSeparaterEnvKey specifies how to split the extra args values. If unset,
+	// it will default to splitting by spaces.
+	extraArgsSeparaterEnvKey = "E2E_EXTRA_ARGS_SEP"
+
+	defaultSkip         = ""
+	defaultFocus        = "[Conformance]"
+	defaultProvider     = "local"
+	defaultParallel     = "1"
+	defaultResultsDir   = "/tmp/results"
+	defaultGinkgoBinary = "/usr/local/bin/ginkgo"
+	defaultTestBinary   = "/usr/local/bin/e2e.test"
+
+	// serialTestsRegexp is the default skip value if running in parallel. Will not
+	// override an explicit E2E_SKIP value.
+	serialTestsRegexp = "[Serial]"
+)

cluster/images/conformance/go-runner/e2erunner.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+func main() {
+	env := envWithDefaults(map[string]string{
+		resultsDirEnvKey: defaultResultsDir,
+		skipEnvKey:       defaultSkip,
+		focusEnvKey:      defaultFocus,
+		providerEnvKey:   defaultProvider,
+		parallelEnvKey:   defaultParallel,
+		ginkgoEnvKey:     defaultGinkgoBinary,
+		testBinEnvKey:    defaultTestBinary,
+	})
+
+	if err := configureAndRunWithEnv(env); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// configureAndRunWithEnv uses the given environment to configure and then start the test run.
+// It will handle TERM signals gracefully and kill the test process and will
+// save the logs/results to the location specified via the RESULTS_DIR environment
+// variable.
+func configureAndRunWithEnv(env Getenver) error {
+	// Ensure we save results regardless of other errors. This helps any
+	// consumer who may be polling for the results.
+	resultsDir := env.Getenv(resultsDirEnvKey)
+	defer saveResults(resultsDir)
+
+	// Print the output to stdout and a logfile which will be returned
+	// as part of the results tarball.
+	logFilePath := filepath.Join(resultsDir, logFileName)
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create log file %v", logFilePath)
+	}
+	mw := io.MultiWriter(os.Stdout, logFile)
+	cmd := getCmd(env, mw)
+
+	log.Printf("Running command:\n%v\n", cmdInfo(cmd))
+	err = cmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "starting command")
+	}
+
+	// Handle signals and shutdown process gracefully.
+	go setupSigHandler(cmd.Process.Pid)
+	return errors.Wrap(cmd.Wait(), "running command")
+}
+
+// setupSigHandler will kill the process identified by the given PID if it
+// gets a TERM signal.
+func setupSigHandler(pid int) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+
+	// Block until a signal is received.
+	log.Println("Now listening for interrupts")
+	s := <-c
+	log.Printf("Got signal: %v. Shutting down test process (PID: %v)\n", s, pid)
+	p, err := os.FindProcess(pid)
+	if err != nil {
+		log.Printf("Could not find process %v to shut down.\n", pid)
+		return
+	}
+	if err := p.Signal(s); err != nil {
+		log.Printf("Failed to signal test process to terminate: %v\n", err)
+		return
+	}
+	log.Printf("Signalled process %v to terminate successfully.\n", pid)
+}
+
+// saveResults will tar the results directory and write the resulting tarball path
+// into the donefile.
+func saveResults(resultsDir string) error {
+	log.Printf("Saving results at %v\n", resultsDir)
+
+	err := tarDir(resultsDir, filepath.Join(resultsDir, resultsTarballName))
+	if err != nil {
+		return errors.Wrapf(err, "tar directory %v", resultsDir)
+	}
+
+	doneFile := filepath.Join(resultsDir, doneFileName)
+
+	resultsTarball := filepath.Join(resultsDir, resultsTarballName)
+	resultsTarball, err = filepath.Abs(resultsTarball)
+	if err != nil {
+		return errors.Wrapf(err, "failed to find absolute path for %v", resultsTarball)
+	}
+
+	return errors.Wrap(
+		ioutil.WriteFile(doneFile, []byte(resultsTarball), os.FileMode(0777)),
+		"writing donefile",
+	)
+}

cluster/images/conformance/go-runner/env.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+)
+
+// Getenver is the interface we use to mock out the env for easier testing. OS env
+// vars can't be as easily tested since internally it uses sync.Once.
+type Getenver interface {
+	Getenv(string) string
+}
+
+// osEnv uses the actual os.Getenv methods to lookup values.
+type osEnv struct{}
+
+// Getenv gets the value of the requested environment variable.
+func (*osEnv) Getenv(s string) string {
+	return os.Getenv(s)
+}
+
+// explicitEnv uses a map instead of os.Getenv methods to lookup values.
+type explicitEnv struct {
+	vals map[string]string
+}
+
+// Getenv returns the value of the requested environment variable (in this
+// implementation, really just a map lookup).
+func (e *explicitEnv) Getenv(s string) string {
+	return e.vals[s]
+}
+
+// defaultOSEnv uses a Getenver to lookup values but if it does
+// not have that value, it falls back to its internal set of defaults.
+type defaultEnver struct {
+	firstChoice Getenver
+	defaults    map[string]string
+}
+
+// Getenv returns the value of the environment variable or its default if unset.
+func (e *defaultEnver) Getenv(s string) string {
+	v := e.firstChoice.Getenv(s)
+	if len(v) == 0 {
+		return e.defaults[s]
+	}
+	return v
+}
+
+func envWithDefaults(defaults map[string]string) Getenver {
+	return &defaultEnver{firstChoice: &osEnv{}, defaults: defaults}
+}

cluster/images/conformance/go-runner/env_test.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+	"testing"
+)
+
+func TestEnv(t *testing.T) {
+	testCases := []struct {
+		desc    string
+		preHook func()
+		env     Getenver
+		expect  map[string]string
+	}{
+		{
+			desc: "OS env",
+			env:  &osEnv{},
+			preHook: func() {
+				os.Setenv("key1", "1")
+			},
+			expect: map[string]string{"key1": "1"},
+		}, {
+			desc: "OS env falls defaults to empty",
+			env:  &osEnv{},
+			preHook: func() {
+				os.Unsetenv("key1")
+			},
+			expect: map[string]string{"key1": ""},
+		}, {
+			desc: "First choice of env respected",
+			env: &defaultEnver{
+				firstChoice: &explicitEnv{
+					vals: map[string]string{
+						"key1": "1",
+					},
+				},
+				defaults: map[string]string{
+					"key1": "default1",
+					"key2": "default2",
+				},
+			},
+			expect: map[string]string{
+				"key1": "1",
+				"key2": "default2",
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			for k, expectVal := range tc.expect {
+				if tc.preHook != nil {
+					tc.preHook()
+				}
+				val := tc.env.Getenv(k)
+				if val != expectVal {
+					t.Errorf("Expected %q but got %q", expectVal, val)
+				}
+			}
+		})
+	}
+}

cluster/images/conformance/go-runner/tar.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// tarDir takes a source and variable writers and walks 'source' writing each file
+// found to the tar writer.
+func tarDir(dir, outpath string) error {
+	// ensure the src actually exists before trying to tar it
+	if _, err := os.Stat(dir); err != nil {
+		return errors.Wrapf(err, "tar unable to stat directory %v", dir)
+	}
+
+	outfile, err := os.Create(outpath)
+	if err != nil {
+		return errors.Wrapf(err, "creating tarball %v", outpath)
+	}
+	defer outfile.Close()
+
+	gzw := gzip.NewWriter(outfile)
+	defer gzw.Close()
+
+	tw := tar.NewWriter(gzw)
+	defer tw.Close()
+
+	return filepath.Walk(dir, func(file string, fi os.FileInfo, err error) error {
+		// Return on any error.
+		if err != nil {
+			return err
+		}
+
+		// Only write regular files and don't include the archive itself.
+		if !fi.Mode().IsRegular() || filepath.Join(dir, fi.Name()) == outpath {
+			return nil
+		}
+
+		// Create a new dir/file header.
+		header, err := tar.FileInfoHeader(fi, fi.Name())
+		if err != nil {
+			return errors.Wrapf(err, "creating file info header %v", fi.Name())
+		}
+
+		// Update the name to correctly reflect the desired destination when untaring.
+		header.Name = strings.TrimPrefix(strings.Replace(file, dir, "", -1), string(filepath.Separator))
+		if err := tw.WriteHeader(header); err != nil {
+			return errors.Wrapf(err, "writing header for tarball %v", header.Name)
+		}
+
+		// Open files, copy into tarfile, and close.
+		f, err := os.Open(file)
+		if err != nil {
+			return errors.Wrapf(err, "opening file %v for writing into tarball", file)
+		}
+		defer f.Close()
+
+		_, err = io.Copy(tw, f)
+		return errors.Wrapf(err, "creating file %v contents into tarball", file)
+	})
+}

cluster/images/conformance/go-runner/tar_test.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+func TestTar(t *testing.T) {
+	testCases := []struct {
+		desc      string
+		dir       string
+		outpath   string
+		expectErr string
+		expect    map[string]string
+	}{
+		{
+			desc:    "Contents preserved and no self-reference",
+			dir:     "testdata/tartest",
+			outpath: "testdata/tartest/out.tar.gz",
+			expect: map[string]string{
+				"file1":        "file1 data",
+				"file2":        "file2 data",
+				"subdir/file4": "file4 data",
+			},
+		}, {
+			desc:      "Errors if directory does not exist",
+			dir:       "testdata/does-not-exist",
+			outpath:   "testdata/tartest/out.tar.gz",
+			expectErr: "tar unable to stat directory",
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			err := tarDir(tc.dir, tc.outpath)
+			switch {
+			case err != nil && len(tc.expectErr) == 0:
+				t.Fatalf("Expected nil error but got %q", err)
+			case err != nil && len(tc.expectErr) > 0:
+				if !strings.Contains(fmt.Sprint(err), tc.expectErr) {
+					t.Errorf("Expected error \n\t%q\nbut got\n\t%q", tc.expectErr, err)
+				}
+				return
+			case err == nil && len(tc.expectErr) > 0:
+				t.Fatalf("Expected error %q but got nil", tc.expectErr)
+			default:
+				// No error
+			}
+
+			data, err := readAllTar(tc.outpath)
+			if !reflect.DeepEqual(data, tc.expect) {
+				t.Errorf("Expected data %v but got %v", tc.expect, data)
+			}
+		})
+	}
+}
+
+// readAllTar walks all of the files in the archive. It returns a map
+// of filenames and their contents and any error encountered.
+func readAllTar(tarPath string) (map[string]string, error) {
+	tarPath, err := filepath.Abs(tarPath)
+	if err != nil {
+		return nil, err
+	}
+
+	fileReader, err := os.Open(tarPath)
+	if err != nil {
+		return nil, err
+	}
+	defer fileReader.Close()
+
+	gzStream, err := gzip.NewReader(fileReader)
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't uncompress reader")
+	}
+	defer gzStream.Close()
+
+	// Open and iterate through the files in the archive.
+	tr := tar.NewReader(gzStream)
+	fileData := map[string]string{}
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break // End of archive
+		}
+		if err != nil {
+
+			return nil, err
+		}
+
+		b, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return nil, err
+		}
+		fileData[filepath.ToSlash(hdr.Name)] = string(b)
+	}
+	return fileData, nil
+}

cluster/images/conformance/go-runner/testdata/tartest/file1

@@ -0,0 +1 @@
+file1 data

cluster/images/conformance/go-runner/testdata/tartest/file2

@@ -0,0 +1 @@
+file2 data

cluster/images/conformance/go-runner/testdata/tartest/subdir/file4

@@ -0,0 +1 @@
+file4 data

cluster/images/conformance/run_e2e.sh

@@ -36,6 +36,14 @@ saveResults() {
     echo -n "${RESULTS_DIR}/e2e.tar.gz" > "${RESULTS_DIR}/done"
 }
 
+# Optional Golang runner alternative to the bash script.
+# Entry provided via env var to simplify invocation. 
+if [[ -n ${E2E_USE_GO_RUNNER:-} ]]; then
+    set -x
+    /gorunner
+    exit $?
+fi
+
 # We get the TERM from kubernetes and handle it gracefully
 trap shutdown TERM
 

hack/dev-push-conformance.sh

@@ -42,7 +42,7 @@ IMAGE="${REGISTRY}/conformance-amd64:${VERSION}"
 
 kube::build::verify_prereqs
 kube::build::build_image
-kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl"
+kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl cluster/images/conformance/go-runner"
 kube::build::copy_output
 
 make -C "${KUBE_ROOT}/cluster/images/conformance" build

hack/lib/golang.sh

@@ -264,6 +264,7 @@ kube::golang::test_targets() {
     cmd/linkcheck
     vendor/github.com/onsi/ginkgo/ginkgo
     test/e2e/e2e.test
+    cluster/images/conformance/go-runner
   )
   echo "${targets[@]}"
 }