e2e node: support running the test binary under a debugger

Single-stepping interactively through a test can be useful to understand what's
happening and to investigate the state at each step.

Similar support was added early to hack/ginkgo-e2e.sh, so the same env variable
is used again.

build/root/Makefile

@@ -269,6 +269,9 @@ define TEST_E2E_NODE_HELP_INFO
 #  INSTANCE_TYPE: For REMOTE=true only. Machine type to use.
 #  NODE_ENV: For REMOTE=true only. Additional metadata keys to add the instance.
 #  RUNTIME_CONFIG: The runtime configuration for the API server on the node e2e tests.
+#  E2E_TEST_DEBUG_TOOL: one of dlv/delve/gdb. Runs the test/e2e_node test binary
+#    interactively under the chosen debugger. Only works for REMOTE=false and
+#    in combination with DBG=1.
 #
 # Example:
 #   make test-e2e-node FOCUS=Kubelet SKIP=container

hack/make-rules/test-e2e-node.sh

@@ -53,6 +53,18 @@ ssh_key=${SSH_KEY:-}
 ssh_options=${SSH_OPTIONS:-}
 kubelet_config_file=${KUBELET_CONFIG_FILE:-"test/e2e_node/jenkins/default-kubelet-config.yaml"}
 
+# If set, the command executed will be:
+# - `dlv exec` if set to "delve"
+# - `gdb` if set to "gdb"
+# NOTE: for this to work the e2e_node.test binary has to be compiled with DBG=1.
+#
+# The name of this variable is the same as in ginkgo-e2e.sh.
+debug_tool=${E2E_TEST_DEBUG_TOOL:-}
+if [ "${remote}" = true ] && [ -n "${debug_tool}" ]; then
+    echo "Support for E2E_TEST_DEBUG_TOOL=${debug_tool} is only implemented for REMOTE=false."
+    exit 1
+fi
+
 # Parse the flags to pass to ginkgo
 ginkgoflags="-timeout=24h"
 if [[ ${parallelism} -gt 1 ]]; then
@@ -247,6 +259,7 @@ else
   # Test using the host the script was run on
   # Provided for backwards compatibility
   go run test/e2e_node/runner/local/run_local.go \
+    --debug-tool="${debug_tool}" \
     --system-spec-name="${system_spec_name}" --extra-envs="${extra_envs}" \
     --ginkgo-flags="${ginkgoflags}" \
     --test-flags="--v 4 --report-dir=${artifacts} --node-name $(hostname) ${test_args}" \

test/e2e_node/runner/local/run_local.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"k8s.io/kubernetes/test/e2e_node/builder"
@@ -32,12 +33,13 @@ import (
 )
 
 var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.")
-var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
+var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner, with shell-style quoting and escaping.")
-var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.")
+var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test, with shell-style quoting and escaping.")
 var systemSpecName = flag.String("system-spec-name", "", fmt.Sprintf("The name of the system spec used for validating the image in the node conformance test. The specs are at %s. If unspecified, the default built-in spec (system.DefaultSpec) will be used.", system.SystemSpecPath))
 var extraEnvs = flag.String("extra-envs", "", "The extra environment variables needed for node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2")
 var runtimeConfig = flag.String("runtime-config", "", "The runtime configuration for the API server on the node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2")
 var kubeletConfigFile = flag.String("kubelet-config-file", "", "The KubeletConfiguration file that should be applied to the kubelet")
+var debugTool = flag.String("debug-tool", "", "'delve', 'dlv' or 'gdb': run e2e_node.test under that debugger")
 
 func main() {
 	klog.InitFlags(nil)
@@ -56,10 +58,34 @@ func main() {
 		klog.Fatalf("Failed to get build output directory: %v", err)
 	}
 	klog.Infof("Got build output dir: %v", outputDir)
-	ginkgo := filepath.Join(outputDir, "ginkgo")
 	test := filepath.Join(outputDir, "e2e_node.test")
+	interactive := false
+	var cmd string
+	var args []string
+	switch *debugTool {
+	case "":
+		// No debugger, run gingko directly.
+		cmd = filepath.Join(outputDir, "ginkgo")
+		args = []string{*ginkgoFlags, test, "--"}
+	case "delve", "dlv":
+		dlv, err := exec.LookPath("dlv")
+		if err != nil {
+			klog.Fatalf("'dlv' not found: %v", err)
+		}
+		interactive = true
+		cmd = dlv
+		args = []string{"exec", test, "--", addGinkgoArgPrefix(*ginkgoFlags)}
+	case "gdb":
+		gdb, err := exec.LookPath("gdb")
+		if err != nil {
+			klog.Fatalf("'gdb' not found: %v", err)
+		}
+		interactive = true
+		cmd = gdb
+		args = []string{test, "--", addGinkgoArgPrefix(*ginkgoFlags)}
+	}
 
-	args := []string{*ginkgoFlags, test, "--", *testFlags, fmt.Sprintf("--runtime-config=%s", *runtimeConfig)}
+	args = append(args, *testFlags, fmt.Sprintf("--runtime-config=%s", *runtimeConfig))
 	if *systemSpecName != "" {
 		rootDir, err := utils.GetK8sRootDir()
 		if err != nil {
@@ -71,16 +97,40 @@ func main() {
 	if *kubeletConfigFile != "" {
 		args = append(args, fmt.Sprintf("--kubelet-config-file=\"%s\"", *kubeletConfigFile))
 	}
-	if err := runCommand(ginkgo, args...); err != nil {
+	if err := runCommand(interactive, cmd, args...); err != nil {
 		klog.Exitf("Test failed: %v", err)
 	}
 	return
 }
 
-func runCommand(name string, args ...string) error {
+func addGinkgoArgPrefix(ginkgoFlags string) string {
+	// Warning, hack! This simplistic search/replace assumes that hyphens do not appear
+	// inside argument values.
+	//
+	// The right solution would be to use github.com/anmitsu/go-shlex to split
+	// the -ginkgo-flags and -test-flags strings into individual arguments, then invoke
+	// exec.Command with the resulting string slice instead of passing a single string
+	// to sh. But github.com/anmitsu/go-shlex is not a Kubernetes dependency and not
+	// worth adding.
+
+	ginkgoFlags = regexp.MustCompile(`(^| )--?`).ReplaceAllString(ginkgoFlags, `$1--ginkgo.`)
+	return ginkgoFlags
+}
+
+func runCommand(interactive bool, name string, args ...string) error {
 	klog.Infof("Running command: %v %v", name, strings.Join(args, " "))
+	// Using sh is necessary because the args are using POSIX quoting.
+	// sh has to parse that.
 	cmd := exec.Command("sudo", "sh", "-c", strings.Join(append([]string{name}, args...), " "))
-	cmd.Stdout = os.Stdout
+	if interactive {
-	cmd.Stderr = os.Stderr
+		// stdin must be a console.
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdin
+		cmd.Stderr = os.Stdin
+	} else {
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
+
 	return cmd.Run()
 }