e2e test layout changes for vsphere (#398)

hack/.golint_failures

@@ -796,6 +796,8 @@ test/e2e/scalability
 test/e2e/scheduling
 test/e2e/servicecatalog
 test/e2e/storage
+test/e2e/storage/utils
+test/e2e/storage/vsphere
 test/e2e/ui
 test/e2e/upgrades
 test/e2e/upgrades/apps

test/e2e/storage/BUILD

@@ -10,66 +10,42 @@ go_library(
     srcs = [
         "empty_dir_wrapper.go",
         "flexvolume.go",
-        "framework.go",
         "pd.go",
         "persistent_volumes.go",
         "persistent_volumes-disruptive.go",
         "persistent_volumes-gce.go",
         "persistent_volumes-local.go",
-        "persistent_volumes-vsphere.go",
-        "pv_reclaimpolicy.go",
-        "pvc_label_selector.go",
         "pvc_protection.go",
         "volume_expand.go",
         "volume_io.go",
         "volume_metrics.go",
         "volume_provisioning.go",
         "volumes.go",
-        "vsphere_scale.go",
-        "vsphere_statefulsets.go",
-        "vsphere_stress.go",
-        "vsphere_utils.go",
-        "vsphere_volume_cluster_ds.go",
-        "vsphere_volume_datastore.go",
-        "vsphere_volume_diskformat.go",
-        "vsphere_volume_disksize.go",
-        "vsphere_volume_fstype.go",
-        "vsphere_volume_master_restart.go",
-        "vsphere_volume_node_poweroff.go",
-        "vsphere_volume_ops_storm.go",
-        "vsphere_volume_perf.go",
-        "vsphere_volume_placement.go",
-        "vsphere_volume_vsan_policy.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/storage",
     deps = [
         "//pkg/api/testapi:go_default_library",
         "//pkg/apis/core/v1/helper:go_default_library",
         "//pkg/apis/storage/v1/util:go_default_library",
-        "//pkg/cloudprovider/providers/vsphere:go_default_library",
-        "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",
         "//pkg/util/slice:go_default_library",
         "//pkg/volume/util:go_default_library",
-        "//pkg/volume/util/volumehelper:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
         "//test/e2e/generated:go_default_library",
+        "//test/e2e/storage/utils:go_default_library",
+        "//test/e2e/storage/vsphere:go_default_library",
         "//test/utils/image:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
-        "//vendor/github.com/vmware/govmomi/find:go_default_library",
-        "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library",
-        "//vendor/golang.org/x/net/context:go_default_library",
         "//vendor/google.golang.org/api/googleapi:go_default_library",
         "//vendor/k8s.io/api/apps/v1beta1:go_default_library",
         "//vendor/k8s.io/api/batch/v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
         "//vendor/k8s.io/api/policy/v1beta1:go_default_library",
         "//vendor/k8s.io/api/rbac/v1beta1:go_default_library",
         "//vendor/k8s.io/api/storage/v1:go_default_library",
@@ -101,6 +77,10 @@ filegroup(
 
 filegroup(
     name = "all-srcs",
-    srcs = [":package-srcs"],
+    srcs = [
+        ":package-srcs",
+        "//test/e2e/storage/utils:all-srcs",
+        "//test/e2e/storage/vsphere:all-srcs",
+    ],
     tags = ["automanaged"],
 )

test/e2e/storage/empty_dir_wrapper.go

@@ -30,6 +30,8 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -51,7 +53,7 @@ const (
 	wrappedVolumeRaceRCNamePrefix            = "wrapped-volume-race-"
 )
 
-var _ = SIGDescribe("EmptyDir wrapper volumes", func() {
+var _ = utils.SIGDescribe("EmptyDir wrapper volumes", func() {
 	f := framework.NewDefaultFramework("emptydir-wrapper")
 
 	It("should not conflict", func() {

test/e2e/storage/flexvolume.go

@@ -28,6 +28,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/generated"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -136,7 +137,7 @@ func sshAndLog(cmd, host string) {
 	}
 }
 
-var _ = SIGDescribe("Flexvolumes [Disruptive] [Feature:FlexVolume]", func() {
+var _ = utils.SIGDescribe("Flexvolumes [Disruptive] [Feature:FlexVolume]", func() {
 	f := framework.NewDefaultFramework("flexvolume")
 
 	// note that namespace deletion is handled by delete-namespace flag

test/e2e/storage/pd.go

@@ -40,6 +40,7 @@ import (
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -52,7 +53,7 @@ const (
 	minNodes            = 2
 )
 
-var _ = SIGDescribe("Pod Disks", func() {
+var _ = utils.SIGDescribe("Pod Disks", func() {
 	var (
 		ns         string
 		cs         clientset.Interface

test/e2e/storage/persistent_volumes-disruptive.go

@@ -18,18 +18,17 @@ package storage
 
 import (
 	"fmt"
-	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/api/core/v1"
-	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 type testBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume)
@@ -37,17 +36,12 @@ type disruptiveTest struct {
 	testItStmt string
 	runTest    testBody
 }
-type kubeletOpt string
 
 const (
 	MinNodes = 2
-	NodeStateTimeout            = 1 * time.Minute
-	kStart           kubeletOpt = "start"
-	kStop            kubeletOpt = "stop"
-	kRestart         kubeletOpt = "restart"
 )
 
-var _ = SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() {
+var _ = utils.SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() {
 
 	f := framework.NewDefaultFramework("disruptive-pv")
 	var (
@@ -223,11 +217,11 @@ var _ = SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() {
 		disruptiveTestTable := []disruptiveTest{
 			{
 				testItStmt: "Should test that a file written to the mount before kubelet restart is readable after restart.",
-				runTest:    testKubeletRestartsAndRestoresMount,
+				runTest:    utils.TestKubeletRestartsAndRestoresMount,
 			},
 			{
 				testItStmt: "Should test that a volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns.",
-				runTest:    testVolumeUnmountsFromDeletedPod,
+				runTest:    utils.TestVolumeUnmountsFromDeletedPod,
 			},
 		}
 
@@ -243,61 +237,6 @@ var _ = SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() {
 	})
 })
 
-// testKubeletRestartsAndRestoresMount tests that a volume mounted to a pod remains mounted after a kubelet restarts
-func testKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
-	By("Writing to the volume.")
-	file := "/mnt/_SUCCESS"
-	out, err := podExec(clientPod, fmt.Sprintf("touch %s", file))
-	framework.Logf(out)
-	Expect(err).NotTo(HaveOccurred())
-
-	By("Restarting kubelet")
-	kubeletCommand(kRestart, c, clientPod)
-
-	By("Testing that written file is accessible.")
-	out, err = podExec(clientPod, fmt.Sprintf("cat %s", file))
-	framework.Logf(out)
-	Expect(err).NotTo(HaveOccurred())
-	framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, file)
-}
-
-// testVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down.
-func testVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
-	nodeIP, err := framework.GetHostExternalAddress(c, clientPod)
-	Expect(err).NotTo(HaveOccurred())
-	nodeIP = nodeIP + ":22"
-
-	By("Expecting the volume mount to be found.")
-	result, err := framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider)
-	framework.LogSSHResult(result)
-	Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
-	Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
-
-	By("Stopping the kubelet.")
-	kubeletCommand(kStop, c, clientPod)
-	defer func() {
-		if err != nil {
-			kubeletCommand(kStart, c, clientPod)
-		}
-	}()
-	By(fmt.Sprintf("Deleting Pod %q", clientPod.Name))
-	err = c.CoreV1().Pods(clientPod.Namespace).Delete(clientPod.Name, &metav1.DeleteOptions{})
-	Expect(err).NotTo(HaveOccurred())
-	By("Starting the kubelet and waiting for pod to delete.")
-	kubeletCommand(kStart, c, clientPod)
-	err = f.WaitForPodTerminated(clientPod.Name, "")
-	if !apierrs.IsNotFound(err) && err != nil {
-		Expect(err).NotTo(HaveOccurred(), "Expected pod to terminate.")
-	}
-
-	By("Expecting the volume mount not to be found.")
-	result, err = framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider)
-	framework.LogSSHResult(result)
-	Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
-	Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no mount found).")
-	framework.Logf("Volume unmounted on node %s", clientPod.Spec.NodeName)
-}
-
 // initTestCase initializes spec resources (pv, pvc, and pod) and returns pointers to be consumed
 // by the test.
 func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig framework.PersistentVolumeConfig, pvcConfig framework.PersistentVolumeClaimConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
@@ -339,101 +278,3 @@ func tearDownTestCase(c clientset.Interface, f *framework.Framework, ns string,
 	framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
 	framework.DeletePersistentVolume(c, pv.Name)
 }
-
-// kubeletCommand performs `start`, `restart`, or `stop` on the kubelet running on the node of the target pod and waits
-// for the desired statues..
-// - First issues the command via `systemctl`
-// - If `systemctl` returns stderr "command not found, issues the command via `service`
-// - If `service` also returns stderr "command not found", the test is aborted.
-// Allowed kubeletOps are `kStart`, `kStop`, and `kRestart`
-func kubeletCommand(kOp kubeletOpt, c clientset.Interface, pod *v1.Pod) {
-	command := ""
-	sudoPresent := false
-	systemctlPresent := false
-	kubeletPid := ""
-
-	nodeIP, err := framework.GetHostExternalAddress(c, pod)
-	Expect(err).NotTo(HaveOccurred())
-	nodeIP = nodeIP + ":22"
-
-	framework.Logf("Checking if sudo command is present")
-	sshResult, err := framework.SSH("sudo --version", nodeIP, framework.TestContext.Provider)
-	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
-	if !strings.Contains(sshResult.Stderr, "command not found") {
-		sudoPresent = true
-	}
-
-	framework.Logf("Checking if systemctl command is present")
-	sshResult, err = framework.SSH("systemctl --version", nodeIP, framework.TestContext.Provider)
-	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
-	if !strings.Contains(sshResult.Stderr, "command not found") {
-		command = fmt.Sprintf("systemctl %s kubelet", string(kOp))
-		systemctlPresent = true
-	} else {
-		command = fmt.Sprintf("service kubelet %s", string(kOp))
-	}
-	if sudoPresent {
-		command = fmt.Sprintf("sudo %s", command)
-	}
-
-	if kOp == kRestart {
-		kubeletPid = getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent)
-	}
-
-	framework.Logf("Attempting `%s`", command)
-	sshResult, err = framework.SSH(command, nodeIP, framework.TestContext.Provider)
-	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
-	framework.LogSSHResult(sshResult)
-	Expect(sshResult.Code).To(BeZero(), "Failed to [%s] kubelet:\n%#v", string(kOp), sshResult)
-
-	if kOp == kStop {
-		if ok := framework.WaitForNodeToBeNotReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok {
-			framework.Failf("Node %s failed to enter NotReady state", pod.Spec.NodeName)
-		}
-	}
-	if kOp == kRestart {
-		// Wait for a minute to check if kubelet Pid is getting changed
-		isPidChanged := false
-		for start := time.Now(); time.Since(start) < 1*time.Minute; time.Sleep(2 * time.Second) {
-			kubeletPidAfterRestart := getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent)
-			if kubeletPid != kubeletPidAfterRestart {
-				isPidChanged = true
-				break
-			}
-		}
-		Expect(isPidChanged).To(BeTrue(), "Kubelet PID remained unchanged after restarting Kubelet")
-		framework.Logf("Noticed that kubelet PID is changed. Waiting for 30 Seconds for Kubelet to come back")
-		time.Sleep(30 * time.Second)
-	}
-	if kOp == kStart || kOp == kRestart {
-		// For kubelet start and restart operations, Wait until Node becomes Ready
-		if ok := framework.WaitForNodeToBeReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok {
-			framework.Failf("Node %s failed to enter Ready state", pod.Spec.NodeName)
-		}
-	}
-}
-
-// return the Main PID of the Kubelet Process
-func getKubeletMainPid(nodeIP string, sudoPresent bool, systemctlPresent bool) string {
-	command := ""
-	if systemctlPresent {
-		command = "systemctl status kubelet | grep 'Main PID'"
-	} else {
-		command = "service kubelet status | grep 'Main PID'"
-	}
-	if sudoPresent {
-		command = fmt.Sprintf("sudo %s", command)
-	}
-	framework.Logf("Attempting `%s`", command)
-	sshResult, err := framework.SSH(command, nodeIP, framework.TestContext.Provider)
-	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", nodeIP))
-	framework.LogSSHResult(sshResult)
-	Expect(sshResult.Code).To(BeZero(), "Failed to get kubelet PID")
-	Expect(sshResult.Stdout).NotTo(BeEmpty(), "Kubelet Main PID should not be Empty")
-	return sshResult.Stdout
-}
-
-// podExec wraps RunKubectl to execute a bash cmd in target pod
-func podExec(pod *v1.Pod, bashExec string) (string, error) {
-	return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--", "/bin/sh", "-c", bashExec)
-}

test/e2e/storage/persistent_volumes-gce.go

@@ -26,6 +26,7 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 // verifyGCEDiskAttached performs a sanity check to verify the PD attached to the node
@@ -51,7 +52,7 @@ func initializeGCETestSpec(c clientset.Interface, ns string, pvConfig framework.
 }
 
 // Testing configurations of single a PV/PVC pair attached to a GCE PD
-var _ = SIGDescribe("PersistentVolumes GCEPD", func() {
+var _ = utils.SIGDescribe("PersistentVolumes GCEPD", func() {
 	var (
 		c         clientset.Interface
 		diskName  string

test/e2e/storage/persistent_volumes-local.go

@@ -44,6 +44,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 )
 
@@ -138,7 +139,7 @@ var (
 		Level: "s0:c0,c1"}
 )
 
-var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [Serial]", func() {
+var _ = utils.SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [Serial]", func() {
 	f := framework.NewDefaultFramework("persistent-local-volumes-test")
 
 	var (
@@ -835,7 +836,7 @@ func createFileDoesntExistCmd(testFileDir string, testFile string) string {
 // Execute a read or write command in a pod.
 // Fail on error
 func podRWCmdExec(pod *v1.Pod, cmd string) string {
-	out, err := podExec(pod, cmd)
+	out, err := utils.PodExec(pod, cmd)
 	Expect(err).NotTo(HaveOccurred())
 	return out
 }

test/e2e/storage/persistent_volumes.go

@@ -29,6 +29,7 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 // Validate PV/PVC, create and verify writer pod, delete the PVC, and validate the PV's
@@ -85,7 +86,7 @@ func completeMultiTest(f *framework.Framework, c clientset.Interface, ns string,
 	return nil
 }
 
-var _ = SIGDescribe("PersistentVolumes", func() {
+var _ = utils.SIGDescribe("PersistentVolumes", func() {
 
 	// global vars for the Context()s and It()'s below
 	f := framework.NewDefaultFramework("pv")

test/e2e/storage/pvc_protection.go

@@ -30,9 +30,10 @@ import (
 	"k8s.io/kubernetes/pkg/util/slice"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
-var _ = SIGDescribe("PVC Protection [Feature:PVCProtection]", func() {
+var _ = utils.SIGDescribe("PVC Protection [Feature:PVCProtection]", func() {
 	var (
 		client                  clientset.Interface
 		nameSpace               string

test/e2e/storage/utils/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "framework.go",
+        "utils.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/e2e/storage/utils",
+    deps = [
+        "//test/e2e/framework:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

test/e2e/storage/utils/framework.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package utils
 
 import "github.com/onsi/ginkgo"
 

test/e2e/storage/utils/utils.go

@@ -0,0 +1,240 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+type KubeletOpt string
+
+const (
+	NodeStateTimeout            = 1 * time.Minute
+	KStart           KubeletOpt = "start"
+	KStop            KubeletOpt = "stop"
+	KRestart         KubeletOpt = "restart"
+)
+
+// PodExec wraps RunKubectl to execute a bash cmd in target pod
+func PodExec(pod *v1.Pod, bashExec string) (string, error) {
+	return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--", "/bin/sh", "-c", bashExec)
+}
+
+// KubeletCommand performs `start`, `restart`, or `stop` on the kubelet running on the node of the target pod and waits
+// for the desired statues..
+// - First issues the command via `systemctl`
+// - If `systemctl` returns stderr "command not found, issues the command via `service`
+// - If `service` also returns stderr "command not found", the test is aborted.
+// Allowed kubeletOps are `KStart`, `KStop`, and `KRestart`
+func KubeletCommand(kOp KubeletOpt, c clientset.Interface, pod *v1.Pod) {
+	command := ""
+	sudoPresent := false
+	systemctlPresent := false
+	kubeletPid := ""
+
+	nodeIP, err := framework.GetHostExternalAddress(c, pod)
+	Expect(err).NotTo(HaveOccurred())
+	nodeIP = nodeIP + ":22"
+
+	framework.Logf("Checking if sudo command is present")
+	sshResult, err := framework.SSH("sudo --version", nodeIP, framework.TestContext.Provider)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
+	if !strings.Contains(sshResult.Stderr, "command not found") {
+		sudoPresent = true
+	}
+
+	framework.Logf("Checking if systemctl command is present")
+	sshResult, err = framework.SSH("systemctl --version", nodeIP, framework.TestContext.Provider)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
+	if !strings.Contains(sshResult.Stderr, "command not found") {
+		command = fmt.Sprintf("systemctl %s kubelet", string(kOp))
+		systemctlPresent = true
+	} else {
+		command = fmt.Sprintf("service kubelet %s", string(kOp))
+	}
+	if sudoPresent {
+		command = fmt.Sprintf("sudo %s", command)
+	}
+
+	if kOp == KRestart {
+		kubeletPid = getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent)
+	}
+
+	framework.Logf("Attempting `%s`", command)
+	sshResult, err = framework.SSH(command, nodeIP, framework.TestContext.Provider)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
+	framework.LogSSHResult(sshResult)
+	Expect(sshResult.Code).To(BeZero(), "Failed to [%s] kubelet:\n%#v", string(kOp), sshResult)
+
+	if kOp == KStop {
+		if ok := framework.WaitForNodeToBeNotReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok {
+			framework.Failf("Node %s failed to enter NotReady state", pod.Spec.NodeName)
+		}
+	}
+	if kOp == KRestart {
+		// Wait for a minute to check if kubelet Pid is getting changed
+		isPidChanged := false
+		for start := time.Now(); time.Since(start) < 1*time.Minute; time.Sleep(2 * time.Second) {
+			kubeletPidAfterRestart := getKubeletMainPid(nodeIP, sudoPresent, systemctlPresent)
+			if kubeletPid != kubeletPidAfterRestart {
+				isPidChanged = true
+				break
+			}
+		}
+		Expect(isPidChanged).To(BeTrue(), "Kubelet PID remained unchanged after restarting Kubelet")
+		framework.Logf("Noticed that kubelet PID is changed. Waiting for 30 Seconds for Kubelet to come back")
+		time.Sleep(30 * time.Second)
+	}
+	if kOp == KStart || kOp == KRestart {
+		// For kubelet start and restart operations, Wait until Node becomes Ready
+		if ok := framework.WaitForNodeToBeReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok {
+			framework.Failf("Node %s failed to enter Ready state", pod.Spec.NodeName)
+		}
+	}
+}
+
+// getKubeletMainPid return the Main PID of the Kubelet Process
+func getKubeletMainPid(nodeIP string, sudoPresent bool, systemctlPresent bool) string {
+	command := ""
+	if systemctlPresent {
+		command = "systemctl status kubelet | grep 'Main PID'"
+	} else {
+		command = "service kubelet status | grep 'Main PID'"
+	}
+	if sudoPresent {
+		command = fmt.Sprintf("sudo %s", command)
+	}
+	framework.Logf("Attempting `%s`", command)
+	sshResult, err := framework.SSH(command, nodeIP, framework.TestContext.Provider)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", nodeIP))
+	framework.LogSSHResult(sshResult)
+	Expect(sshResult.Code).To(BeZero(), "Failed to get kubelet PID")
+	Expect(sshResult.Stdout).NotTo(BeEmpty(), "Kubelet Main PID should not be Empty")
+	return sshResult.Stdout
+}
+
+// TestKubeletRestartsAndRestoresMount tests that a volume mounted to a pod remains mounted after a kubelet restarts
+func TestKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
+	By("Writing to the volume.")
+	file := "/mnt/_SUCCESS"
+	out, err := PodExec(clientPod, fmt.Sprintf("touch %s", file))
+	framework.Logf(out)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Restarting kubelet")
+	KubeletCommand(KRestart, c, clientPod)
+
+	By("Testing that written file is accessible.")
+	out, err = PodExec(clientPod, fmt.Sprintf("cat %s", file))
+	framework.Logf(out)
+	Expect(err).NotTo(HaveOccurred())
+	framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, file)
+}
+
+// TestVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down.
+func TestVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
+	nodeIP, err := framework.GetHostExternalAddress(c, clientPod)
+	Expect(err).NotTo(HaveOccurred())
+	nodeIP = nodeIP + ":22"
+
+	By("Expecting the volume mount to be found.")
+	result, err := framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider)
+	framework.LogSSHResult(result)
+	Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
+	Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
+
+	By("Stopping the kubelet.")
+	KubeletCommand(KStop, c, clientPod)
+	defer func() {
+		if err != nil {
+			KubeletCommand(KStart, c, clientPod)
+		}
+	}()
+	By(fmt.Sprintf("Deleting Pod %q", clientPod.Name))
+	err = c.CoreV1().Pods(clientPod.Namespace).Delete(clientPod.Name, &metav1.DeleteOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	By("Starting the kubelet and waiting for pod to delete.")
+	KubeletCommand(KStart, c, clientPod)
+	err = f.WaitForPodTerminated(clientPod.Name, "")
+	if !apierrs.IsNotFound(err) && err != nil {
+		Expect(err).NotTo(HaveOccurred(), "Expected pod to terminate.")
+	}
+
+	By("Expecting the volume mount not to be found.")
+	result, err = framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider)
+	framework.LogSSHResult(result)
+	Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
+	Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no mount found).")
+	framework.Logf("Volume unmounted on node %s", clientPod.Spec.NodeName)
+}
+
+// RunInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
+func RunInPodWithVolume(c clientset.Interface, ns, claimName, command string) {
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-volume-tester-",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:    "volume-tester",
+					Image:   "busybox",
+					Command: []string{"/bin/sh"},
+					Args:    []string{"-c", command},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "my-volume",
+							MountPath: "/mnt/test",
+						},
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+			Volumes: []v1.Volume{
+				{
+					Name: "my-volume",
+					VolumeSource: v1.VolumeSource{
+						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+							ClaimName: claimName,
+							ReadOnly:  false,
+						},
+					},
+				},
+			},
+		},
+	}
+	pod, err := c.CoreV1().Pods(ns).Create(pod)
+	framework.ExpectNoError(err, "Failed to create pod: %v", err)
+	defer func() {
+		framework.DeletePodOrFail(c, ns, pod.Name)
+	}()
+	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
+}

test/e2e/storage/volume_expand.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -38,7 +39,7 @@ const (
 	totalResizeWaitPeriod = 20 * time.Minute
 )
 
-var _ = SIGDescribe("Volume expand [Feature:ExpandPersistentVolumes] [Slow]", func() {
+var _ = utils.SIGDescribe("Volume expand [Feature:ExpandPersistentVolumes] [Slow]", func() {
 	var (
 		c           clientset.Interface
 		ns          string

test/e2e/storage/volume_io.go

@@ -40,6 +40,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -126,7 +127,7 @@ func writeToFile(pod *v1.Pod, fpath, dd_input string, fsize int64) error {
 	By(fmt.Sprintf("writing %d bytes to test file %s", fsize, fpath))
 	loopCnt := fsize / minFileSize
 	writeCmd := fmt.Sprintf("i=0; while [ $i -lt %d ]; do dd if=%s bs=%d >>%s 2>/dev/null; let i+=1; done", loopCnt, dd_input, minFileSize, fpath)
-	_, err := podExec(pod, writeCmd)
+	_, err := utils.PodExec(pod, writeCmd)
 
 	return err
 }
@@ -134,7 +135,7 @@ func writeToFile(pod *v1.Pod, fpath, dd_input string, fsize int64) error {
 // Verify that the test file is the expected size and contains the expected content.
 func verifyFile(pod *v1.Pod, fpath string, expectSize int64, dd_input string) error {
 	By("verifying file size")
-	rtnstr, err := podExec(pod, fmt.Sprintf("stat -c %%s %s", fpath))
+	rtnstr, err := utils.PodExec(pod, fmt.Sprintf("stat -c %%s %s", fpath))
 	if err != nil || rtnstr == "" {
 		return fmt.Errorf("unable to get file size via `stat %s`: %v", fpath, err)
 	}
@@ -147,7 +148,7 @@ func verifyFile(pod *v1.Pod, fpath string, expectSize int64, dd_input string) er
 	}
 
 	By("verifying file hash")
-	rtnstr, err = podExec(pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath))
+	rtnstr, err = utils.PodExec(pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath))
 	if err != nil {
 		return fmt.Errorf("unable to test file hash via `md5sum %s`: %v", fpath, err)
 	}
@@ -168,7 +169,7 @@ func verifyFile(pod *v1.Pod, fpath string, expectSize int64, dd_input string) er
 // Delete `fpath` to save some disk space on host. Delete errors are logged but ignored.
 func deleteFile(pod *v1.Pod, fpath string) {
 	By(fmt.Sprintf("deleting test file %s...", fpath))
-	_, err := podExec(pod, fmt.Sprintf("rm -f %s", fpath))
+	_, err := utils.PodExec(pod, fmt.Sprintf("rm -f %s", fpath))
 	if err != nil {
 		// keep going, the test dir will be deleted when the volume is unmounted
 		framework.Logf("unable to delete test file %s: %v\nerror ignored, continuing test", fpath, err)
@@ -237,7 +238,7 @@ func testVolumeIO(f *framework.Framework, cs clientset.Interface, config framewo
 
 // These tests need privileged containers which are disabled by default.
 // TODO: support all of the plugins tested in storage/volumes.go
-var _ = SIGDescribe("Volume plugin streaming [Slow]", func() {
+var _ = utils.SIGDescribe("Volume plugin streaming [Slow]", func() {
 	f := framework.NewDefaultFramework("volume-io")
 	var (
 		config    framework.VolumeTestConfig

test/e2e/storage/volume_metrics.go

@@ -30,11 +30,12 @@ import (
 	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/framework/metrics"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 // This test needs to run in serial because other tests could interfere
 // with metrics being tested here.
-var _ = SIGDescribe("[Serial] Volume metrics", func() {
+var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 	var (
 		c              clientset.Interface
 		ns             string

test/e2e/storage/volume_provisioning.go

@@ -45,6 +45,7 @@ import (
 	storageutil "k8s.io/kubernetes/pkg/apis/storage/v1/util"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 type storageClassTest struct {
@@ -229,7 +230,7 @@ func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
 	return nil
 }
 
-var _ = SIGDescribe("Dynamic Provisioning", func() {
+var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 	f := framework.NewDefaultFramework("volume-provisioning")
 
 	// filled in BeforeEach

test/e2e/storage/volumes.go

@@ -55,6 +55,8 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
+	vspheretest "k8s.io/kubernetes/test/e2e/storage/vsphere"
 )
 
 func DeleteCinderVolume(name string) error {
@@ -79,7 +81,7 @@ func DeleteCinderVolume(name string) error {
 }
 
 // These tests need privileged containers, which are disabled by default.
-var _ = SIGDescribe("Volumes", func() {
+var _ = utils.SIGDescribe("Volumes", func() {
 	f := framework.NewDefaultFramework("volume")
 
 	// note that namespace deletion is handled by delete-namespace flag
@@ -510,10 +512,10 @@ var _ = SIGDescribe("Volumes", func() {
 			if err != nil {
 				return
 			}
-			vsp, err := getVSphere(c)
+			vsp, err := vspheretest.GetVSphere(c)
 			Expect(err).NotTo(HaveOccurred())
 
-			volumePath, err = createVSphereVolume(vsp, nil)
+			volumePath, err = vspheretest.CreateVSphereVolume(vsp, nil)
 			Expect(err).NotTo(HaveOccurred())
 
 			defer func() {

test/e2e/storage/vsphere/BUILD

@@ -0,0 +1,67 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "persistent_volumes-vsphere.go",
+        "pv_reclaimpolicy.go",
+        "pvc_label_selector.go",
+        "vsphere_scale.go",
+        "vsphere_statefulsets.go",
+        "vsphere_stress.go",
+        "vsphere_utils.go",
+        "vsphere_volume_cluster_ds.go",
+        "vsphere_volume_datastore.go",
+        "vsphere_volume_diskformat.go",
+        "vsphere_volume_disksize.go",
+        "vsphere_volume_fstype.go",
+        "vsphere_volume_master_restart.go",
+        "vsphere_volume_node_poweroff.go",
+        "vsphere_volume_ops_storm.go",
+        "vsphere_volume_perf.go",
+        "vsphere_volume_placement.go",
+        "vsphere_volume_vsan_policy.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/e2e/storage/vsphere",
+    deps = [
+        "//pkg/cloudprovider/providers/vsphere:go_default_library",
+        "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
+        "//pkg/volume/util/volumehelper:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/storage/utils:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/github.com/vmware/govmomi/find:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/api/storage/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

test/e2e/storage/vsphere/persistent_volumes-vsphere.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"time"
@@ -28,10 +28,11 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 // Testing configurations of single a PV/PVC pair attached to a vSphere Disk
-var _ = SIGDescribe("PersistentVolumes:vsphere", func() {
+var _ = utils.SIGDescribe("PersistentVolumes:vsphere", func() {
 	var (
 		c          clientset.Interface
 		ns         string
@@ -182,7 +183,7 @@ var _ = SIGDescribe("PersistentVolumes:vsphere", func() {
 		3. Verify that written file is accessible after kubelet restart
 	*/
 	It("should test that a file written to the vspehre volume mount before kubelet restart can be read after restart [Disruptive]", func() {
-		testKubeletRestartsAndRestoresMount(c, f, clientPod, pvc, pv)
+		utils.TestKubeletRestartsAndRestoresMount(c, f, clientPod, pvc, pv)
 	})
 
 	/*
@@ -197,7 +198,7 @@ var _ = SIGDescribe("PersistentVolumes:vsphere", func() {
 		5. Verify that volume mount not to be found.
 	*/
 	It("should test that a vspehre volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns [Disruptive]", func() {
-		testVolumeUnmountsFromDeletedPod(c, f, clientPod, pvc, pv)
+		utils.TestVolumeUnmountsFromDeletedPod(c, f, clientPod, pvc, pv)
 	})
 
 	/*

test/e2e/storage/vsphere/pv_reclaimpolicy.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"strconv"
@@ -29,9 +29,10 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
-var _ = SIGDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func() {
+var _ = utils.SIGDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func() {
 	f := framework.NewDefaultFramework("persistentvolumereclaim")
 	var (
 		c          clientset.Interface
@@ -47,7 +48,7 @@ var _ = SIGDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func() {
 		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
 	})
 
-	SIGDescribe("persistentvolumereclaim:vsphere", func() {
+	utils.SIGDescribe("persistentvolumereclaim:vsphere", func() {
 		BeforeEach(func() {
 			framework.SkipUnlessProviderIs("vsphere")
 			pv = nil

test/e2e/storage/vsphere/pvc_label_selector.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"time"
@@ -24,6 +24,7 @@ import (
 	"k8s.io/api/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -43,7 +44,7 @@ import (
    9. delete pvc_vvol
 
 */
-var _ = SIGDescribe("PersistentVolumes [Feature:LabelSelector]", func() {
+var _ = utils.SIGDescribe("PersistentVolumes [Feature:LabelSelector]", func() {
 	f := framework.NewDefaultFramework("pvclabelselector")
 	var (
 		c          clientset.Interface
@@ -68,7 +69,7 @@ var _ = SIGDescribe("PersistentVolumes [Feature:LabelSelector]", func() {
 
 	})
 
-	SIGDescribe("Selector-Label Volume Binding:vsphere", func() {
+	utils.SIGDescribe("Selector-Label Volume Binding:vsphere", func() {
 		AfterEach(func() {
 			By("Running clean up actions")
 			if framework.ProviderIs("vsphere") {

test/e2e/storage/vsphere/vsphere_scale.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -30,6 +30,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -52,7 +53,7 @@ type NodeSelector struct {
 	labelValue string
 }
 
-var _ = SIGDescribe("vcp at scale [Feature:vsphere] ", func() {
+var _ = utils.SIGDescribe("vcp at scale [Feature:vsphere] ", func() {
 	f := framework.NewDefaultFramework("vcp-at-scale")
 
 	var (

test/e2e/storage/vsphere/vsphere_statefulsets.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -49,7 +50,7 @@ const (
 	storageclassname = "nginx-sc"
 )
 
-var _ = SIGDescribe("vsphere statefulset", func() {
+var _ = utils.SIGDescribe("vsphere statefulset", func() {
 	f := framework.NewDefaultFramework("vsphere-statefulset")
 	var (
 		namespace string

test/e2e/storage/vsphere/vsphere_stress.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -31,6 +31,7 @@ import (
 	k8stype "k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -43,7 +44,7 @@ import (
 	4. Each instance of routine iterates for n times, where n is read from system env - VCP_STRESS_ITERATIONS
 	5. Each iteration creates 1 PVC, 1 POD using the provisioned PV, Verify disk is attached to the node, Verify pod can access the volume, delete the pod and finally delete the PVC.
 */
-var _ = SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("vcp-stress")
 	var (
 		client        clientset.Interface

test/e2e/storage/vsphere/vsphere_utils.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -36,6 +36,7 @@ import (
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
 	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -250,15 +251,20 @@ func createVSphereVolume(vsp *vsphere.VSphere, volumeOptions *vclib.VolumeOption
 	return volumePath, nil
 }
 
+// CreateVSphereVolume creates a vmdk volume
+func CreateVSphereVolume(vsp *vsphere.VSphere, volumeOptions *vclib.VolumeOptions) (string, error) {
+	return createVSphereVolume(vsp, volumeOptions)
+}
+
 // function to write content to the volume backed by given PVC
 func writeContentToVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) {
-	runInPodWithVolume(client, pvc.Namespace, pvc.Name, "echo "+expectedContent+" > /mnt/test/data")
+	utils.RunInPodWithVolume(client, pvc.Namespace, pvc.Name, "echo "+expectedContent+" > /mnt/test/data")
 	framework.Logf("Done with writing content to volume")
 }
 
 // function to verify content is matching on the volume backed for given PVC
 func verifyContentOfVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) {
-	runInPodWithVolume(client, pvc.Namespace, pvc.Name, "grep '"+expectedContent+"' /mnt/test/data")
+	utils.RunInPodWithVolume(client, pvc.Namespace, pvc.Name, "grep '"+expectedContent+"' /mnt/test/data")
 	framework.Logf("Successfully verified content of the volume")
 }
 
@@ -457,3 +463,8 @@ func getVSphere(c clientset.Interface) (*vsphere.VSphere, error) {
 	addNodesToVCP(vsp, c)
 	return vsp, nil
 }
+
+// GetVSphere returns vsphere cloud provider
+func GetVSphere(c clientset.Interface) (*vsphere.VSphere, error) {
+	return getVSphere(c)
+}

test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -27,6 +27,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -39,7 +40,7 @@ import (
 	1. CLUSTER_DATASTORE which should be set to clustered datastore
 	2. VSPHERE_SPBM_POLICY_DS_CLUSTER which should be set to a tag based spbm policy tagged to a clustered datastore
 */
-var _ = SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("volume-provision")
 
 	var client clientset.Interface

test/e2e/storage/vsphere/vsphere_volume_datastore.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -27,6 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -44,7 +45,7 @@ const (
 	4. Verify the error returned on PVC failure is the correct.
 */
 
-var _ = SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("volume-datastore")
 	var (
 		client       clientset.Interface

test/e2e/storage/vsphere/vsphere_volume_diskformat.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"os"
@@ -32,6 +32,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -52,7 +53,7 @@ import (
 	11. Delete PVC, PV and Storage Class
 */
 
-var _ = SIGDescribe("Volume Disk Format [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("Volume Disk Format [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("volume-disk-format")
 	var (
 		client            clientset.Interface

test/e2e/storage/vsphere/vsphere_volume_disksize.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -28,6 +28,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -44,7 +45,7 @@ const (
 	4. Verify the error returned on PVC failure is the correct.
 */
 
-var _ = SIGDescribe("Volume Disk Size [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("Volume Disk Size [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("volume-disksize")
 	var (
 		client       clientset.Interface

test/e2e/storage/vsphere/vsphere_volume_fstype.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"strings"
@@ -28,6 +28,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -63,7 +64,7 @@ const (
 	7. Verify if the MountVolume.MountDevice fails because it is unable to find the file system executable file on the node.
 */
 
-var _ = SIGDescribe("Volume FStype [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("Volume FStype [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("volume-fstype")
 	var (
 		client    clientset.Interface

test/e2e/storage/vsphere/vsphere_volume_master_restart.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -41,7 +42,7 @@ import (
 	6. Delete the pod and wait for the volume to be detached
 	7. Delete the volume
 */
-var _ = SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disruptive]", func() {
+var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disruptive]", func() {
 	f := framework.NewDefaultFramework("restart-master")
 
 	const labelKey = "vsphere_e2e_label"

test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -36,6 +36,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -43,7 +44,7 @@ import (
 	1. Verify the pod got provisioned on a different node with volume attached to it
 	2. Verify the volume is detached from the powered off node
 */
-var _ = SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", func() {
+var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]", func() {
 	f := framework.NewDefaultFramework("node-poweroff")
 	var (
 		client     clientset.Interface

test/e2e/storage/vsphere/vsphere_volume_ops_storm.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -29,6 +29,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /*
@@ -47,7 +48,7 @@ import (
 		10. Delete storage class.
 */
 
-var _ = SIGDescribe("Volume Operations Storm [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("Volume Operations Storm [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("volume-ops-storm")
 	const DEFAULT_VOLUME_OPS_SCALE = 30
 	var (

test/e2e/storage/vsphere/vsphere_volume_perf.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 /* This test calculates latency numbers for volume lifecycle operations
@@ -48,7 +49,7 @@ const (
 	DeleteOp                  = "DeleteOp"
 )
 
-var _ = SIGDescribe("vcp-performance [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("vcp-performance [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("vcp-performance")
 
 	var (

test/e2e/storage/vsphere/vsphere_volume_placement.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -31,9 +31,10 @@ import (
 	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
-var _ = SIGDescribe("Volume Placement", func() {
+var _ = utils.SIGDescribe("Volume Placement", func() {
 	f := framework.NewDefaultFramework("volume-placement")
 	var (
 		c                  clientset.Interface

test/e2e/storage/vsphere/vsphere_volume_vsan_policy.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package storage
+package vsphere
 
 import (
 	"fmt"
@@ -26,6 +26,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vmware/govmomi/find"
 	"golang.org/x/net/context"
 	"k8s.io/api/core/v1"
@@ -34,6 +35,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
 const (
@@ -90,7 +92,7 @@ const (
 
 */
 
-var _ = SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsphere]", func() {
+var _ = utils.SIGDescribe("Storage Policy Based Volume Provisioning [Feature:vsphere]", func() {
 	f := framework.NewDefaultFramework("volume-vsan-policy")
 	var (
 		client       clientset.Interface