e2e test: test azure disk volume

Signed-off-by: Huamin Chen <hchen@redhat.com>

hack/ginkgo-e2e.sh

@@ -29,6 +29,7 @@ e2e_test=$(kube::util::find-binary "e2e.test")
 # --- Setup some env vars.
 
 GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel
+CLOUD_CONFIG=${CLOUD_CONFIG:-""}
 
 # If 'y', Ginkgo's reporter will not print out in color when tests are run
 # in parallel
@@ -97,6 +98,13 @@ if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
   NODE_INSTANCE_GROUP=$(kube::util::join , "${NODE_INSTANCE_GROUPS[@]}")
 fi
 
+if [[ "${KUBERNETES_PROVIDER}" == "azure" ]]; then
+    if [[ ${CLOUD_CONFIG} == "" ]]; then
+        echo "Missing azure cloud config"
+        exit 1
+    fi
+fi
+
 ginkgo_args=()
 if [[ -n "${CONFORMANCE_TEST_SKIP_REGEX:-}" ]]; then
   ginkgo_args+=("--skip=${CONFORMANCE_TEST_SKIP_REGEX}")
@@ -137,6 +145,7 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}"
   --gke-cluster="${CLUSTER_NAME:-}" \
   --kube-master="${KUBE_MASTER:-}" \
   --cluster-tag="${CLUSTER_ID:-}" \
+  --cloud-config-file="${CLOUD_CONFIG:-}" \
   --repo-root="${KUBE_ROOT}" \
   --node-instance-group="${NODE_INSTANCE_GROUP:-}" \
   --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \

hack/verify-flags/known-flags.txt

@@ -97,6 +97,7 @@ clientset-name
 clientset-only
 clientset-path
 cloud-config
+cloud-config-file
 cloud-provider
 cloud-provider-gce-lb-src-cidrs
 cluster-cidr

test/e2e/BUILD

@@ -130,6 +130,7 @@ go_library(
         "//pkg/client/clientset_generated/clientset/typed/extensions/v1beta1:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/cloudprovider:go_default_library",
+        "//pkg/cloudprovider/providers/azure:go_default_library",
         "//pkg/cloudprovider/providers/gce:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/controller/daemon:go_default_library",

test/e2e/e2e.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
 	"k8s.io/kubernetes/pkg/util/logs"
 	commontest "k8s.io/kubernetes/test/e2e/common"
@@ -85,6 +86,16 @@ func setupProviderConfig() error {
 		if cloudConfig.Zone == "" {
 			return fmt.Errorf("gce-zone must be specified for AWS")
 		}
+	case "azure":
+		if cloudConfig.ConfigFile == "" {
+			return fmt.Errorf("config-file must be specified for Azure")
+		}
+		config, err := os.Open(cloudConfig.ConfigFile)
+		if err != nil {
+			framework.Logf("Couldn't open cloud provider configuration %s: %#v",
+				cloudConfig.ConfigFile, err)
+		}
+		cloudConfig.Provider, err = azure.NewCloud(config)
 	}
 
 	return nil

test/e2e/framework/BUILD

@@ -64,6 +64,7 @@ go_library(
         "//pkg/client/unversioned/remotecommand:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/cloudprovider/providers/aws:go_default_library",
+        "//pkg/cloudprovider/providers/azure:go_default_library",
         "//pkg/cloudprovider/providers/gce:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/controller/deployment/util:go_default_library",

test/e2e/framework/pv_util.go

@@ -689,6 +689,19 @@ func createPD() (string, error) {
 
 		volumeName := "aws://" + az + "/" + awsID
 		return volumeName, nil
+	} else if TestContext.Provider == "azure" {
+		pdName := fmt.Sprintf("%s-%s", TestContext.Prefix, string(uuid.NewUUID()))
+		azureCloud, err := GetAzureCloud()
+		if err != nil {
+			return "", err
+		}
+
+		_, diskUri, _, err := azureCloud.CreateVolume(pdName, "" /* account */, "" /* sku */, "" /* location */, 1 /* sizeGb */)
+		if err != nil {
+			return "", err
+		}
+		return diskUri, nil
+
 	} else {
 		return "", fmt.Errorf("provider does not support volume creation")
 	}
@@ -728,6 +741,18 @@ func deletePD(pdName string) error {
 			}
 		}
 		return nil
+	} else if TestContext.Provider == "azure" {
+		azureCloud, err := GetAzureCloud()
+		if err != nil {
+			return err
+		}
+		diskName := pdName[(strings.LastIndex(pdName, "/") + 1):]
+		err = azureCloud.DeleteVolume(diskName, pdName)
+		if err != nil {
+			Logf("failed to delete Azure volume %q: %v", pdName, err)
+			return err
+		}
+		return nil
 	} else {
 		return fmt.Errorf("provider does not support volume deletion")
 	}

test/e2e/framework/test_context.go

@@ -141,6 +141,7 @@ type CloudConfig struct {
 	NumNodes          int
 	ClusterTag        string
 	Network           string
+	ConfigFile        string // for azure and openstack
 
 	Provider cloudprovider.Interface
 }
@@ -209,6 +210,7 @@ func RegisterClusterFlags() {
 	flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster")
 
 	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")
+	flag.StringVar(&cloudConfig.ConfigFile, "cloud-config-file", "", "Cloud config file.  Only required if provider is azure.")
 	flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
 	flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.")
 	flag.DurationVar(&TestContext.NodeSchedulableTimeout, "node-schedulable-timeout", 4*time.Hour, "Timeout for waiting for all nodes to be schedulable.")

test/e2e/framework/util.go

@@ -83,6 +83,7 @@ import (
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/client/conditions"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
 	"k8s.io/kubernetes/pkg/controller"
 	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
@@ -5510,3 +5511,12 @@ func CreateEmptyFileOnPod(namespace string, podName string, filePath string) err
 	_, err := RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, "--", "/bin/sh", "-c", fmt.Sprintf("touch %s", filePath))
 	return err
 }
+
+// GetAzureCloud returns azure cloud provider
+func GetAzureCloud() (*azure.Cloud, error) {
+	cloud, ok := TestContext.CloudConfig.Provider.(*azure.Cloud)
+	if !ok {
+		return nil, fmt.Errorf("failed to convert CloudConfig.Provider to Azure: %#v", TestContext.CloudConfig.Provider)
+	}
+	return cloud, nil
+}

test/e2e/storage/volume_provisioning.go

@@ -356,6 +356,15 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
 					"1.5Gi",
 					nil,
 				},
+				{
+					"Azure disk volume with empty sku and location",
+					[]string{"azure"},
+					"kubernetes.io/azure-disk",
+					map[string]string{},
+					"1Gi",
+					"1Gi",
+					nil,
+				},
 			}
 
 			var betaTest *storageClassTest
@@ -463,7 +472,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
 			// not being deleted.
 			// NOTE:  Polls until no PVs are detected, times out at 5 minutes.
 
-			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
+			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
 
 			const raceAttempts int = 100
 			var residualPVs []*v1.PersistentVolume
@@ -561,7 +570,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
 
 		// Modifying the default storage class can be disruptive to other tests that depend on it
 		It("should be disabled by changing the default annotation[Slow] [Serial] [Disruptive] [Volume]", func() {
-			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
+			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
 			scName := getDefaultStorageClassName(c)
 			test := storageClassTest{
 				name:      "default",
@@ -592,7 +601,7 @@ var _ = framework.KubeDescribe("Dynamic Provisioning", func() {
 
 		// Modifying the default storage class can be disruptive to other tests that depend on it
 		It("should be disabled by removing the default annotation[Slow] [Serial] [Disruptive] [Volume]", func() {
-			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
+			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
 			scName := getDefaultStorageClassName(c)
 			test := storageClassTest{
 				name:      "default",
@@ -753,6 +762,8 @@ func getDefaultPluginName() string {
 		return "kubernetes.io/cinder"
 	case framework.ProviderIs("vsphere"):
 		return "kubernetes.io/vsphere-volume"
+	case framework.ProviderIs("azure"):
+		return "kubernetes.io/azure-disk"
 	}
 	return ""
 }

test/e2e/storage/volumes.go

@@ -682,6 +682,56 @@ var _ = framework.KubeDescribe("Volumes [Volume]", func() {
 
 			framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
 
+			fsGroup := int64(1234)
+			framework.TestVolumeClient(cs, config, &fsGroup, tests)
+		})
+	})
+	////////////////////////////////////////////////////////////////////////
+	// Azure Disk
+	////////////////////////////////////////////////////////////////////////
+	framework.KubeDescribe("Azure Disk [Feature:Volumes]", func() {
+		It("should be mountable [Slow]", func() {
+			framework.SkipUnlessProviderIs("azure")
+			config := framework.VolumeTestConfig{
+				Namespace: namespace.Name,
+				Prefix:    "azure",
+			}
+
+			By("creating a test azure disk volume")
+			volumeName, err := framework.CreatePDWithRetry()
+			Expect(err).NotTo(HaveOccurred())
+			defer func() {
+				framework.DeletePDWithRetry(volumeName)
+			}()
+
+			defer func() {
+				if clean {
+					framework.Logf("Running volumeTestCleanup")
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+			fsType := "ext4"
+			readOnly := false
+			diskName := volumeName[(strings.LastIndex(volumeName, "/") + 1):]
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						AzureDisk: &v1.AzureDiskVolumeSource{
+							DiskName:    diskName,
+							DataDiskURI: volumeName,
+							FSType:      &fsType,
+							ReadOnly:    &readOnly,
+						},
+					},
+					File: "index.html",
+					// Randomize index.html to make sure we don't see the
+					// content from previous test runs.
+					ExpectedContent: "Hello from Azure from namespace " + volumeName,
+				},
+			}
+
+			framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
+
 			fsGroup := int64(1234)
 			framework.TestVolumeClient(cs, config, &fsGroup, tests)
 		})