Perform resize of mounted volume if necessary

Add e2e test for mounted volume resize

pkg/util/resizefs/BUILD

@@ -56,7 +56,6 @@ go_library(
         "@io_bazel_rules_go//go/platform:linux": [
             "//pkg/util/mount:go_default_library",
             "//vendor/github.com/golang/glog:go_default_library",
-            "//vendor/k8s.io/utils/exec:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
             "//pkg/util/mount:go_default_library",

pkg/util/resizefs/resizefs_linux.go

@@ -23,14 +23,6 @@ import (
 
 	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/util/mount"
-	utilexec "k8s.io/utils/exec"
-)
-
-const (
-	// 'fsck' found errors and corrected them
-	fsckErrorsCorrected = 1
-	// 'fsck' found errors but exited without correcting them
-	fsckErrorsUncorrected = 4
 )
 
 // ResizeFs Provides support for resizing file systems
@@ -44,11 +36,11 @@ func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs {
 }
 
 // Resize perform resize of file system
-func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) {
+func (resizefs *ResizeFs) Resize(devicePath string, deviceMountPath string) (bool, error) {
 	format, err := resizefs.mounter.GetDiskFormat(devicePath)
 
 	if err != nil {
-		formatErr := fmt.Errorf("error checking format for device %s: %v", devicePath, err)
+		formatErr := fmt.Errorf("ResizeFS.Resize - error checking format for device %s: %v", devicePath, err)
 		return false, formatErr
 	}
 
@@ -58,63 +50,14 @@ func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) {
 		return false, nil
 	}
 
-	deviceOpened, err := resizefs.mounter.DeviceOpened(devicePath)
-
-	if err != nil {
-		deviceOpenErr := fmt.Errorf("error verifying if device %s is open: %v", devicePath, err)
-		return false, deviceOpenErr
-	}
-
-	if deviceOpened {
-		deviceAlreadyOpenErr := fmt.Errorf("the device %s is already in use", devicePath)
-		return false, deviceAlreadyOpenErr
-	}
-
+	glog.V(3).Infof("ResizeFS.Resize - Expanding mounted volume %s", devicePath)
 	switch format {
 	case "ext3", "ext4":
-		fsckErr := resizefs.extFsck(devicePath, format)
-		if fsckErr != nil {
-			return false, fsckErr
-		}
 		return resizefs.extResize(devicePath)
 	case "xfs":
-		fsckErr := resizefs.fsckDevice(devicePath)
-		if fsckErr != nil {
-			return false, fsckErr
-		}
-		return resizefs.xfsResize(devicePath)
+		return resizefs.xfsResize(deviceMountPath)
 	}
-	return false, fmt.Errorf("resize of format %s is not supported for device %s", format, devicePath)
-}
-
-func (resizefs *ResizeFs) fsckDevice(devicePath string) error {
-	glog.V(4).Infof("Checking for issues with fsck on device: %s", devicePath)
-	args := []string{"-a", devicePath}
-	out, err := resizefs.mounter.Exec.Run("fsck", args...)
-	if err != nil {
-		ee, isExitError := err.(utilexec.ExitError)
-		switch {
-		case err == utilexec.ErrExecutableNotFound:
-			glog.Warningf("'fsck' not found on system; continuing resizing without running 'fsck'.")
-		case isExitError && ee.ExitStatus() == fsckErrorsCorrected:
-			glog.V(2).Infof("Device %s has errors which were corrected by fsck: %s", devicePath, string(out))
-		case isExitError && ee.ExitStatus() == fsckErrorsUncorrected:
-			return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", devicePath, string(out))
-		case isExitError && ee.ExitStatus() > fsckErrorsUncorrected:
-			glog.Infof("`fsck` error %s", string(out))
-		}
-	}
-	return nil
-}
-
-func (resizefs *ResizeFs) extFsck(devicePath string, fsType string) error {
-	glog.V(4).Infof("Checking for issues with fsck.%s on device: %s", fsType, devicePath)
-	args := []string{"-f", "-y", devicePath}
-	out, err := resizefs.mounter.Run("fsck."+fsType, args...)
-	if err != nil {
-		return fmt.Errorf("running fsck.%s failed on %s with error: %v\n Output: %s", fsType, devicePath, err, string(out))
-	}
-	return nil
+	return false, fmt.Errorf("ResizeFS.Resize - resize of format %s is not supported for device %s mounted at %s", format, devicePath, deviceMountPath)
 }
 
 func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) {
@@ -129,15 +72,15 @@ func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) {
 
 }
 
-func (resizefs *ResizeFs) xfsResize(devicePath string) (bool, error) {
-	args := []string{"-d", devicePath}
+func (resizefs *ResizeFs) xfsResize(deviceMountPath string) (bool, error) {
+	args := []string{"-d", deviceMountPath}
 	output, err := resizefs.mounter.Exec.Run("xfs_growfs", args...)
 
 	if err == nil {
-		glog.V(2).Infof("Device %s resized successfully", devicePath)
+		glog.V(2).Infof("Device %s resized successfully", deviceMountPath)
 		return true, nil
 	}
 
-	resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", devicePath, err, string(output))
+	resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", deviceMountPath, err, string(output))
 	return false, resizeError
 }

pkg/util/resizefs/resizefs_unsupported.go

@@ -35,6 +35,6 @@ func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs {
 }
 
 // Resize perform resize of file system
-func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) {
+func (resizefs *ResizeFs) Resize(devicePath string, deviceMountPath string) (bool, error) {
 	return false, fmt.Errorf("Resize is not supported for this build")
 }

pkg/volume/util/operationexecutor/operation_generator.go

@@ -495,14 +495,6 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 
 			glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath)))
 
-			// resizeFileSystem will resize the file system if user has requested a resize of
-			// underlying persistent volume and is allowed to do so.
-			resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, volumePlugin.GetPluginName())
-
-			if resizeSimpleError != nil || resizeDetailedError != nil {
-				return resizeSimpleError, resizeDetailedError
-			}
-
 			deviceMountPath, err :=
 				volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec)
 			if err != nil {
@@ -529,6 +521,15 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 				// On failure, return error. Caller will log and retry.
 				return volumeToMount.GenerateError("MountVolume.MarkDeviceAsMounted failed", markDeviceMountedErr)
 			}
+
+			// resizeFileSystem will resize the file system if user has requested a resize of
+			// underlying persistent volume and is allowed to do so.
+			resizeSimpleError, resizeDetailedError := og.resizeFileSystem(volumeToMount, devicePath, deviceMountPath, volumePlugin.GetPluginName())
+
+			if resizeSimpleError != nil || resizeDetailedError != nil {
+				return resizeSimpleError, resizeDetailedError
+			}
+
 		}
 
 		if og.checkNodeCapabilitiesBeforeMount {
@@ -586,7 +587,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 	}, nil
 }
 
-func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath string, pluginName string) (simpleErr, detailedErr error) {
+func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath, deviceMountPath, pluginName string) (simpleErr, detailedErr error) {
 	if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
 		glog.V(6).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName)
 		return nil, nil
@@ -626,7 +627,7 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi
 			}
 
 			resizer := resizefs.NewResizeFs(diskFormatter)
-			resizeStatus, resizeErr := resizer.Resize(devicePath)
+			resizeStatus, resizeErr := resizer.Resize(devicePath, deviceMountPath)
 
 			if resizeErr != nil {
 				return volumeToMount.GenerateError("MountVolume.resizeFileSystem failed", resizeErr)

test/e2e/framework/deployment_util.go

@@ -216,8 +216,8 @@ func CheckDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName,
 	return testutils.CheckDeploymentRevisionAndImage(c, ns, deploymentName, revision, image)
 }
 
-func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, command string) (*extensions.Deployment, error) {
-	deploymentSpec := MakeDeployment(replicas, podLabels, namespace, pvclaims, false, command)
+func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, command string) (*extensions.Deployment, error) {
+	deploymentSpec := MakeDeployment(replicas, podLabels, nodeSelector, namespace, pvclaims, false, command)
 	deployment, err := client.ExtensionsV1beta1().Deployments(namespace).Create(deploymentSpec)
 	if err != nil {
 		return nil, fmt.Errorf("deployment %q Create API error: %v", deploymentSpec.Name, err)
@@ -232,7 +232,7 @@ func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[
 
 // MakeDeployment creates a deployment definition based on the namespace. The deployment references the PVC's
 // name.  A slice of BASH commands can be supplied as args to be run by the pod
-func MakeDeployment(replicas int32, podLabels map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *extensions.Deployment {
+func MakeDeployment(replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *extensions.Deployment {
 	if len(command) == 0 {
 		command = "while true; do sleep 1; done"
 	}
@@ -276,6 +276,9 @@ func MakeDeployment(replicas int32, podLabels map[string]string, namespace strin
 	}
 	deploymentSpec.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
 	deploymentSpec.Spec.Template.Spec.Volumes = volumes
+	if nodeSelector != nil {
+		deploymentSpec.Spec.Template.Spec.NodeSelector = nodeSelector
+	}
 	return deploymentSpec
 }
 

test/e2e/storage/BUILD

@@ -12,6 +12,7 @@ go_library(
         "csi_volumes.go",
         "empty_dir_wrapper.go",
         "flexvolume.go",
+        "mounted_volume_resize.go",
         "pd.go",
         "persistent_volumes.go",
         "persistent_volumes-disruptive.go",
@@ -29,6 +30,7 @@ go_library(
         "//pkg/api/testapi:go_default_library",
         "//pkg/apis/core/v1/helper:go_default_library",
         "//pkg/apis/storage/v1/util:go_default_library",
+        "//pkg/client/conditions:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",
         "//pkg/util/slice:go_default_library",

test/e2e/storage/mounted_volume_resize.go

@@ -0,0 +1,172 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
+	storage "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/client/conditions"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
+)
+
+var _ = utils.SIGDescribe("Mounted volume expand [Feature:ExpandPersistentVolumes] [Slow]", func() {
+	var (
+		c                 clientset.Interface
+		ns                string
+		err               error
+		pvc               *v1.PersistentVolumeClaim
+		resizableSc       *storage.StorageClass
+		nodeName          string
+		isNodeLabeled     bool
+		nodeKeyValueLabel map[string]string
+		nodeLabelValue    string
+		nodeKey           string
+	)
+
+	f := framework.NewDefaultFramework("mounted-volume-expand")
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("aws", "gce")
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+
+		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		if len(nodeList.Items) != 0 {
+			nodeName = nodeList.Items[0].Name
+		} else {
+			framework.Failf("Unable to find ready and schedulable Node")
+		}
+
+		nodeKey = "mounted_volume_expand"
+
+		if !isNodeLabeled {
+			nodeLabelValue = ns
+			nodeKeyValueLabel = make(map[string]string)
+			nodeKeyValueLabel[nodeKey] = nodeLabelValue
+			framework.AddOrUpdateLabelOnNode(c, nodeName, nodeKey, nodeLabelValue)
+			isNodeLabeled = true
+		}
+
+		test := storageClassTest{
+			name:      "default",
+			claimSize: "2Gi",
+		}
+		resizableSc, err = createResizableStorageClass(test, ns, "resizing", c)
+		Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class")
+		Expect(*resizableSc.AllowVolumeExpansion).To(BeTrue())
+
+		pvc = newClaim(test, ns, "default")
+		pvc.Spec.StorageClassName = &resizableSc.Name
+		pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc)
+		Expect(err).NotTo(HaveOccurred(), "Error creating pvc")
+	})
+
+	framework.AddCleanupAction(func() {
+		if len(nodeLabelValue) > 0 {
+			framework.RemoveLabelOffNode(c, nodeName, nodeKey)
+		}
+	})
+
+	AfterEach(func() {
+		framework.Logf("AfterEach: Cleaning up resources for mounted volume resize")
+
+		if c != nil {
+			if errs := framework.PVPVCCleanup(c, ns, nil, pvc); len(errs) > 0 {
+				framework.Failf("AfterEach: Failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs))
+			}
+			pvc, nodeName, isNodeLabeled, nodeLabelValue = nil, "", false, ""
+			nodeKeyValueLabel = make(map[string]string)
+		}
+	})
+
+	It("Should verify mounted devices can be resized", func() {
+		By("Waiting for PVC to be in bound phase")
+		pvcClaims := []*v1.PersistentVolumeClaim{pvc}
+		pvs, err := framework.WaitForPVClaimBoundPhase(c, pvcClaims, framework.ClaimProvisionTimeout)
+		Expect(err).NotTo(HaveOccurred(), "Failed waiting for PVC to be bound %v", err)
+		Expect(len(pvs)).To(Equal(1))
+
+		By("Creating a deployment with the provisioned volume")
+		deployment, err := framework.CreateDeployment(c, int32(1), map[string]string{"test": "app"}, nodeKeyValueLabel, ns, pvcClaims, "")
+		defer c.ExtensionsV1beta1().Deployments(ns).Delete(deployment.Name, &metav1.DeleteOptions{})
+
+		By("Expanding current pvc")
+		newSize := resource.MustParse("6Gi")
+		pvc, err = expandPVCSize(pvc, newSize, c)
+		Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size")
+		Expect(pvc).NotTo(BeNil())
+
+		pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage]
+		if pvcSize.Cmp(newSize) != 0 {
+			framework.Failf("error updating pvc size %q", pvc.Name)
+		}
+
+		By("Waiting for cloudprovider resize to finish")
+		err = waitForControllerVolumeResize(pvc, c)
+		Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish")
+
+		By("Getting a pod from deployment")
+		podList, err := framework.GetPodsForDeployment(c, deployment)
+		Expect(podList.Items).NotTo(BeEmpty())
+		pod := podList.Items[0]
+
+		By("Deleting the pod from deployment")
+		err = framework.DeletePodWithWait(f, c, &pod)
+		Expect(err).NotTo(HaveOccurred(), "while deleting pod for resizing")
+
+		By("Waiting for deployment to create new pod")
+		pod, err = waitForDeploymentToRecreatePod(c, deployment)
+		Expect(err).NotTo(HaveOccurred(), "While waiting for pod to be recreated")
+
+		By("Waiting for file system resize to finish")
+		pvc, err = waitForFSResize(pvc, c)
+		Expect(err).NotTo(HaveOccurred(), "while waiting for fs resize to finish")
+
+		pvcConditions := pvc.Status.Conditions
+		Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions")
+	})
+})
+
+func waitForDeploymentToRecreatePod(client clientset.Interface, deployment *extensions.Deployment) (v1.Pod, error) {
+	var runningPod v1.Pod
+	waitErr := wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) {
+		podList, err := framework.GetPodsForDeployment(client, deployment)
+		for _, pod := range podList.Items {
+			switch pod.Status.Phase {
+			case v1.PodRunning:
+				runningPod = pod
+				return true, nil
+			case v1.PodFailed, v1.PodSucceeded:
+				return false, conditions.ErrPodCompleted
+			}
+			return false, nil
+		}
+		return false, err
+	})
+	return runningPod, waitErr
+}

test/e2e/storage/vsphere/vsphere_volume_node_poweroff.go

@@ -97,7 +97,7 @@ var _ = utils.SIGDescribe("Node Poweroff [Feature:vsphere] [Slow] [Disruptive]",
 		volumePath := pvs[0].Spec.VsphereVolume.VolumePath
 
 		By("Creating a Deployment")
-		deployment, err := framework.CreateDeployment(client, int32(1), map[string]string{"test": "app"}, namespace, pvclaims, "")
+		deployment, err := framework.CreateDeployment(client, int32(1), map[string]string{"test": "app"}, nil, namespace, pvclaims, "")
 		defer client.ExtensionsV1beta1().Deployments(namespace).Delete(deployment.Name, &metav1.DeleteOptions{})
 
 		By("Get pod from the deployement")