Use new size returned by nodeExpander for recording in ASOW

This commit is contained in:
Hemant Kumar
2024-08-14 09:22:46 -04:00
parent 0e5dd6b7c9
commit 99fc7cc7b1
6 changed files with 29 additions and 23 deletions

View File

@@ -960,6 +960,8 @@ func (asw *actualStateOfWorld) volumeNeedsExpansion(volumeObj attachedVolume, de
return currentSize, false return currentSize, false
} }
klog.V(5).Infof("NodeExpandVolume checking size, actual size %s, desired size %s, for volume %s", volumeObj.persistentVolumeSize.String(), desiredVolumeSize.String(), volumeObj.volumeName)
if desiredVolumeSize.Cmp(*volumeObj.persistentVolumeSize) > 0 { if desiredVolumeSize.Cmp(*volumeObj.persistentVolumeSize) > 0 {
volumePlugin, err := asw.volumePluginMgr.FindNodeExpandablePluginBySpec(volumeObj.spec) volumePlugin, err := asw.volumePluginMgr.FindNodeExpandablePluginBySpec(volumeObj.spec)
if err != nil || volumePlugin == nil { if err != nil || volumePlugin == nil {

View File

@@ -380,10 +380,11 @@ func (dswp *desiredStateOfWorldPopulator) checkVolumeFSResize(
klog.V(5).InfoS("Skip file system resize check for the volume, as the volume is mounted as readonly", "pod", klog.KObj(pod), "volumeName", podVolume.Name) klog.V(5).InfoS("Skip file system resize check for the volume, as the volume is mounted as readonly", "pod", klog.KObj(pod), "volumeName", podVolume.Name)
return return
} }
pvCap := volumeSpec.PersistentVolume.Spec.Capacity.Storage() pvCap := volumeSpec.PersistentVolume.Spec.Capacity.Storage()
pvcStatusCap := pvc.Status.Capacity.Storage() pvcStatusCap := pvc.Status.Capacity.Storage()
dswp.desiredStateOfWorld.UpdatePersistentVolumeSize(uniqueVolumeName, pvCap) dswp.desiredStateOfWorld.UpdatePersistentVolumeSize(uniqueVolumeName, pvCap)
klog.V(5).Infof("NodeExpandVolume updating size, actual size %s, desired size %s, for volume %s", pvcStatusCap.String(), pvCap.String(), uniqueVolumeName)
// in case the actualStateOfWorld was rebuild after kubelet restart ensure that claimSize is set to accurate value // in case the actualStateOfWorld was rebuild after kubelet restart ensure that claimSize is set to accurate value
dswp.actualStateOfWorld.InitializeClaimSize(klog.TODO(), uniqueVolumeName, pvcStatusCap) dswp.actualStateOfWorld.InitializeClaimSize(klog.TODO(), uniqueVolumeName, pvcStatusCap)
} }

View File

@@ -115,11 +115,11 @@ func (ne *NodeExpander) runPreCheck() bool {
return false return false
} }
func (ne *NodeExpander) expandOnPlugin() (bool, error, testResponseData) { func (ne *NodeExpander) expandOnPlugin() (bool, resource.Quantity, error, testResponseData) {
allowExpansion := ne.runPreCheck() allowExpansion := ne.runPreCheck()
if !allowExpansion { if !allowExpansion {
klog.V(3).Infof("NodeExpandVolume is not allowed to proceed for volume %s with resizeStatus %s", ne.vmt.VolumeName, ne.resizeStatus) klog.V(3).Infof("NodeExpandVolume is not allowed to proceed for volume %s with resizeStatus %s", ne.vmt.VolumeName, ne.resizeStatus)
return false, nil, testResponseData{false, true} return false, ne.pluginResizeOpts.OldSize, nil, testResponseData{false, true}
} }
var err error var err error
@@ -131,7 +131,7 @@ func (ne *NodeExpander) expandOnPlugin() (bool, error, testResponseData) {
if err != nil { if err != nil {
msg := ne.vmt.GenerateErrorDetailed("MountVolume.NodeExpandVolume failed to mark node expansion in progress: %v", err) msg := ne.vmt.GenerateErrorDetailed("MountVolume.NodeExpandVolume failed to mark node expansion in progress: %v", err)
klog.Errorf(msg.Error()) klog.Errorf(msg.Error())
return false, err, testResponseData{} return false, ne.pluginResizeOpts.OldSize, err, testResponseData{}
} }
} }
_, resizeErr := ne.volumePlugin.NodeExpand(ne.pluginResizeOpts) _, resizeErr := ne.volumePlugin.NodeExpand(ne.pluginResizeOpts)
@@ -158,9 +158,9 @@ func (ne *NodeExpander) expandOnPlugin() (bool, error, testResponseData) {
if volumetypes.IsFailedPreconditionError(resizeErr) { if volumetypes.IsFailedPreconditionError(resizeErr) {
ne.actualStateOfWorld.MarkForInUseExpansionError(ne.vmt.VolumeName) ne.actualStateOfWorld.MarkForInUseExpansionError(ne.vmt.VolumeName)
klog.Errorf(ne.vmt.GenerateErrorDetailed("MountVolume.NodeExapndVolume failed with %v", resizeErr).Error()) klog.Errorf(ne.vmt.GenerateErrorDetailed("MountVolume.NodeExapndVolume failed with %v", resizeErr).Error())
return false, nil, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true} return false, ne.pluginResizeOpts.OldSize, nil, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true}
} }
return false, resizeErr, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true} return false, ne.pluginResizeOpts.OldSize, resizeErr, testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: true}
} }
simpleMsg, detailedMsg := ne.vmt.GenerateMsg("MountVolume.NodeExpandVolume succeeded", nodeName) simpleMsg, detailedMsg := ne.vmt.GenerateMsg("MountVolume.NodeExpandVolume succeeded", nodeName)
ne.recorder.Eventf(ne.vmt.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg) ne.recorder.Eventf(ne.vmt.Pod, v1.EventTypeNormal, kevents.FileSystemResizeSuccess, simpleMsg)
@@ -169,13 +169,13 @@ func (ne *NodeExpander) expandOnPlugin() (bool, error, testResponseData) {
// no need to update PVC object if we already updated it // no need to update PVC object if we already updated it
if ne.pvcAlreadyUpdated { if ne.pvcAlreadyUpdated {
return true, nil, testResponseData{true, true} return true, ne.pluginResizeOpts.NewSize, nil, testResponseData{true, true}
} }
// File system resize succeeded, now update the PVC's Capacity to match the PV's // File system resize succeeded, now update the PVC's Capacity to match the PV's
ne.pvc, err = util.MarkFSResizeFinished(ne.pvc, ne.pluginResizeOpts.NewSize, ne.kubeClient) ne.pvc, err = util.MarkFSResizeFinished(ne.pvc, ne.pluginResizeOpts.NewSize, ne.kubeClient)
if err != nil { if err != nil {
return true, fmt.Errorf("mountVolume.NodeExpandVolume update pvc status failed: %v", err), testResponseData{true, true} return true, ne.pluginResizeOpts.NewSize, fmt.Errorf("mountVolume.NodeExpandVolume update pvc status failed: %v", err), testResponseData{true, true}
} }
return true, nil, testResponseData{true, true} return true, ne.pluginResizeOpts.NewSize, nil, testResponseData{true, true}
} }

View File

@@ -162,7 +162,7 @@ func TestNodeExpander(t *testing.T) {
ogInstance, _ := og.(*operationGenerator) ogInstance, _ := og.(*operationGenerator)
nodeExpander := newNodeExpander(resizeOp, ogInstance.kubeClient, ogInstance.recorder) nodeExpander := newNodeExpander(resizeOp, ogInstance.kubeClient, ogInstance.recorder)
_, err, expansionResponse := nodeExpander.expandOnPlugin() _, _, err, expansionResponse := nodeExpander.expandOnPlugin()
pvc = nodeExpander.pvc pvc = nodeExpander.pvc
pvcStatusCap := pvc.Status.Capacity[v1.ResourceStorage] pvcStatusCap := pvc.Status.Capacity[v1.ResourceStorage]

View File

@@ -1957,14 +1957,14 @@ func (og *operationGenerator) doOnlineExpansion(volumeToMount VolumeToMount,
actualStateOfWorld ActualStateOfWorldMounterUpdater, actualStateOfWorld ActualStateOfWorldMounterUpdater,
resizeOptions volume.NodeResizeOptions) (bool, error, error) { resizeOptions volume.NodeResizeOptions) (bool, error, error) {
resizeDone, err := og.nodeExpandVolume(volumeToMount, actualStateOfWorld, resizeOptions) resizeDone, newSize, err := og.nodeExpandVolume(volumeToMount, actualStateOfWorld, resizeOptions)
if err != nil { if err != nil {
e1, e2 := volumeToMount.GenerateError("NodeExpandVolume.NodeExpandVolume failed", err) e1, e2 := volumeToMount.GenerateError("NodeExpandVolume.NodeExpandVolume failed", err)
klog.Errorf(e2.Error()) klog.Errorf(e2.Error())
return false, e1, e2 return false, e1, e2
} }
if resizeDone { if resizeDone {
markingDone := actualStateOfWorld.MarkVolumeAsResized(volumeToMount.VolumeName, &resizeOptions.NewSize) markingDone := actualStateOfWorld.MarkVolumeAsResized(volumeToMount.VolumeName, &newSize)
if !markingDone { if !markingDone {
// On failure, return error. Caller will log and retry. // On failure, return error. Caller will log and retry.
genericFailureError := fmt.Errorf("unable to mark volume as resized") genericFailureError := fmt.Errorf("unable to mark volume as resized")
@@ -2013,7 +2013,7 @@ func (og *operationGenerator) expandVolumeDuringMount(volumeToMount VolumeToMoun
rsOpts.NewSize = pvc.Status.AllocatedResources[v1.ResourceStorage] rsOpts.NewSize = pvc.Status.AllocatedResources[v1.ResourceStorage]
resizeOp.pluginResizeOpts = rsOpts resizeOp.pluginResizeOpts = rsOpts
nodeExpander := newNodeExpander(resizeOp, og.kubeClient, og.recorder) nodeExpander := newNodeExpander(resizeOp, og.kubeClient, og.recorder)
resizeFinished, err, _ := nodeExpander.expandOnPlugin() resizeFinished, _, err, _ := nodeExpander.expandOnPlugin()
return resizeFinished, err return resizeFinished, err
} else { } else {
return og.legacyCallNodeExpandOnPlugin(resizeOp) return og.legacyCallNodeExpandOnPlugin(resizeOp)
@@ -2044,7 +2044,7 @@ func (og *operationGenerator) checkIfSupportsNodeExpansion(volumeToMount VolumeT
func (og *operationGenerator) nodeExpandVolume( func (og *operationGenerator) nodeExpandVolume(
volumeToMount VolumeToMount, volumeToMount VolumeToMount,
actualStateOfWorld ActualStateOfWorldMounterUpdater, actualStateOfWorld ActualStateOfWorldMounterUpdater,
rsOpts volume.NodeResizeOptions) (bool, error) { rsOpts volume.NodeResizeOptions) (bool, resource.Quantity, error) {
supportsExpansion, expandableVolumePlugin := og.checkIfSupportsNodeExpansion(volumeToMount) supportsExpansion, expandableVolumePlugin := og.checkIfSupportsNodeExpansion(volumeToMount)
@@ -2053,9 +2053,10 @@ func (og *operationGenerator) nodeExpandVolume(
if rsOpts.NewSize.Cmp(rsOpts.OldSize) > 0 { if rsOpts.NewSize.Cmp(rsOpts.OldSize) > 0 {
pv := volumeToMount.VolumeSpec.PersistentVolume pv := volumeToMount.VolumeSpec.PersistentVolume
pvc, err := og.kubeClient.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace).Get(context.TODO(), pv.Spec.ClaimRef.Name, metav1.GetOptions{}) pvc, err := og.kubeClient.CoreV1().PersistentVolumeClaims(pv.Spec.ClaimRef.Namespace).Get(context.TODO(), pv.Spec.ClaimRef.Name, metav1.GetOptions{})
currentSize := pvc.Status.Capacity.Storage()
if err != nil { if err != nil {
// Return error rather than leave the file system un-resized, caller will log and retry // Return error rather than leave the file system un-resized, caller will log and retry
return false, fmt.Errorf("mountVolume.NodeExpandVolume get PVC failed : %v", err) return false, *currentSize, fmt.Errorf("mountVolume.NodeExpandVolume get PVC failed : %v", err)
} }
if volumeToMount.VolumeSpec.ReadOnly { if volumeToMount.VolumeSpec.ReadOnly {
@@ -2063,7 +2064,7 @@ func (og *operationGenerator) nodeExpandVolume(
klog.Warningf(detailedMsg) klog.Warningf(detailedMsg)
og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg)
og.recorder.Eventf(pvc, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) og.recorder.Eventf(pvc, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg)
return true, nil return true, *currentSize, nil
} }
resizeOp := nodeResizeOperationOpts{ resizeOp := nodeResizeOperationOpts{
vmt: volumeToMount, vmt: volumeToMount,
@@ -2076,17 +2077,19 @@ func (og *operationGenerator) nodeExpandVolume(
if og.checkForRecoveryFromExpansion(pvc, volumeToMount) { if og.checkForRecoveryFromExpansion(pvc, volumeToMount) {
// if recovery feature is enabled, we can use allocated size from PVC status as new size // if recovery feature is enabled, we can use allocated size from PVC status as new size
rsOpts.NewSize = pvc.Status.AllocatedResources[v1.ResourceStorage] newSize := pvc.Status.AllocatedResources[v1.ResourceStorage]
resizeOp.pluginResizeOpts = rsOpts rsOpts.NewSize = newSize
resizeOp.pluginResizeOpts.NewSize = newSize
nodeExpander := newNodeExpander(resizeOp, og.kubeClient, og.recorder) nodeExpander := newNodeExpander(resizeOp, og.kubeClient, og.recorder)
resizeFinished, err, _ := nodeExpander.expandOnPlugin() resizeFinished, newSize, err, _ := nodeExpander.expandOnPlugin()
return resizeFinished, err return resizeFinished, newSize, err
} else { } else {
return og.legacyCallNodeExpandOnPlugin(resizeOp) resizeFinished, err := og.legacyCallNodeExpandOnPlugin(resizeOp)
return resizeFinished, rsOpts.NewSize, err
} }
} }
} }
return true, nil return true, rsOpts.OldSize, nil
} }
func (og *operationGenerator) checkForRecoveryFromExpansion(pvc *v1.PersistentVolumeClaim, volumeToMount VolumeToMount) bool { func (og *operationGenerator) checkForRecoveryFromExpansion(pvc *v1.PersistentVolumeClaim, volumeToMount VolumeToMount) bool {

View File

@@ -305,7 +305,7 @@ func TestOperationGenerator_nodeExpandVolume(t *testing.T) {
asow := newFakeActualStateOfWorld() asow := newFakeActualStateOfWorld()
ogInstance, _ := og.(*operationGenerator) ogInstance, _ := og.(*operationGenerator)
_, err := ogInstance.nodeExpandVolume(vmt, asow, pluginResizeOpts) _, _, err := ogInstance.nodeExpandVolume(vmt, asow, pluginResizeOpts)
if !test.expectError && err != nil { if !test.expectError && err != nil {
t.Errorf("For test %s, expected no error got: %v", test.name, err) t.Errorf("For test %s, expected no error got: %v", test.name, err)