scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-11-04 04:08:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #33067 from justinsb/better_aws_logging

Browse Source 
Automatic merge from submit-queue

Better AWS logging around volumes
This commit is contained in:
Kubernetes Submit Queue
2016-09-28 00:20:56 -07:00
committed by GitHub
parent 8ce107cbff 310423a4f9
commit c20965c652
1 changed files with 19 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
pkg/cloudprovider/providers/aws/aws.go
View File

@@ -1254,7 +1254,11 @@ func (c *Cloud) endAttaching(i *awsInstance, volumeID string, mountDevice mountD
return false return false
} }
if volumeID != existingVolumeID { if volumeID != existingVolumeID {
glog.Errorf("endAttaching on device assigned to different volume") // This actually can happen, because getMountDevice combines the attaching map with the volumes
// attached to the instance (as reported by the EC2 API). So if endAttaching comes after
// a 10 second poll delay, we might well have had a concurrent request to allocate a mountpoint,
// which because we allocate sequentially is _very_ likely to get the immediately freed volume
glog.Infof("endAttaching on device %q assigned to different volume: %q vs %q", mountDevice, volumeID, existingVolumeID)
return false return false
} }
glog.V(2).Infof("Releasing in-process attachment entry: %s -> volume %s", mountDevice, volumeID) glog.V(2).Infof("Releasing in-process attachment entry: %s -> volume %s", mountDevice, volumeID)
@@ -1318,13 +1322,13 @@ func (d *awsDisk) describeVolume() (*ec2.Volume, error) {
volumes, err := d.ec2.DescribeVolumes(request) volumes, err := d.ec2.DescribeVolumes(request)
if err != nil { if err != nil {
return nil, fmt.Errorf("error querying ec2 for volume info: %v", err) return nil, fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
} }
if len(volumes) == 0 { if len(volumes) == 0 {
return nil, fmt.Errorf("no volumes found for volume: %s", d.awsID) return nil, fmt.Errorf("no volumes found for volume %q", volumeID)
} }
if len(volumes) > 1 { if len(volumes) > 1 {
return nil, fmt.Errorf("multiple volumes found for volume: %s", d.awsID) return nil, fmt.Errorf("multiple volumes found for volume %q", volumeID)
} }
return volumes[0], nil return volumes[0], nil
} }
@@ -1357,21 +1361,21 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment,
} }
if len(info.Attachments) > 1 { if len(info.Attachments) > 1 {
// Shouldn't happen; log so we know if it is // Shouldn't happen; log so we know if it is
glog.Warningf("Found multiple attachments for volume: %v", info) glog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info)
} }
var attachment *ec2.VolumeAttachment var attachment *ec2.VolumeAttachment
attachmentStatus := "" attachmentStatus := ""
for _, a := range info.Attachments { for _, a := range info.Attachments {
if attachmentStatus != "" { if attachmentStatus != "" {
// Shouldn't happen; log so we know if it is // Shouldn't happen; log so we know if it is
glog.Warningf("Found multiple attachments: %v", info) glog.Warningf("Found multiple attachments for volume %q: %v", d.awsID, info)
} }
if a.State != nil { if a.State != nil {
attachment = a attachment = a
attachmentStatus = *a.State attachmentStatus = *a.State
} else { } else {
// Shouldn't happen; log so we know if it is // Shouldn't happen; log so we know if it is
glog.Warningf("Ignoring nil attachment state: %v", a) glog.Warningf("Ignoring nil attachment state for volume %q: %v", d.awsID, a)
} }
} }
if attachmentStatus == "" { if attachmentStatus == "" {
@@ -1382,11 +1386,11 @@ func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment,
} }
if time.Now().Unix() > timeoutAt { if time.Now().Unix() > timeoutAt {
glog.Warningf("Timeout waiting for volume state: actual=%s, desired=%s", attachmentStatus, status) glog.Warningf("Timeout waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status)
return nil, fmt.Errorf("Timeout waiting for volume state: actual=%s, desired=%s", attachmentStatus, status) return nil, fmt.Errorf("Timeout waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status)
} }
glog.V(2).Infof("Waiting for volume state: actual=%s, desired=%s", attachmentStatus, status) glog.V(2).Infof("Waiting for volume %q state: actual=%s, desired=%s", d.awsID, attachmentStatus, status)
time.Sleep(volumeAttachmentStatusPollInterval) time.Sleep(volumeAttachmentStatusPollInterval)
} }
@@ -1405,7 +1409,7 @@ func (d *awsDisk) deleteVolume() (bool, error) {
return false, volume.NewDeletedVolumeInUseError(err.Error()) return false, volume.NewDeletedVolumeInUseError(err.Error())
} }
} }
return false, fmt.Errorf("error deleting EBS volumes: %v", err) return false, fmt.Errorf("error deleting EBS volume %q: %v", d.awsID, err)
} }
return true, nil return true, nil
} }
@@ -1482,7 +1486,7 @@ func (c *Cloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bo
defer func() { defer func() {
if attachEnded { if attachEnded {
if !c.endAttaching(awsInstance, disk.awsID, mountDevice) { if !c.endAttaching(awsInstance, disk.awsID, mountDevice) {
glog.Errorf("endAttaching called when attach not in progress") glog.Errorf("endAttaching called for disk %q when attach not in progress", disk.awsID)
} }
} }
}() }()
@@ -1509,10 +1513,10 @@ func (c *Cloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bo
if err != nil { if err != nil {
attachEnded = true attachEnded = true
// TODO: Check if the volume was concurrently attached? // TODO: Check if the volume was concurrently attached?
return "", fmt.Errorf("Error attaching EBS volume: %v", err) return "", fmt.Errorf("Error attaching EBS volume %q to instance %q: %v", disk.awsID, awsInstance.awsID, err)
} }
glog.V(2).Infof("AttachVolume request returned %v", attachResponse) glog.V(2).Infof("AttachVolume volume=%q instance=%q request returned %v", disk.awsID, awsInstance.awsID, attachResponse)
} }
attachment, err := disk.waitForAttachmentStatus("attached") attachment, err := disk.waitForAttachmentStatus("attached")
@@ -1578,7 +1582,7 @@ func (c *Cloud) DetachDisk(diskName string, nodeName types.NodeName) (string, er
response, err := c.ec2.DetachVolume(&request) response, err := c.ec2.DetachVolume(&request)
if err != nil { if err != nil {
return "", fmt.Errorf("error detaching EBS volume: %v", err) return "", fmt.Errorf("error detaching EBS volume %q from %q: %v", disk.awsID, awsInstance.awsID, err)
} }
if response == nil { if response == nil {
return "", errors.New("no response from DetachVolume") return "", errors.New("no response from DetachVolume")