add conditions for remaining object totals during ns termination

pkg/controller/namespace/deletion/namespaced_resources_deleter.go

@@ -421,33 +421,42 @@ func (d *namespacedResourcesDeleter) deleteEachItem(gvr schema.GroupVersionResou
 	return nil
 }
 
+type gvrDeletionMetadata struct {
+	// finalizerEstimateSeconds is an estimate of how much longer to wait
+	finalizerEstimateSeconds int64
+	// numRemaining is how many instances of the gvr remain
+	numRemaining int
+	// finalizersToNumRemaining maps finalizers to how many resources are stuck on them
+	finalizersToNumRemaining map[string]int
+}
+
 // deleteAllContentForGroupVersionResource will use the dynamic client to delete each resource identified in gvr.
 // It returns an estimate of the time remaining before the remaining resources are deleted.
 // If estimate > 0, not all resources are guaranteed to be gone.
 func (d *namespacedResourcesDeleter) deleteAllContentForGroupVersionResource(
 	gvr schema.GroupVersionResource, namespace string,
-	namespaceDeletedAt metav1.Time) (int64, error) {
+	namespaceDeletedAt metav1.Time) (gvrDeletionMetadata, error) {
 	klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - namespace: %s, gvr: %v", namespace, gvr)
 
 	// estimate how long it will take for the resource to be deleted (needed for objects that support graceful delete)
 	estimate, err := d.estimateGracefulTermination(gvr, namespace, namespaceDeletedAt)
 	if err != nil {
 		klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - unable to estimate - namespace: %s, gvr: %v, err: %v", namespace, gvr, err)
-		return estimate, err
+		return gvrDeletionMetadata{}, err
 	}
 	klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - estimate - namespace: %s, gvr: %v, estimate: %v", namespace, gvr, estimate)
 
 	// first try to delete the entire collection
 	deleteCollectionSupported, err := d.deleteCollection(gvr, namespace)
 	if err != nil {
-		return estimate, err
+		return gvrDeletionMetadata{finalizerEstimateSeconds: estimate}, err
 	}
 
 	// delete collection was not supported, so we list and delete each item...
 	if !deleteCollectionSupported {
 		err = d.deleteEachItem(gvr, namespace)
 		if err != nil {
-			return estimate, err
+			return gvrDeletionMetadata{finalizerEstimateSeconds: estimate}, err
 		}
 	}
 
@@ -457,24 +466,56 @@ func (d *namespacedResourcesDeleter) deleteAllContentForGroupVersionResource(
 	unstructuredList, listSupported, err := d.listCollection(gvr, namespace)
 	if err != nil {
 		klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - error verifying no items in namespace: %s, gvr: %v, err: %v", namespace, gvr, err)
-		return estimate, err
+		return gvrDeletionMetadata{finalizerEstimateSeconds: estimate}, err
 	}
 	if !listSupported {
-		return estimate, nil
+		return gvrDeletionMetadata{finalizerEstimateSeconds: estimate}, nil
 	}
 	klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - items remaining - namespace: %s, gvr: %v, items: %v", namespace, gvr, len(unstructuredList.Items))
-	if len(unstructuredList.Items) != 0 && estimate == int64(0) {
-		// if any item has a finalizer, we treat that as a normal condition, and use a default estimation to allow for GC to complete.
+	if len(unstructuredList.Items) == 0 {
+		// we're done
+		return gvrDeletionMetadata{finalizerEstimateSeconds: 0, numRemaining: 0}, nil
+	}
+
+	// use the list to find the finalizers
+	finalizersToNumRemaining := map[string]int{}
 	for _, item := range unstructuredList.Items {
-			if len(item.GetFinalizers()) > 0 {
-				klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - items remaining with finalizers - namespace: %s, gvr: %v, finalizers: %v", namespace, gvr, item.GetFinalizers())
-				return finalizerEstimateSeconds, nil
+		for _, finalizer := range item.GetFinalizers() {
+			finalizersToNumRemaining[finalizer] = finalizersToNumRemaining[finalizer] + 1
 		}
 	}
+
+	if estimate != int64(0) {
+		klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - estimate is present - namespace: %s, gvr: %v, finalizers: %v", namespace, gvr, finalizersToNumRemaining)
+		return gvrDeletionMetadata{
+			finalizerEstimateSeconds: estimate,
+			numRemaining:             len(unstructuredList.Items),
+			finalizersToNumRemaining: finalizersToNumRemaining,
+		}, nil
+	}
+
+	// if any item has a finalizer, we treat that as a normal condition, and use a default estimation to allow for GC to complete.
+	if len(finalizersToNumRemaining) > 0 {
+		klog.V(5).Infof("namespace controller - deleteAllContentForGroupVersionResource - items remaining with finalizers - namespace: %s, gvr: %v, finalizers: %v", namespace, gvr, finalizersToNumRemaining)
+		return gvrDeletionMetadata{
+			finalizerEstimateSeconds: finalizerEstimateSeconds,
+			numRemaining:             len(unstructuredList.Items),
+			finalizersToNumRemaining: finalizersToNumRemaining,
+		}, nil
+	}
+
 	// nothing reported a finalizer, so something was unexpected as it should have been deleted.
-		return estimate, fmt.Errorf("unexpected items still remain in namespace: %s for gvr: %v", namespace, gvr)
-	}
-	return estimate, nil
+	return gvrDeletionMetadata{
+		finalizerEstimateSeconds: estimate,
+		numRemaining:             len(unstructuredList.Items),
+	}, fmt.Errorf("unexpected items still remain in namespace: %s for gvr: %v", namespace, gvr)
+}
+
+type allGVRDeletionMetadata struct {
+	// gvrToNumRemaining is how many instances of the gvr remain
+	gvrToNumRemaining map[schema.GroupVersionResource]int
+	// finalizersToNumRemaining maps finalizers to how many resources are stuck on them
+	finalizersToNumRemaining map[string]int
 }
 
 // deleteAllContent will use the dynamic client to delete each resource identified in groupVersionResources.
@@ -502,18 +543,33 @@ func (d *namespacedResourcesDeleter) deleteAllContent(ns *v1.Namespace) (int64,
 		errs = append(errs, err)
 		conditionUpdater.ProcessGroupVersionErr(err)
 	}
+
+	numRemainingTotals := allGVRDeletionMetadata{
+		gvrToNumRemaining:        map[schema.GroupVersionResource]int{},
+		finalizersToNumRemaining: map[string]int{},
+	}
 	for gvr := range groupVersionResources {
-		gvrEstimate, err := d.deleteAllContentForGroupVersionResource(gvr, namespace, namespaceDeletedAt)
+		gvrDeletionMetadata, err := d.deleteAllContentForGroupVersionResource(gvr, namespace, namespaceDeletedAt)
 		if err != nil {
 			// If there is an error, hold on to it but proceed with all the remaining
 			// groupVersionResources.
 			errs = append(errs, err)
 			conditionUpdater.ProcessDeleteContentErr(err)
 		}
-		if gvrEstimate > estimate {
-			estimate = gvrEstimate
+		if gvrDeletionMetadata.finalizerEstimateSeconds > estimate {
+			estimate = gvrDeletionMetadata.finalizerEstimateSeconds
+		}
+		if gvrDeletionMetadata.numRemaining > 0 {
+			numRemainingTotals.gvrToNumRemaining[gvr] = gvrDeletionMetadata.numRemaining
+			for finalizer, numRemaining := range gvrDeletionMetadata.finalizersToNumRemaining {
+				if numRemaining == 0 {
+					continue
+				}
+				numRemainingTotals.finalizersToNumRemaining[finalizer] = numRemainingTotals.finalizersToNumRemaining[finalizer] + numRemaining
 			}
 		}
+	}
+	conditionUpdater.ProcessContentTotals(numRemainingTotals)
 
 	// we always want to update the conditions because if we have set a condition to "it worked" after it was previously, "it didn't work",
 	// we need to reflect that information.  Recall that additional finalizers can be set on namespaces, so this finalizer may clear itself and
@@ -540,7 +596,7 @@ func (d *namespacedResourcesDeleter) estimateGracefulTermination(gvr schema.Grou
 		estimate, err = d.estimateGracefulTerminationForPods(ns)
 	}
 	if err != nil {
-		return estimate, err
+		return 0, err
 	}
 	// determine if the estimate is greater than the deletion timestamp
 	duration := time.Since(namespaceDeletedAt.Time)
@@ -557,11 +613,11 @@ func (d *namespacedResourcesDeleter) estimateGracefulTerminationForPods(ns strin
 	estimate := int64(0)
 	podsGetter := d.podsGetter
 	if podsGetter == nil || reflect.ValueOf(podsGetter).IsNil() {
-		return estimate, fmt.Errorf("unexpected: podsGetter is nil. Cannot estimate grace period seconds for pods")
+		return 0, fmt.Errorf("unexpected: podsGetter is nil. Cannot estimate grace period seconds for pods")
 	}
 	items, err := podsGetter.Pods(ns).List(metav1.ListOptions{})
 	if err != nil {
-		return estimate, err
+		return 0, err
 	}
 	for i := range items.Items {
 		pod := items.Items[i]

pkg/controller/namespace/deletion/status_condition_utils.go

@@ -48,16 +48,22 @@ var (
 		v1.NamespaceDeletionDiscoveryFailure,
 		v1.NamespaceDeletionGVParsingFailure,
 		v1.NamespaceDeletionContentFailure,
+		v1.NamespaceContentRemaining,
+		v1.NamespaceFinalizersRemaining,
 	}
 	okMessages = map[v1.NamespaceConditionType]string{
 		v1.NamespaceDeletionDiscoveryFailure: "All resources successfully discovered",
 		v1.NamespaceDeletionGVParsingFailure: "All legacy kube types successfully parsed",
-		v1.NamespaceDeletionContentFailure:   "All content successfully deleted",
+		v1.NamespaceDeletionContentFailure:   "All content successfully deleted, may be waiting on finalization",
+		v1.NamespaceContentRemaining:         "All content successfully removed",
+		v1.NamespaceFinalizersRemaining:      "All content successfully removed",
 	}
 	okReasons = map[v1.NamespaceConditionType]string{
 		v1.NamespaceDeletionDiscoveryFailure: "ResourcesDiscovered",
 		v1.NamespaceDeletionGVParsingFailure: "ParsedGroupVersions",
 		v1.NamespaceDeletionContentFailure:   "ContentDeleted",
+		v1.NamespaceContentRemaining:         "ContentRemoved",
+		v1.NamespaceFinalizersRemaining:      "ContentRemoved",
 	}
 )
 
@@ -92,6 +98,47 @@ func (u *namespaceConditionUpdater) ProcessDiscoverResourcesErr(err error) {
 
 }
 
+// ProcessContentTotals may create conditions for NamespaceContentRemaining and NamespaceFinalizersRemaining.
+func (u *namespaceConditionUpdater) ProcessContentTotals(contentTotals allGVRDeletionMetadata) {
+	if len(contentTotals.gvrToNumRemaining) != 0 {
+		remainingResources := []string{}
+		for gvr, numRemaining := range contentTotals.gvrToNumRemaining {
+			if numRemaining == 0 {
+				continue
+			}
+			remainingResources = append(remainingResources, fmt.Sprintf("%s.%s has %d resource instances", gvr.Resource, gvr.Group, numRemaining))
+		}
+		// sort for stable updates
+		sort.Strings(remainingResources)
+		u.newConditions = append(u.newConditions, v1.NamespaceCondition{
+			Type:               v1.NamespaceContentRemaining,
+			Status:             v1.ConditionTrue,
+			LastTransitionTime: metav1.Now(),
+			Reason:             "SomeResourcesRemain",
+			Message:            fmt.Sprintf("Some resources are remaining: %s", strings.Join(remainingResources, ", ")),
+		})
+	}
+
+	if len(contentTotals.finalizersToNumRemaining) != 0 {
+		remainingByFinalizer := []string{}
+		for finalizer, numRemaining := range contentTotals.finalizersToNumRemaining {
+			if numRemaining == 0 {
+				continue
+			}
+			remainingByFinalizer = append(remainingByFinalizer, fmt.Sprintf("%s in %d resource instances", finalizer, numRemaining))
+		}
+		// sort for stable updates
+		sort.Strings(remainingByFinalizer)
+		u.newConditions = append(u.newConditions, v1.NamespaceCondition{
+			Type:               v1.NamespaceFinalizersRemaining,
+			Status:             v1.ConditionTrue,
+			LastTransitionTime: metav1.Now(),
+			Reason:             "SomeFinalizerRemain",
+			Message:            fmt.Sprintf("Some finalizers are remaining: %s", strings.Join(remainingByFinalizer, ", ")),
+		})
+	}
+}
+
 // ProcessDeleteContentErr creates error condition from multiple delete content errors.
 func (u *namespaceConditionUpdater) ProcessDeleteContentErr(err error) {
 	u.deleteContentErrors = append(u.deleteContentErrors, err)

pkg/controller/namespace/deletion/status_condition_utils_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
 func TestUpdateConditions(t *testing.T) {
@@ -46,6 +47,8 @@ func TestUpdateConditions(t *testing.T) {
 				*newSuccessfulCondition(v1.NamespaceDeletionDiscoveryFailure),
 				*newSuccessfulCondition(v1.NamespaceDeletionGVParsingFailure),
 				*newSuccessfulCondition(v1.NamespaceDeletionContentFailure),
+				*newSuccessfulCondition(v1.NamespaceContentRemaining),
+				*newSuccessfulCondition(v1.NamespaceFinalizersRemaining),
 			},
 		},
 		{
@@ -61,6 +64,8 @@ func TestUpdateConditions(t *testing.T) {
 				*newSuccessfulCondition(v1.NamespaceDeletionDiscoveryFailure),
 				*newSuccessfulCondition(v1.NamespaceDeletionGVParsingFailure),
 				*newSuccessfulCondition(v1.NamespaceDeletionContentFailure),
+				*newSuccessfulCondition(v1.NamespaceContentRemaining),
+				*newSuccessfulCondition(v1.NamespaceFinalizersRemaining),
 			},
 		},
 		{
@@ -77,6 +82,8 @@ func TestUpdateConditions(t *testing.T) {
 				*newSuccessfulCondition(v1.NamespaceDeletionGVParsingFailure),
 				*newSuccessfulCondition(v1.NamespaceDeletionDiscoveryFailure),
 				*newSuccessfulCondition(v1.NamespaceDeletionContentFailure),
+				*newSuccessfulCondition(v1.NamespaceContentRemaining),
+				*newSuccessfulCondition(v1.NamespaceFinalizersRemaining),
 			},
 		},
 		{
@@ -95,6 +102,8 @@ func TestUpdateConditions(t *testing.T) {
 				{Type: v1.NamespaceDeletionGVParsingFailure, Status: v1.ConditionTrue, Reason: "foo", Message: "bar"},
 				*newSuccessfulCondition(v1.NamespaceDeletionDiscoveryFailure),
 				*newSuccessfulCondition(v1.NamespaceDeletionContentFailure),
+				*newSuccessfulCondition(v1.NamespaceContentRemaining),
+				*newSuccessfulCondition(v1.NamespaceFinalizersRemaining),
 			},
 		},
 		{
@@ -112,6 +121,8 @@ func TestUpdateConditions(t *testing.T) {
 				*newSuccessfulCondition(v1.NamespaceDeletionDiscoveryFailure),
 				{Type: v1.NamespaceDeletionGVParsingFailure, Status: v1.ConditionTrue, Reason: "foo", Message: "bar"},
 				*newSuccessfulCondition(v1.NamespaceDeletionContentFailure),
+				*newSuccessfulCondition(v1.NamespaceContentRemaining),
+				*newSuccessfulCondition(v1.NamespaceFinalizersRemaining),
 			},
 		},
 	}
@@ -135,3 +146,89 @@ func TestUpdateConditions(t *testing.T) {
 		})
 	}
 }
+
+func TestProcessContentTotals(t *testing.T) {
+	tests := []struct {
+		name string
+
+		contentTotals allGVRDeletionMetadata
+		expecteds     []v1.NamespaceCondition
+	}{
+		{
+			name: "nothing",
+
+			contentTotals: allGVRDeletionMetadata{
+				gvrToNumRemaining:        map[schema.GroupVersionResource]int{},
+				finalizersToNumRemaining: map[string]int{},
+			},
+			expecteds: []v1.NamespaceCondition{},
+		},
+		{
+			name: "just remaining",
+
+			contentTotals: allGVRDeletionMetadata{
+				gvrToNumRemaining: map[schema.GroupVersionResource]int{
+					{Group: "apps.k8s.io", Resource: "daemonsets"}:  5,
+					{Group: "apps.k8s.io", Resource: "deployments"}: 5,
+				},
+				finalizersToNumRemaining: map[string]int{},
+			},
+			expecteds: []v1.NamespaceCondition{
+				{Type: v1.NamespaceContentRemaining, Status: v1.ConditionTrue, Reason: "SomeResourcesRemain", Message: `Some resources are remaining: daemonsets.apps.k8s.io has 5 resource instances, deployments.apps.k8s.io has 5 resource instances`},
+			},
+		},
+		{
+			name: "just finalizers", // this shouldn't happen
+
+			contentTotals: allGVRDeletionMetadata{
+				gvrToNumRemaining: map[schema.GroupVersionResource]int{},
+				finalizersToNumRemaining: map[string]int{
+					"service-catalog": 6,
+					"kubedb":          5,
+				},
+			},
+			expecteds: []v1.NamespaceCondition{
+				{Type: v1.NamespaceFinalizersRemaining, Status: v1.ConditionTrue, Reason: "SomeFinalizerRemain", Message: `Some finalizers are remaining: kubedb in 5 resource instances, service-catalog in 6 resource instances`},
+			},
+		},
+		{
+			name: "both",
+
+			contentTotals: allGVRDeletionMetadata{
+				gvrToNumRemaining: map[schema.GroupVersionResource]int{
+					{Group: "apps.k8s.io", Resource: "daemonsets"}:  5,
+					{Group: "apps.k8s.io", Resource: "deployments"}: 5,
+				},
+				finalizersToNumRemaining: map[string]int{
+					"service-catalog": 6,
+					"kubedb":          5,
+				},
+			},
+			expecteds: []v1.NamespaceCondition{
+				{Type: v1.NamespaceContentRemaining, Status: v1.ConditionTrue, Reason: "SomeResourcesRemain", Message: `Some resources are remaining: daemonsets.apps.k8s.io has 5 resource instances, deployments.apps.k8s.io has 5 resource instances`},
+				{Type: v1.NamespaceFinalizersRemaining, Status: v1.ConditionTrue, Reason: "SomeFinalizerRemain", Message: `Some finalizers are remaining: kubedb in 5 resource instances, service-catalog in 6 resource instances`},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			u := namespaceConditionUpdater{}
+
+			u.ProcessContentTotals(test.contentTotals)
+
+			actuals := u.newConditions
+			if len(actuals) != len(test.expecteds) {
+				t.Fatal(actuals)
+			}
+			for i := range actuals {
+				actual := actuals[i]
+				expected := test.expecteds[i]
+				expected.LastTransitionTime = actual.LastTransitionTime
+				if !reflect.DeepEqual(expected, actual) {
+					t.Error(actual)
+				}
+			}
+		})
+	}
+}

staging/src/k8s.io/api/core/v1/types.go

@@ -4670,6 +4670,10 @@ const (
 	NamespaceDeletionContentFailure NamespaceConditionType = "NamespaceDeletionContentFailure"
 	// NamespaceDeletionGVParsingFailure contains information about namespace deleter errors parsing GV for legacy types.
 	NamespaceDeletionGVParsingFailure NamespaceConditionType = "NamespaceDeletionGroupVersionParsingFailure"
+	// NamespaceContentRemaining contains information about resources remaining in a namespace.
+	NamespaceContentRemaining NamespaceConditionType = "NamespaceContentRemaining"
+	// NamespaceFinalizersRemaining contains information about which finalizers are on resources remaining in a namespace.
+	NamespaceFinalizersRemaining NamespaceConditionType = "NamespaceFinalizersRemaining"
 )
 
 // NamespaceCondition contains details about state of namespace.

test/integration/namespace/ns_conditions_test.go

@@ -87,23 +87,27 @@ func TestNamespaceCondition(t *testing.T) {
 			return false, err
 		}
 
-		foundContentCondition := false
-		foundFinalizerCondition := false
-
+		conditionsFound := 0
 		for _, condition := range curr.Status.Conditions {
 			if condition.Type == corev1.NamespaceDeletionGVParsingFailure && condition.Message == `All legacy kube types successfully parsed` {
-				foundContentCondition = true
+				conditionsFound++
 			}
 			if condition.Type == corev1.NamespaceDeletionDiscoveryFailure && condition.Message == `All resources successfully discovered` {
-				foundFinalizerCondition = true
+				conditionsFound++
 			}
-			if condition.Type == corev1.NamespaceDeletionContentFailure && condition.Message == `All content successfully deleted` {
-				foundFinalizerCondition = true
+			if condition.Type == corev1.NamespaceDeletionContentFailure && condition.Message == `All content successfully deleted, may be waiting on finalization` {
+				conditionsFound++
+			}
+			if condition.Type == corev1.NamespaceContentRemaining && condition.Message == `Some resources are remaining: deployments.apps has 1 resource instances` {
+				conditionsFound++
+			}
+			if condition.Type == corev1.NamespaceFinalizersRemaining && condition.Message == `Some finalizers are remaining: custom.io/finalizer in 1 resource instances` {
+				conditionsFound++
 			}
 		}
 
 		t.Log(spew.Sdump(curr))
-		return foundContentCondition && foundFinalizerCondition, nil
+		return conditionsFound == 5, nil
 	})
 	if err != nil {
 		t.Fatal(err)