From 8bc5cb01a9c1e7698ddd641963da08d09410dcfc Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Tue, 25 Feb 2020 22:24:28 -0500 Subject: [PATCH] kubelet: Clear the podStatusChannel before invoking syncBatch The status manager syncBatch() method processes the current state of the cache, which should include all entries in the channel. Flush the channel before we call a batch to avoid unnecessary work and to unblock pod workers when the node is congested. Discovered while investigating long shutdown intervals on the node where the status channel stayed full for tens of seconds. Add a for loop around the select statement to avoid unnecessary invocations of the wait.Forever closure each time. --- pkg/kubelet/status/status_manager.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/kubelet/status/status_manager.go b/pkg/kubelet/status/status_manager.go index f42aed3f070..92d00d495df 100644 --- a/pkg/kubelet/status/status_manager.go +++ b/pkg/kubelet/status/status_manager.go @@ -160,13 +160,20 @@ func (m *manager) Start() { syncTicker := time.Tick(syncPeriod) // syncPod and syncBatch share the same go routine to avoid sync races. go wait.Forever(func() { - select { - case syncRequest := <-m.podStatusChannel: - klog.V(5).Infof("Status Manager: syncing pod: %q, with status: (%d, %v) from podStatusChannel", - syncRequest.podUID, syncRequest.status.version, syncRequest.status.status) - m.syncPod(syncRequest.podUID, syncRequest.status) - case <-syncTicker: - m.syncBatch() + for { + select { + case syncRequest := <-m.podStatusChannel: + klog.V(5).Infof("Status Manager: syncing pod: %q, with status: (%d, %v) from podStatusChannel", + syncRequest.podUID, syncRequest.status.version, syncRequest.status.status) + m.syncPod(syncRequest.podUID, syncRequest.status) + case <-syncTicker: + klog.V(5).Infof("Status Manager: syncing batch") + // remove any entries in the status channel since the batch will handle them + for i := len(m.podStatusChannel); i > 0; i-- { + <-m.podStatusChannel + } + m.syncBatch() + } } }, 0) }