scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-11-28 20:33:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Promote Local storage capacity isolation feature to GA

Browse Source 
This change is to promote local storage capacity isolation feature to GA

At the same time, to allow rootless system disable this feature due to
unable to get root fs, this change introduced a new kubelet config
"localStorageCapacityIsolation". By default it is set to true. For
rootless systems, they can set this configuration to false to disable
the feature. Once it is set, user cannot set ephemeral-storage
request/limit because capacity and allocatable will not be set.

Change-Id: I48a52e737c6a09e9131454db6ad31247b56c000a
This commit is contained in:
jinxu
2022-07-28 08:03:20 -07:00
parent bc4c4930ff
commit 0064010cdd
42 changed files with 267 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/kubelet/cm/container_manager.go
View File

@@ -47,7 +47,7 @@ type ContainerManager interface {
// Runs the container manager's housekeeping.
// - Ensures that the Docker daemon is in a container.
// - Creates the system container where all non-containerized processes run.
Start(*v1.Node, ActivePodsFunc, config.SourcesReady, status.PodStatusProvider, internalapi.RuntimeService) error
Start(*v1.Node, ActivePodsFunc, config.SourcesReady, status.PodStatusProvider, internalapi.RuntimeService, bool) error
// SystemCgroupsLimit returns resources allocated to system cgroups in the machine.
// These cgroups include the system and Kubernetes services.
@@ -73,7 +73,7 @@ type ContainerManager interface {
GetNodeAllocatableReservation() v1.ResourceList
// GetCapacity returns the amount of compute resources tracked by container manager available on the node.
GetCapacity() v1.ResourceList
GetCapacity(localStorageCapacityIsolation bool) v1.ResourceList
// GetDevicePluginResourceCapacity returns the node capacity (amount of total device plugin resources),
// node allocatable (amount of total healthy resources reported by device plugin),

9
pkg/kubelet/cm/container_manager_linux.go
View File

@@ -554,7 +554,8 @@ func (cm *containerManagerImpl) Start(node *v1.Node,
activePods ActivePodsFunc,
sourcesReady config.SourcesReady,
podStatusProvider status.PodStatusProvider,
runtimeService internalapi.RuntimeService) error {
runtimeService internalapi.RuntimeService,
localStorageCapacityIsolation bool) error {
// Initialize CPU manager
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CPUManager) {
@@ -578,7 +579,7 @@ func (cm *containerManagerImpl) Start(node *v1.Node,
// allocatable of the node
cm.nodeInfo = node
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LocalStorageCapacityIsolation) {
if localStorageCapacityIsolation {
rootfs, err := cm.cadvisorInterface.RootFsInfo()
if err != nil {
return fmt.Errorf("failed to get rootfs info: %v", err)
@@ -915,8 +916,8 @@ func isKernelPid(pid int) bool {
// GetCapacity returns node capacity data for "cpu", "memory", "ephemeral-storage", and "huge-pages*"
// At present this method is only invoked when introspecting ephemeral storage
func (cm *containerManagerImpl) GetCapacity() v1.ResourceList {
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LocalStorageCapacityIsolation) {
func (cm *containerManagerImpl) GetCapacity(localStorageCapacityIsolation bool) v1.ResourceList {
if localStorageCapacityIsolation {
// We store allocatable ephemeral-storage in the capacity property once we Start() the container manager
if _, ok := cm.capacity[v1.ResourceEphemeralStorage]; !ok {
// If we haven't yet stored the capacity for ephemeral-storage, we can try to fetch it directly from cAdvisor,

42
pkg/kubelet/cm/container_manager_linux_test.go
View File

@@ -28,9 +28,6 @@ import (
gomock "github.com/golang/mock/gomock"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
kubefeatures "k8s.io/kubernetes/pkg/features"
"github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/stretchr/testify/assert"
@@ -193,11 +190,11 @@ func TestGetCapacity(t *testing.T) {
mockCadvisorError := cadvisortest.NewMockInterface(mockCtrlError)
mockCadvisorError.EXPECT().RootFsInfo().Return(cadvisorapiv2.FsInfo{}, errors.New("Unable to get rootfs data from cAdvisor interface"))
cases := []struct {
name string
cm *containerManagerImpl
expectedResourceQuantity *resource.Quantity
expectedNoEphemeralStorage bool
enableLocalStorageCapacityIsolation bool
name string
cm *containerManagerImpl
expectedResourceQuantity *resource.Quantity
expectedNoEphemeralStorage bool
disablelocalStorageCapacityIsolation bool
}{
{
name: "capacity property has ephemeral-storage",
@@ -207,9 +204,8 @@ func TestGetCapacity(t *testing.T) {
v1.ResourceEphemeralStorage: *resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI),
},
},
expectedResourceQuantity: resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI),
expectedNoEphemeralStorage: false,
enableLocalStorageCapacityIsolation: true,
expectedResourceQuantity: resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI),
expectedNoEphemeralStorage: false,
},
{
name: "capacity property does not have ephemeral-storage",
@@ -217,9 +213,8 @@ func TestGetCapacity(t *testing.T) {
cadvisorInterface: mockCadvisor,
capacity: v1.ResourceList{},
},
expectedResourceQuantity: resource.NewQuantity(ephemeralStorageFromCadvisor, resource.BinarySI),
expectedNoEphemeralStorage: false,
enableLocalStorageCapacityIsolation: true,
expectedResourceQuantity: resource.NewQuantity(ephemeralStorageFromCadvisor, resource.BinarySI),
expectedNoEphemeralStorage: false,
},
{
name: "capacity property does not have ephemeral-storage, error from rootfs",
@@ -227,8 +222,7 @@ func TestGetCapacity(t *testing.T) {
cadvisorInterface: mockCadvisorError,
capacity: v1.ResourceList{},
},
expectedNoEphemeralStorage: true,
enableLocalStorageCapacityIsolation: true,
expectedNoEphemeralStorage: true,
},
{
name: "capacity property does not have ephemeral-storage, cadvisor interface is nil",
@@ -236,26 +230,24 @@ func TestGetCapacity(t *testing.T) {
cadvisorInterface: nil,
capacity: v1.ResourceList{},
},
expectedNoEphemeralStorage: true,
enableLocalStorageCapacityIsolation: true,
expectedNoEphemeralStorage: true,
},
{
name: "LocalStorageCapacityIsolation feature flag is disabled",
name: "capacity property has ephemeral-storage, but localStorageCapacityIsolation is disabled",
cm: &containerManagerImpl{
cadvisorInterface: mockCadvisor,
capacity: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("4"),
v1.ResourceMemory: resource.MustParse("16G"),
v1.ResourceEphemeralStorage: *resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI),
},
},
expectedNoEphemeralStorage: true,
enableLocalStorageCapacityIsolation: false,
expectedResourceQuantity: resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI),
expectedNoEphemeralStorage: true,
disablelocalStorageCapacityIsolation: true,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, kubefeatures.LocalStorageCapacityIsolation, c.enableLocalStorageCapacityIsolation)()
ret := c.cm.GetCapacity()
ret := c.cm.GetCapacity(!c.disablelocalStorageCapacityIsolation)
if v, exists := ret[v1.ResourceEphemeralStorage]; !exists {
if !c.expectedNoEphemeralStorage {
t.Errorf("did not get any ephemeral storage data")

7
pkg/kubelet/cm/container_manager_stub.go
View File

@@ -41,7 +41,7 @@ type containerManagerStub struct {
var _ ContainerManager = &containerManagerStub{}
func (cm *containerManagerStub) Start(_ *v1.Node, _ ActivePodsFunc, _ config.SourcesReady, _ status.PodStatusProvider, _ internalapi.RuntimeService) error {
func (cm *containerManagerStub) Start(_ *v1.Node, _ ActivePodsFunc, _ config.SourcesReady, _ status.PodStatusProvider, _ internalapi.RuntimeService, _ bool) error {
klog.V(2).InfoS("Starting stub container manager")
return nil
}
@@ -74,7 +74,10 @@ func (cm *containerManagerStub) GetNodeAllocatableReservation() v1.ResourceList
return nil
}
func (cm *containerManagerStub) GetCapacity() v1.ResourceList {
func (cm *containerManagerStub) GetCapacity(localStorageCapacityIsolation bool) v1.ResourceList {
if !localStorageCapacityIsolation {
return v1.ResourceList{}
}
c := v1.ResourceList{
v1.ResourceEphemeralStorage: *resource.NewQuantity(
int64(0),

2
pkg/kubelet/cm/container_manager_unsupported.go
View File

@@ -38,7 +38,7 @@ type unsupportedContainerManager struct {
var _ ContainerManager = &unsupportedContainerManager{}
func (unsupportedContainerManager) Start(_ *v1.Node, _ ActivePodsFunc, _ config.SourcesReady, _ status.PodStatusProvider, _ internalapi.RuntimeService) error {
func (unsupportedContainerManager) Start(_ *v1.Node, _ ActivePodsFunc, _ config.SourcesReady, _ status.PodStatusProvider, _ internalapi.RuntimeService, _ bool) error {
return fmt.Errorf("Container Manager is unsupported in this build")
}

9
pkg/kubelet/cm/container_manager_windows.go
View File

@@ -30,11 +30,9 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/tools/record"
internalapi "k8s.io/cri-api/pkg/apis"
podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1"
kubefeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/cm/admission"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
@@ -72,10 +70,11 @@ func (cm *containerManagerImpl) Start(node *v1.Node,
activePods ActivePodsFunc,
sourcesReady config.SourcesReady,
podStatusProvider status.PodStatusProvider,
runtimeService internalapi.RuntimeService) error {
runtimeService internalapi.RuntimeService,
localStorageCapacityIsolation bool) error {
klog.V(2).InfoS("Starting Windows container manager")
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LocalStorageCapacityIsolation) {
if localStorageCapacityIsolation {
rootfs, err := cm.cadvisorInterface.RootFsInfo()
if err != nil {
return fmt.Errorf("failed to get rootfs info: %v", err)
@@ -171,7 +170,7 @@ func (cm *containerManagerImpl) GetNodeAllocatableReservation() v1.ResourceList
return result
}
func (cm *containerManagerImpl) GetCapacity() v1.ResourceList {
func (cm *containerManagerImpl) GetCapacity(localStorageCapacityIsolation bool) v1.ResourceList {
return cm.capacity
}

7
pkg/kubelet/cm/fake_container_manager.go
View File

@@ -50,7 +50,7 @@ func NewFakeContainerManager() *FakeContainerManager {
}
}
func (cm *FakeContainerManager) Start(_ *v1.Node, _ ActivePodsFunc, _ config.SourcesReady, _ status.PodStatusProvider, _ internalapi.RuntimeService) error {
func (cm *FakeContainerManager) Start(_ *v1.Node, _ ActivePodsFunc, _ config.SourcesReady, _ status.PodStatusProvider, _ internalapi.RuntimeService, _ bool) error {
cm.Lock()
defer cm.Unlock()
cm.CalledFunctions = append(cm.CalledFunctions, "Start")
@@ -106,10 +106,13 @@ func (cm *FakeContainerManager) GetNodeAllocatableReservation() v1.ResourceList
return nil
}
func (cm *FakeContainerManager) GetCapacity() v1.ResourceList {
func (cm *FakeContainerManager) GetCapacity(localStorageCapacityIsolation bool) v1.ResourceList {
cm.Lock()
defer cm.Unlock()
cm.CalledFunctions = append(cm.CalledFunctions, "GetCapacity")
if !localStorageCapacityIsolation {
return v1.ResourceList{}
}
c := v1.ResourceList{
v1.ResourceEphemeralStorage: *resource.NewQuantity(
int64(0),