feat: cleanup pod critical pod annotations feature

cluster/addons/calico-policy-controller/calico-node-daemonset.yaml

@@ -17,8 +17,6 @@ spec:
     metadata:
       labels:
         k8s-app: calico-node
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       nodeSelector:

cluster/addons/calico-policy-controller/calico-node-vertical-autoscaler-deployment.yaml

@@ -16,8 +16,6 @@ spec:
     metadata:
       labels:
         k8s-app: calico-node-autoscaler
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-cluster-critical
       containers:

cluster/addons/calico-policy-controller/typha-deployment.yaml

@@ -16,8 +16,6 @@ spec:
     metadata:
       labels:
         k8s-app: calico-typha
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-cluster-critical
       tolerations:

cluster/addons/calico-policy-controller/typha-horizontal-autoscaler-deployment.yaml

@@ -16,8 +16,6 @@ spec:
     metadata:
       labels:
         k8s-app: calico-typha-autoscaler
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-cluster-critical
       securityContext:

cluster/addons/calico-policy-controller/typha-vertical-autoscaler-deployment.yaml

@@ -16,8 +16,6 @@ spec:
     metadata:
       labels:
         k8s-app: calico-typha-autoscaler
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-cluster-critical
       containers:

cluster/addons/cluster-monitoring/google/heapster-controller.yaml

@@ -51,7 +51,6 @@ spec:
         k8s-app: heapster
         version: v1.6.0-beta.1
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml

@@ -51,7 +51,6 @@ spec:
         k8s-app: heapster
         version: v1.6.0-beta.1
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml

@@ -51,7 +51,6 @@ spec:
         k8s-app: heapster
         version: v1.6.0-beta.1
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml

@@ -19,7 +19,6 @@ spec:
         k8s-app: influxGrafana
         version: v4
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml

@@ -39,7 +39,6 @@ spec:
         k8s-app: heapster
         version: v1.6.0-beta.1
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml

@@ -39,7 +39,6 @@ spec:
         k8s-app: heapster
         version: v1.6.0-beta.1
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/dashboard/dashboard-controller.yaml

@@ -24,7 +24,6 @@ spec:
       labels:
         k8s-app: kubernetes-dashboard
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml

@@ -14,8 +14,6 @@ spec:
     metadata:
       labels:
         k8s-app: nvidia-gpu-device-plugin
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       affinity:

cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml

@@ -76,7 +76,6 @@ spec:
       labels:
         k8s-app: kube-dns-autoscaler
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/dns/kube-dns/kube-dns.yaml.base

@@ -82,7 +82,6 @@ spec:
       labels:
         k8s-app: kube-dns
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
         prometheus.io/port: "10054"
         prometheus.io/scrape: "true"

cluster/addons/dns/kube-dns/kube-dns.yaml.in

@@ -82,7 +82,6 @@ spec:
       labels:
         k8s-app: kube-dns
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
         prometheus.io/port: "10054"
         prometheus.io/scrape: "true"

cluster/addons/dns/kube-dns/kube-dns.yaml.sed

@@ -82,7 +82,6 @@ spec:
       labels:
         k8s-app: kube-dns
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
         prometheus.io/port: "10054"
         prometheus.io/scrape: "true"

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -65,7 +65,6 @@ spec:
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-node-critical

cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml

@@ -21,11 +21,6 @@ spec:
         k8s-app: fluentd-gcp
         kubernetes.io/cluster-service: "true"
         version: {{ fluentd_gcp_yaml_version }}
-      # This annotation ensures that fluentd does not get evicted if the node
-      # supports critical pod annotation based priority scheme.
-      # Note that this does not guarantee admission on the nodes (#40573).
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       serviceAccountName: fluentd-gcp

cluster/addons/ip-masq-agent/ip-masq-agent.yaml

@@ -24,8 +24,6 @@ spec:
     metadata:
       labels:
         k8s-app: ip-masq-agent
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       serviceAccountName: ip-masq-agent

cluster/addons/kube-proxy/kube-proxy-ds.yaml

@@ -21,8 +21,6 @@ spec:
     metadata:
       labels:
         k8s-app: kube-proxy
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       hostNetwork: true

cluster/addons/metadata-proxy/gce/metadata-proxy.yaml

@@ -31,11 +31,6 @@ spec:
         k8s-app: metadata-proxy
         kubernetes.io/cluster-service: "true"
         version: v0.1
-      # This annotation ensures that the proxy does not get evicted if the node
-      # supports critical pod annotation based priority scheme.
-      # Note that this does not guarantee admission on the nodes (#40573).
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       serviceAccountName: metadata-proxy

cluster/addons/metrics-server/metrics-server-deployment.yaml

@@ -42,7 +42,6 @@ spec:
         k8s-app: metrics-server
         version: v0.3.3
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       priorityClassName: system-cluster-critical

cluster/addons/prometheus/alertmanager-deployment.yaml

@@ -19,8 +19,6 @@ spec:
       labels:
         k8s-app: alertmanager
         version: v0.14.0
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-cluster-critical
       containers:

cluster/addons/prometheus/kube-state-metrics-deployment.yaml

@@ -19,8 +19,6 @@ spec:
       labels:
         k8s-app: kube-state-metrics
         version: v1.3.0
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: kube-state-metrics

cluster/addons/prometheus/node-exporter-ds.yml

@@ -20,8 +20,6 @@ spec:
       labels:
         k8s-app: node-exporter
         version: v0.15.2
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       containers:

cluster/addons/prometheus/prometheus-statefulset.yaml

@@ -21,8 +21,6 @@ spec:
     metadata:
       labels:
         k8s-app: prometheus
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: prometheus

cluster/gce/addons/node-termination-handler/daemonset.yaml

@@ -17,8 +17,6 @@ spec:
     metadata:
       labels:
         k8s-app: node-termination-handler
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       # Necessary to reboot node

cluster/gce/config-default.sh

@@ -250,10 +250,14 @@ if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then
 fi
 
 # Optional: set feature gates
-FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
+FEATURE_GATES="${KUBE_FEATURE_GATES:-}"
 
 if [[ ! -z "${NODE_ACCELERATORS}" ]]; then
-    FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true"
+    if [[ -z "${FEATURE_GATES:-}" ]]; then
+        FEATURE_GATES="DevicePlugins=true"
+    else
+        FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true"
+    fi
     if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then
         NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}"
     fi

cluster/gce/config-test.sh

@@ -139,7 +139,7 @@ if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then
 fi
 
 # Optional: set feature gates
-FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
+FEATURE_GATES="${KUBE_FEATURE_GATES:-}"
 
 TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}
 
@@ -283,7 +283,11 @@ if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then
 fi
 
 if [[ ! -z "${NODE_ACCELERATORS}" ]]; then
-    FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true"
+    if [[ -z "${FEATURE_GATES:-}" ]]; then
+        FEATURE_GATES="DevicePlugins=true"
+    else
+        FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true"
+    fi
     if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then
         NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}"
     fi

cluster/gce/manifests/etcd-empty-dir-cleanup.yaml

@@ -4,7 +4,6 @@ metadata:
   name: etcd-empty-dir-cleanup
   namespace: kube-system
   annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
     seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
   labels:
     k8s-app: etcd-empty-dir-cleanup

cluster/gce/manifests/etcd.manifest

@@ -5,11 +5,11 @@
   "name":"etcd-server{{ suffix }}",
   "namespace": "kube-system",
   "annotations": {
-    "scheduler.alpha.kubernetes.io/critical-pod": "",
     "seccomp.security.alpha.kubernetes.io/pod": "docker/default"
   }
 },
 "spec":{
+"priorityClass": "system-node-critical",
 "hostNetwork": true,
 "containers":[
     {

cluster/gce/manifests/glbc.manifest

@@ -4,13 +4,13 @@ metadata:
   name: l7-lb-controller-v1.2.3
   namespace: kube-system
   annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
     seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
   labels:
     k8s-app: gcp-lb-controller
     version: v1.2.3
     kubernetes.io/name: "GLBC"
 spec:
+  priorityClassName: system-node-critical
   terminationGracePeriodSeconds: 600
   hostNetwork: true
   containers:

cluster/gce/manifests/kube-addon-manager.yaml

@@ -4,11 +4,11 @@ metadata:
   name: kube-addon-manager
   namespace: kube-system
   annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
     seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
   labels:
     component: kube-addon-manager
 spec:
+  priorityClassName: system-node-critical
   hostNetwork: true
   containers:
   - name: kube-addon-manager

cluster/gce/manifests/kube-apiserver.manifest

@@ -5,7 +5,6 @@
   "name":"kube-apiserver",
   "namespace": "kube-system",
   "annotations": {
-    "scheduler.alpha.kubernetes.io/critical-pod": "",
     "seccomp.security.alpha.kubernetes.io/pod": "docker/default"
   },
   "labels": {
@@ -14,6 +13,7 @@
   }
 },
 "spec":{
+"priorityClass": "system-node-critical",
 "hostNetwork": true,
 "containers":[
     {

cluster/gce/manifests/kube-controller-manager.manifest

@@ -5,7 +5,6 @@
   "name":"kube-controller-manager",
   "namespace": "kube-system",
   "annotations": {
-    "scheduler.alpha.kubernetes.io/critical-pod": "",
     "seccomp.security.alpha.kubernetes.io/pod": "docker/default"
   },
   "labels": {
@@ -14,6 +13,7 @@
   }
 },
 "spec":{
+"priorityClass": "system-node-critical",
 "hostNetwork": true,
 "containers":[
     {

cluster/gce/manifests/kube-proxy.manifest

@@ -3,12 +3,6 @@ kind: Pod
 metadata:
   name: kube-proxy
   namespace: kube-system
-  # This annotation ensures that kube-proxy does not get evicted if the node
-  # supports critical pod annotation based priority scheme.
-  # Note that kube-proxy runs as a static pod so this annotation does NOT have
-  # any effect on default scheduler which scheduling kube-proxy.
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
   labels:
     tier: node
     component: kube-proxy

cluster/gce/manifests/kube-scheduler.manifest

@@ -5,7 +5,6 @@
   "name":"kube-scheduler",
   "namespace": "kube-system",
   "annotations": {
-    "scheduler.alpha.kubernetes.io/critical-pod": "",
     "seccomp.security.alpha.kubernetes.io/pod": "docker/default"
   },
   "labels": {
@@ -14,6 +13,7 @@
   }
 },
 "spec":{
+"priorityClass": "system-node-critical",
 "hostNetwork": true,
 "containers":[
     {

cluster/gce/windows/k8s-node-setup.psm1

@@ -973,7 +973,6 @@ function Start-WorkerServices {
   #   kube-proxy --master=https://35.239.84.171
   #   --kubeconfig=/var/lib/kube-proxy/kubeconfig --cluster-cidr=10.64.0.0/14
   #   --oom-score-adj=-998 --v=2
-  #   --feature-gates=ExperimentalCriticalPodAnnotation=true
   #   --iptables-sync-period=1m --iptables-min-sync-period=10s
   #   --ipvs-sync-period=1m --ipvs-min-sync-period=10s
   # And also with various volumeMounts and "securityContext: privileged: true".

pkg/controller/daemon/BUILD

@@ -66,9 +66,9 @@ go_test(
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/api/v1/pod:go_default_library",
         "//pkg/apis/core:go_default_library",
+        "//pkg/apis/scheduling:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/features:go_default_library",
-        "//pkg/kubelet/types:go_default_library",
         "//pkg/scheduler/api:go_default_library",
         "//pkg/securitycontext:go_default_library",
         "//pkg/util/labels:go_default_library",

pkg/controller/daemon/daemon_controller_test.go

@@ -46,9 +46,9 @@ import (
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/apis/scheduling"
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/features"
-	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	"k8s.io/kubernetes/pkg/securitycontext"
 	labelsutil "k8s.io/kubernetes/pkg/util/labels"
@@ -1815,6 +1815,34 @@ func TestTaintPressureNodeDaemonLaunchesPod(t *testing.T) {
 // When ScheduleDaemonSetPods is disabled, DaemonSet should launch a critical pod even when the node has insufficient free resource.
 func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) {
 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, false)()
+	for _, strategy := range updateStrategies() {
+		podSpec := resourcePodSpec("too-much-mem", "75M", "75m")
+		ds := newDaemonSet("critical")
+		ds.Spec.UpdateStrategy = *strategy
+		ds.Spec.Template.Spec = podSpec
+
+		manager, podControl, _, err := newTestController(ds)
+		if err != nil {
+			t.Fatalf("error creating DaemonSets controller: %v", err)
+		}
+		node := newNode("too-much-mem", nil)
+		node.Status.Allocatable = allocatableResources("100M", "200m")
+		manager.nodeStore.Add(node)
+		manager.podStore.Add(&v1.Pod{
+			Spec: podSpec,
+		})
+
+		manager.dsStore.Add(ds)
+		switch strategy.Type {
+		case apps.OnDeleteDaemonSetStrategyType:
+			syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 2)
+		case apps.RollingUpdateDaemonSetStrategyType:
+			syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 3)
+		default:
+			t.Fatalf("unexpected UpdateStrategy %+v", strategy)
+		}
+	}
+
 	for _, strategy := range updateStrategies() {
 		podSpec := resourcePodSpec("too-much-mem", "75M", "75m")
 		ds := newDaemonSet("critical")
@@ -1833,25 +1861,13 @@ func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) {
 			Spec: podSpec,
 		})
 
-		// Without enabling critical pod annotation feature gate, we shouldn't create critical pod
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, false)()
 		manager.dsStore.Add(ds)
-		switch strategy.Type {
-		case apps.OnDeleteDaemonSetStrategyType:
-			syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 2)
-		case apps.RollingUpdateDaemonSetStrategyType:
-			syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 3)
-		default:
-			t.Fatalf("unexpected UpdateStrategy %+v", strategy)
-		}
 
-		// Enabling critical pod annotation feature gate should create critical pod
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
 		switch strategy.Type {
 		case apps.OnDeleteDaemonSetStrategyType:
-			syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 2)
+			syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0)
 		case apps.RollingUpdateDaemonSetStrategyType:
-			syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 3)
+			syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0)
 		default:
 			t.Fatalf("unexpected UpdateStrategy %+v", strategy)
 		}
@@ -1880,7 +1896,6 @@ func TestPortConflictNodeDaemonDoesNotLaunchCriticalPod(t *testing.T) {
 			Spec: podSpec,
 		})
 
-		defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
 		ds := newDaemonSet("critical")
 		ds.Spec.UpdateStrategy = *strategy
 		ds.Spec.Template.Spec = podSpec
@@ -1895,7 +1910,8 @@ func setDaemonSetCritical(ds *apps.DaemonSet) {
 	if ds.Spec.Template.ObjectMeta.Annotations == nil {
 		ds.Spec.Template.ObjectMeta.Annotations = make(map[string]string)
 	}
-	ds.Spec.Template.ObjectMeta.Annotations[kubelettypes.CriticalPodAnnotationKey] = ""
+	podPriority := scheduling.SystemCriticalPriority
+	ds.Spec.Template.Spec.Priority = &podPriority
 }
 
 func TestNodeShouldRunDaemonPod(t *testing.T) {

pkg/features/kube_features.go

@@ -48,15 +48,6 @@ const (
 	// SYS_TIME). This should only be enabled if user namespace remapping is enabled in the docker daemon.
 	ExperimentalHostUserNamespaceDefaultingGate featuregate.Feature = "ExperimentalHostUserNamespaceDefaulting"
 
-	// owner: @vishh
-	// alpha: v1.5
-	//
-	// DEPRECATED - This feature is deprecated by Pod Priority and Preemption as of Kubernetes 1.13.
-	// Ensures guaranteed scheduling of pods marked with a special pod annotation `scheduler.alpha.kubernetes.io/critical-pod`
-	// and also prevents them from being evicted from a node.
-	// Note: This feature is not supported for `BestEffort` pods.
-	ExperimentalCriticalPodAnnotation featuregate.Feature = "ExperimentalCriticalPodAnnotation"
-
 	// owner: @jiayingz
 	// beta: v1.10
 	//
@@ -472,65 +463,64 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	AppArmor:             {Default: true, PreRelease: featuregate.Beta},
 	DynamicKubeletConfig: {Default: true, PreRelease: featuregate.Beta},
 	ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: featuregate.Beta},
-	ExperimentalCriticalPodAnnotation:           {Default: false, PreRelease: featuregate.Alpha},
+	DevicePlugins:                       {Default: true, PreRelease: featuregate.Beta},
-	DevicePlugins:                               {Default: true, PreRelease: featuregate.Beta},
+	TaintBasedEvictions:                 {Default: true, PreRelease: featuregate.Beta},
-	TaintBasedEvictions:                         {Default: true, PreRelease: featuregate.Beta},
+	RotateKubeletServerCertificate:      {Default: true, PreRelease: featuregate.Beta},
-	RotateKubeletServerCertificate:              {Default: true, PreRelease: featuregate.Beta},
+	RotateKubeletClientCertificate:      {Default: true, PreRelease: featuregate.Beta},
-	RotateKubeletClientCertificate:              {Default: true, PreRelease: featuregate.Beta},
+	PersistentLocalVolumes:              {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.17
-	PersistentLocalVolumes:                      {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.17
+	LocalStorageCapacityIsolation:       {Default: true, PreRelease: featuregate.Beta},
-	LocalStorageCapacityIsolation:               {Default: true, PreRelease: featuregate.Beta},
+	Sysctls:                             {Default: true, PreRelease: featuregate.Beta},
-	Sysctls:                                     {Default: true, PreRelease: featuregate.Beta},
+	DebugContainers:                     {Default: false, PreRelease: featuregate.Alpha},
-	DebugContainers:                             {Default: false, PreRelease: featuregate.Alpha},
+	PodShareProcessNamespace:            {Default: true, PreRelease: featuregate.Beta},
-	PodShareProcessNamespace:                    {Default: true, PreRelease: featuregate.Beta},
+	PodPriority:                         {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.18
-	PodPriority:                                 {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.18
+	TaintNodesByCondition:               {Default: true, PreRelease: featuregate.Beta},
-	TaintNodesByCondition:                       {Default: true, PreRelease: featuregate.Beta},
+	QOSReserved:                         {Default: false, PreRelease: featuregate.Alpha},
-	QOSReserved:                                 {Default: false, PreRelease: featuregate.Alpha},
+	ExpandPersistentVolumes:             {Default: true, PreRelease: featuregate.Beta},
-	ExpandPersistentVolumes:                     {Default: true, PreRelease: featuregate.Beta},
+	ExpandInUsePersistentVolumes:        {Default: true, PreRelease: featuregate.Beta},
-	ExpandInUsePersistentVolumes:                {Default: true, PreRelease: featuregate.Beta},
+	ExpandCSIVolumes:                    {Default: false, PreRelease: featuregate.Alpha},
-	ExpandCSIVolumes:                            {Default: false, PreRelease: featuregate.Alpha},
+	AttachVolumeLimit:                   {Default: true, PreRelease: featuregate.Beta},
-	AttachVolumeLimit:                           {Default: true, PreRelease: featuregate.Beta},
+	CPUManager:                          {Default: true, PreRelease: featuregate.Beta},
-	CPUManager:                                  {Default: true, PreRelease: featuregate.Beta},
+	CPUCFSQuotaPeriod:                   {Default: false, PreRelease: featuregate.Alpha},
-	CPUCFSQuotaPeriod:                           {Default: false, PreRelease: featuregate.Alpha},
+	TopologyManager:                     {Default: false, PreRelease: featuregate.Alpha},
-	TopologyManager:                             {Default: false, PreRelease: featuregate.Alpha},
+	ServiceNodeExclusion:                {Default: false, PreRelease: featuregate.Alpha},
-	ServiceNodeExclusion:                        {Default: false, PreRelease: featuregate.Alpha},
+	MountContainers:                     {Default: false, PreRelease: featuregate.Alpha},
-	MountContainers:                             {Default: false, PreRelease: featuregate.Alpha},
+	CSIDriverRegistry:                   {Default: true, PreRelease: featuregate.Beta},
-	CSIDriverRegistry:                           {Default: true, PreRelease: featuregate.Beta},
+	CSINodeInfo:                         {Default: true, PreRelease: featuregate.Beta},
-	CSINodeInfo:                                 {Default: true, PreRelease: featuregate.Beta},
+	BlockVolume:                         {Default: true, PreRelease: featuregate.Beta},
-	BlockVolume:                                 {Default: true, PreRelease: featuregate.Beta},
+	StorageObjectInUseProtection:        {Default: true, PreRelease: featuregate.GA},
-	StorageObjectInUseProtection:                {Default: true, PreRelease: featuregate.GA},
+	ResourceLimitsPriorityFunction:      {Default: false, PreRelease: featuregate.Alpha},
-	ResourceLimitsPriorityFunction:              {Default: false, PreRelease: featuregate.Alpha},
+	SupportIPVSProxyMode:                {Default: true, PreRelease: featuregate.GA},
-	SupportIPVSProxyMode:                        {Default: true, PreRelease: featuregate.GA},
+	SupportPodPidsLimit:                 {Default: true, PreRelease: featuregate.Beta},
-	SupportPodPidsLimit:                         {Default: true, PreRelease: featuregate.Beta},
+	SupportNodePidsLimit:                {Default: true, PreRelease: featuregate.Beta},
-	SupportNodePidsLimit:                        {Default: true, PreRelease: featuregate.Beta},
+	HyperVContainer:                     {Default: false, PreRelease: featuregate.Alpha},
-	HyperVContainer:                             {Default: false, PreRelease: featuregate.Alpha},
+	ScheduleDaemonSetPods:               {Default: true, PreRelease: featuregate.Beta},
-	ScheduleDaemonSetPods:                       {Default: true, PreRelease: featuregate.Beta},
+	TokenRequest:                        {Default: true, PreRelease: featuregate.Beta},
-	TokenRequest:                                {Default: true, PreRelease: featuregate.Beta},
+	TokenRequestProjection:              {Default: true, PreRelease: featuregate.Beta},
-	TokenRequestProjection:                      {Default: true, PreRelease: featuregate.Beta},
+	BoundServiceAccountTokenVolume:      {Default: false, PreRelease: featuregate.Alpha},
-	BoundServiceAccountTokenVolume:              {Default: false, PreRelease: featuregate.Alpha},
+	CRIContainerLogRotation:             {Default: true, PreRelease: featuregate.Beta},
-	CRIContainerLogRotation:                     {Default: true, PreRelease: featuregate.Beta},
+	deprecatedGCERegionalPersistentDisk: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.17
-	deprecatedGCERegionalPersistentDisk:         {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.17
+	CSIMigration:                        {Default: false, PreRelease: featuregate.Alpha},
-	CSIMigration:                                {Default: false, PreRelease: featuregate.Alpha},
+	CSIMigrationGCE:                     {Default: false, PreRelease: featuregate.Alpha},
-	CSIMigrationGCE:                             {Default: false, PreRelease: featuregate.Alpha},
+	CSIMigrationAWS:                     {Default: false, PreRelease: featuregate.Alpha},
-	CSIMigrationAWS:                             {Default: false, PreRelease: featuregate.Alpha},
+	CSIMigrationAzureDisk:               {Default: false, PreRelease: featuregate.Alpha},
-	CSIMigrationAzureDisk:                       {Default: false, PreRelease: featuregate.Alpha},
+	CSIMigrationAzureFile:               {Default: false, PreRelease: featuregate.Alpha},
-	CSIMigrationAzureFile:                       {Default: false, PreRelease: featuregate.Alpha},
+	RunAsGroup:                          {Default: true, PreRelease: featuregate.Beta},
-	RunAsGroup:                                  {Default: true, PreRelease: featuregate.Beta},
+	CSIMigrationOpenStack:               {Default: false, PreRelease: featuregate.Alpha},
-	CSIMigrationOpenStack:                       {Default: false, PreRelease: featuregate.Alpha},
+	VolumeSubpath:                       {Default: true, PreRelease: featuregate.GA},
-	VolumeSubpath:                               {Default: true, PreRelease: featuregate.GA},
+	BalanceAttachedNodeVolumes:          {Default: false, PreRelease: featuregate.Alpha},
-	BalanceAttachedNodeVolumes:                  {Default: false, PreRelease: featuregate.Alpha},
+	VolumeSubpathEnvExpansion:           {Default: true, PreRelease: featuregate.Beta},
-	VolumeSubpathEnvExpansion:                   {Default: true, PreRelease: featuregate.Beta},
+	ResourceQuotaScopeSelectors:         {Default: true, PreRelease: featuregate.Beta},
-	ResourceQuotaScopeSelectors:                 {Default: true, PreRelease: featuregate.Beta},
+	CSIBlockVolume:                      {Default: true, PreRelease: featuregate.Beta},
-	CSIBlockVolume:                              {Default: true, PreRelease: featuregate.Beta},
+	CSIInlineVolume:                     {Default: false, PreRelease: featuregate.Alpha},
-	CSIInlineVolume:                             {Default: false, PreRelease: featuregate.Alpha},
+	RuntimeClass:                        {Default: true, PreRelease: featuregate.Beta},
-	RuntimeClass:                                {Default: true, PreRelease: featuregate.Beta},
+	NodeLease:                           {Default: true, PreRelease: featuregate.Beta},
-	NodeLease:                                   {Default: true, PreRelease: featuregate.Beta},
+	SCTPSupport:                         {Default: false, PreRelease: featuregate.Alpha},
-	SCTPSupport:                                 {Default: false, PreRelease: featuregate.Alpha},
+	VolumeSnapshotDataSource:            {Default: false, PreRelease: featuregate.Alpha},
-	VolumeSnapshotDataSource:                    {Default: false, PreRelease: featuregate.Alpha},
+	ProcMountType:                       {Default: false, PreRelease: featuregate.Alpha},
-	ProcMountType:                               {Default: false, PreRelease: featuregate.Alpha},
+	TTLAfterFinished:                    {Default: false, PreRelease: featuregate.Alpha},
-	TTLAfterFinished:                            {Default: false, PreRelease: featuregate.Alpha},
+	KubeletPodResources:                 {Default: true, PreRelease: featuregate.Beta},
-	KubeletPodResources:                         {Default: true, PreRelease: featuregate.Beta},
+	WindowsGMSA:                         {Default: false, PreRelease: featuregate.Alpha},
-	WindowsGMSA:                                 {Default: false, PreRelease: featuregate.Alpha},
+	ServiceLoadBalancerFinalizer:        {Default: false, PreRelease: featuregate.Alpha},
-	ServiceLoadBalancerFinalizer:                {Default: false, PreRelease: featuregate.Alpha},
 	LocalStorageCapacityIsolationFSQuotaMonitoring: {Default: false, PreRelease: featuregate.Alpha},
 	NonPreemptingPriority:                          {Default: false, PreRelease: featuregate.Alpha},
 	VolumePVCDataSource:                            {Default: false, PreRelease: featuregate.Alpha},

pkg/kubelet/eviction/BUILD

@@ -17,6 +17,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",
+        "//pkg/apis/scheduling:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
         "//pkg/kubelet/eviction/api:go_default_library",

pkg/kubelet/eviction/eviction_manager_test.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	kubeapi "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/apis/scheduling"
 	"k8s.io/kubernetes/pkg/features"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
@@ -1132,12 +1133,12 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
 	}
 }
 
-// TestCriticalPodsAreNotEvicted
+// TestStaticCriticalPodsAreNotEvicted
-func TestCriticalPodsAreNotEvicted(t *testing.T) {
+func TestStaticCriticalPodsAreNotEvicted(t *testing.T) {
 	podMaker := makePodWithMemoryStats
 	summaryStatsMaker := makeMemoryStats
 	podsToMake := []podToMake{
-		{name: "critical", priority: defaultPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "800Mi"},
+		{name: "critical", priority: scheduling.SystemCriticalPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "800Mi"},
 	}
 	pods := []*v1.Pod{}
 	podStats := map[*v1.Pod]statsapi.PodStats{}
@@ -1147,11 +1148,12 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
 		podStats[pod] = podStat
 	}
 
-	// Mark the pod as critical
 	pods[0].Annotations = map[string]string{
-		kubelettypes.CriticalPodAnnotationKey:  "",
 		kubelettypes.ConfigSourceAnnotationKey: kubelettypes.FileSource,
 	}
+	// Mark the pod as critical
+	podPriority := scheduling.SystemCriticalPriority
+	pods[0].Spec.Priority = &podPriority
 	pods[0].Namespace = kubeapi.NamespaceSystem
 
 	podToEvict := pods[0]
@@ -1208,9 +1210,6 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
 		thresholdsFirstObservedAt:    thresholdsObservedAt{},
 	}
 
-	// Enable critical pod annotation feature gate
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
-	// induce soft threshold
 	fakeClock.Step(1 * time.Minute)
 	summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
 	manager.synchronize(diskInfoProvider, activePodsFunc)
@@ -1253,8 +1252,11 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
 		t.Errorf("Manager should not report memory pressure")
 	}
 
-	// Disable critical pod annotation feature gate
+	pods[0].Annotations = map[string]string{
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, false)()
+		kubelettypes.ConfigSourceAnnotationKey: kubelettypes.FileSource,
+	}
+	pods[0].Spec.Priority = nil
+	pods[0].Namespace = kubeapi.NamespaceSystem
 
 	// induce memory pressure!
 	fakeClock.Step(1 * time.Minute)

pkg/kubelet/preemption/BUILD

@@ -45,13 +45,9 @@ go_test(
     deps = [
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/scheduling:go_default_library",
-        "//pkg/features:go_default_library",
-        "//pkg/kubelet/types:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
-        "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
     ],
 )

pkg/kubelet/preemption/preemption_test.go

@@ -23,17 +23,12 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/record"
-	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	kubeapi "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/scheduling"
-	"k8s.io/kubernetes/pkg/features"
-	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
 const (
-	critical              = "critical"
 	clusterCritical       = "cluster-critical"
 	nodeCritical          = "node-critical"
 	bestEffort            = "bestEffort"
@@ -96,7 +91,6 @@ func getTestCriticalPodAdmissionHandler(podProvider *fakePodProvider, podKiller
 }
 
 func TestEvictPodsToFreeRequestsWithError(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
 	type testRun struct {
 		testName              string
 		inputPods             []*v1.Pod
@@ -112,7 +106,7 @@ func TestEvictPodsToFreeRequestsWithError(t *testing.T) {
 		{
 			testName: "multiple pods eviction error",
 			inputPods: []*v1.Pod{
-				allPods[critical], allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable],
+				allPods[clusterCritical], allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable],
 				allPods[guaranteed], allPods[highRequestGuaranteed]},
 			insufficientResources: getAdmissionRequirementList(0, 550, 0),
 			expectErr:             false,
@@ -121,7 +115,7 @@ func TestEvictPodsToFreeRequestsWithError(t *testing.T) {
 	}
 	for _, r := range runs {
 		podProvider.setPods(r.inputPods)
-		outErr := criticalPodAdmissionHandler.evictPodsToFreeRequests(allPods[critical], r.insufficientResources)
+		outErr := criticalPodAdmissionHandler.evictPodsToFreeRequests(allPods[clusterCritical], r.insufficientResources)
 		outputPods := podKiller.getKilledPods()
 		if !r.expectErr && outErr != nil {
 			t.Errorf("evictPodsToFreeRequests returned an unexpected error during the %s test.  Err: %v", r.testName, outErr)
@@ -135,7 +129,6 @@ func TestEvictPodsToFreeRequestsWithError(t *testing.T) {
 }
 
 func TestEvictPodsToFreeRequests(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
 	type testRun struct {
 		testName              string
 		inputPods             []*v1.Pod
@@ -150,7 +143,7 @@ func TestEvictPodsToFreeRequests(t *testing.T) {
 	runs := []testRun{
 		{
 			testName:              "critical pods cannot be preempted",
-			inputPods:             []*v1.Pod{allPods[critical]},
+			inputPods:             []*v1.Pod{allPods[clusterCritical]},
 			insufficientResources: getAdmissionRequirementList(0, 0, 1),
 			expectErr:             true,
 			expectedOutput:        nil,
@@ -165,7 +158,7 @@ func TestEvictPodsToFreeRequests(t *testing.T) {
 		{
 			testName: "multiple pods evicted",
 			inputPods: []*v1.Pod{
-				allPods[critical], allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable],
+				allPods[clusterCritical], allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable],
 				allPods[guaranteed], allPods[highRequestGuaranteed]},
 			insufficientResources: getAdmissionRequirementList(0, 550, 0),
 			expectErr:             false,
@@ -174,7 +167,7 @@ func TestEvictPodsToFreeRequests(t *testing.T) {
 	}
 	for _, r := range runs {
 		podProvider.setPods(r.inputPods)
-		outErr := criticalPodAdmissionHandler.evictPodsToFreeRequests(allPods[critical], r.insufficientResources)
+		outErr := criticalPodAdmissionHandler.evictPodsToFreeRequests(allPods[clusterCritical], r.insufficientResources)
 		outputPods := podKiller.getKilledPods()
 		if !r.expectErr && outErr != nil {
 			t.Errorf("evictPodsToFreeRequests returned an unexpected error during the %s test.  Err: %v", r.testName, outErr)
@@ -203,7 +196,6 @@ func BenchmarkGetPodsToPreempt(t *testing.B) {
 }
 
 func TestGetPodsToPreempt(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
 	type testRun struct {
 		testName              string
 		preemptor             *v1.Pod
@@ -216,7 +208,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 	runs := []testRun{
 		{
 			testName:              "no requirements",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{},
 			insufficientResources: getAdmissionRequirementList(0, 0, 0),
 			expectErr:             false,
@@ -224,7 +216,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "no pods",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{},
 			insufficientResources: getAdmissionRequirementList(0, 0, 1),
 			expectErr:             true,
@@ -232,7 +224,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "equal pods and resources requirements",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{allPods[burstable]},
 			insufficientResources: getAdmissionRequirementList(100, 100, 1),
 			expectErr:             false,
@@ -240,7 +232,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "higher requirements than pod requests",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{allPods[burstable]},
 			insufficientResources: getAdmissionRequirementList(200, 200, 2),
 			expectErr:             true,
@@ -248,7 +240,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "choose between bestEffort and burstable",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{allPods[burstable], allPods[bestEffort]},
 			insufficientResources: getAdmissionRequirementList(0, 0, 1),
 			expectErr:             false,
@@ -256,7 +248,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "choose between burstable and guaranteed",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{allPods[burstable], allPods[guaranteed]},
 			insufficientResources: getAdmissionRequirementList(0, 0, 1),
 			expectErr:             false,
@@ -264,7 +256,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "choose lower request burstable if it meets requirements",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{allPods[bestEffort], allPods[highRequestBurstable], allPods[burstable]},
 			insufficientResources: getAdmissionRequirementList(100, 100, 0),
 			expectErr:             false,
@@ -272,7 +264,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "choose higher request burstable if lower does not meet requirements",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable]},
 			insufficientResources: getAdmissionRequirementList(150, 150, 0),
 			expectErr:             false,
@@ -280,7 +272,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "multiple pods required",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable], allPods[guaranteed], allPods[highRequestGuaranteed]},
 			insufficientResources: getAdmissionRequirementList(350, 350, 0),
 			expectErr:             false,
@@ -288,7 +280,7 @@ func TestGetPodsToPreempt(t *testing.T) {
 		},
 		{
 			testName:              "evict guaranteed when we have to, and dont evict the extra burstable",
-			preemptor:             allPods[critical],
+			preemptor:             allPods[clusterCritical],
 			inputPods:             []*v1.Pod{allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable], allPods[guaranteed], allPods[highRequestGuaranteed]},
 			insufficientResources: getAdmissionRequirementList(0, 550, 0),
 			expectErr:             false,
@@ -423,12 +415,6 @@ func getTestPods() map[string]*v1.Pod {
 			},
 		}),
 		bestEffort: getPodWithResources(bestEffort, v1.ResourceRequirements{}),
-		critical: getPodWithResources(critical, v1.ResourceRequirements{
-			Requests: v1.ResourceList{
-				v1.ResourceCPU:    resource.MustParse("100m"),
-				v1.ResourceMemory: resource.MustParse("100Mi"),
-			},
-		}),
 		clusterCritical: getPodWithResources(clusterCritical, v1.ResourceRequirements{
 			Requests: v1.ResourceList{
 				v1.ResourceCPU:    resource.MustParse("100m"),
@@ -474,9 +460,6 @@ func getTestPods() map[string]*v1.Pod {
 			},
 		}),
 	}
-	allPods[critical].Namespace = kubeapi.NamespaceSystem
-	allPods[critical].Annotations[kubetypes.CriticalPodAnnotationKey] = ""
-
 	allPods[clusterCritical].Namespace = kubeapi.NamespaceSystem
 	allPods[clusterCritical].Spec.PriorityClassName = scheduling.SystemClusterCritical
 	clusterPriority := scheduling.SystemCriticalPriority

pkg/kubelet/types/BUILD

@@ -18,13 +18,10 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/types",
     deps = [
-        "//pkg/apis/core:go_default_library",
         "//pkg/apis/scheduling:go_default_library",
-        "//pkg/features:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 
@@ -38,11 +35,8 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
-        "//pkg/features:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
-        "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
     ],

pkg/kubelet/types/pod_update.go

@@ -19,12 +19,9 @@ package types
 import (
 	"fmt"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	kubeapi "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/scheduling"
-	"k8s.io/kubernetes/pkg/features"
 )
 
 const (
@@ -32,7 +29,6 @@ const (
 	ConfigMirrorAnnotationKey    = v1.MirrorPodAnnotationKey
 	ConfigFirstSeenAnnotationKey = "kubernetes.io/config.seen"
 	ConfigHashAnnotationKey      = "kubernetes.io/config.hash"
-	CriticalPodAnnotationKey     = "scheduler.alpha.kubernetes.io/critical-pod"
 )
 
 // PodOperation defines what changes will be made on a pod configuration.
@@ -142,18 +138,11 @@ func (sp SyncPodType) String() string {
 	}
 }
 
-// IsCriticalPod returns true if the pod bears the critical pod annotation key or if pod's priority is greater than
+// IsCriticalPod returns true if pod's priority is greater than or equal to SystemCriticalPriority.
-// or equal to SystemCriticalPriority. Both the default scheduler and the kubelet use this function
-// to make admission and scheduling decisions.
 func IsCriticalPod(pod *v1.Pod) bool {
 	if pod.Spec.Priority != nil && IsCriticalPodBasedOnPriority(*pod.Spec.Priority) {
 		return true
 	}
-	if utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) {
-		if IsCritical(pod.Namespace, pod.Annotations) {
-			return true
-		}
-	}
 	return false
 }
 
@@ -171,21 +160,6 @@ func Preemptable(preemptor, preemptee *v1.Pod) bool {
 	return false
 }
 
-// IsCritical returns true if parameters bear the critical pod annotation
-// key. The DaemonSetController use this key directly to make scheduling decisions.
-// TODO: @ravig - Deprecated. Remove this when we move to resolving critical pods based on priorityClassName.
-func IsCritical(ns string, annotations map[string]string) bool {
-	// Critical pods are restricted to "kube-system" namespace as of now.
-	if ns != kubeapi.NamespaceSystem {
-		return false
-	}
-	val, ok := annotations[CriticalPodAnnotationKey]
-	if ok && val == "" {
-		return true
-	}
-	return false
-}
-
 // IsCriticalPodBasedOnPriority checks if the given pod is a critical pod based on priority resolved from pod Spec.
 func IsCriticalPodBasedOnPriority(priority int32) bool {
 	if priority >= scheduling.SystemCriticalPriority {

pkg/kubelet/types/pod_update_test.go

@@ -23,9 +23,6 @@ import (
 	"github.com/stretchr/testify/require"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	featuregatetesting "k8s.io/component-base/featuregate/testing"
-	"k8s.io/kubernetes/pkg/features"
 )
 
 func TestGetValidatedSources(t *testing.T) {
@@ -117,70 +114,6 @@ func TestString(t *testing.T) {
 	}
 }
 
-func TestIsCriticalPod(t *testing.T) {
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
-	cases := []struct {
-		pod      v1.Pod
-		expected bool
-	}{
-		{
-			pod: v1.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "pod1",
-					Namespace: "ns",
-					Annotations: map[string]string{
-						"scheduler.alpha.kubernetes.io/critical-pod": "",
-					},
-				},
-			},
-			expected: false,
-		},
-		{
-			pod: v1.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "pod2",
-					Namespace: "ns",
-					Annotations: map[string]string{
-						"scheduler.alpha.kubernetes.io/critical-pod": "abc",
-					},
-				},
-			},
-			expected: false,
-		},
-		{
-			pod: v1.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "pod3",
-					Namespace: "kube-system",
-					Annotations: map[string]string{
-						"scheduler.alpha.kubernetes.io/critical-pod": "abc",
-					},
-				},
-			},
-			expected: false,
-		},
-		{
-			pod: v1.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      "pod4",
-					Namespace: "kube-system",
-					Annotations: map[string]string{
-						"scheduler.alpha.kubernetes.io/critical-pod": "",
-					},
-				},
-			},
-			expected: true,
-		},
-	}
-	for i, data := range cases {
-		actual := IsCriticalPod(&data.pod)
-		if actual != data.expected {
-			t.Errorf("IsCriticalPod result wrong:\nexpected: %v\nactual: %v for test[%d] with Annotations: %v",
-				data.expected, actual, i, data.pod.Annotations)
-		}
-	}
-}
-
 func TestIsCriticalPodBasedOnPriority(t *testing.T) {
 	tests := []struct {
 		priority    int32

plugin/pkg/admission/priority/BUILD

@@ -36,7 +36,6 @@ go_library(
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/scheduling:go_default_library",
         "//pkg/features:go_default_library",
-        "//pkg/kubelet/types:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/scheduling/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",

plugin/pkg/admission/priority/admission.go

@@ -35,7 +35,6 @@ import (
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/scheduling"
 	"k8s.io/kubernetes/pkg/features"
-	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 )
 
 const (
@@ -178,13 +177,6 @@ func (p *priorityPlugin) admitPod(a admission.Attributes) error {
 	if operation == admission.Create {
 		var priority int32
 		var preemptionPolicy *apiv1.PreemptionPolicy
-		// TODO: @ravig - This is for backwards compatibility to ensure that critical pods with annotations just work fine.
-		// Remove when no longer needed.
-		if len(pod.Spec.PriorityClassName) == 0 &&
-			utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) &&
-			kubelettypes.IsCritical(a.GetNamespace(), pod.Annotations) {
-			pod.Spec.PriorityClassName = scheduling.SystemClusterCritical
-		}
 		if len(pod.Spec.PriorityClassName) == 0 {
 			var err error
 			var pcName string

plugin/pkg/admission/priority/admission_test.go

@@ -422,23 +422,7 @@ func TestPodAdmission(t *testing.T) {
 				Priority:          &intPriority,
 			},
 		},
-		// pod[7]: Pod with a critical priority annotation. This needs to be automatically assigned
+		// pod[7]: Pod with a system priority class name in non-system namespace
-		// system-cluster-critical
-		{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:        "pod-w-system-priority",
-				Namespace:   "kube-system",
-				Annotations: map[string]string{"scheduler.alpha.kubernetes.io/critical-pod": ""},
-			},
-			Spec: api.PodSpec{
-				Containers: []api.Container{
-					{
-						Name: containerName,
-					},
-				},
-			},
-		},
-		// pod[8]: Pod with a system priority class name in non-system namespace
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "pod-w-system-priority-in-nonsystem-namespace",
@@ -453,7 +437,7 @@ func TestPodAdmission(t *testing.T) {
 				PriorityClassName: scheduling.SystemClusterCritical,
 			},
 		},
-		// pod[9]: Pod with a priority value that matches the resolved priority
+		// pod[8]: Pod with a priority value that matches the resolved priority
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "pod-w-zero-priority-in-nonsystem-namespace",
@@ -468,7 +452,7 @@ func TestPodAdmission(t *testing.T) {
 				Priority: &zeroPriority,
 			},
 		},
-		// pod[10]: Pod with a priority value that matches the resolved default priority
+		// pod[9]: Pod with a priority value that matches the resolved default priority
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "pod-w-priority-matching-default-priority",
@@ -483,7 +467,7 @@ func TestPodAdmission(t *testing.T) {
 				Priority: &defaultClass2.Value,
 			},
 		},
-		// pod[11]: Pod with a priority value that matches the resolved priority
+		// pod[10]: Pod with a priority value that matches the resolved priority
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "pod-w-priority-matching-resolved-default-priority",
@@ -499,7 +483,7 @@ func TestPodAdmission(t *testing.T) {
 				Priority:          &systemClusterCritical.Value,
 			},
 		},
-		// pod[12]: Pod without a preemption policy that matches the resolved preemption policy
+		// pod[11]: Pod without a preemption policy that matches the resolved preemption policy
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "pod-never-preemption-policy-matching-resolved-preemption-policy",
@@ -516,7 +500,7 @@ func TestPodAdmission(t *testing.T) {
 				PreemptionPolicy:  nil,
 			},
 		},
-		// pod[13]: Pod with a preemption policy that matches the resolved preemption policy
+		// pod[12]: Pod with a preemption policy that matches the resolved preemption policy
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "pod-preemption-policy-matching-resolved-preemption-policy",
@@ -533,7 +517,7 @@ func TestPodAdmission(t *testing.T) {
 				PreemptionPolicy:  &preemptLowerPriority,
 			},
 		},
-		// pod[14]: Pod with a preemption policy that does't match the resolved preemption policy
+		// pod[13]: Pod with a preemption policy that does't match the resolved preemption policy
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "pod-preemption-policy-not-matching-resolved-preemption-policy",
@@ -551,8 +535,6 @@ func TestPodAdmission(t *testing.T) {
 			},
 		},
 	}
-	// Enable ExperimentalCriticalPodAnnotation feature gate.
-	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)()
 	// Enable NonPreemptingPriority feature gate.
 	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NonPreemptingPriority, true)()
 	tests := []struct {
@@ -638,18 +620,10 @@ func TestPodAdmission(t *testing.T) {
 			true,
 			nil,
 		},
-		{
-			"pod with critical pod annotation",
-			[]*scheduling.PriorityClass{systemClusterCritical},
-			*pods[7],
-			scheduling.SystemCriticalPriority,
-			false,
-			nil,
-		},
 		{
 			"pod with system critical priority in non-system namespace",
 			[]*scheduling.PriorityClass{systemClusterCritical},
-			*pods[8],
+			*pods[7],
 			scheduling.SystemCriticalPriority,
 			true,
 			nil,
@@ -657,7 +631,7 @@ func TestPodAdmission(t *testing.T) {
 		{
 			"pod with priority that matches computed priority",
 			[]*scheduling.PriorityClass{nondefaultClass1},
-			*pods[9],
+			*pods[8],
 			0,
 			false,
 			nil,
@@ -665,7 +639,7 @@ func TestPodAdmission(t *testing.T) {
 		{
 			"pod with priority that matches default priority",
 			[]*scheduling.PriorityClass{defaultClass2},
-			*pods[10],
+			*pods[9],
 			defaultClass2.Value,
 			false,
 			nil,
@@ -673,7 +647,7 @@ func TestPodAdmission(t *testing.T) {
 		{
 			"pod with priority that matches resolved priority",
 			[]*scheduling.PriorityClass{systemClusterCritical},
-			*pods[11],
+			*pods[10],
 			systemClusterCritical.Value,
 			false,
 			nil,
@@ -681,7 +655,7 @@ func TestPodAdmission(t *testing.T) {
 		{
 			"pod with nil preemtpion policy",
 			[]*scheduling.PriorityClass{preemptionPolicyClass},
-			*pods[12],
+			*pods[11],
 			preemptionPolicyClass.Value,
 			false,
 			nil,
@@ -689,7 +663,7 @@ func TestPodAdmission(t *testing.T) {
 		{
 			"pod with preemtpion policy that matches resolved preemtpion policy",
 			[]*scheduling.PriorityClass{preemptionPolicyClass},
-			*pods[13],
+			*pods[12],
 			preemptionPolicyClass.Value,
 			false,
 			&preemptLowerPriority,
@@ -697,7 +671,7 @@ func TestPodAdmission(t *testing.T) {
 		{
 			"pod with preemtpion policy that does't matches resolved preemtpion policy",
 			[]*scheduling.PriorityClass{preemptionPolicyClass},
-			*pods[14],
+			*pods[13],
 			preemptionPolicyClass.Value,
 			true,
 			&preemptLowerPriority,

test/e2e/testing-manifests/sample-device-plugin.yaml

@@ -14,7 +14,6 @@ spec:
       labels:
         k8s-app: sample-device-plugin
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       tolerations:

test/e2e_node/BUILD

@@ -126,6 +126,7 @@ go_test(
     tags = ["e2e"],
     deps = [
         "//pkg/apis/core:go_default_library",
+        "//pkg/apis/scheduling:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/kubelet:go_default_library",
         "//pkg/kubelet/apis/config:go_default_library",

test/e2e_node/critical_pod_test.go

@@ -23,8 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeapi "k8s.io/kubernetes/pkg/apis/core"
-	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/apis/scheduling"
-	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/test/e2e/framework"
 	imageutils "k8s.io/kubernetes/test/utils/image"
@@ -44,13 +43,6 @@ var _ = framework.KubeDescribe("CriticalPod [Serial] [Disruptive] [NodeFeature:C
 	f := framework.NewDefaultFramework("critical-pod-test")
 
 	Context("when we need to admit a critical pod", func() {
-		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-			if initialConfig.FeatureGates == nil {
-				initialConfig.FeatureGates = make(map[string]bool)
-			}
-			initialConfig.FeatureGates[string(features.ExperimentalCriticalPodAnnotation)] = true
-		})
-
 		It("should be able to create and delete a critical pod", func() {
 			configEnabled, err := isKubeletConfigEnabled(f)
 			framework.ExpectNoError(err)
@@ -142,12 +134,11 @@ func getTestPod(critical bool, name string, resources v1.ResourceRequirements) *
 	}
 	if critical {
 		pod.ObjectMeta.Namespace = kubeapi.NamespaceSystem
-		pod.ObjectMeta.Annotations = map[string]string{
+		pod.ObjectMeta.Annotations = map[string]string{}
-			kubelettypes.CriticalPodAnnotationKey: "",
+		pod.Spec.PriorityClassName = scheduling.SystemClusterCritical
-		}
+		Expect(kubelettypes.IsCriticalPod(pod)).To(BeTrue(), "pod should be a critical pod")
-		Expect(kubelettypes.IsCritical(pod.Namespace, pod.Annotations)).To(BeTrue(), "pod should be a critical pod")
 	} else {
-		Expect(kubelettypes.IsCritical(pod.Namespace, pod.Annotations)).To(BeFalse(), "pod should not be a critical pod")
+		Expect(kubelettypes.IsCriticalPod(pod)).To(BeFalse(), "pod should not be a critical pod")
 	}
 	return pod
 }

test/kubemark/resources/kube_dns_template.yaml

@@ -57,9 +57,8 @@ spec:
     metadata:
       labels:
         k8s-app: kube-dns
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      priorityClassName: system-node-critical
       tolerations:
       - key: "CriticalAddonsOnly"
         operator: "Exists"