Kubelet changes for Windows GMSA support

This patch comprises the kubelet changes outlined in the GMSA KEP
(https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/20181221-windows-group-managed-service-accounts-for-container-identity.md)
to add GMSA support to Windows workloads.

More precisely, it includes the logic proposed in the KEP to resolve
which GMSA spec should be applied to which containers, and changes
`dockershim` to copy the relevant GMSA credential specs to Windows
registry values prior to creating the container, passing them down
to docker itself, and finally removing the values from the registry
afterwards; both these changes need to be activated with the `WindowsGMSA`
feature gate.

Includes unit tests.

Signed-off-by: Jean Rouge <rougej+github@gmail.com>

pkg/features/kube_features.go

@@ -404,6 +404,12 @@ const (
 	//
 	// Enables the AWS EBS in-tree driver to AWS EBS CSI Driver migration feature.
 	CSIMigrationAWS utilfeature.Feature = "CSIMigrationAWS"
+
+	// owner: @wk8
+	// alpha: v1.14
+	//
+	// Enables GMSA support for Windows workloads.
+	WindowsGMSA utilfeature.Feature = "WindowsGMSA"
 )
 
 func init() {

pkg/kubelet/dockershim/BUILD

@@ -7,6 +7,8 @@ go_library(
         "doc.go",
         "docker_checkpoint.go",
         "docker_container.go",
+        "docker_container_unsupported.go",
+        "docker_container_windows.go",
         "docker_image.go",
         "docker_image_linux.go",
         "docker_image_unsupported.go",
@@ -71,8 +73,11 @@ go_library(
         "//vendor/k8s.io/utils/exec:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:windows": [
+            "//pkg/features:go_default_library",
             "//pkg/kubelet/apis:go_default_library",
             "//pkg/kubelet/winstats:go_default_library",
+            "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+            "//vendor/golang.org/x/sys/windows/registry:go_default_library",
         ],
         "//conditions:default": [],
     }),
@@ -84,6 +89,7 @@ go_test(
         "convert_test.go",
         "docker_checkpoint_test.go",
         "docker_container_test.go",
+        "docker_container_windows_test.go",
         "docker_image_test.go",
         "docker_sandbox_test.go",
         "docker_service_test.go",
@@ -118,6 +124,9 @@ go_test(
         "@io_bazel_rules_go//go/platform:linux": [
             "//staging/src/k8s.io/api/core/v1:go_default_library",
         ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//vendor/golang.org/x/sys/windows/registry:go_default_library",
+        ],
         "//conditions:default": [],
     }),
 )

pkg/kubelet/dockershim/docker_container.go

@@ -162,11 +162,20 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create
 
 	hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)
 
+	cleanupInfo, err := ds.applyPlatformSpecificDockerConfig(r, &createConfig)
+	if err != nil {
+		return nil, err
+	}
+
 	createResp, err := ds.client.CreateContainer(createConfig)
 	if err != nil {
 		createResp, err = recoverFromCreationConflictIfNeeded(ds.client, createConfig, err)
 	}
 
+	if err = ds.performPlatformSpecificContainerCreationCleanup(cleanupInfo); err != nil {
+		return nil, err
+	}
+
 	if createResp != nil {
 		return &runtimeapi.CreateContainerResponse{ContainerId: createResp.ID}, nil
 	}

pkg/kubelet/dockershim/docker_container_unsupported.go

@@ -0,0 +1,39 @@
+// +build !windows
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	dockertypes "github.com/docker/docker/api/types"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+)
+
+type containerCreationCleanupInfo struct{}
+
+// applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct.
+// The containerCreationCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCreationCleanup
+// after the container has been created.
+func (ds *dockerService) applyPlatformSpecificDockerConfig(*runtimeapi.CreateContainerRequest, *dockertypes.ContainerCreateConfig) (*containerCreationCleanupInfo, error) {
+	return nil, nil
+}
+
+// performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup
+// after a container creation.
+func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(*containerCreationCleanupInfo) error {
+	return nil
+}

pkg/kubelet/dockershim/docker_container_windows.go

@@ -0,0 +1,180 @@
+// +build windows
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+
+	"golang.org/x/sys/windows/registry"
+
+	dockertypes "github.com/docker/docker/api/types"
+	dockercontainer "github.com/docker/docker/api/types/container"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/klog"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
+)
+
+type containerCreationCleanupInfo struct {
+	gMSARegistryValueName string
+}
+
+// applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct.
+// The containerCreationCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCreationCleanup
+// after the container has been created.
+func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.CreateContainerRequest, createConfig *dockertypes.ContainerCreateConfig) (*containerCreationCleanupInfo, error) {
+	cleanupInfo := &containerCreationCleanupInfo{}
+
+	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) {
+		if err := applyGMSAConfig(request, createConfig, cleanupInfo); err != nil {
+			return nil, err
+		}
+	}
+
+	return cleanupInfo, nil
+}
+
+// applyGMSAConfig looks at the kuberuntime.GMSASpecContainerAnnotationKey container annotation; if present,
+// it copies its contents to a unique registry value, and sets a SecurityOpt on the config pointing to that registry value.
+// We use registry values instead of files since their location cannot change - as opposed to credential spec files,
+// whose location could potentially change down the line, or even be unknown (eg if docker is not installed on the
+// C: drive)
+// When docker supports passing a credential spec's contents directly, we should switch to using that
+// as it will avoid cluttering the registry.
+func applyGMSAConfig(request *runtimeapi.CreateContainerRequest, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCreationCleanupInfo) error {
+	config := request.GetConfig()
+	credSpec := config.Annotations[kuberuntime.GMSASpecContainerAnnotationKey]
+	if credSpec == "" {
+		return nil
+	}
+
+	valueName, err := copyGMSACredSpecToRegistryValue(credSpec, makeContainerName(request.GetSandboxConfig(), config))
+	if err != nil {
+		return err
+	}
+
+	if createConfig.HostConfig == nil {
+		createConfig.HostConfig = &dockercontainer.HostConfig{}
+	}
+
+	createConfig.HostConfig.SecurityOpt = append(createConfig.HostConfig.SecurityOpt, "credentialspec=registry://"+valueName)
+	cleanupInfo.gMSARegistryValueName = valueName
+
+	return nil
+}
+
+const (
+	registryNamePrefix = "k8s-cred-spec-"
+	// same as https://github.com/moby/moby/blob/93d994e29c9cc8d81f1b0477e28d705fa7e2cd72/daemon/oci_windows.go#L23
+	credentialSpecRegistryLocation = `SOFTWARE\Microsoft\Windows NT\CurrentVersion\Virtualization\Containers\CredentialSpecs`
+)
+
+// useful to allow mocking the registry in tests
+type registryKey interface {
+	SetStringValue(name, value string) error
+	DeleteValue(name string) error
+	Close() error
+}
+
+var registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) {
+	return registry.CreateKey(baseKey, path, access)
+}
+
+// and same for random
+var randomReader = rand.Reader
+
+// copyGMSACredSpecToRegistryKey copies the credential specs to a unique registry value, and returns its name.
+// To avoid leaking registry keys over the life of the node, we generate a unique name for that value, and clean
+// it up after creating the container.
+func copyGMSACredSpecToRegistryValue(credSpec string, dockerContainerName string) (string, error) {
+	valueName, err := gMSARegistryValueName(credSpec, dockerContainerName)
+	if err != nil {
+		return "", err
+	}
+
+	// write to the registry
+	key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
+	if err != nil {
+		return "", fmt.Errorf("unable to open registry key %s: %v", credentialSpecRegistryLocation, err)
+	}
+	defer key.Close()
+	if err = key.SetStringValue(valueName, credSpec); err != nil {
+		return "", fmt.Errorf("unable to write into registry value %s/%s: %v", credentialSpecRegistryLocation, valueName, err)
+	}
+
+	return valueName, nil
+}
+
+// gMSARegistryValueName computes the name of the registry value where to store the GMSA cred spec contents.
+// The value's name is computed by concatenating the docker container's name (guaranteed to be unique over the
+// container's lifetime), the value itself, and an additional 64 random bytes.
+func gMSARegistryValueName(inputs ...string) (string, error) {
+	hasher := sha256.New()
+	for _, s := range inputs {
+		// according to the doc, that can never return an error
+		io.WriteString(hasher, s)
+	}
+	randBytes := make([]byte, 64)
+	if _, err := randomReader.Read(randBytes); err != nil {
+		return "", fmt.Errorf("unable to generate random string: %v", err)
+	}
+	hasher.Write(randBytes)
+
+	return registryNamePrefix + hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
+// performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup
+// after a container creation.
+func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(cleanupInfo *containerCreationCleanupInfo) error {
+	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) {
+		// this is best effort, we don't bubble errors upstream as failing to remove the GMSA registry keys shouldn't
+		// prevent k8s from working correctly, and the leaked registry keys are not a major concern anyway:
+		// they don't contain any secret, and they're sufficiently random to prevent collisions with
+		// future ones
+		if err := removeGMSARegistryValue(cleanupInfo); err != nil {
+			klog.Warningf("won't remove GMSA cred spec registry value: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// removeGMSARegistryValue removes the registry value containing the GMSA cred spec for this container, if any.
+func removeGMSARegistryValue(cleanupInfo *containerCreationCleanupInfo) error {
+	if cleanupInfo.gMSARegistryValueName == "" {
+		return nil
+	}
+
+	key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
+	if err != nil {
+		return fmt.Errorf("unable to open registry key %s: %v", credentialSpecRegistryLocation, err)
+	}
+	defer key.Close()
+	if err = key.DeleteValue(cleanupInfo.gMSARegistryValueName); err != nil {
+		return fmt.Errorf("unable to remove registry value %s/%s: %v", credentialSpecRegistryLocation, cleanupInfo.gMSARegistryValueName, err)
+	}
+
+	return nil
+}

pkg/kubelet/dockershim/docker_container_windows_test.go

@@ -0,0 +1,233 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dockershim
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+
+	dockertypes "github.com/docker/docker/api/types"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/sys/windows/registry"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+)
+
+type dummyRegistryKey struct {
+	setStringError    error
+	stringValues      [][]string
+	deleteValueError  error
+	deletedValueNames []string
+	closed            bool
+}
+
+func (k *dummyRegistryKey) SetStringValue(name, value string) error {
+	k.stringValues = append(k.stringValues, []string{name, value})
+	return k.setStringError
+}
+
+func (k *dummyRegistryKey) DeleteValue(name string) error {
+	k.deletedValueNames = append(k.deletedValueNames, name)
+	return k.deleteValueError
+}
+
+func (k *dummyRegistryKey) Close() error {
+	k.closed = true
+	return nil
+}
+
+func TestApplyGMSAConfig(t *testing.T) {
+	dummyCredSpec := "test cred spec contents"
+	randomBytes := []byte{85, 205, 157, 137, 41, 50, 187, 175, 242, 115, 92, 212, 181, 70, 56, 20, 172, 17, 100, 178, 19, 42, 217, 177, 240, 37, 127, 123, 53, 250, 61, 157, 11, 41, 69, 160, 117, 163, 51, 118, 53, 86, 167, 111, 137, 78, 195, 229, 50, 144, 178, 209, 66, 107, 144, 165, 184, 92, 10, 17, 229, 163, 194, 12}
+	expectedHash := "8975ef53024af213c1aca6dfc6e2e48f42c3a984a79e67b140627b8d96007c2a"
+	expectedValueName := "k8s-cred-spec-" + expectedHash
+
+	sandboxConfig := &runtimeapi.PodSandboxConfig{
+		Metadata: &runtimeapi.PodSandboxMetadata{
+			Namespace: "namespace",
+			Uid:       "uid",
+		},
+	}
+
+	containerMeta := &runtimeapi.ContainerMetadata{
+		Name:    "container_name",
+		Attempt: 12,
+	}
+
+	requestWithoutGMSAAnnotation := &runtimeapi.CreateContainerRequest{
+		Config:        &runtimeapi.ContainerConfig{Metadata: containerMeta},
+		SandboxConfig: sandboxConfig,
+	}
+
+	requestWithGMSAAnnotation := &runtimeapi.CreateContainerRequest{
+		Config: &runtimeapi.ContainerConfig{
+			Metadata:    containerMeta,
+			Annotations: map[string]string{"container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec},
+		},
+		SandboxConfig: sandboxConfig,
+	}
+
+	t.Run("happy path", func(t *testing.T) {
+		key := &dummyRegistryKey{}
+		defer setRegistryCreateKeyFunc(t, key)()
+		defer setRandomReader(randomBytes)()
+
+		createConfig := &dockertypes.ContainerCreateConfig{}
+		cleanupInfo := &containerCreationCleanupInfo{}
+		err := applyGMSAConfig(requestWithGMSAAnnotation, createConfig, cleanupInfo)
+
+		assert.Nil(t, err)
+
+		// the registry key should have been properly created
+		assert.Equal(t, 1, len(key.stringValues))
+		assert.Equal(t, []string{expectedValueName, dummyCredSpec}, key.stringValues[0])
+		assert.True(t, key.closed)
+
+		// the create config's security opt should have been populated
+		assert.Equal(t, createConfig.HostConfig.SecurityOpt, []string{"credentialspec=registry://" + expectedValueName})
+
+		// and the name of that value should have been saved to the cleanup info
+		assert.Equal(t, expectedValueName, cleanupInfo.gMSARegistryValueName)
+	})
+	t.Run("happy path with a truly random string", func(t *testing.T) {
+		defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{})()
+
+		createConfig := &dockertypes.ContainerCreateConfig{}
+		cleanupInfo := &containerCreationCleanupInfo{}
+		err := applyGMSAConfig(requestWithGMSAAnnotation, createConfig, cleanupInfo)
+
+		assert.Nil(t, err)
+
+		secOpt := createConfig.HostConfig.SecurityOpt[0]
+
+		expectedPrefix := "credentialspec=registry://k8s-cred-spec-"
+		assert.Equal(t, expectedPrefix, secOpt[:len(expectedPrefix)])
+
+		hash := secOpt[len(expectedPrefix):]
+		hexRegex, _ := regexp.Compile("^[0-9a-f]{64}$")
+		assert.True(t, hexRegex.MatchString(hash))
+		assert.NotEqual(t, expectedHash, hash)
+
+		assert.Equal(t, "k8s-cred-spec-"+hash, cleanupInfo.gMSARegistryValueName)
+	})
+	t.Run("if there's an error opening the registry key", func(t *testing.T) {
+		defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))()
+
+		err := applyGMSAConfig(requestWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
+
+		assert.NotNil(t, err)
+		assert.True(t, strings.Contains(err.Error(), "unable to open registry key"))
+	})
+	t.Run("if there's an error writing the registry key", func(t *testing.T) {
+		key := &dummyRegistryKey{}
+		key.setStringError = fmt.Errorf("dummy error")
+		defer setRegistryCreateKeyFunc(t, key)()
+
+		err := applyGMSAConfig(requestWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
+
+		assert.NotNil(t, err)
+		assert.True(t, strings.Contains(err.Error(), "unable to write into registry value"))
+		assert.True(t, key.closed)
+	})
+	t.Run("if there is no GMSA annotation", func(t *testing.T) {
+		createConfig := &dockertypes.ContainerCreateConfig{}
+
+		err := applyGMSAConfig(requestWithoutGMSAAnnotation, createConfig, &containerCreationCleanupInfo{})
+
+		assert.Nil(t, err)
+		assert.Nil(t, createConfig.HostConfig)
+	})
+}
+
+func TestRemoveGMSARegistryValue(t *testing.T) {
+	emptyCleanupInfo := &containerCreationCleanupInfo{}
+
+	valueName := "k8s-cred-spec-8975ef53024af213c1aca6dfc6e2e48f42c3a984a79e67b140627b8d96007c2a"
+	cleanupInfoWithValue := &containerCreationCleanupInfo{gMSARegistryValueName: valueName}
+
+	t.Run("it does remove the registry value", func(t *testing.T) {
+		key := &dummyRegistryKey{}
+		defer setRegistryCreateKeyFunc(t, key)()
+
+		err := removeGMSARegistryValue(cleanupInfoWithValue)
+
+		assert.Nil(t, err)
+
+		// the registry key should have been properly deleted
+		assert.Equal(t, 1, len(key.deletedValueNames))
+		assert.Equal(t, []string{valueName}, key.deletedValueNames)
+		assert.True(t, key.closed)
+	})
+	t.Run("if there's an error opening the registry key", func(t *testing.T) {
+		defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))()
+
+		err := removeGMSARegistryValue(cleanupInfoWithValue)
+
+		assert.NotNil(t, err)
+		assert.True(t, strings.Contains(err.Error(), "unable to open registry key"))
+	})
+	t.Run("if there's an error writing the registry key", func(t *testing.T) {
+		key := &dummyRegistryKey{}
+		key.deleteValueError = fmt.Errorf("dummy error")
+		defer setRegistryCreateKeyFunc(t, key)()
+
+		err := removeGMSARegistryValue(cleanupInfoWithValue)
+
+		assert.NotNil(t, err)
+		assert.True(t, strings.Contains(err.Error(), "unable to remove registry value"))
+		assert.True(t, key.closed)
+	})
+	t.Run("if there's no registry value to be removed", func(t *testing.T) {
+		err := removeGMSARegistryValue(emptyCleanupInfo)
+
+		assert.Nil(t, err)
+	})
+}
+
+// setRegistryCreateKeyFunc replaces the registryCreateKeyFunc package variable, and returns a function
+// to be called to revert the change when done with testing.
+func setRegistryCreateKeyFunc(t *testing.T, key *dummyRegistryKey, err ...error) func() {
+	previousRegistryCreateKeyFunc := registryCreateKeyFunc
+
+	registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) {
+		// this should always be called with exactly the same arguments
+		assert.Equal(t, registry.LOCAL_MACHINE, baseKey)
+		assert.Equal(t, credentialSpecRegistryLocation, path)
+		assert.Equal(t, uint32(registry.SET_VALUE), access)
+
+		if len(err) > 0 {
+			return nil, false, err[0]
+		}
+		return key, false, nil
+	}
+
+	return func() {
+		registryCreateKeyFunc = previousRegistryCreateKeyFunc
+	}
+}
+
+// setRandomReader replaces the randomReader package variable with a dummy reader that returns the provided
+// byte slice, and returns a function to be called to revert the change when done with testing.
+func setRandomReader(b []byte) func() {
+	previousRandomReader := randomReader
+	randomReader = bytes.NewReader(b)
+	return func() {
+		randomReader = previousRandomReader
+	}
+}

pkg/kubelet/dockershim/naming.go

@@ -66,6 +66,7 @@ func makeSandboxName(s *runtimeapi.PodSandboxConfig) string {
 	}, nameDelimiter)
 }
 
+// makeContainerName generates a container name that's guaranteed to be unique on its host.
 func makeContainerName(s *runtimeapi.PodSandboxConfig, c *runtimeapi.ContainerConfig) string {
 	return strings.Join([]string{
 		kubePrefix,                            // 0

pkg/kubelet/kuberuntime/BUILD

@@ -89,6 +89,7 @@ go_test(
         "instrumented_services_test.go",
         "kuberuntime_container_linux_test.go",
         "kuberuntime_container_test.go",
+        "kuberuntime_container_windows_test.go",
         "kuberuntime_gc_test.go",
         "kuberuntime_image_test.go",
         "kuberuntime_manager_test.go",

pkg/kubelet/kuberuntime/kuberuntime_container_windows.go

@@ -23,6 +23,8 @@ import (
 	"github.com/docker/docker/pkg/sysinfo"
 
 	"k8s.io/api/core/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	"k8s.io/kubernetes/pkg/securitycontext"
@@ -35,6 +37,10 @@ func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config
 		return err
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) {
+		determineEffectiveSecurityContext(config, container, pod)
+	}
+
 	config.Windows = windowsConfig
 	return nil
 }
@@ -97,3 +103,40 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
 
 	return wc, nil
 }
+
+const (
+	// GMSASpecContainerAnnotationKey is the container annotation where we store the contents of the GMSA credential spec to use.
+	GMSASpecContainerAnnotationKey = "container.alpha.windows.kubernetes.io/gmsa-credential-spec"
+	// gMSAContainerSpecPodAnnotationKeySuffix is the suffix of the pod annotation where the GMSA webhook admission controller
+	// stores the contents of the GMSA credential spec for a given container (the full annotation being the container's name
+	// with this suffix appended).
+	gMSAContainerSpecPodAnnotationKeySuffix = "." + GMSASpecContainerAnnotationKey
+	// gMSAPodSpecPodAnnotationKey is the pod annotation where the GMSA webhook admission controller stores the contents of the GMSA
+	// credential spec to use for containers that do not have their own specific GMSA cred spec set via a
+	// gMSAContainerSpecPodAnnotationKeySuffix annotation as explained above
+	gMSAPodSpecPodAnnotationKey = "pod.alpha.windows.kubernetes.io/gmsa-credential-spec"
+)
+
+// determineEffectiveSecurityContext determines the effective GMSA credential spec and, if any, copies it to the container's
+// GMSASpecContainerAnnotationKey annotation.
+func determineEffectiveSecurityContext(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod) {
+	var containerCredSpec string
+
+	containerGMSAPodAnnotation := container.Name + gMSAContainerSpecPodAnnotationKeySuffix
+	if pod.Annotations[containerGMSAPodAnnotation] != "" {
+		containerCredSpec = pod.Annotations[containerGMSAPodAnnotation]
+	} else if pod.Annotations[gMSAPodSpecPodAnnotationKey] != "" {
+		containerCredSpec = pod.Annotations[gMSAPodSpecPodAnnotationKey]
+	}
+
+	if containerCredSpec != "" {
+		if config.Annotations == nil {
+			config.Annotations = make(map[string]string)
+		}
+		config.Annotations[GMSASpecContainerAnnotationKey] = containerCredSpec
+	} else {
+		// the annotation shouldn't be present, but let's err on the side of caution:
+		// it should only be set here and nowhere else
+		delete(config.Annotations, GMSASpecContainerAnnotationKey)
+	}
+}

pkg/kubelet/kuberuntime/kuberuntime_container_windows_test.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kuberuntime
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+)
+
+func TestDetermineEffectiveSecurityContext(t *testing.T) {
+	containerName := "container_name"
+	container := &corev1.Container{Name: containerName}
+	dummyCredSpec := "test cred spec contents"
+
+	buildPod := func(annotations map[string]string) *corev1.Pod {
+		return &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Annotations: annotations,
+			},
+		}
+	}
+
+	t.Run("when there's a specific GMSA for that pod, and no pod-wide GMSA", func(t *testing.T) {
+		containerConfig := &runtimeapi.ContainerConfig{}
+
+		pod := buildPod(map[string]string{
+			"container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
+		})
+
+		determineEffectiveSecurityContext(containerConfig, container, pod)
+
+		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
+	})
+	t.Run("when there's a specific GMSA for that pod, and a pod-wide GMSA", func(t *testing.T) {
+		containerConfig := &runtimeapi.ContainerConfig{}
+
+		pod := buildPod(map[string]string{
+			"container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
+			"pod.alpha.windows.kubernetes.io/gmsa-credential-spec":                      "should be ignored",
+		})
+
+		determineEffectiveSecurityContext(containerConfig, container, pod)
+
+		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
+	})
+	t.Run("when there's no specific GMSA for that pod, and a pod-wide GMSA", func(t *testing.T) {
+		containerConfig := &runtimeapi.ContainerConfig{}
+
+		pod := buildPod(map[string]string{
+			"pod.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
+		})
+
+		determineEffectiveSecurityContext(containerConfig, container, pod)
+
+		assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
+	})
+	t.Run("when there's no specific GMSA for that pod, and no pod-wide GMSA", func(t *testing.T) {
+		containerConfig := &runtimeapi.ContainerConfig{}
+
+		determineEffectiveSecurityContext(containerConfig, container, &corev1.Pod{})
+
+		assert.Nil(t, containerConfig.Annotations)
+	})
+}