Merge pull request #40843 from luomiao/photon-cloud-provider-authentication-update

Automatic merge from submit-queue (batch tested with PRs 41756, 36344, 34259, 40843, 41526) Update Photon Controller cloud provider for authentication support Resolve Issue: [#40755](https://github.com/kubernetes/kubernetes/issues/40755) 1. Update the configuration file for Photon Controller cloud provider 2. Only master nodes can communicate with Photon Controller endpoint 3. Enable support for authentication-enabled Photon Controller endpoint 4. Update NodeAddresses function for query from local node New format of photon controller config file: ``` [Global] target = https://[LOAD_BALANCER_IP]:443 project = [PROJECT ID] overrideIP = true vmID = [LOCAL VM ID] authentication = true ``` This config file will be automatically created by Photon Controller cluster management. If authentication file is set to true, then a pc_login_info file should be placed under /etc/kubernetes with username and password. This file can be created by user directly. Or the user can choose to use kubernetes secret and a handling pod to avoid directly login to master nodes. This usage will be available with Photon Controller 1.2. This is a temporary solution before metadata service becomes available in Photon Controller.
2025-11-04 04:08:16 +00:00 · 2017-02-20 13:39:39 -08:00
parent 8738e36c70 1aa3bba72b
commit 9c0e46bdff
2 changed files with 207 additions and 96 deletions
--- a/pkg/cloudprovider/providers/photon/photon.go
+++ b/pkg/cloudprovider/providers/photon/photon.go
@@ -24,10 +24,13 @@ limitations under the License.
 package photon

 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"io"
 	"log"
+	"net"
+	"os"
 	"os/exec"
 	"strings"

@@ -42,12 +45,10 @@ import (
 const (
 	ProviderName = "photon"
 	DiskSpecKind = "persistent-disk"
+	MAC_OUI_VC   = "00:50:56"
+	MAC_OUI_ESX  = "00:0c:29"
 )

-// Global variable pointing to photon client
-var photonClient *photon.Client
-var logger *log.Logger = nil
-
 // overrideIP indicates if the hostname is overriden by IP address, such as when
 // running multi-node kubernetes using docker. In this case the user should set
 // overrideIP = true in cloud config file. Default value is false.
@@ -66,22 +67,23 @@ type PCCloud struct {
 	// in the environment per current Photon Controller deployment methodology.
 	projID string
 	cloudprovider.Zone
+	photonClient *photon.Client
+	logger       *log.Logger
 }

 type PCConfig struct {
 	Global struct {
 		// the Photon Controller endpoint IP address
 		CloudTarget string `gcfg:"target"`
-		// when the Photon Controller authentication is enabled, set to true;
-		// otherwise, set to false.
-		IgnoreCertificate bool `gcfg:"ignoreCertificate"`
-		// Photon Controller tenant name
-		Tenant string `gcfg:"tenant"`
 		// Photon Controller project name
 		Project string `gcfg:"project"`
 		// when kubelet is started with '--hostname-override=${IP_ADDRESS}', set to true;
 		// otherwise, set to false.
 		OverrideIP bool `gcfg:"overrideIP"`
+		// VM ID for this node
+		VMID string `gcfg:"vmID"`
+		// Authentication enabled or not
+		AuthEnabled bool `gcfg:"authentication"`
 	}
 }

@@ -140,10 +142,16 @@ func init() {
 }

 // Retrieve the Photon VM ID from the Photon Controller endpoint based on the node name
-func getVMIDbyNodename(project string, nodeName string) (string, error) {
-	vmList, err := photonClient.Projects.GetVMs(project, nil)
+func getVMIDbyNodename(pc *PCCloud, nodeName string) (string, error) {
+	photonClient, err := getPhotonClient(pc)
 	if err != nil {
-		glog.Errorf("Photon Cloud Provider: Failed to GetVMs from project %s with nodeName %s, error: [%v]", project, nodeName, err)
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err)
+		return "", err
+	}
+
+	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to GetVMs from project %s with nodeName %s, error: [%v]", pc.projID, nodeName, err)
 		return "", err
 	}

@@ -157,10 +165,16 @@ func getVMIDbyNodename(project string, nodeName string) (string, error) {
 }

 // Retrieve the Photon VM ID from the Photon Controller endpoint based on the IP address
-func getVMIDbyIP(project string, IPAddress string) (string, error) {
-	vmList, err := photonClient.Projects.GetVMs(project, nil)
+func getVMIDbyIP(pc *PCCloud, IPAddress string) (string, error) {
+	photonClient, err := getPhotonClient(pc)
 	if err != nil {
-		glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. error: [%v]", project, err)
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err)
+		return "", err
+	}
+
+	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. error: [%v]", pc.projID, err)
 		return "", err
 	}

@@ -191,59 +205,67 @@ func getVMIDbyIP(project string, IPAddress string) (string, error) {
 	return "", fmt.Errorf("No matching VM is found with IP %s", IPAddress)
 }

-// Retrieve the the Photon project ID from the Photon Controller endpoint based on the project name
-func getProjIDbyName(tenantName, projName string) (string, error) {
-	tenants, err := photonClient.Tenants.GetAll()
-	if err != nil {
-		glog.Errorf("Photon Cloud Provider: GetAll tenants failed with error [%v].", err)
-		return "", err
-	}
-
-	for _, tenant := range tenants.Items {
-		if tenant.Name == tenantName {
-			projects, err := photonClient.Tenants.GetProjects(tenant.ID, nil)
-			if err != nil {
-				glog.Errorf("Photon Cloud Provider: Failed to GetProjects for tenant %s. error [%v]", tenantName, err)
-				return "", err
-			}
-
-			for _, project := range projects.Items {
-				if project.Name == projName {
-					return project.ID, nil
-				}
-			}
-		}
-	}
-
-	return "", fmt.Errorf("No matching tenant/project name is found with %s/%s", tenantName, projName)
-}
-
-func newPCCloud(cfg PCConfig) (*PCCloud, error) {
-	if len(cfg.Global.CloudTarget) == 0 {
+func getPhotonClient(pc *PCCloud) (*photon.Client, error) {
+	var err error
+	if len(pc.cfg.Global.CloudTarget) == 0 {
 		return nil, fmt.Errorf("Photon Controller endpoint was not specified.")
 	}

-	// Currently we support Photon Controller endpoint with authentication disabled.
 	options := &photon.ClientOptions{
-		IgnoreCertificate: cfg.Global.IgnoreCertificate,
+		IgnoreCertificate: true,
 	}

-	photonClient = photon.NewClient(cfg.Global.CloudTarget, options, logger)
-	status, err := photonClient.Status.Get()
+	pc.photonClient = photon.NewClient(pc.cfg.Global.CloudTarget, options, pc.logger)
+	if pc.cfg.Global.AuthEnabled == true {
+		// work around before metadata is available
+		file, err := os.Open("/etc/kubernetes/pc_login_info")
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: Authentication is enabled but found no username/password at /etc/kubernetes/pc_login_info. Error[%v]", err)
+			return nil, err
+		}
+		defer file.Close()
+		scanner := bufio.NewScanner(file)
+		if !scanner.Scan() {
+			glog.Errorf("Photon Cloud Provider: Empty username inside /etc/kubernetes/pc_login_info.")
+			return nil, fmt.Errorf("Failed to create authentication enabled client with invalid username")
+		}
+		username := scanner.Text()
+		if !scanner.Scan() {
+			glog.Errorf("Photon Cloud Provider: Empty password set inside /etc/kubernetes/pc_login_info.")
+			return nil, fmt.Errorf("Failed to create authentication enabled client with invalid password")
+		}
+		password := scanner.Text()
+
+		token_options, err := pc.photonClient.Auth.GetTokensByPassword(username, password)
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: failed to get tokens by password")
+			return nil, err
+		}
+
+		options = &photon.ClientOptions{
+			IgnoreCertificate: true,
+			TokenOptions: &photon.TokenOptions{
+				AccessToken: token_options.AccessToken,
+			},
+		}
+		pc.photonClient = photon.NewClient(pc.cfg.Global.CloudTarget, options, pc.logger)
+	}
+
+	status, err := pc.photonClient.Status.Get()
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: new client creation failed. Error[%v]", err)
 		return nil, err
 	}
 	glog.V(2).Info("Photon Cloud Provider: Status of the new photon controller client: %v", status)

-	// Get Photon Controller project ID for future use
-	projID, err := getProjIDbyName(cfg.Global.Tenant, cfg.Global.Project)
-	if err != nil {
-		glog.Errorf("Photon Cloud Provider: getProjIDbyName failed when creating new Photon Controller client. Error[%v]", err)
-		return nil, err
-	}
+	return pc.photonClient, nil
+}

-	// Get local hostname for localInstanceID
+func newPCCloud(cfg PCConfig) (*PCCloud, error) {
+	projID := cfg.Global.Project
+	vmID := cfg.Global.VMID
+
+	// Get local hostname
 	cmd := exec.Command("bash", "-c", `echo $HOSTNAME`)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
@@ -255,11 +277,6 @@ func newPCCloud(cfg PCConfig) (*PCCloud, error) {
 		return nil, fmt.Errorf("unable to retrieve hostname for Instance ID")
 	}
 	hostname := strings.TrimRight(string(out), "\n")
-	vmID, err := getVMIDbyNodename(projID, hostname)
-	if err != nil {
-		glog.Errorf("Photon Cloud Provider: getVMIDbyNodename failed when creating new Photon Controller client. Error[%v]", err)
-		return nil, err
-	}

 	pc := PCCloud{
 		cfg:              &cfg,
@@ -286,26 +303,68 @@ func (pc *PCCloud) List(filter string) ([]k8stypes.NodeName, error) {

 // NodeAddresses is an implementation of Instances.NodeAddresses.
 func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress, error) {
-	addrs := []v1.NodeAddress{}
+	nodeAddrs := []v1.NodeAddress{}
 	name := string(nodeName)

-	var vmID string
-	var err error
 	if name == pc.localK8sHostname {
-		vmID = pc.localInstanceID
+		ifaces, err := net.Interfaces()
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: net.Interfaces() failed for NodeAddresses. Error[%v]", err)
+			return nodeAddrs, err
+		}
+
+		for _, i := range ifaces {
+			addrs, err := i.Addrs()
+			if err != nil {
+				glog.Warningf("Photon Cloud Provider: Failed to extract addresses for NodeAddresses. Error[%v]", err)
 			} else {
-		vmID, err = getInstanceID(name, pc.projID)
+				for _, addr := range addrs {
+					if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+						if ipnet.IP.To4() != nil {
+							// Filter external IP by MAC address OUIs from vCenter and from ESX
+							if strings.HasPrefix(i.HardwareAddr.String(), MAC_OUI_VC) ||
+								strings.HasPrefix(i.HardwareAddr.String(), MAC_OUI_ESX) {
+								v1.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeExternalIP,
+										Address: ipnet.IP.String(),
+									},
+								)
+							} else {
+								v1.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeInternalIP,
+										Address: ipnet.IP.String(),
+									},
+								)
+							}
+						}
+					}
+				}
+			}
+		}
+		return nodeAddrs, nil
+	}
+
+	// Inquiring IP addresses from photon controller endpoint only for a node other than this node.
+	// This is assumed to be done by master only.
+	vmID, err := getInstanceID(pc, name)
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: getInstanceID failed for NodeAddresses. Error[%v]", err)
-			return addrs, err
+		return nodeAddrs, err
 	}
+
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for NodeAddresses, error: [%v]", err)
+		return nodeAddrs, err
 	}

 	// Retrieve the Photon VM's IP addresses from the Photon Controller endpoint based on the VM ID
 	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. Error[%v]", pc.projID, err)
-		return addrs, err
+		return nodeAddrs, err
 	}

 	for _, vm := range vmList.Items {
@@ -313,23 +372,37 @@ func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress,
 			task, err := photonClient.VMs.GetNetworks(vm.ID)
 			if err != nil {
 				glog.Errorf("Photon Cloud Provider: GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err)
-				return addrs, err
+				return nodeAddrs, err
 			} else {
 				task, err = photonClient.Tasks.Wait(task.ID)
 				if err != nil {
 					glog.Errorf("Photon Cloud Provider: Wait task for GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err)
-					return addrs, err
+					return nodeAddrs, err
 				} else {
 					networkConnections := task.ResourceProperties.(map[string]interface{})
 					networks := networkConnections["networkConnections"].([]interface{})
 					for _, nt := range networks {
+						ipAddr := "-"
+						macAddr := "-"
 						network := nt.(map[string]interface{})
 						if val, ok := network["ipAddress"]; ok && val != nil {
-							ipAddr := val.(string)
+							ipAddr = val.(string)
+						}
+						if val, ok := network["macAddress"]; ok && val != nil {
+							macAddr = val.(string)
+						}
 						if ipAddr != "-" {
-								v1.AddToNodeAddresses(&addrs,
+							if strings.HasPrefix(macAddr, MAC_OUI_VC) ||
+								strings.HasPrefix(macAddr, MAC_OUI_ESX) {
+								v1.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeExternalIP,
+										Address: ipAddr,
+									},
+								)
+							} else {
+								v1.AddToNodeAddresses(&nodeAddrs,
 									v1.NodeAddress{
-										// TODO: figure out the type of the IP
 										Type:    v1.NodeInternalIP,
 										Address: ipAddr,
 									},
@@ -337,14 +410,14 @@ func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress,
 							}
 						}
 					}
-					return addrs, nil
+					return nodeAddrs, nil
 				}
 			}
 		}
 	}

 	glog.Errorf("Failed to find the node %s from Photon Controller endpoint", name)
-	return addrs, fmt.Errorf("Failed to find the node %s from Photon Controller endpoint", name)
+	return nodeAddrs, fmt.Errorf("Failed to find the node %s from Photon Controller endpoint", name)
 }

 func (pc *PCCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
@@ -356,14 +429,14 @@ func (pc *PCCloud) CurrentNodeName(hostname string) (k8stypes.NodeName, error) {
 	return k8stypes.NodeName(hostname), nil
 }

-func getInstanceID(name string, projID string) (string, error) {
+func getInstanceID(pc *PCCloud, name string) (string, error) {
 	var vmID string
 	var err error

 	if overrideIP == true {
-		vmID, err = getVMIDbyIP(projID, name)
+		vmID, err = getVMIDbyIP(pc, name)
 	} else {
-		vmID, err = getVMIDbyNodename(projID, name)
+		vmID, err = getVMIDbyNodename(pc, name)
 	}
 	if err != nil {
 		return "", err
@@ -382,7 +455,8 @@ func (pc *PCCloud) ExternalID(nodeName k8stypes.NodeName) (string, error) {
 	if name == pc.localK8sHostname {
 		return pc.localInstanceID, nil
 	} else {
-		ID, err := getInstanceID(name, pc.projID)
+		// We assume only master need to get InstanceID of a node other than itself
+		ID, err := getInstanceID(pc, name)
 		if err != nil {
 			glog.Errorf("Photon Cloud Provider: getInstanceID failed for ExternalID. Error[%v]", err)
 			return ID, err
@@ -398,7 +472,8 @@ func (pc *PCCloud) InstanceID(nodeName k8stypes.NodeName) (string, error) {
 	if name == pc.localK8sHostname {
 		return pc.localInstanceID, nil
 	} else {
-		ID, err := getInstanceID(name, pc.projID)
+		// We assume only master need to get InstanceID of a node other than itself
+		ID, err := getInstanceID(pc, name)
 		if err != nil {
 			glog.Errorf("Photon Cloud Provider: getInstanceID failed for InstanceID. Error[%v]", err)
 			return ID, err
@@ -447,6 +522,12 @@ func (pc *PCCloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []st

 // Attaches given virtual disk volume to the compute running kubelet.
 func (pc *PCCloud) AttachDisk(pdID string, nodeName k8stypes.NodeName) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for AttachDisk, error: [%v]", err)
+		return err
+	}
+
 	operation := &photon.VmDiskOperation{
 		DiskID: pdID,
 	}
@@ -474,6 +555,12 @@ func (pc *PCCloud) AttachDisk(pdID string, nodeName k8stypes.NodeName) error {

 // Detaches given virtual disk volume from the compute running kubelet.
 func (pc *PCCloud) DetachDisk(pdID string, nodeName k8stypes.NodeName) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DetachDisk, error: [%v]", err)
+		return err
+	}
+
 	operation := &photon.VmDiskOperation{
 		DiskID: pdID,
 	}
@@ -501,6 +588,12 @@ func (pc *PCCloud) DetachDisk(pdID string, nodeName k8stypes.NodeName) error {

 // DiskIsAttached returns if disk is attached to the VM using controllers supported by the plugin.
 func (pc *PCCloud) DiskIsAttached(pdID string, nodeName k8stypes.NodeName) (bool, error) {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DiskIsAttached, error: [%v]", err)
+		return false, err
+	}
+
 	disk, err := photonClient.Disks.Get(pdID)
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: Failed to Get disk with pdID %s. Error[%v]", pdID, err)
@@ -525,6 +618,12 @@ func (pc *PCCloud) DiskIsAttached(pdID string, nodeName k8stypes.NodeName) (bool
 // DisksAreAttached returns if disks are attached to the VM using controllers supported by the plugin.
 func (pc *PCCloud) DisksAreAttached(pdIDs []string, nodeName k8stypes.NodeName) (map[string]bool, error) {
 	attached := make(map[string]bool)
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DisksAreAttached, error: [%v]", err)
+		return attached, err
+	}
+
 	for _, pdID := range pdIDs {
 		attached[pdID] = false
 	}
@@ -553,6 +652,12 @@ func (pc *PCCloud) DisksAreAttached(pdIDs []string, nodeName k8stypes.NodeName)

 // Create a volume of given size (in GB).
 func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err error) {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for CreateDisk, error: [%v]", err)
+		return "", err
+	}
+
 	diskSpec := photon.DiskCreateSpec{}
 	diskSpec.Name = volumeOptions.Name
 	diskSpec.Flavor = volumeOptions.Flavor
@@ -576,6 +681,12 @@ func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err er

 // Deletes a volume given volume name.
 func (pc *PCCloud) DeleteDisk(pdID string) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DeleteDisk, error: [%v]", err)
+		return err
+	}
+
 	task, err := photonClient.Disks.Delete(pdID)
 	if err != nil {
 		glog.Errorf("Photon Cloud Provider: Failed to DeleteDisk. Error[%v]", err)
--- a/pkg/cloudprovider/providers/photon/photon_test.go
+++ b/pkg/cloudprovider/providers/photon/photon_test.go
@@ -29,21 +29,21 @@ import (
 )

 func configFromEnv() (TestVM string, TestFlavor string, cfg PCConfig, ok bool) {
-	var IgnoreCertificate bool
+	var AuthEnabled bool
 	var OverrideIP bool
 	var err error
 	cfg.Global.CloudTarget = os.Getenv("PHOTON_TARGET")
-	cfg.Global.Tenant = os.Getenv("PHOTON_TENANT")
 	cfg.Global.Project = os.Getenv("PHOTON_PROJECT")
-	if os.Getenv("PHOTON_IGNORE_CERTIFICATE") != "" {
-		IgnoreCertificate, err = strconv.ParseBool(os.Getenv("PHOTON_IGNORE_CERTIFICATE"))
+	cfg.Global.VMID = os.Getenv("PHOTON_VMID")
+	if os.Getenv("PHOTON_AUTH_ENABLED") != "" {
+		AuthEnabled, err = strconv.ParseBool(os.Getenv("PHOTON_AUTH_ENABLED"))
 	} else {
-		IgnoreCertificate = false
+		AuthEnabled = false
 	}
 	if err != nil {
 		log.Fatal(err)
 	}
-	cfg.Global.IgnoreCertificate = IgnoreCertificate
+	cfg.Global.AuthEnabled = AuthEnabled
 	if os.Getenv("PHOTON_OVERRIDE_IP") != "" {
 		OverrideIP, err = strconv.ParseBool(os.Getenv("PHOTON_OVERRIDE_IP"))
 	} else {
@@ -65,8 +65,8 @@ func configFromEnv() (TestVM string, TestFlavor string, cfg PCConfig, ok bool) {
 	}

 	ok = (cfg.Global.CloudTarget != "" &&
-		cfg.Global.Tenant != "" &&
 		cfg.Global.Project != "" &&
+		cfg.Global.VMID != "" &&
 		TestVM != "")

 	return
@@ -81,10 +81,10 @@ func TestReadConfig(t *testing.T) {
 	cfg, err := readConfig(strings.NewReader(`
 [Global]
 target = 0.0.0.0
-ignoreCertificate = true
-tenant = tenant
 project = project
-overrideIP = false
+overrideIP = true
+vmID = vmid
+authentication = false
 `))
 	if err != nil {
 		t.Fatalf("Should succeed when a valid config is provided: %s", err)
@@ -94,13 +94,13 @@ overrideIP = false
 		t.Errorf("incorrect photon target ip: %s", cfg.Global.CloudTarget)
 	}

-	if cfg.Global.Tenant != "tenant" {
-		t.Errorf("incorrect tenant: %s", cfg.Global.Tenant)
-	}
-
 	if cfg.Global.Project != "project" {
 		t.Errorf("incorrect project: %s", cfg.Global.Project)
 	}
+
+	if cfg.Global.VMID != "vmid" {
+		t.Errorf("incorrect vmid: %s", cfg.Global.VMID)
+	}
 }

 func TestNewPCCloud(t *testing.T) {