Merge pull request #47881 from cadmuxe/endpoint

Automatic merge from submit-queue (batch tested with PRs 47918, 47964, 48151, 47881, 48299)

Add ApiEndpoint support to GCE config.

**What this PR does / why we need it**:
Add the ability to change ApiEndpoint  for GCE.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

**Special notes for your reviewer**:

**Release note**:
```release-note
None
```

cluster/common.sh

@@ -665,6 +665,7 @@ ENABLE_APISERVER_BASIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_BASIC_AUDIT:-})
 ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-})
 ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false})
 ADVANCED_AUDIT_BACKEND: $(yaml-quote ${ADVANCED_AUDIT_BACKEND:-log})
+GCE_API_ENDPOINT: $(yaml-quote ${GCE_API_ENDPOINT:-})
 EOF
   if [ -n "${KUBELET_PORT:-}" ]; then
     cat >>$file <<EOF

cluster/gce/config-default.sh

@@ -72,6 +72,7 @@ GCI_VERSION=${KUBE_GCI_VERSION:-cos-stable-59-9460-64-0}
 MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-}
 MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
 NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}}
+GCE_API_ENDPOINT=${KUBE_GCE_API_ENDPOINT:-}
 NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-cos-cloud}
 CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker}
 RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0}
@@ -129,7 +130,7 @@ HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
 # TODO(piosz): remove this in 1.8
 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
 
-# To avoid running Calico on a node that is not configured appropriately, 
+# To avoid running Calico on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
 if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
 	NODE_LABELS="${NODE_LABELS},projectcalico.org/ds-ready=true"

cluster/gce/gci/configure-helper.sh

@@ -379,6 +379,11 @@ function create-master-auth {
   cat <<EOF >/etc/gce.conf
 [global]
 EOF
+  if [[ -n "${GCE_API_ENDPOINT:-}" ]]; then
+    cat <<EOF >>/etc/gce.conf
+api-endpoint = ${GCE_API_ENDPOINT}
+EOF
+  fi
   if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then
     use_cloud_config="true"
     cat <<EOF >>/etc/gce.conf
@@ -1706,7 +1711,7 @@ function start-kube-addons {
     sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}"
     sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}"
   else
-    # If not configured to use Calico, the set the typha replica count to 0, but only if the 
+    # If not configured to use Calico, the set the typha replica count to 0, but only if the
     # addon is present.
     local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml"
     if [[ -e $typha_dep_file ]]; then

hack/verify-flags/known-flags.txt

@@ -298,6 +298,7 @@ gather-logs-sizes
 gather-metrics-at-teardown
 gather-resource-usage
 gather-suite-metrics-at-teardown
+gce-api-endpoint
 gce-multizone
 gce-project
 gce-service-account

pkg/cloudprovider/providers/gce/gce.go

@@ -137,6 +137,7 @@ type Config struct {
 		NodeTags           []string `gcfg:"node-tags"`
 		NodeInstancePrefix string   `gcfg:"node-instance-prefix"`
 		Multizone          bool     `gcfg:"multizone"`
+		ApiEndpoint        string   `gcfg:"api-endpoint"`
 	}
 }
 
@@ -155,6 +156,7 @@ func (g *GCECloud) GetComputeService() *compute.Service {
 
 // newGCECloud creates a new instance of GCECloud.
 func newGCECloud(config io.Reader) (*GCECloud, error) {
+	apiEndpoint := ""
 	projectID, zone, err := getProjectAndZone()
 	if err != nil {
 		return nil, err
@@ -169,7 +171,7 @@ func newGCECloud(config io.Reader) (*GCECloud, error) {
 	if err != nil {
 		return nil, err
 	}
-	networkURL := gceNetworkURL(projectID, networkName)
+	networkURL := gceNetworkURL(apiEndpoint, projectID, networkName)
 	subnetworkURL := ""
 
 	// By default, Kubernetes clusters only run against one zone
@@ -185,22 +187,23 @@ func newGCECloud(config io.Reader) (*GCECloud, error) {
 			return nil, err
 		}
 		glog.Infof("Using GCE provider config %+v", cfg)
+		if cfg.Global.ApiEndpoint != "" {
+			apiEndpoint = cfg.Global.ApiEndpoint
+		}
 		if cfg.Global.ProjectID != "" {
 			projectID = cfg.Global.ProjectID
 		}
-		if cfg.Global.NetworkName != "" {
-			if strings.Contains(cfg.Global.NetworkName, "/") {
-				networkURL = cfg.Global.NetworkName
-			} else {
-				networkURL = gceNetworkURL(cfg.Global.ProjectID, cfg.Global.NetworkName)
-			}
+
+		if cfg.Global.NetworkName != "" && strings.Contains(cfg.Global.NetworkName, "/") {
+			networkURL = cfg.Global.NetworkName
+		} else {
+			networkURL = gceNetworkURL(apiEndpoint, projectID, networkName)
 		}
-		if cfg.Global.SubnetworkName != "" {
-			if strings.Contains(cfg.Global.SubnetworkName, "/") {
-				subnetworkURL = cfg.Global.SubnetworkName
-			} else {
-				subnetworkURL = gceSubnetworkURL(cfg.Global.ProjectID, region, cfg.Global.SubnetworkName)
-			}
+
+		if cfg.Global.SubnetworkName != "" && strings.Contains(cfg.Global.SubnetworkName, "/") {
+			subnetworkURL = cfg.Global.SubnetworkName
+		} else {
+			subnetworkURL = gceSubnetworkURL(apiEndpoint, cfg.Global.ProjectID, region, cfg.Global.SubnetworkName)
 		}
 		if cfg.Global.TokenURL != "" {
 			tokenSource = NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)
@@ -212,7 +215,7 @@ func newGCECloud(config io.Reader) (*GCECloud, error) {
 		}
 	}
 
-	return CreateGCECloud(projectID, region, zone, managedZones, networkURL, subnetworkURL,
+	return CreateGCECloud(apiEndpoint, projectID, region, zone, managedZones, networkURL, subnetworkURL,
 		nodeTags, nodeInstancePrefix, tokenSource, true /* useMetadataServer */)
 }
 
@@ -220,7 +223,7 @@ func newGCECloud(config io.Reader) (*GCECloud, error) {
 // If no networkUrl is specified, loads networkName via rest call.
 // If no tokenSource is specified, uses oauth2.DefaultTokenSource.
 // If managedZones is nil / empty all zones in the region will be managed.
-func CreateGCECloud(projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string,
+func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string,
 	nodeInstancePrefix string, tokenSource oauth2.TokenSource, useMetadataServer bool) (*GCECloud, error) {
 
 	client, err := newOauthClient(tokenSource)
@@ -233,6 +236,10 @@ func CreateGCECloud(projectID, region, zone string, managedZones []string, netwo
 		return nil, err
 	}
 
+	if apiEndpoint != "" {
+		service.BasePath = fmt.Sprintf("%sprojects/", apiEndpoint)
+	}
+
 	client, err = newOauthClient(tokenSource)
 	serviceBeta, err := computebeta.New(client)
 	if err != nil {
@@ -249,7 +256,7 @@ func CreateGCECloud(projectID, region, zone string, managedZones []string, netwo
 		if err != nil {
 			return nil, err
 		}
-		networkURL = gceNetworkURL(projectID, networkName)
+		networkURL = gceNetworkURL(apiEndpoint, projectID, networkName)
 	}
 
 	networkProjectID, err := getProjectIDInURL(networkURL)
@@ -365,12 +372,18 @@ func (gce *GCECloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []
 // GCECloud implements cloudprovider.Interface.
 var _ cloudprovider.Interface = (*GCECloud)(nil)
 
-func gceNetworkURL(project, network string) string {
-	return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/%s", project, network)
+func gceNetworkURL(api_endpoint, project, network string) string {
+	if api_endpoint == "" {
+		api_endpoint = "https://www.googleapis.com/compute/v1/"
+	}
+	return fmt.Sprintf("%sprojects/%s/global/networks/%s", api_endpoint, project, network)
 }
 
-func gceSubnetworkURL(project, region, subnetwork string) string {
-	return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", project, region, subnetwork)
+func gceSubnetworkURL(api_endpoint, project, region, subnetwork string) string {
+	if api_endpoint == "" {
+		api_endpoint = "https://www.googleapis.com/compute/v1/"
+	}
+	return fmt.Sprintf("%sprojects/%s/regions/%s/subnetworks/%s", api_endpoint, project, region, subnetwork)
 }
 
 // getProjectIDInURL parses typical full resource URLS and shorter URLS

test/e2e/e2e.go

@@ -76,7 +76,8 @@ func setupProviderConfig() error {
 		if !framework.TestContext.CloudConfig.MultiZone {
 			managedZones = []string{zone}
 		}
-		cloudConfig.Provider, err = gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ProjectID,
+		cloudConfig.Provider, err = gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ApiEndpoint,
+			framework.TestContext.CloudConfig.ProjectID,
 			region, zone, managedZones, "" /* networkUrl */, "" /* subnetworkUrl */, nil, /* nodeTags */
 			"" /* nodeInstancePerfix */, nil /* tokenSource */, false /* useMetadataServer */)
 		if err != nil {

test/e2e/framework/test_context.go

@@ -133,6 +133,7 @@ type NodeTestContextType struct {
 }
 
 type CloudConfig struct {
+	ApiEndpoint       string
 	ProjectID         string
 	Zone              string
 	MultiZone         bool
@@ -206,6 +207,7 @@ func RegisterClusterFlags() {
 	// TODO: Flags per provider?  Rename gce-project/gce-zone?
 	cloudConfig := &TestContext.CloudConfig
 	flag.StringVar(&cloudConfig.MasterName, "kube-master", "", "Name of the kubernetes master. Only required if provider is gce or gke")
+	flag.StringVar(&cloudConfig.ApiEndpoint, "gce-api-endpoint", "", "The GCE ApiEndpoint being used, if applicable")
 	flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable")
 	flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable")
 	flag.BoolVar(&cloudConfig.MultiZone, "gce-multizone", false, "If true, start GCE cloud provider with multizone support.")