AWS: Configure minion routes dynamically

We need to implement the Routes interface, and then enable the functionality in the cluster scripts.

cluster/aws/config-default.sh

@@ -41,10 +41,10 @@ MASTER_NAME="${INSTANCE_PREFIX}-master"
 MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
 MASTER_TAG="${INSTANCE_PREFIX}-master"
 MINION_TAG="${INSTANCE_PREFIX}-minion"
-MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
 MINION_SCOPES=""
 POLL_SLEEP_INTERVAL=3
 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"  # formerly PORTAL_NET
+CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}"
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 # If set to Elastic IP, master instance will be associated with this IP.
 # If set to auto, a new Elastic IP will be aquired

cluster/aws/config-test.sh

@@ -37,10 +37,10 @@ MASTER_NAME="${INSTANCE_PREFIX}-master"
 MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
 MASTER_TAG="${INSTANCE_PREFIX}-master"
 MINION_TAG="${INSTANCE_PREFIX}-minion"
-MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
 MINION_SCOPES=""
 POLL_SLEEP_INTERVAL=3
 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"  # formerly PORTAL_NET
+CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.245.0.0/16}"
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 # If set to Elastic IP, master instance will be associated with this IP.
 # If set to auto, a new Elastic IP will be aquired

cluster/aws/coreos/util.sh

@@ -30,8 +30,8 @@ function detect-minion-image (){
 
 function generate-minion-user-data() {
   i=$1
+  # TODO(bakins): Is this actually used?
   MINION_PRIVATE_IP=$INTERNAL_IP_BASE.1${i}
-  MINION_IP_RANGE=${MINION_IP_RANGES[$i]}
 
   # this is a bit of a hack. We make all of our "variables" in
   # our cloud config controlled by env vars from this script
@@ -44,7 +44,6 @@ function generate-minion-user-data() {
       DNS_SERVER_IP=$(yaml-quote ${DNS_SERVER_IP:-})
       DNS_DOMAIN=$(yaml-quote ${DNS_DOMAIN:-})
       MASTER_IP=$(yaml-quote ${MASTER_INTERNAL_IP})
-      MINION_IP_RANGE=$(yaml-quote ${MINION_IP_RANGE})
       MINION_IP=$(yaml-quote ${MINION_PRIVATE_IP})
       KUBELET_TOKEN=$(yaml-quote ${KUBELET_TOKEN:-})
       KUBE_PROXY_TOKEN=$(yaml-quote ${KUBE_PROXY_TOKEN:-})

cluster/aws/templates/create-dynamic-salt-files.sh

@@ -22,6 +22,8 @@ mkdir -p /srv/salt-overlay/pillar
 cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
 instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
 node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")'
+cluster_cidr: '$(echo "$CLUSTER_IP_RANGE" | sed -e "s/'/''/g")'
+allocate_node_cidrs: '$(echo "$ALLOCATE_NODE_CIDRS" | sed -e "s/'/''/g")'
 service_cluster_ip_range: '$(echo "$SERVICE_CLUSTER_IP_RANGE" | sed -e "s/'/''/g")'
 enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")'
 enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")'

cluster/aws/templates/salt-minion.sh

@@ -26,7 +26,7 @@ cat <<EOF >/etc/salt/minion.d/grains.conf
 grains:
   roles:
     - kubernetes-pool
-  cbr-cidr: $MINION_IP_RANGE
+  cbr-cidr: 10.123.45.0/30
   cloud: aws
 EOF
 

cluster/aws/ubuntu/common.sh

@@ -29,7 +29,6 @@ function generate-minion-user-data {
   # We pipe this to the ami as a startup script in the user-data field.  Requires a compatible ami
   echo "#! /bin/bash"
   echo "SALT_MASTER='${MASTER_INTERNAL_IP}'"
-  echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'"
   echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'"
   echo "readonly DOCKER_STORAGE='${DOCKER_STORAGE:-}'"
   grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"

cluster/aws/util.sh

@@ -22,6 +22,8 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/cluster/aws/${KUBE_CONFIG_FILE-"config-default.sh"}"
 source "${KUBE_ROOT}/cluster/common.sh"
 
+ALLOCATE_NODE_CIDRS=true
+
 case "${KUBE_OS_DISTRIBUTION}" in
   ubuntu|wheezy|coreos)
     source "${KUBE_ROOT}/cluster/aws/${KUBE_OS_DISTRIBUTION}/util.sh"
@@ -695,6 +697,8 @@ function kube-up {
     echo "readonly SALT_MASTER='${MASTER_INTERNAL_IP}'"
     echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
     echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'"
+    echo "readonly CLUSTER_IP_RANGE='${CLUSTER_IP_RANGE}'"
+    echo "readonly ALLOCATE_NODE_CIDRS='${ALLOCATE_NODE_CIDRS}'"
     echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
     echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
     echo "readonly ZONE='${ZONE}'"
@@ -854,7 +858,8 @@ function kube-up {
     MINION_IDS[$i]=$minion_id
   done
 
-  # Add routes to minions
+  # Configure minion networking
+  # TODO(justinsb): Check if we can change source-dest-check before instance fully running
   for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
     # We are not able to add a route to the instance until that instance is in "running" state.
     # This is quite an ugly solution to this problem. In Bash 4 we could use assoc. arrays to do this for
@@ -864,7 +869,6 @@ function kube-up {
     echo "Minion ${MINION_NAMES[$i]} running"
     sleep 10
     $AWS_CMD modify-instance-attribute --instance-id $minion_id --source-dest-check '{"Value": false}' > $LOG
-    $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MINION_IP_RANGES[$i]} --instance-id $minion_id > $LOG
   done
 
   FAIL=0

pkg/cloudprovider/aws/aws.go

@@ -86,6 +86,10 @@ type EC2 interface {
 	DescribeSubnets(*ec2.DescribeSubnetsInput) ([]*ec2.Subnet, error)
 
 	CreateTags(*ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error)
+
+	DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error)
+	CreateRoute(request *ec2.CreateRouteInput) (*ec2.CreateRouteOutput, error)
+	DeleteRoute(request *ec2.DeleteRouteInput) (*ec2.DeleteRouteOutput, error)
 }
 
 // This is a simple pass-through of the ELB client interface, which allows for testing
@@ -393,6 +397,23 @@ func (s *awsSdkEC2) CreateTags(request *ec2.CreateTagsInput) (*ec2.CreateTagsOut
 	return s.ec2.CreateTags(request)
 }
 
+func (s *awsSdkEC2) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) {
+	// Not paged
+	response, err := s.ec2.DescribeRouteTables(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing AWS route tables: %v", err)
+	}
+	return response.RouteTables, nil
+}
+
+func (s *awsSdkEC2) CreateRoute(request *ec2.CreateRouteInput) (*ec2.CreateRouteOutput, error) {
+	return s.ec2.CreateRoute(request)
+}
+
+func (s *awsSdkEC2) DeleteRoute(request *ec2.DeleteRouteInput) (*ec2.DeleteRouteOutput, error) {
+	return s.ec2.DeleteRoute(request)
+}
+
 func init() {
 	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
 		creds := credentials.NewChainCredentials(
@@ -550,7 +571,7 @@ func (aws *AWSCloud) Zones() (cloudprovider.Zones, bool) {
 
 // Routes returns an implementation of Routes for Amazon Web Services.
 func (aws *AWSCloud) Routes() (cloudprovider.Routes, bool) {
-	return nil, false
+	return aws, true
 }
 
 // NodeAddresses is an implementation of Instances.NodeAddresses.

pkg/cloudprovider/aws/aws_routes.go

@@ -0,0 +1,114 @@
+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws_cloud
+
+import (
+	"fmt"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+func (s *AWSCloud) findRouteTable(clusterName string) (*ec2.RouteTable, error) {
+	request := &ec2.DescribeRouteTablesInput{}
+	filters := []*ec2.Filter{}
+	// This should be unnecessary (we already filter on TagNameKubernetesCluster,
+	// and something is broken if cluster name doesn't match, but anyway...
+	// TODO: All clouds should be cluster-aware by default
+	filters = append(filters, newEc2Filter("tag:"+TagNameKubernetesCluster, clusterName))
+	request.Filters = s.addFilters(filters)
+
+	tables, err := s.ec2.DescribeRouteTables(request)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(tables) == 0 {
+		return nil, fmt.Errorf("unable to find route table for AWS cluster: %s", clusterName)
+	}
+
+	if len(tables) != 1 {
+		return nil, fmt.Errorf("found multiple matching AWS route tables for AWS cluster: %s", clusterName)
+	}
+	return tables[0], nil
+}
+
+// ListRoutes implements Routes.ListRoutes
+// List all routes that match the filter
+func (s *AWSCloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) {
+	table, err := s.findRouteTable(clusterName)
+	if err != nil {
+		return nil, err
+	}
+
+	var routes []*cloudprovider.Route
+	for _, r := range table.Routes {
+		instanceID := orEmpty(r.InstanceID)
+		destinationCIDR := orEmpty(r.DestinationCIDRBlock)
+
+		if instanceID == "" || destinationCIDR == "" {
+			continue
+		}
+
+		routeName := clusterName + "-" + destinationCIDR
+		routes = append(routes, &cloudprovider.Route{routeName, instanceID, destinationCIDR})
+	}
+
+	return routes, nil
+}
+
+// CreateRoute implements Routes.CreateRoute
+// Create the described route
+func (s *AWSCloud) CreateRoute(clusterName string, nameHint string, route *cloudprovider.Route) error {
+	table, err := s.findRouteTable(clusterName)
+	if err != nil {
+		return err
+	}
+
+	request := &ec2.CreateRouteInput{}
+	// TODO: use ClientToken for idempotency?
+	request.DestinationCIDRBlock = aws.String(route.DestinationCIDR)
+	request.InstanceID = aws.String(route.TargetInstance)
+	request.RouteTableID = table.RouteTableID
+
+	_, err = s.ec2.CreateRoute(request)
+	if err != nil {
+		return fmt.Errorf("error creating AWS route (%s): %v", route.DestinationCIDR, err)
+	}
+
+	return nil
+}
+
+// DeleteRoute implements Routes.DeleteRoute
+// Delete the specified route
+func (s *AWSCloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
+	table, err := s.findRouteTable(clusterName)
+	if err != nil {
+		return err
+	}
+
+	request := &ec2.DeleteRouteInput{}
+	request.DestinationCIDRBlock = aws.String(route.DestinationCIDR)
+	request.RouteTableID = table.RouteTableID
+
+	_, err = s.ec2.DeleteRoute(request)
+	if err != nil {
+		return fmt.Errorf("error deleting AWS route (%s): %v", route.DestinationCIDR, err)
+	}
+
+	return nil
+}

pkg/cloudprovider/aws/aws_test.go

@@ -360,6 +360,18 @@ func (ec2 *FakeEC2) CreateTags(*ec2.CreateTagsInput) (*ec2.CreateTagsOutput, err
 	panic("Not implemented")
 }
 
+func (s *FakeEC2) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) {
+	panic("Not implemented")
+}
+
+func (s *FakeEC2) CreateRoute(request *ec2.CreateRouteInput) (*ec2.CreateRouteOutput, error) {
+	panic("Not implemented")
+}
+
+func (s *FakeEC2) DeleteRoute(request *ec2.DeleteRouteInput) (*ec2.DeleteRouteOutput, error) {
+	panic("Not implemented")
+}
+
 type FakeELB struct {
 	aws *FakeAWSServices
 }