Merge pull request #3922 from ZJU-SEL/ubuntu-cluster

Scripts to install k8s on ubuntu cluster with flannel network

.gitignore

@@ -42,3 +42,5 @@ network_closure.sh
 
 # also ignore etcd installed by hack/install-etcd.sh
 /third_party/etcd*
+
+.tags*

cluster/ubuntu-cluster/build.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# simple use the sed to replace some ip settings on user's demand
+# Run as root only
+
+# author @resouer
+set -e
+
+# check root
+if [ "$(id -u)" != "0" ]; then
+    echo >&2 "Please run as root"
+    exit 1
+fi
+
+mkdir -p binaries
+
+# flannel
+echo "Download & build flanneld ..."
+apt-get install linux-libc-dev
+if [ ! -d flannel ] ; then
+    echo "flannel does not exsit, cloning ..."
+    git clone https://github.com/coreos/flannel.git
+fi
+
+pushd flannel
+docker run -v `pwd`:/opt/flannel -i -t google/golang /bin/bash -c "cd /opt/flannel && ./build"
+popd
+cp flannel/bin/flanneld binaries/
+
+# ectd
+echo "Download etcd release ..."
+ETCD_V="v2.0.0"
+ETCD="etcd-${ETCD_V}-linux-amd64"
+if [ ! -f etcd.tar.gz ] ; then
+    curl -L  https://github.com/coreos/etcd/releases/download/$ETCD_V/$ETCD.tar.gz -o etcd.tar.gz
+    tar xzf etcd.tar.gz
+fi
+cp $ETCD/etcd $ETCD/etcdctl binaries
+
+# kuber
+echo "Download kubernetes release ..."
+if [ ! -f kubernetes.tar.gz ] ; then
+    curl -L https://github.com/GoogleCloudPlatform/kubernetes/releases/download/v0.10.1/kubernetes.tar.gz -o kubernetes.tar.gz
+    tar xzf kubernetes.tar.gz
+fi
+pushd kubernetes/server
+tar xzf kubernetes-server-linux-amd64.tar.gz
+popd
+cp kubernetes/server/kubernetes/server/bin/* binaries/
+
+rm -rf flannel kubernetes* etcd*
+echo "Done! All your commands locate in ./binaries dir"

cluster/ubuntu-cluster/configure.sh

@@ -0,0 +1,201 @@
+#!/bin/bash
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# simple use the sed to replace some ip settings on user's demand
+# Run as root only
+
+# author @WIZARD-CXY @resouer
+
+set -e
+
+function cpMaster(){
+	# copy /etc/init files
+    cp init_conf/etcd.conf /etc/init/
+    cp init_conf/kube-apiserver.conf /etc/init/
+    cp init_conf/kube-controller-manager.conf /etc/init/
+    cp init_conf/kube-scheduler.conf /etc/init/
+
+    # copy /etc/initd/ files
+    cp initd_scripts/etcd /etc/init.d/
+    cp initd_scripts/kube-apiserver /etc/init.d/
+    cp initd_scripts/kube-controller-manager /etc/init.d/
+    cp initd_scripts/kube-scheduler /etc/init.d/
+
+    # copy default configs
+    cp default_scripts/etcd /etc/default/
+    cp default_scripts/kube-apiserver /etc/default/
+    cp default_scripts/kube-scheduler /etc/default/
+    cp default_scripts/kube-controller-manager /etc/default/
+}
+
+function cpMinion(){
+	# copy /etc/init files
+    cp init_conf/etcd.conf /etc/init/
+    cp init_conf/kubelet.conf /etc/init/
+    cp init_conf/flanneld.conf /etc/init/
+    cp init_conf/kube-proxy.conf /etc/init/
+
+    # copy /etc/initd/ files
+    cp initd_scripts/etcd /etc/init.d/
+    cp initd_scripts/flanneld /etc/init.d/
+    cp initd_scripts/kubelet /etc/init.d/
+    cp initd_scripts/kube-proxy /etc/init.d/
+
+    # copy default configs
+    cp default_scripts/etcd /etc/default/
+    cp default_scripts/flanneld /etc/default/
+    cp default_scripts/kube-proxy /etc/default
+    cp default_scripts/kubelet /etc/default/
+}
+
+# check if input IP in machine list
+function inList(){
+	if [ "$#" -eq 1 ]; then
+		echo -e "\e[0;31mERROR\e[0m: "$1" is not in your machine list."
+		exit 1
+	fi
+}
+
+# set values in ETCD_OPTS
+function configEtcd(){
+    echo ETCD_OPTS=\"-name $1 -initial-advertise-peer-urls http://$2:2380 -listen-peer-urls http://$2:2380 -initial-cluster-token etcd-cluster-1 -initial-cluster $3 -initial-cluster-state new\" > default_scripts/etcd	
+}
+
+# check root
+if [ "$(id -u)" != "0" ]; then
+    echo >&2 "Please run as root"
+    exit 1
+fi
+
+echo "Welcome to use this script to configure k8s setup"
+
+echo
+
+PATH=$PATH:/opt/bin
+
+# use ubuntu
+if ! $(grep Ubuntu /etc/lsb-release > /dev/null 2>&1)
+then
+    echo "warning: not detecting a ubuntu system"
+    exit 1
+fi
+
+# check etcd
+if ! $(which etcd > /dev/null)
+then
+    echo "warning: etcd binary is not found in the PATH: $PATH"
+    exit 1
+fi
+
+# check kube commands
+if ! $(which kube-apiserver > /dev/null) && ! $(which kubelet > /dev/null)
+then
+    echo "warning: kube binaries are not found in the $PATH"
+    exit 1
+fi
+
+# detect the etcd version, we support only etcd 2.0.
+etcdVersion=$(/opt/bin/etcd --version | awk '{print $3}')
+
+if [ "$etcdVersion" != "2.0.0" ]; then
+	echo "We only support 2.0.0 version of etcd"
+	exit 1
+fi
+
+
+# use an array to record name and ip
+declare -A mm
+ii=1
+# we use static etcd configuration 
+# see https://github.com/coreos/etcd/blob/master/Documentation/clustering.md#static
+echo "Please enter all your cluster node ips, MASTER node comes first"
+read -p "And separated with blank space like \"<ip_1> <ip_2> <ip_3>\": " etcdIPs
+
+for i in $etcdIPs
+do
+    name="infra"$ii
+    item="$name=http://$i:2380"
+    if [ "$ii" == 1 ]; then 
+        cluster=$item
+    else
+        cluster="$cluster,$item"
+        if [ "$ii" -gt 2 ]; then
+        	    minionIPs="$minionIPs,$i"
+        else
+        	    minionIPs="$i"
+        fi
+    fi
+    mm[$i]=$name
+    let ii++
+done
+echo 
+
+# input node IPs
+while true; do
+    echo "This machine acts as"
+    echo -e "  both MASTER and MINION:      \033[1m1\033[0m"
+    echo -e "  only MASTER:                 \033[1m2\033[0m"
+    echo -e "  only MINION:                 \033[1m3\033[0m"
+	read -p "Please choose a role > " option 
+    echo
+
+	case $option in
+	    [2] )
+        	read -p "IP address of this machine > " myIP
+            echo
+            etcdName=${mm[$myIP]}
+            inList $etcdName $myIP 
+            configEtcd $etcdName $myIP $cluster
+            # set MINION IPs in kube-controller-manager
+            sed -i "s/MINION_IPS/${minionIPs}/g" default_scripts/kube-controller-manager  
+	        cpMaster
+	        break
+	        ;;
+	    [3] )     
+        	read -p "IP address of this machine > " myIP
+            echo           
+            etcdName=${mm[$myIP]}
+            inList $etcdName $myIP 
+            configEtcd $etcdName $myIP $cluster
+            # set MINION IP in default_scripts/kubelet
+            sed -i "s/MY_IP/${myIP}/g" default_scripts/kubelet
+	        cpMinion
+	        break
+	        ;;
+	    [1] )
+        	read -p "IP address of this machine > " myIP
+            echo 
+            etcdName=${mm[$myIP]}
+            inList $etcdName $myIP 
+            configEtcd $etcdName $myIP $cluster
+            # For minion set MINION IP in default_scripts/kubelet
+            sed -i "s/MY_IP/${myIP}/g" default_scripts/kubelet 
+            
+            # For master set MINION IPs in kube-controller-manager
+	        minionIPs="$minionIPs,$myIP"       
+	        sed -i "s/MINION_IPS/${minionIPs}/g" default_scripts/kube-controller-manager
+	        
+	        cpMaster
+	        cpMinion
+	        break
+	        ;;
+	    * )
+	        echo "Please choose 1 or 2 or 3."
+	        ;;
+	esac
+done
+
+echo -e "\e[0;32mConfigure Success\033[0m"

cluster/ubuntu-cluster/default_scripts/etcd

@@ -0,0 +1 @@
+ETCD_OPTS="-name infra1 -initial-advertise-peer-urls http://10.10.103.250:2380 -listen-peer-urls http://10.10.103.250:2380 -initial-cluster-token etcd-cluster-1 -initial-cluster infra1=http://10.10.103.250:2380,infra2=http://10.10.103.223:2380,infra3=http://10.10.103.224:2380 -initial-cluster-state new"

cluster/ubuntu-cluster/default_scripts/flanneld

@@ -0,0 +1,7 @@
+# flannel Upstart and SysVinit configuration file
+
+# Customize kube-apiserver binary location 
+# FLANNEL="/opt/bin/flanneld"
+
+# Use FLANNEL_OPTS to modify the start/restart options
+FLANNEL_OPTS=""

cluster/ubuntu-cluster/default_scripts/kube-apiserver

@@ -0,0 +1,14 @@
+# Kube-Apiserver Upstart and SysVinit configuration file
+
+# Customize kube-apiserver binary location 
+# KUBE_APISERVER="/opt/bin/kube-apiserver"
+
+# Use KUBE_APISERVER_OPTS to modify the start/restart options
+
+KUBE_APISERVER_OPTS="--address=0.0.0.0 \
+--port=8080 \
+--etcd_servers=http://127.0.0.1:4001 \
+--logtostderr=true \
+--portal_net=11.1.1.0/24"
+
+# Add more envionrment settings used by kube-apiserver here

cluster/ubuntu-cluster/default_scripts/kube-controller-manager

@@ -0,0 +1,11 @@
+# Kube-Controller-Manager Upstart and SysVinit configuration file
+
+# Customize kube-controller-manager binary location 
+# KUBE_CONTROLLER_MANAGER="/opt/bin/kube-controller-manager"
+
+# Use KUBE_CONTROLLER_MANAGER_OPTS to modify the start/restart options
+KUBE_CONTROLLER_MANAGER_OPTS="--master=127.0.0.1:8080 \
+--machines=MINION_IPS \
+--logtostderr=true"
+
+# Add more envionrment settings used by kube-controller-manager here

cluster/ubuntu-cluster/default_scripts/kube-proxy

@@ -0,0 +1,10 @@
+# Kube-Proxy Upstart and SysVinit configuration file
+
+# Customize kube-proxy binary location 
+# KUBE_PROXY="/opt/bin/kube-proxy"
+
+# Use KUBE_PROXY_OPTS to modify the start/restart options
+KUBE_PROXY_OPTS="--etcd_servers=http://127.0.0.1:4001 \
+--logtostderr=true"
+
+# Add more envionrment settings used by kube-apiserver here

cluster/ubuntu-cluster/default_scripts/kube-scheduler

@@ -0,0 +1,11 @@
+# Kube-Scheduler Upstart and SysVinit configuration file
+
+# Customize kube-apiserver binary location 
+
+# KUBE_SCHEDULER="/opt/bin/kube-scheduler"
+
+# Use KUBE_SCHEDULER_OPTS to modify the start/restart options
+KUBE_SCHEDULER_OPTS="--logtostderr=true \
+--master=127.0.0.1:8080"
+
+# Add more envionrment settings used by kube-scheduler here

cluster/ubuntu-cluster/default_scripts/kubelet

@@ -0,0 +1,14 @@
+# Kubelet Upstart and SysVinit configuration file
+
+# Customize kubelet binary location 
+# KUBELET="/opt/bin/kubelet"
+
+# Use KUBELET_OPTS to modify the start/restart options
+
+KUBELET_OPTS="--address=0.0.0.0 \
+--port=10250 \
+--hostname_override=MY_IP \
+--etcd_servers=http://127.0.0.1:4001 \
+--logtostderr=true"
+
+# Add more envionrment settings used by kube-scheduler here

cluster/ubuntu-cluster/init_conf/etcd.conf

@@ -0,0 +1,27 @@
+description "Etcd service"
+author "@jainvipin"
+
+respawn
+
+pre-start script
+	# see also https://github.com/jainvipin/kubernetes-ubuntu-start
+	ETCD=/opt/bin/$UPSTART_JOB
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	if [ -f $ETCD ]; then
+		exit 0
+	fi
+    echo "$ETCD binary not found, exiting"
+    exit 22
+end script
+
+script
+	# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
+	ETCD=/opt/bin/$UPSTART_JOB
+	ETCD_OPTS=""
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	exec "$ETCD" $ETCD_OPTS
+end script

cluster/ubuntu-cluster/init_conf/flanneld.conf

@@ -0,0 +1,29 @@
+description "Flannel service"
+author "@chenxingyu"
+
+# respawn
+
+# start in conjunction with etcd
+start on started etcd
+stop on stopping etcd
+
+pre-start script
+	FLANNEL=/opt/bin/$UPSTART_JOB
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	if [ -f $FLANNEL ]; then
+		exit 0
+	fi
+    exit 22
+end script
+
+script
+	# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
+	FLANNEL=/opt/bin/$UPSTART_JOB
+	FLANNEL_OPTS=""
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	exec "$FLANNEL" $FLANNEL_OPTS
+end script

cluster/ubuntu-cluster/init_conf/kube-apiserver.conf

@@ -0,0 +1,30 @@
+description "Kube-Apiserver service"
+author "@jainvipin"
+
+# respawn
+
+# start in conjunction with etcd
+start on started etcd
+stop on stopping etcd
+
+pre-start script
+	# see also https://github.com/jainvipin/kubernetes-start
+	KUBE_APISERVER=/opt/bin/$UPSTART_JOB
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	if [ -f $KUBE_APISERVER ]; then
+		exit 0
+	fi
+    exit 22
+end script
+
+script
+	# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
+	KUBE_APISERVER=/opt/bin/$UPSTART_JOB
+	KUBE_APISERVER_OPTS=""
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	exec "$KUBE_APISERVER" $KUBE_APISERVER_OPTS
+end script

cluster/ubuntu-cluster/init_conf/kube-controller-manager.conf

@@ -0,0 +1,30 @@
+description "Kube-Controller-Manager service"
+author "@jainvipin"
+
+# respawn
+
+# start in conjunction with etcd
+start on started etcd
+stop on stopping etcd
+
+pre-start script
+	# see also https://github.com/jainvipin/kubernetes-ubuntu-start
+	KUBE_CONTROLLER_MANAGER=/opt/bin/$UPSTART_JOB
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	if [ -f $KUBE_CONTROLLER_MANAGER ]; then
+		exit 0
+	fi
+    exit 22
+end script
+
+script
+	# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
+	KUBE_CONTROLLER_MANAGER=/opt/bin/$UPSTART_JOB
+	KUBE_CONTROLLER_MANAGER_OPTS=""
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	exec "$KUBE_CONTROLLER_MANAGER" $KUBE_CONTROLLER_MANAGER_OPTS
+end script

cluster/ubuntu-cluster/init_conf/kube-proxy.conf

@@ -0,0 +1,30 @@
+description "Kube-Proxy service"
+author "@jainvipin"
+
+# respawn
+
+# start in conjunction with etcd
+start on started etcd
+stop on stopping etcd
+
+pre-start script
+	# see also https://github.com/jainvipin/kubernetes-start
+	KUBE_PROXY=/opt/bin/$UPSTART_JOB
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	if [ -f $KUBE_PROXY ]; then
+		exit 0
+	fi
+    exit 22
+end script
+
+script
+	# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
+	KUBE_PROXY=/opt/bin/$UPSTART_JOB
+	KUBE_PROXY_OPTS=""
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	exec "$KUBE_PROXY" $KUBE_PROXY_OPTS
+end script

cluster/ubuntu-cluster/init_conf/kube-scheduler.conf

@@ -0,0 +1,30 @@
+description "Kube-Scheduler service"
+author "@jainvipin"
+
+# respawn
+
+# start in conjunction with etcd
+start on started etcd
+stop on stopping etcd
+
+pre-start script
+	# see also https://github.com/jainvipin/kubernetes-start
+	KUBE_SCHEDULER=/opt/bin/$UPSTART_JOB
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	if [ -f $KUBE_SCHEDULER ]; then
+		exit 0
+	fi
+    exit 22
+end script
+
+script
+	# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
+	KUBE_SCHEDULER=/opt/bin/$UPSTART_JOB
+	KUBE_SCHEDULER_OPTS=""
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	exec "$KUBE_SCHEDULER" $KUBE_SCHEDULER_OPTS
+end script

cluster/ubuntu-cluster/init_conf/kubelet.conf

@@ -0,0 +1,30 @@
+description "Kubelet service"
+author "@jainvipin"
+
+# respawn
+
+# start in conjunction with etcd
+start on started etcd
+stop on stopping etcd
+
+pre-start script
+	# see also https://github.com/jainvipin/kubernetes-ubuntu-start
+	KUBELET=/opt/bin/$UPSTART_JOB
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	if [ -f $KUBELET ]; then
+		exit 0
+	fi
+    exit 22
+end script
+
+script
+	# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
+	KUBELET=/opt/bin/$UPSTART_JOB
+	KUBELET_OPTS=""
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	exec "$KUBELET" $KUBELET_OPTS
+end script

cluster/ubuntu-cluster/initd_scripts/etcd

@@ -0,0 +1,100 @@
+#!/bin/sh
+set -e
+
+### BEGIN INIT INFO
+# Provides:           etcd
+# Required-Start:     $docker
+# Required-Stop:      
+# Should-Start:       
+# Should-Stop:        
+# Default-Start:      
+# Default-Stop:       
+# Short-Description:  Start distrubted key/value pair service
+# Description:
+#  http://www.github.com/coreos/etcd
+### END INIT INFO
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/bin:
+
+BASE=$(basename $0)
+
+# modify these in /etc/default/$BASE (/etc/default/etcd)
+ETCD=/opt/bin/$BASE
+# This is the pid file managed by etcd itself
+ETCD_PIDFILE=/var/run/$BASE.pid
+ETCD_LOGFILE=/var/log/$BASE.log
+ETCD_OPTS=""
+ETCD_DESC="Etcd"
+
+# Get lsb functions
+. /lib/lsb/init-functions
+
+if [ -f /etc/default/$BASE ]; then
+	. /etc/default/$BASE
+fi
+
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if false && [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
+	log_failure_msg "$ETCD_DESC is managed via upstart, try using service $BASE $1"
+	exit 1
+fi
+
+# Check etcd is present
+if [ ! -x $ETCD ]; then
+	log_failure_msg "$ETCD not present or not executable"
+	exit 1
+fi
+
+fail_unless_root() {
+	if [ "$(id -u)" != '0' ]; then
+		log_failure_msg "$ETCD_DESC must be run as root"
+		exit 1
+	fi
+}
+
+ETCD_START="start-stop-daemon \
+--start \
+--background \
+--quiet \
+--exec $ETCD \
+--make-pidfile \
+--pidfile $ETCD_PIDFILE \
+-- $ETCD_OPTS \
+>> $ETCD_LOGFILE 2>&1"
+
+ETCD_STOP="start-stop-daemon \
+--stop \
+--pidfile $ETCD_PIDFILE"
+
+case "$1" in
+	start)
+		fail_unless_root
+		log_begin_msg "Starting $ETCD_DESC: $BASE"
+        $ETCD_START
+		log_end_msg $?
+		;;
+
+	stop)
+		fail_unless_root
+		log_begin_msg "Stopping $ETCD_DESC: $BASE"
+        $ETCD_STOP
+		log_end_msg $?
+		;;
+
+	restart | force-reload)
+		fail_unless_root
+		log_begin_msg "Restarting $ETCD_DESC: $BASE"
+        $ETCD_STOP
+        $ETCD_START
+		log_end_msg $?
+		;;
+
+	status)
+		status_of_proc -p "$ETCD_PIDFILE" "$ETCD" "$ETCD_DESC"
+		;;
+
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac

cluster/ubuntu-cluster/initd_scripts/flanneld

@@ -0,0 +1,99 @@
+#!/bin/sh
+set -e
+
+### BEGIN INIT INFO
+# Provides:           flannel
+# Required-Start:     $etcd
+# Required-Stop:      
+# Should-Start:       
+# Should-Stop:        
+# Default-Start:      
+# Default-Stop:       
+# Short-Description:  Start flannel networking service
+# Description:
+#  https://github.com/coreos/flannel
+### END INIT INFO
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/bin:
+
+BASE=$(basename $0)
+
+# modify these in /etc/default/$BASE (/etc/default/flannel)
+FLANNEL=/opt/bin/$BASE
+# This is the pid file managed by kube-apiserver itself
+FLANNEL_PIDFILE=/var/run/$BASE.pid
+FLANNEL_LOGFILE=/var/log/$BASE.log
+FLANNEL_OPTS=""
+FLANNEL_DESC="Flannel"
+
+# Get lsb functions
+. /lib/lsb/init-functions
+
+if [ -f /etc/default/$BASE ]; then
+	. /etc/default/$BASE
+fi
+
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
+	log_failure_msg "$FLANNEL_DESC is managed via upstart, try using service $BASE $1"
+	exit 1
+fi
+
+# Check flanneld is present
+if [ ! -x $FLANNEL ]; then
+	log_failure_msg "$FLANNEL not present or not executable"
+	exit 1
+fi
+
+fail_unless_root() {
+	if [ "$(id -u)" != '0' ]; then
+		log_failure_msg "$FLANNEL_DESC must be run as root"
+		exit 1
+	fi
+}
+
+FLANNEL_START="start-stop-daemon \
+--start \
+--background \
+--quiet \
+--exec $FLANNEL \
+--make-pidfile --pidfile $FLANNEL_PIDFILE \
+-- $FLANNEL_OPTS \
+>> $FLANNEL_LOGFILE 2>&1"
+
+FLANNEL_STOP="start-stop-daemon \
+--stop \
+--pidfile $FLANNEL_PIDFILE"
+
+case "$1" in
+	start)
+		fail_unless_root
+		log_begin_msg "Starting $FLANNEL_DESC: $BASE"
+        $KUBE_APISERVER_START
+		log_end_msg $?
+		;;
+
+	stop)
+		fail_unless_root
+		log_begin_msg "Stopping $FLANNEL_DESC: $BASE"
+		$KUBE_APISERVER_STOP
+		log_end_msg $?
+		;;
+
+	restart | force-reload)
+		fail_unless_root
+		log_begin_msg "Stopping $FLANNEL_DESC: $BASE"
+		$KUBE_APISERVER_STOP
+		$KUBE_APISERVER_START
+		log_end_msg $?
+		;;
+
+	status)
+		status_of_proc -p "$FLANNEL_DESC" "$FLANNEL" "$FLANNEL_DESC"
+		;;
+
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac

cluster/ubuntu-cluster/initd_scripts/kube-apiserver

@@ -0,0 +1,99 @@
+#!/bin/sh
+set -e
+
+### BEGIN INIT INFO
+# Provides:           kube-apiserver
+# Required-Start:     $etcd
+# Required-Stop:      
+# Should-Start:       
+# Should-Stop:        
+# Default-Start:      
+# Default-Stop:       
+# Short-Description:  Start distrubted key/value pair service
+# Description:
+#  http://www.github.com/GoogleCloudPlatform/Kubernetes
+### END INIT INFO
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/bin:
+
+BASE=$(basename $0)
+
+# modify these in /etc/default/$BASE (/etc/default/kube-apiserver)
+KUBE_APISERVER=/opt/bin/$BASE
+# This is the pid file managed by kube-apiserver itself
+KUBE_APISERVER_PIDFILE=/var/run/$BASE.pid
+KUBE_APISERVER_LOGFILE=/var/log/$BASE.log
+KUBE_APISERVER_OPTS=""
+KUBE_APISERVER_DESC="Kube-Apiserver"
+
+# Get lsb functions
+. /lib/lsb/init-functions
+
+if [ -f /etc/default/$BASE ]; then
+	. /etc/default/$BASE
+fi
+
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
+	log_failure_msg "$KUBE_APISERVER_DESC is managed via upstart, try using service $BASE $1"
+	exit 1
+fi
+
+# Check kube-apiserver is present
+if [ ! -x $KUBE_APISERVER ]; then
+	log_failure_msg "$KUBE_APISERVER not present or not executable"
+	exit 1
+fi
+
+fail_unless_root() {
+	if [ "$(id -u)" != '0' ]; then
+		log_failure_msg "$KUBE_APISERVER_DESC must be run as root"
+		exit 1
+	fi
+}
+
+KUBE_APISERVER_START="start-stop-daemon \
+--start \
+--background \
+--quiet \
+--exec $KUBE_APISERVER \
+--make-pidfile --pidfile $KUBE_APISERVER_PIDFILE \
+-- $KUBE_APISERVER_OPTS \
+>> $KUBE_APISERVER_LOGFILE 2>&1"
+
+KUBE_APISERVER_STOP="start-stop-daemon \
+--stop \
+--pidfile $KUBE_APISERVER_PIDFILE"
+
+case "$1" in
+	start)
+		fail_unless_root
+		log_begin_msg "Starting $KUBE_APISERVER_DESC: $BASE"
+        $KUBE_APISERVER_START
+		log_end_msg $?
+		;;
+
+	stop)
+		fail_unless_root
+		log_begin_msg "Stopping $KUBE_APISERVER_DESC: $BASE"
+		$KUBE_APISERVER_STOP
+		log_end_msg $?
+		;;
+
+	restart | force-reload)
+		fail_unless_root
+		log_begin_msg "Stopping $KUBE_APISERVER_DESC: $BASE"
+		$KUBE_APISERVER_STOP
+		$KUBE_APISERVER_START
+		log_end_msg $?
+		;;
+
+	status)
+		status_of_proc -p "$KUBE_APISERVER_PIDFILE" "$KUBE_APISERVER" "$KUBE_APISERVER_DESC"
+		;;
+
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac

cluster/ubuntu-cluster/initd_scripts/kube-controller-manager

@@ -0,0 +1,99 @@
+#!/bin/sh
+set -e
+
+### BEGIN INIT INFO
+# Provides:           kube-controller-manager
+# Required-Start:     $etcd
+# Required-Stop:      
+# Should-Start:       
+# Should-Stop:        
+# Default-Start:      
+# Default-Stop:       
+# Short-Description:  Start distrubted key/value pair service
+# Description:
+#  http://www.github.com/GoogleCloudPlatform/Kubernetes
+### END INIT INFO
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/bin:
+
+BASE=$(basename $0)
+
+# modify these in /etc/default/$BASE (/etc/default/kube-controller-manager)
+KUBE_CONTROLLER_MANAGER=/opt/bin/$BASE
+# This is the pid file managed by kube-controller-manager itself
+KUBE_CONTROLLER_MANAGER_PIDFILE=/var/run/$BASE.pid
+KUBE_CONTROLLER_MANAGER_LOGFILE=/var/log/$BASE.log
+KUBE_CONTROLLER_MANAGER_OPTS=""
+KUBE_CONTROLLER_MANAGER_DESC="Kube-Controller-Manager"
+
+# Get lsb functions
+. /lib/lsb/init-functions
+
+if [ -f /etc/default/$BASE ]; then
+	. /etc/default/$BASE
+fi
+
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
+	log_failure_msg "$KUBE_CONTROLLER_MANAGER_DESC is managed via upstart, try using service $BASE $1"
+	exit 1
+fi
+
+# Check kube-controller-manager is present
+if [ ! -x $KUBE_CONTROLLER_MANAGER ]; then
+	log_failure_msg "$KUBE_CONTROLLER_MANAGER not present or not executable"
+	exit 1
+fi
+
+fail_unless_root() {
+	if [ "$(id -u)" != '0' ]; then
+		log_failure_msg "$KUBE_CONTROLLER_MANAGER_DESC must be run as root"
+		exit 1
+	fi
+}
+
+KUBE_CONTROLLER_MANAGER_START="start-stop-daemon 
+--start --background \
+--quiet \
+--exec $KUBE_CONTROLLER_MANAGER \
+--make-pidfile \
+--pidfile $KUBE_CONTROLLER_MANAGER_PIDFILE \
+-- $KUBE_CONTROLLER_MANAGER_OPTS \
+>> "$KUBE_CONTROLLER_MANAGER_LOGFILE" 2>&1
+
+KUBE_CONTROLLER_MANAGER_STOP="start-stop-daemon \
+--stop \
+--pidfile $KUBE_CONTROLLER_MANAGER_PIDFILE"
+
+case "$1" in
+	start)
+		fail_unless_root
+		log_begin_msg "Starting $KUBE_CONTROLLER_MANAGER_DESC: $BASE"
+        $KUBE_CONTROLLER_MANAGER_START
+		log_end_msg $?
+		;;
+
+	stop)
+		fail_unless_root
+		log_begin_msg "Stopping $KUBE_CONTROLLER_MANAGER_DESC: $BASE"
+        $KUBE_CONTROLLER_MANAGER_STOP
+		log_end_msg $?
+		;;
+
+	restart | force-reload)
+		fail_unless_root
+        log_daemon_message "Restarting $KUBE_CONTROLLER_MANAGER" || true
+        $KUBE_CONTROLLER_MANAGER_STOP
+        $KUBE_CONTROLLER_MANAGER_START
+		log_end_msg $?
+		;;
+
+	status)
+		status_of_proc -p "$KUBE_CONTROLLER_MANAGER_PIDFILE" "$KUBE_CONTROLLER_MANAGER" "$KUBE_CONTROLLER_MANAGER_DESC"
+		;;
+
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac

cluster/ubuntu-cluster/initd_scripts/kube-proxy

@@ -0,0 +1,99 @@
+#!/bin/sh
+set -e
+
+### BEGIN INIT INFO
+# Provides:           kube-proxy
+# Required-Start:     $etcd
+# Required-Stop:      
+# Should-Start:       
+# Should-Stop:        
+# Default-Start:      
+# Default-Stop:       
+# Short-Description:  Start kube-proxy service
+# Description:
+#  http://www.github.com/GoogleCloudPlatform/Kubernetes
+### END INIT INFO
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/bin:
+
+BASE=$(basename $0)
+
+# modify these in /etc/default/$BASE (/etc/default/kube-proxy)
+KUBE_PROXY=/opt/bin/$BASE
+# This is the pid file managed by kube-proxy itself
+KUBE_PROXY_PIDFILE=/var/run/$BASE.pid
+KUBE_PROXY_LOGFILE=/var/log/$BASE.log
+KUBE_PROXY_OPTS=""
+KUBE_PROXY_DESC="Kube-Proxy"
+
+# Get lsb functions
+. /lib/lsb/init-functions
+
+if [ -f /etc/default/$BASE ]; then
+	. /etc/default/$BASE
+fi
+
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
+	log_failure_msg "$KUBE_PROXY_DESC is managed via upstart, try using service $BASE $1"
+	exit 1
+fi
+
+# Check kube-proxy is present
+if [ ! -x $KUBE_PROXY ]; then
+	log_failure_msg "$KUBE_PROXY not present or not executable"
+	exit 1
+fi
+
+fail_unless_root() {
+	if [ "$(id -u)" != '0' ]; then
+		log_failure_msg "$KUBE_PROXY_DESC must be run as root"
+		exit 1
+	fi
+}
+
+KUBE_PROXY_START="start-stop-daemon \
+--start \
+--background \
+--quiet \
+--exec $KUBE_PROXY \
+--make-pidfile --pidfile $KUBE_PROXY_PIDFILE \
+-- $KUBE_PROXY_OPTS \
+>> $KUBE_PROXY_LOGFILE 2>&1"
+
+KUBE_PROXY_STOP="start-stop-daemon \
+--stop \
+--pidfile $KUBE_PROXY_PIDFILE"
+
+case "$1" in
+	start)
+		fail_unless_root
+		log_begin_msg "Starting $KUBE_PROXY_DESC: $BASE"
+        $KUBE_PROXY_START
+		log_end_msg $?
+		;;
+
+	stop)
+		fail_unless_root
+		log_begin_msg "Stopping $KUBE_PROXY_DESC: $BASE"
+		$KUBE_PROXY_STOP
+		log_end_msg $?
+		;;
+
+	restart | force-reload)
+		fail_unless_root
+		log_begin_msg "Stopping $KUBE_PROXY_DESC: $BASE"
+		$KUBE_PROXY_STOP
+		$KUBE_PROXY_START
+		log_end_msg $?
+		;;
+
+	status)
+		status_of_proc -p "$KUBE_PROXY_PIDFILE" "$KUBE_PROXY" "$KUBE_PROXY_DESC"
+		;;
+
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac

cluster/ubuntu-cluster/initd_scripts/kube-scheduler

@@ -0,0 +1,99 @@
+#!/bin/sh
+set -e
+
+### BEGIN INIT INFO
+# Provides:           kube-scheduler
+# Required-Start:     $etcd
+# Required-Stop:      
+# Should-Start:       
+# Should-Stop:        
+# Default-Start:      
+# Default-Stop:       
+# Short-Description:  Start kube-proxy service
+# Description:
+#  http://www.github.com/GoogleCloudPlatform/Kubernetes
+### END INIT INFO
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/bin:
+
+BASE=$(basename $0)
+
+# modify these in /etc/default/$BASE (/etc/default/kube-scheduler)
+KUBE_SCHEDULER=/opt/bin/$BASE
+# This is the pid file managed by kube-scheduler itself
+KUBE_SCHEDULER_PIDFILE=/var/run/$BASE.pid
+KUBE_SCHEDULER_LOGFILE=/var/log/$BASE.log
+KUBE_SCHEDULER_OPTS=""
+KUBE_SCHEDULER_DESC="Kube-Scheduler"
+
+# Get lsb functions
+. /lib/lsb/init-functions
+
+if [ -f /etc/default/$BASE ]; then
+	. /etc/default/$BASE
+fi
+
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
+	log_failure_msg "$KUBE_SCHEDULER_DESC is managed via upstart, try using service $BASE $1"
+	exit 1
+fi
+
+# Check kube-scheduler is present
+if [ ! -x $KUBE_SCHEDULER ]; then
+	log_failure_msg "$KUBE_SCHEDULER not present or not executable"
+	exit 1
+fi
+
+fail_unless_root() {
+	if [ "$(id -u)" != '0' ]; then
+		log_failure_msg "$KUBE_SCHEDULER_DESC must be run as root"
+		exit 1
+	fi
+}
+
+KUBE_SCHEDULER_START="start-stop-daemon \
+--start \
+--background \
+--quiet \
+--exec $KUBE_SCHEDULER \
+--make-pidfile --pidfile $KUBE_SCHEDULER_PIDFILE \
+-- $KUBE_SCHEDULER_OPTS \
+>> $KUBE_SCHEDULER_LOGFILE 2>&1"
+
+KUBE_SCHEDULER_STOP="start-stop-daemon \
+--stop \
+--pidfile $KUBE_SCHEDULER_PIDFILE"
+
+case "$1" in
+	start)
+		fail_unless_root
+		log_begin_msg "Starting $KUBE_SCHEDULER_DESC: $BASE"
+        $KUBE_SCHEDULER_START
+		log_end_msg $?
+		;;
+
+	stop)
+		fail_unless_root
+		log_begin_msg "Stopping $KUBE_SCHEDULER_DESC: $BASE"
+		$KUBE_SCHEDULER_STOP
+		log_end_msg $?
+		;;
+
+	restart | force-reload)
+		fail_unless_root
+		log_begin_msg "Restarting $KUBE_SCHEDULER_DESC: $BASE"
+		$KUBE_SCHEDULER_STOP
+		$KUBE_SCHEDULER_START
+		log_end_msg $?
+		;;
+
+	status)
+		status_of_proc -p "$KUBE_SCHEDULER_PIDFILE" "$KUBE_SCHEDULER" "$KUBE_SCHEDULER_DESC"
+		;;
+
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac

cluster/ubuntu-cluster/initd_scripts/kubelet

@@ -0,0 +1,99 @@
+#!/bin/sh
+set -e
+
+### BEGIN INIT INFO
+# Provides:           kubelet
+# Required-Start:     $etcd
+# Required-Stop:      
+# Should-Start:       
+# Should-Stop:        
+# Default-Start:      
+# Default-Stop:       
+# Short-Description:  Start kubelet service
+# Description:
+#  http://www.github.com/GoogleCloudPlatform/Kubernetes
+### END INIT INFO
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/bin:
+
+BASE=$(basename $0)
+
+# modify these in /etc/default/$BASE (/etc/default/kube-apiserver)
+KUBELET=/opt/bin/$BASE
+# This is the pid file managed by kube-apiserver itself
+KUBELET_PIDFILE=/var/run/$BASE.pid
+KUBELET_LOGFILE=/var/log/$BASE.log
+KUBELET_OPTS=""
+KUBELET_DESC="Kube-Apiserver"
+
+# Get lsb functions
+. /lib/lsb/init-functions
+
+if [ -f /etc/default/$BASE ]; then
+	. /etc/default/$BASE
+fi
+
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
+	log_failure_msg "$KUBELET_DESC is managed via upstart, try using service $BASE $1"
+	exit 1
+fi
+
+# Check kube-apiserver is present
+if [ ! -x $KUBELET ]; then
+	log_failure_msg "$KUBELET not present or not executable"
+	exit 1
+fi
+
+fail_unless_root() {
+	if [ "$(id -u)" != '0' ]; then
+		log_failure_msg "$KUBELET_DESC must be run as root"
+		exit 1
+	fi
+}
+
+KUBELET_START="start-stop-daemon \
+--start \
+--background \
+--quiet \
+--exec $KUBELET \
+--make-pidfile --pidfile $KUBELET_PIDFILE \
+-- $KUBELET_OPTS \
+>> $KUBELET_LOGFILE 2>&1"
+
+KUBELET_STOP="start-stop-daemon \
+--stop \
+--pidfile $KUBELET_PIDFILE"
+
+case "$1" in
+	start)
+		fail_unless_root
+		log_begin_msg "Starting $KUBELET_DESC: $BASE"
+        $KUBELET_START
+		log_end_msg $?
+		;;
+
+	stop)
+		fail_unless_root
+		log_begin_msg "Stopping $KUBELET_DESC: $BASE"
+		$KUBELET_STOP
+		log_end_msg $?
+		;;
+
+	restart | force-reload)
+		fail_unless_root
+		log_begin_msg "Stopping $KUBELET_DESC: $BASE"
+		$KUBELET_STOP
+		$KUBELET_START
+		log_end_msg $?
+		;;
+
+	status)
+		status_of_proc -p "$KUBELET_PIDFILE" "$KUBELET" "$KUBELET_DESC"
+		;;
+
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac

cluster/ubuntu-cluster/reconfigureDocker.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# script to reconfigue the docker daemon network settings 
+
+# Run as root only
+if [ "$(id -u)" != "0" ]; then
+    echo >&2 "Please run as root"
+    exit 1
+fi
+
+ip link set dev docker0 down
+brctl delbr docker0
+
+source /run/flannel/subnet.env
+
+echo DOCKER_OPTS=\"-H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock \
+    --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}\" > /etc/default/docker
+
+service docker restart

cluster/ubuntu/default_scripts/kube-scheduler

@@ -1,7 +1,7 @@
 # Kube-Scheduler Upstart and SysVinit configuration file
 
 # Customize kube-apiserver binary location 
-# KUBE_SCHEDULER="/opt/bin/kube-apiserver"
+# KUBE_SCHEDULER="/opt/bin/kube-scheduler"
 
 # Use KUBE_SCHEDULER_OPTS to modify the start/restart options
 KUBE_SCHEDULER_OPTS="--logtostderr=true \

docs/getting-started-guides/ubuntu_multinodes_cluster.md

@@ -0,0 +1,161 @@
+# Kubernetes deployed on multiple ubuntu nodes
+
+This document describes how to deploy kubernetes on multiple ubuntu nodes, including 1 master node and 3 minion nodes, and people uses this approach can scale to **any number of minion nodes** by changing some settings with ease. Although there exists saltstack based ubuntu k8s installation ,  it may be tedious and hard for a guy that knows little about saltstack but want to build a really distributed k8s cluster. This approach is inspired by [k8s deploy on a single node](https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/getting-started-guides/ubuntu_single_node.md).       [Cloud team from ZJU](https://github.com/ZJU-SEL) will keep updating this work.
+
+### **Prerequisites：**
+*1 The minion nodes have installed docker version 1.2+* 
+
+*2 All machines can communicate with each orther, no need to connect Internet (should use private docker registry in this case)*
+
+*3 These guide is tested OK on Ubuntu 14.04 LTS 64bit server, but it should also work on most Ubuntu versions*
+
+*4 Dependences of this guide: etcd-2.0.0, flannel-0.2.0, k8s-0.10.1, but it should also work on higher versions*
+
+
+### **Main Steps**
+#### I. Make *kubernetes* , *etcd* and *flanneld* binaries
+
+On your laptop, copy `cluster/ubuntu-cluster` directory to your workspace.
+
+The `build.sh` will download and build all the needed binaries into `./binaries`.
+```
+$ cd cluster/ubuntu-cluster
+$ sudo ./build.sh
+```
+Please copy all the files in `./binaries` into `/opt/bin` of every machine you want to run as Kubernetes cluster node.
+
+
+Alternatively, if your Kubernetes nodes have access to Internet, you can copy `cluster/ubuntu-cluster` directory to every node and run:
+```
+# in every node
+$ cd cluster/ubuntu-cluster
+$ sudo ./build.sh
+$ sudo cp ./binaries/* /opt/bin
+```
+
+
+> We used flannel here because we want to use overlay network, but please remember it is not the only choice, and it is also not a k8s' necessary dependence. Actually you can just build up k8s cluster natively, or use flannel, Open vSwitch or any other SDN tool you like, we just choose flannel here as a example.
+
+#### II. Configue and install every components upstart script
+The example cluster is listed as below:
+
+| IP Address|Role |      
+|---------|------|
+|10.10.103.223| minion|
+|10.10.103.224| minion|
+|10.10.103.162| minion|
+|10.10.103.250| master|
+
+First of all, make sure `cluster/ubuntu-cluster` exists on this node，and run `configue.sh`.
+
+On master( infra1 10.10.103.250 ) node:
+
+```
+# in cluster/ubuntu-cluster
+$ sudo ./configure.sh
+Welcome to use this script to configure k8s setup
+
+Please enter all your cluster node ips, MASTER node comes first
+And separated with blank space like "<ip_1> <ip2> <ip3> 10.10.103.250 10.10.103.223 10.10.103.224 10.10.103.162
+
+This machine acts as
+  both MASTER and MINION:      1
+  only MASTER:                 2
+  only MINION:                 3
+Please choose a role > 2
+
+IP address of this machine > 10.10.103.250
+
+Configure Success
+```
+
+On every minion ( e.g.  10.10.103.224 ) node:
+
+
+```
+# in cluster/ubuntu-cluster
+$ sudo ./configure.sh 
+Welcome to use this script to configure k8s setup
+
+Please enter all your cluster node ips, MASTER node comes first
+And separated with blank space like "<ip_1> <ip2> <ip3> 10.10.103.250 10.10.103.223 10.10.103.224 10.10.103.162
+
+This machine acts as
+  both MASTER and MINION:      1
+  only MASTER:                 2
+  only MINION:                 3
+Please choose a role > 3
+
+IP address of this machine > 10.10.103.224
+
+Configure Success
+```
+
+If you want a node acts as **both running the master and minion**, please choose option 1.
+
+#### III. Start all components
+1. On the master node:
+  
+	`$ sudo service etcd start`
+	
+	Then on every minion node:
+	
+	`$ sudo service etcd start`
+	
+	> The kubernetes commands will be started automatically after etcd
+  
+2. On any node:
+	
+	`$ /opt/bin/etcdctl mk /coreos.com/network/config '{"Network":"10.0.0.0/16"}'`
+	
+	> You can use the below command on another node to comfirm if the network setting is correct.
+	
+	> `$ /opt/bin/etcdctl get /coreos.com/network/config`
+	
+	> If you got `{"Network":"10.0.0.0/16"}`，then etcd cluster is working in good condition. **Victory is in sight！**
+	> If not , you should check` /var/log/upstart/etcd.log` to resolve etcd problem before going forward.
+  
+  
+3. On every minion node
+  
+	> You can use ifconfig to see if there is a new network interface named `flannel0` coming up.
+	
+	> Make sure you have `brctl` installed on every minion, otherwise run `sudo apt-get install bridge-utils`
+	
+	`$ sudo ./reconfigureDocker.sh`
+	
+	This will make the docker daemon aware of flannel network.
+ 
+
+**All done !**
+
+#### IV. Validation
+You can use kubectl command to see if the newly created k8s is working correctly. 
+
+For example , `$ kubectl get minions` to see if you get all your minion nodes comming up. 
+
+Also you can run kubernetes [guest-example](https://github.com/GoogleCloudPlatform/kubernetes/tree/master/examples/guestbook) to build a redis backend cluster on the k8s．
+
+#### V. Trouble Shooting
+
+Generally, what of this guide did is quite simple: 
+
+1. copy bins and files to right dirctories 
+
+2. config etcd using inputed IPs 
+
+3. start flannel network
+
+So, whenver you have problem, do not blame Kubernetes, **check etcd configuration first** 
+
+Please try:
+
+1. Check `/var/log/upstart/etcd.log` for suspicisous etcd log 
+
+2. Check `/etc/default/etcd`, as we do not have much input validation, the right config should be like:
+
+`ETCD_OPTS="-name infra1 -initial-advertise-peer-urls <http://ip_of_this_node:2380> -listen-peer-urls <http://ip_of_this_node:2380> -initial-cluster-token etcd-cluster-1 -initial-cluster infra1=<http://ip_of_this_node:2380>,infra2=<http://ip_of_another_node:2380>,infra3=<http://ip_of_another_node:2380> -initial-cluster-state new"`
+
+3. Remove `data-dir` of etcd and run `reconfigureDocker.sh`again, the default path of `data-dir` is /infra*.etcd/
+
+4. You can also customize your own settings in `/etc/default/{component_name}` after configured success.