Merge pull request #535 from coreos/bootkube-tests

tests: Add cluster tests for bootkube-install (terraform-based)

Jenkinsfile

@@ -46,7 +46,7 @@ parallel (
   "etcd3-terraform": {
     node('fedora && bare-metal') {
       stage('etcd3-terraform') {
-        timeout(time:8, unit:'MINUTES') {
+        timeout(time:10, unit:'MINUTES') {
           checkout scm
           sh '''#!/bin/bash -e
           export ASSETS_DIR=~/assets; export CONFIG_DIR=~/matchbox/examples/etc/matchbox; ./tests/smoke/etcd3-terraform
@@ -54,5 +54,17 @@ parallel (
         }
       }
     }
+  },
+  "bootkube-terraform": {
+    node('fedora && bare-metal') {
+      stage('bootkube-terraform') {
+        timeout(time:15, unit:'MINUTES') {
+          checkout scm
+          sh '''#!/bin/bash -e
+          export ASSETS_DIR=~/assets; export CONFIG_DIR=~/matchbox/examples/etc/matchbox; ./tests/smoke/bootkube-terraform
+          '''
+        }
+      }
+    }
   }
 )

tests/smoke/bootkube-terraform

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TERRAFORM_PID=0
+source "${DIR}/common"
+
+main() {
+  cleanup
+  trap cleanup EXIT
+
+  ./scripts/get-kubectl
+  ./scripts/devnet create
+  
+  # Add fake_rsa to ssh-agent
+  eval `ssh-agent -s`
+  chmod 600 ${DIR}/fake_rsa
+  ssh-add ${DIR}/fake_rsa
+
+  pushd examples/terraform/bootkube-install
+  terraform get
+  (until [[ "$(terraform apply -var-file ${DIR}/bootkube.tfvars)" == "0" ]]; do
+    echo "Wait and retry Terraform apply"
+    sleep 10
+  done) &
+  TERRAFORM_PID=$!
+  popd
+  
+  sleep 10
+  ./scripts/libvirt create
+
+  until kubelet "node1.example.com" \
+    && kubelet "node2.example.com" \
+    && kubelet "node3.example.com"
+  do
+    sleep 10
+    echo "Waiting for Kubelets to start..."
+  done
+
+  until [[ "$(readyNodes)" == "3" ]]; do
+    sleep 5
+    echo "$(readyNodes) of 3 nodes are Ready..."
+  done
+  
+  echo "Getting nodes..."
+  k8s get nodes
+
+  sleep 10
+  echo "Getting pods..."
+  k8s get pods --all-namespaces
+
+  echo "bootkube cluster came up!"
+  echo
+
+  rm -rf assets
+  cleanup
+}
+
+k8s() {
+  ./bin/kubectl --kubeconfig=examples/terraform/bootkube-install/assets/auth/kubeconfig "$@"
+}
+
+readyNodes() {
+  k8s get nodes -o template --template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{.}}{{end}}{{end}}{{end}}' | grep -o -E True | wc -l
+}
+
+cleanup() {
+  ./scripts/libvirt destroy || true
+  ./scripts/devnet destroy || true
+  rkt gc --grace-period=0
+  if [[ -z "$TERRAFORM_PID" ]]; then
+    kill $TERRAFORM_PID
+  fi
+  rm -rf examples/terraform/bootkube-install/assets
+  rm -f examples/terraform/bootkube-install/*.tfstate*
+}
+
+main $@

tests/smoke/bootkube.tfvars

@@ -0,0 +1,23 @@
+matchbox_http_endpoint = "http://matchbox.example.com:8080"
+matchbox_rpc_endpoint = "matchbox.example.com:8081"
+ssh_authorized_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPQFdwVLr+alsWIgYRz9OdqDhnx9jjuFbkdSdpqq4gd9uZApYlivMDD4UgjFazQpezx8DiNhu9ym7i6LgAcdwi+10hE4L9yoJv9uBgbBxOAd65znqLqF91NtV4mlKP5YfJtR7Ehs+pTB+IIC+o5veDbPn+BYgDMJ2x7Osbn1/gFSDken/yoOFbYbRMGMfVEQYjJzC4r/qCKH0bl/xuVNLxf9FkWSTCcQFKGOndwuGITDkshD4r2Kk8gUddXPxoahBv33/2QH0CY5zbKYjhgN6I6WtwO+O1uJwtNeV1AGhYjurdd60qggNwx+W7623uK3nIXvJd3hzDO8u5oa53/tIL fake-test-key"
+
+cluster_name = "example"
+container_linux_version = "1298.7.0"
+container_linux_channel = "stable"
+
+# Machines
+controller_names = ["node1"]
+controller_macs = ["52:54:00:a1:9c:ae"]
+controller_domains = ["node1.example.com"]
+worker_names = ["node2", "node3"]
+worker_macs = ["52:54:00:b2:2f:86", "52:54:00:c3:61:77"]
+worker_domains = ["node2.example.com", "node3.example.com"]
+
+# Bootkube
+k8s_domain_name = "cluster.example.com"
+asset_dir = "assets"
+
+# Optional
+# container_linux_oem = ""
+# experimental_self_hosted_etcd = "true"