diff --git a/Documentation/bootkube.md b/Documentation/bootkube.md index 616f4d0b..1cae9a17 100644 --- a/Documentation/bootkube.md +++ b/Documentation/bootkube.md @@ -47,7 +47,7 @@ Add your SSH public key to each machine group definition [as shown](../examples/ Use the `bootkube` tool to render Kubernetes manifests and credentials into an `--asset-dir`. Later, `bootkube` will schedule these manifests during bootstrapping and the credentials will be used to access your cluster. ```sh -$ bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=http://127.0.0.1:2379 +bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379 ``` ## Containers @@ -60,6 +60,15 @@ Client machines should boot and provision themselves. Local client VMs should ne We're ready to use bootkube to create a temporary control plane and bootstrap a self-hosted Kubernetes cluster. +Secure copy the etcd TLS assets to `/etc/ssl/etcd/*` on **every** node. + +```bash +for node in 'node1' 'node2' 'node3'; do + scp -r assets/tls/etcd-* core@$node.example.com:/home/core/ + ssh core@$node.example.com 'sudo mkdir -p /etc/ssl/etcd && sudo mv etcd-* /etc/ssl/etcd/ && sudo chown -R etcd:etcd /etc/ssl/etcd && sudo chmod -R 500 /etc/ssl/etcd/' +done +``` + Secure copy the `kubeconfig` to `/etc/kubernetes/kubeconfig` on **every** node which will path activate the `kubelet.service`. ```bash @@ -72,8 +81,8 @@ done Secure copy the `bootkube` generated assets to any controller node and run `bootkube-start`. ```sh -$ scp -r assets core@node1.example.com:/home/core -$ ssh core@node1.example.com 'sudo mv assets /opt/bootkube/assets && sudo systemctl start bootkube' +scp -r assets core@node1.example.com:/home/core +ssh core@node1.example.com 'sudo mv assets /opt/bootkube/assets && sudo systemctl start bootkube' ``` Optionally watch the Kubernetes control plane bootstrapping with the bootkube temporary api-server. You will see quite a bit of output. diff --git a/examples/groups/bootkube-install/node1.json b/examples/groups/bootkube-install/node1.json index 367096b8..dba26a73 100644 --- a/examples/groups/bootkube-install/node1.json +++ b/examples/groups/bootkube-install/node1.json @@ -8,7 +8,8 @@ }, "metadata": { "domain_name": "node1.example.com", - "etcd_initial_cluster": "node1=http://node1.example.com:2380", + "etcd_initial_cluster": "node1=https://node1.example.com:2380", + "etcd_endpoints": "https://node1.example.com:2379", "etcd_name": "node1", "k8s_dns_service_ip": "10.3.0.10", "ssh_authorized_keys": [ diff --git a/examples/groups/bootkube-install/node2.json b/examples/groups/bootkube-install/node2.json index 1b2581d1..d365f55b 100644 --- a/examples/groups/bootkube-install/node2.json +++ b/examples/groups/bootkube-install/node2.json @@ -8,7 +8,7 @@ }, "metadata": { "domain_name": "node2.example.com", - "etcd_endpoints": "node1.example.com:2379", + "etcd_endpoints": "https://node1.example.com:2379", "k8s_dns_service_ip": "10.3.0.10", "ssh_authorized_keys": [ "ADD ME" diff --git a/examples/groups/bootkube-install/node3.json b/examples/groups/bootkube-install/node3.json index 4e03758d..a4af27a6 100644 --- a/examples/groups/bootkube-install/node3.json +++ b/examples/groups/bootkube-install/node3.json @@ -8,7 +8,7 @@ }, "metadata": { "domain_name": "node3.example.com", - "etcd_endpoints": "node1.example.com:2379", + "etcd_endpoints": "https://node1.example.com:2379", "k8s_dns_service_ip": "10.3.0.10", "ssh_authorized_keys": [ "ADD ME" diff --git a/examples/groups/bootkube/node1.json b/examples/groups/bootkube/node1.json index 8013af97..c8013370 100644 --- a/examples/groups/bootkube/node1.json +++ b/examples/groups/bootkube/node1.json @@ -7,7 +7,8 @@ }, "metadata": { "domain_name": "node1.example.com", - "etcd_initial_cluster": "node1=http://node1.example.com:2380", + "etcd_initial_cluster": "node1=https://node1.example.com:2380", + "etcd_endpoints": "https://node1.example.com:2379", "etcd_name": "node1", "k8s_dns_service_ip": "10.3.0.10", "pxe": "true", diff --git a/examples/groups/bootkube/node2.json b/examples/groups/bootkube/node2.json index e9f1bdd5..805d9287 100644 --- a/examples/groups/bootkube/node2.json +++ b/examples/groups/bootkube/node2.json @@ -7,7 +7,7 @@ }, "metadata": { "domain_name": "node2.example.com", - "etcd_endpoints": "node1.example.com:2379", + "etcd_endpoints": "https://node1.example.com:2379", "k8s_dns_service_ip": "10.3.0.10", "pxe": "true", "ssh_authorized_keys": [ diff --git a/examples/groups/bootkube/node3.json b/examples/groups/bootkube/node3.json index 5cd85e7c..9f8dad22 100644 --- a/examples/groups/bootkube/node3.json +++ b/examples/groups/bootkube/node3.json @@ -7,7 +7,7 @@ }, "metadata": { "domain_name": "node3.example.com", - "etcd_endpoints": "node1.example.com:2379", + "etcd_endpoints": "https://node1.example.com:2379", "k8s_dns_service_ip": "10.3.0.10", "pxe": "true", "ssh_authorized_keys": [ diff --git a/examples/ignition/bootkube-controller.yaml b/examples/ignition/bootkube-controller.yaml index 299055f8..d38160ed 100644 --- a/examples/ignition/bootkube-controller.yaml +++ b/examples/ignition/bootkube-controller.yaml @@ -9,12 +9,19 @@ systemd: [Service] Environment="ETCD_IMAGE_TAG=v3.1.6" Environment="ETCD_NAME={{.etcd_name}}" - Environment="ETCD_ADVERTISE_CLIENT_URLS=http://{{.domain_name}}:2379" - Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{.domain_name}}:2380" - Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379" - Environment="ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380" + Environment="ETCD_ADVERTISE_CLIENT_URLS=https://{{.domain_name}}:2379" + Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://{{.domain_name}}:2380" + Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379" + Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380" Environment="ETCD_INITIAL_CLUSTER={{.etcd_initial_cluster}}" Environment="ETCD_STRICT_RECONFIG_CHECK=true" + Environment="ETCD_SSL_DIR=/etc/ssl/etcd" + Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd-client.crt" + Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd-client.key" + Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd-peer.crt" + Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd-peer.key" + Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd-ca.crt" + Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" - name: docker.service enable: true - name: locksmithd.service @@ -23,6 +30,10 @@ systemd: contents: | [Service] Environment="REBOOT_STRATEGY=etcd-lock" + Environment="LOCKSMITHD_ETCD_CAFILE=/etc/ssl/etcd/etcd-ca.crt" + Environment="LOCKSMITHD_ETCD_CERTFILE=/etc/ssl/etcd/etcd-client.crt" + Environment="LOCKSMITHD_ETCD_KEYFILE=/etc/ssl/etcd/etcd-client.key" + Environment="LOCKSMITHD_ENDPOINT={{.etcd_endpoints}}" - name: kubelet.path enable: true contents: | @@ -119,6 +130,12 @@ storage: inline: | KUBELET_IMAGE_URL=quay.io/coreos/hyperkube KUBELET_IMAGE_TAG=v1.6.4_coreos.0 + - path: /etc/ssl/etcd/.empty + filesystem: root + mode: 0644 + contents: + inline: | + empty - path: /etc/hostname filesystem: root mode: 0644 diff --git a/examples/ignition/bootkube-worker.yaml b/examples/ignition/bootkube-worker.yaml index 88f40697..810b9eef 100644 --- a/examples/ignition/bootkube-worker.yaml +++ b/examples/ignition/bootkube-worker.yaml @@ -1,17 +1,6 @@ --- systemd: units: - - name: etcd-member.service - enable: true - dropins: - - name: 40-etcd-cluster.conf - contents: | - [Service] - Environment="ETCD_IMAGE_TAG=v3.1.6" - ExecStart= - ExecStart=/usr/lib/coreos/etcd-wrapper gateway start \ - --listen-addr=127.0.0.1:2379 \ - --endpoints={{.etcd_endpoints}} - name: docker.service enable: true - name: locksmithd.service @@ -20,6 +9,10 @@ systemd: contents: | [Service] Environment="REBOOT_STRATEGY=etcd-lock" + Environment="LOCKSMITHD_ETCD_CAFILE=/etc/ssl/etcd/etcd-ca.crt" + Environment="LOCKSMITHD_ETCD_CERTFILE=/etc/ssl/etcd/etcd-client.crt" + Environment="LOCKSMITHD_ETCD_KEYFILE=/etc/ssl/etcd/etcd-client.key" + Environment="LOCKSMITHD_ENDPOINT={{.etcd_endpoints}}" - name: kubelet.path enable: true contents: | @@ -108,6 +101,12 @@ storage: inline: | KUBELET_IMAGE_URL=quay.io/coreos/hyperkube KUBELET_IMAGE_TAG=v1.6.4_coreos.0 + - path: /etc/ssl/etcd/.empty + filesystem: root + mode: 0644 + contents: + inline: | + empty - path: /etc/hostname filesystem: root mode: 0644 diff --git a/tests/smoke/bootkube b/tests/smoke/bootkube index ef9bfb2e..a1ff4afb 100755 --- a/tests/smoke/bootkube +++ b/tests/smoke/bootkube @@ -15,7 +15,7 @@ main() { ./scripts/libvirt create echo "bootkube render" - ./bin/bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=http://127.0.0.1:2379 + ./bin/bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379 for i in `seq 1 10`; do ssh node1.example.com -o ConnectTimeout=5 -- 'echo "Connected"' && break @@ -23,6 +23,12 @@ main() { sleep 10 done + echo "Add etcd certs to nodes" + for node in 'node1' 'node2' 'node3'; do + scp -r assets/tls/etcd-* core@$node.example.com:/home/core/ + ssh core@$node.example.com 'sudo mkdir -p /etc/ssl/etcd && sudo mv etcd-* /etc/ssl/etcd/ && sudo chown -R etcd:etcd /etc/ssl/etcd && sudo chmod -R 500 /etc/ssl/etcd/' + done + echo "Add kubeconfig to nodes" for node in 'node1' 'node2' 'node3'; do scp assets/auth/kubeconfig core@${node}.example.com:/home/core/kubeconfig