rkt: Add intial example with rkt, CNI, and ACIs

Documentation/rkt.md

@@ -0,0 +1,54 @@
+
+# rkt Tutorial
+
+**Like the Docker libvirt setup, the rkt libvirt setup is meant for local development and testing**
+
+Get started on your laptop with `rkt`, `libvirt`, and `virt-manager` (tested on Fedora 23).
+
+Install a [rkt](https://github.com/coreos/rkt/releases) and [acbuild](https://github.com/appc/acbuild/releases) release and get the `libvirt` and `virt-manager` packages.
+
+    sudo dnf install virt-manager virt-install
+
+Clone the source.
+
+    git clone https://github.com/coreos/coreos-baremetal.git
+    cd coreos-baremetal
+
+Currently, rkt and the Fedora/RHEL/CentOS SELinux policies aren't supported. See the [issue](https://github.com/coreos/rkt/issues/1727) tracking the work and policy changes. To test these examples on your laptop, temporarily disable SELinux enforcement if you are comfortable (`sudo setenforce 0`). Enable it again when you are finished.
+
+Download the CoreOS network boot images to assets/coreos:
+
+    cd coreos-baremetal
+    ./scripts/get-coreos
+
+Run the config server on an IP which will correspond to the examples and DNS setup.
+
+    sudo rkt --insecure-options=image fetch docker://quay.io/coreos/bootcfg
+
+The insecure flag is needed because Docker images do not support signature verification.
+
+    sudo rkt run --net=metal0:IP=172.15.0.2 --mount volume=assets,target=/assets --volume assets,kind=host,source=$PWD/assets --mount volume=data,target=/data --volume data,kind=host,source=$PWD/examples/etcd-large quay.io/coreos/bootcfg -- -address=0.0.0.0:8080 -log-level=debug
+
+If you get an error about the IP being assigned already.
+
+    sudo rkt gc --grace-period=0
+    sudo rkt list            # should be empty
+
+Create 5 VM nodes which have known "hardware" attributes that match the examples. This also creates a CNI network `metal0` which containers and VMs can share for local development.
+
+    sudo ./scripts/libvirt create-rkt
+    # if you previously tried the docker examples, cleanup first
+    sudo ./scripts/libvirt shutdown
+    sudo ./scripts/libvirt destroy
+
+In your firewall settings, configure the `metal0` interface as trusted.
+
+Build an dnsmasq ACI and run it to create a DNS server, TFTP server, and DHCP server which points network boot clients to the config server started above.
+
+    cd contrib
+    sudo ./acifile
+
+Sign the file if you wish, but we'll run it directly.
+
+    sudo rkt --insecure-options=image run dnsmasq.aci --net=metal0 -- -d -q --dhcp-range=172.15.0.50,172.15.0.99 --enable-tftp --tftp-root=/var/lib/tftpboot --dhcp-userclass=set:ipxe,iPXE --dhcp-boot=tag:#ipxe,undionly.kpxe --dhcp-boot=tag:ipxe,http://bootcfg.example:8080/boot.ipxe --log-queries --log-dhcp --dhcp-option=3,172.15.0.1 --address=/bootcfg.example/172.15.0.2
+

README.md

@@ -9,12 +9,13 @@ CoreOS on Baremetal contains guides for network booting and configuring CoreOS c
 
 * [Network Booting](Documentation/network-booting.md)
 * [Config Service](Documentation/bootcfg.md)
+* [rkt Tutorial](Documentation/rkt.md)
 * [Libvirt Guide](Documentation/virtual-hardware.md)
 * [Baremetal Guide](Documentation/physical-hardware.md)
 
 ## Config Service
 
-The config service provides network boot (PXE, iPXE, Pixiecore), [Ignition](https://coreos.com/ignition/docs/latest/what-is-ignition.html), and [Cloud-Init](https://github.com/coreos/coreos-cloudinit) configs to machines based on hardware attributes (e.g. UUID, MAC, hostname) or free-form tag matchers.
+The config service provides network boot (PXE, iPXE, Pixiecore), [Ignition](https://coreos.com/ignition/docs/latest/what-is-ignition.html), and [Cloud-Init](https://github.com/coreos/coreos-cloudinit) configs to machines based on hardware attributes (e.g. UUID, MAC, hostname) or free-form tags.
 
 * [API](Documentation/api.md)
 * [Flags](Documentation/config.md)

contrib/.gitignore

@@ -0,0 +1,2 @@
+tftpboot
+dnsmasq.aci

contrib/acifile

@@ -0,0 +1,46 @@
+#!/usr/bin/bash -e
+
+if [ "$EUID" -ne 0 ]; then
+  echo "Please run as root"
+  exit 1
+fi
+
+# Start the build with an empty ACI
+acbuild --debug begin
+
+# In the event of the script exiting, end the build
+trap "{ export EXT=$?; acbuild --debug end && exit $EXT; }" EXIT
+
+# Name the ACI
+acbuild --debug set-name coreos/dnsmasq
+
+# Based on alpine
+acbuild --debug dep add quay.io/coreos/alpine-sh
+
+# Install dnsmasq and curl
+acbuild --debug run apk update
+acbuild --debug run apk add dnsmasq curl
+
+# Get the PXE->iPXE chainloader
+if [ ! -d "tftpboot" ]; then
+  mkdir tftpboot
+  curl -s -o tftpboot/undionly.kpxe http://boot.ipxe.org/undionly.kpxe
+  cp tftpboot/undionly.kpxe tftpboot/undionly.kpxe.0
+fi
+acbuild --debug copy tftpboot /var/lib/tftpboot
+
+# Add a port for dhcp discover traffic
+acbuild --debug port add dhcp udp 67
+acbuild --debug port add dhcp udp 53
+
+# Elevate network admin capabilities
+echo "{\"set\": [\"CAP_NET_ADMIN\"]}" | acbuild isolator add os/linux/capabilities-retain-set -
+
+# Run dnsmasq
+acbuild --debug set-exec -- /usr/sbin/dnsmasq -d
+
+# Check that the manifest looks ok
+acbuild --debug cat-manifest
+
+# Save the ACI
+acbuild --debug write --overwrite dnsmasq.aci

examples/etcd-large-rkt/cloud/etcd1.yaml

@@ -0,0 +1,15 @@
+#cloud-config
+coreos:
+  etcd2:
+    name: node1
+    initial-cluster: node1=http://172.15.0.21:2380,node2=http://172.15.0.22:2380,node3=http://172.15.0.23:2380
+    initial-advertise-peer-urls: http://172.15.0.21:2380
+    initial-cluster-state: new
+    advertise-client-urls: http://172.15.0.21:2379
+    listen-client-urls: http://0.0.0.0:2379
+    listen-peer-urls: http://172.15.0.21:2380
+  units:
+    - name: etcd2.service
+      command: start
+    - name: fleet.service
+      command: start

examples/etcd-large-rkt/cloud/etcd2.yaml

@@ -0,0 +1,15 @@
+#cloud-config
+coreos:
+  etcd2:
+    name: node2
+    initial-cluster: node1=http://172.15.0.21:2380,node2=http://172.15.0.22:2380,node3=http://172.15.0.23:2380
+    initial-advertise-peer-urls: http://172.15.0.22:2380
+    initial-cluster-state: new
+    advertise-client-urls: http://172.15.0.22:2379
+    listen-client-urls: http://0.0.0.0:2379
+    listen-peer-urls: http://172.15.0.22:2380
+  units:
+    - name: etcd2.service
+      command: start
+    - name: fleet.service
+      command: start

examples/etcd-large-rkt/cloud/etcd3.yaml

@@ -0,0 +1,15 @@
+#cloud-config
+coreos:
+  etcd2:
+    name: node3
+    initial-cluster: node1=http://172.15.0.21:2380,node2=http://172.15.0.22:2380,node3=http://172.15.0.23:2380
+    initial-advertise-peer-urls: http://172.15.0.23:2380
+    initial-cluster-state: new
+    advertise-client-urls: http://172.15.0.23:2379
+    listen-client-urls: http://0.0.0.0:2379
+    listen-peer-urls: http://172.15.0.23:2380
+  units:
+    - name: etcd2.service
+      command: start
+    - name: fleet.service
+      command: start

examples/etcd-large-rkt/cloud/worker.yaml

@@ -0,0 +1,14 @@
+#cloud-config
+coreos:
+  etcd2:
+    initial-cluster: node1=http://172.15.0.21:2380,node2=http://172.15.0.22:2380,node3=http://172.15.0.23:2380
+    proxy: on 
+    listen-client-urls: http://localhost:2379
+  fleet:
+    metadata: "role=worker"
+    etcd_servers: "http://localhost:2379"
+  units:
+    - name: etcd2.service
+      command: start
+    - name: fleet.service
+      command: start

examples/etcd-large-rkt/config.yaml

@@ -0,0 +1,20 @@
+---
+api_version: v1alpha1
+groups:
+  - name: node1
+    spec: etcd1
+    require:
+      uuid: 16e7d8a7-bfa9-428b-9117-363341bb330b
+
+  - name: node2
+    spec: etcd2
+    require:
+      uuid: 264cd073-ca62-44b3-98c0-50aad5b5f819
+
+  - name: node3
+    spec: etcd3
+    require:
+      uuid: 39d2e747-2648-4d68-ae92-bbc70b245055
+
+  - name: default
+    spec: worker

examples/etcd-large-rkt/ignition/etcd1.json

@@ -0,0 +1,11 @@
+{
+  "ignitionVersion": 1,
+  "networkd": {
+    "units": [
+      {
+        "name": "00-ens3.network",
+        "contents": "[Match]\nName=ens3\n\n[Network]\nGateway=172.15.0.1\nAddress=172.15.0.21"
+      }
+    ]
+  }
+}

examples/etcd-large-rkt/ignition/etcd2.json

@@ -0,0 +1,11 @@
+{
+  "ignitionVersion": 1,
+  "networkd": {
+    "units": [
+      {
+        "name": "00-ens3.network",
+        "contents": "[Match]\nName=ens3\n\n[Network]\nGateway=172.15.0.1\nAddress=172.15.0.22"
+      }
+    ]
+  }
+}

examples/etcd-large-rkt/ignition/etcd3.json

@@ -0,0 +1,11 @@
+{
+  "ignitionVersion": 1,
+  "networkd": {
+    "units": [
+      {
+        "name": "00-ens3.network",
+        "contents": "[Match]\nName=ens3\n\n[Network]\nGateway=172.15.0.1\nAddress=172.15.0.23"
+      }
+    ]
+  }
+}

examples/etcd-large-rkt/specs/etcd1/spec.json

@@ -0,0 +1,15 @@
+{
+  "id": "etcd1",
+  "boot": {
+    "kernel": "/assets/coreos/835.9.0/coreos_production_pxe.vmlinuz",
+    "initrd": ["/assets/coreos/835.9.0/coreos_production_pxe_image.cpio.gz"],
+    "cmdline": {
+      "cloud-config-url": "http://172.15.0.2:8080/cloud?uuid=${uuid}&mac=${net0/mac:hexhyp}",
+      "coreos.config.url": "http://bootcfg.example:8080/ignition?uuid=${uuid}",
+      "coreos.autologin": "",
+      "coreos.first_boot": ""
+    }
+  },
+  "cloud_id": "etcd1.yaml",
+  "ignition_id": "etcd1.json"
+}

examples/etcd-large-rkt/specs/etcd2/spec.json

@@ -0,0 +1,15 @@
+{
+  "id": "etcd2",
+  "boot": {
+    "kernel": "/assets/coreos/835.9.0/coreos_production_pxe.vmlinuz",
+    "initrd": ["/assets/coreos/835.9.0/coreos_production_pxe_image.cpio.gz"],
+    "cmdline": {
+      "cloud-config-url": "http://172.15.0.2:8080/cloud?uuid=${uuid}&mac=${net0/mac:hexhyp}",
+      "coreos.config.url": "http://bootcfg.example:8080/ignition?uuid=${uuid}",
+      "coreos.autologin": "",
+      "coreos.first_boot": ""
+    }
+  },
+  "cloud_id": "etcd2.yaml",
+  "ignition_id": "etcd2.json"
+}

examples/etcd-large-rkt/specs/etcd3/spec.json

@@ -0,0 +1,15 @@
+{
+  "id": "etcd3",
+  "boot": {
+    "kernel": "/assets/coreos/835.9.0/coreos_production_pxe.vmlinuz",
+    "initrd": ["/assets/coreos/835.9.0/coreos_production_pxe_image.cpio.gz"],
+    "cmdline": {
+      "cloud-config-url": "http://172.15.0.2:8080/cloud?uuid=${uuid}&mac=${net0/mac:hexhyp}",
+      "coreos.config.url": "http://bootcfg.example:8080/ignition?uuid=${uuid}",
+      "coreos.autologin": "",
+      "coreos.first_boot": ""
+    }
+  },
+  "cloud_id": "etcd3.yaml",
+  "ignition_id": "etcd3.json"
+}

examples/etcd-large-rkt/specs/worker/spec.json

@@ -0,0 +1,13 @@
+{
+  "id": "worker",
+  "boot": {
+    "kernel": "/assets/coreos/835.9.0/coreos_production_pxe.vmlinuz",
+    "initrd": ["/assets/coreos/835.9.0/coreos_production_pxe_image.cpio.gz"],
+    "cmdline": {
+      "cloud-config-url": "http://172.15.0.2:8080/cloud?uuid=${uuid}&mac=${net0/mac:hexhyp}",
+      "coreos.autologin": ""
+    }
+  },
+  "cloud_id": "worker.yaml",
+  "ignition_id": ""
+}

scripts/libvirt

@@ -8,7 +8,8 @@ fi
 
 function main {
   case "$1" in
-    "create") create;;
+    "create-docker") create;;
+    "create-rkt") create_rkt;;
     "start") start;;
     "reboot") reboot;;
     "shutdown") shutdown;;
@@ -24,7 +25,8 @@ function main {
 function usage {
   echo "USAGE: ${0##*/} <command>"
   echo "Commands:"
-  echo -e "\tcreate\t\tcreate 5 libvirt nodes with specific hardware attributes"
+  echo -e "\tcreate-docker\tcreate 5 libvirt nodes on the docker0 bridge"
+  echo -e "\tcreate-rkt\tcreate 5 libvirt nodes on a rkt CNI metal0 bridge"
   echo -e "\tstart\t\tstart the 5 libvirt nodes"
   echo -e "\treboot\t\treboot the 5 libvirt nodes"
   echo -e "\tshutdown\tshutdown the 5 libvirt nodes"
@@ -40,6 +42,31 @@ function create {
   virt-install --name node5 -u 53683e94-3273-4a49-9a82-d769b88e3ccf --pxe --network=bridge:docker0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole
 }
 
+function create_rkt {
+  metal0
+  virt-install --name node1 -u 16e7d8a7-bfa9-428b-9117-363341bb330b --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole
+  virt-install --name node2 -u 264cd073-ca62-44b3-98c0-50aad5b5f819 --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole
+  virt-install --name node3 -u 39d2e747-2648-4d68-ae92-bbc70b245055 --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole
+  virt-install --name node4 -u 4ed46e8e-db69-471e-b874-0990dd65649d --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole
+  virt-install --name node5 -u 53683e94-3273-4a49-9a82-d769b88e3ccf --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole
+}
+
+function metal0 {
+  cat > /etc/rkt/net.d/20-metal.conf << EOF
+{
+  "name": "metal0",
+  "type": "bridge",
+  "bridge": "metal0",
+  "isGateway": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.15.0.0/16",
+    "routes" : [ { "dst" : "172.15.0.0/16" } ]
+   }
+}
+EOF
+}
+
 nodes=(node1 node2 node3 node4 node5)
 
 function start {

scripts/rkt-gc-force

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+# See https://github.com/coreos/rkt/issues/1572
+rkt gc --grace-period=0s
+for p in `mount | grep -oP "/tmp/[^\s,\)]*"`; do umount -l $p; done
+for p in `mount | grep -oP "/var/lib/rkt/pods/[^\s,\)]*"`; do umount -l $p; done
+rkt gc --grace-period=0s
+for i in `rkt image list | grep -oP '^sha512[^ \t]*'`; do rkt image rm $i; done
+for ns in `ip netns l | grep -o -E '^[[:alnum:]]+'`; do ip netns del $ns; done
+ip l del metal0
+for veth in `ip l show | grep -oE 'veth[^@]+'`; do ip l del $veth; done
+rm -Rf /var/lib/cni/networks/*
+rm -Rf /var/lib/rkt/*