diff --git a/Documentation/rkt.md b/Documentation/rkt.md new file mode 100644 index 00000000..b703311c --- /dev/null +++ b/Documentation/rkt.md @@ -0,0 +1,54 @@ + +# rkt Tutorial + +**Like the Docker libvirt setup, the rkt libvirt setup is meant for local development and testing** + +Get started on your laptop with `rkt`, `libvirt`, and `virt-manager` (tested on Fedora 23). + +Install a [rkt](https://github.com/coreos/rkt/releases) and [acbuild](https://github.com/appc/acbuild/releases) release and get the `libvirt` and `virt-manager` packages. + + sudo dnf install virt-manager virt-install + +Clone the source. + + git clone https://github.com/coreos/coreos-baremetal.git + cd coreos-baremetal + +Currently, rkt and the Fedora/RHEL/CentOS SELinux policies aren't supported. See the [issue](https://github.com/coreos/rkt/issues/1727) tracking the work and policy changes. To test these examples on your laptop, temporarily disable SELinux enforcement if you are comfortable (`sudo setenforce 0`). Enable it again when you are finished. + +Download the CoreOS network boot images to assets/coreos: + + cd coreos-baremetal + ./scripts/get-coreos + +Run the config server on an IP which will correspond to the examples and DNS setup. + + sudo rkt --insecure-options=image fetch docker://quay.io/coreos/bootcfg + +The insecure flag is needed because Docker images do not support signature verification. + + sudo rkt run --net=metal0:IP=172.15.0.2 --mount volume=assets,target=/assets --volume assets,kind=host,source=$PWD/assets --mount volume=data,target=/data --volume data,kind=host,source=$PWD/examples/etcd-large quay.io/coreos/bootcfg -- -address=0.0.0.0:8080 -log-level=debug + +If you get an error about the IP being assigned already. + + sudo rkt gc --grace-period=0 + sudo rkt list # should be empty + +Create 5 VM nodes which have known "hardware" attributes that match the examples. This also creates a CNI network `metal0` which containers and VMs can share for local development. + + sudo ./scripts/libvirt create-rkt + # if you previously tried the docker examples, cleanup first + sudo ./scripts/libvirt shutdown + sudo ./scripts/libvirt destroy + +In your firewall settings, configure the `metal0` interface as trusted. + +Build an dnsmasq ACI and run it to create a DNS server, TFTP server, and DHCP server which points network boot clients to the config server started above. + + cd contrib + sudo ./acifile + +Sign the file if you wish, but we'll run it directly. + + sudo rkt --insecure-options=image run dnsmasq.aci --net=metal0 -- -d -q --dhcp-range=172.15.0.50,172.15.0.99 --enable-tftp --tftp-root=/var/lib/tftpboot --dhcp-userclass=set:ipxe,iPXE --dhcp-boot=tag:#ipxe,undionly.kpxe --dhcp-boot=tag:ipxe,http://bootcfg.example:8080/boot.ipxe --log-queries --log-dhcp --dhcp-option=3,172.15.0.1 --address=/bootcfg.example/172.15.0.2 + diff --git a/README.md b/README.md index afcd11a2..b40795d8 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,13 @@ CoreOS on Baremetal contains guides for network booting and configuring CoreOS c * [Network Booting](Documentation/network-booting.md) * [Config Service](Documentation/bootcfg.md) +* [rkt Tutorial](Documentation/rkt.md) * [Libvirt Guide](Documentation/virtual-hardware.md) * [Baremetal Guide](Documentation/physical-hardware.md) ## Config Service -The config service provides network boot (PXE, iPXE, Pixiecore), [Ignition](https://coreos.com/ignition/docs/latest/what-is-ignition.html), and [Cloud-Init](https://github.com/coreos/coreos-cloudinit) configs to machines based on hardware attributes (e.g. UUID, MAC, hostname) or free-form tag matchers. +The config service provides network boot (PXE, iPXE, Pixiecore), [Ignition](https://coreos.com/ignition/docs/latest/what-is-ignition.html), and [Cloud-Init](https://github.com/coreos/coreos-cloudinit) configs to machines based on hardware attributes (e.g. UUID, MAC, hostname) or free-form tags. * [API](Documentation/api.md) * [Flags](Documentation/config.md) diff --git a/contrib/.gitignore b/contrib/.gitignore new file mode 100644 index 00000000..c2daced8 --- /dev/null +++ b/contrib/.gitignore @@ -0,0 +1,2 @@ +tftpboot +dnsmasq.aci \ No newline at end of file diff --git a/contrib/acifile b/contrib/acifile new file mode 100755 index 00000000..bb197f9a --- /dev/null +++ b/contrib/acifile @@ -0,0 +1,46 @@ +#!/usr/bin/bash -e + +if [ "$EUID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +# Start the build with an empty ACI +acbuild --debug begin + +# In the event of the script exiting, end the build +trap "{ export EXT=$?; acbuild --debug end && exit $EXT; }" EXIT + +# Name the ACI +acbuild --debug set-name coreos/dnsmasq + +# Based on alpine +acbuild --debug dep add quay.io/coreos/alpine-sh + +# Install dnsmasq and curl +acbuild --debug run apk update +acbuild --debug run apk add dnsmasq curl + +# Get the PXE->iPXE chainloader +if [ ! -d "tftpboot" ]; then + mkdir tftpboot + curl -s -o tftpboot/undionly.kpxe http://boot.ipxe.org/undionly.kpxe + cp tftpboot/undionly.kpxe tftpboot/undionly.kpxe.0 +fi +acbuild --debug copy tftpboot /var/lib/tftpboot + +# Add a port for dhcp discover traffic +acbuild --debug port add dhcp udp 67 +acbuild --debug port add dhcp udp 53 + +# Elevate network admin capabilities +echo "{\"set\": [\"CAP_NET_ADMIN\"]}" | acbuild isolator add os/linux/capabilities-retain-set - + +# Run dnsmasq +acbuild --debug set-exec -- /usr/sbin/dnsmasq -d + +# Check that the manifest looks ok +acbuild --debug cat-manifest + +# Save the ACI +acbuild --debug write --overwrite dnsmasq.aci \ No newline at end of file diff --git a/examples/etcd-large-rkt/cloud/etcd1.yaml b/examples/etcd-large-rkt/cloud/etcd1.yaml new file mode 100644 index 00000000..db0fd867 --- /dev/null +++ b/examples/etcd-large-rkt/cloud/etcd1.yaml @@ -0,0 +1,15 @@ +#cloud-config +coreos: + etcd2: + name: node1 + initial-cluster: node1=http://172.15.0.21:2380,node2=http://172.15.0.22:2380,node3=http://172.15.0.23:2380 + initial-advertise-peer-urls: http://172.15.0.21:2380 + initial-cluster-state: new + advertise-client-urls: http://172.15.0.21:2379 + listen-client-urls: http://0.0.0.0:2379 + listen-peer-urls: http://172.15.0.21:2380 + units: + - name: etcd2.service + command: start + - name: fleet.service + command: start diff --git a/examples/etcd-large-rkt/cloud/etcd2.yaml b/examples/etcd-large-rkt/cloud/etcd2.yaml new file mode 100644 index 00000000..f0aab64c --- /dev/null +++ b/examples/etcd-large-rkt/cloud/etcd2.yaml @@ -0,0 +1,15 @@ +#cloud-config +coreos: + etcd2: + name: node2 + initial-cluster: node1=http://172.15.0.21:2380,node2=http://172.15.0.22:2380,node3=http://172.15.0.23:2380 + initial-advertise-peer-urls: http://172.15.0.22:2380 + initial-cluster-state: new + advertise-client-urls: http://172.15.0.22:2379 + listen-client-urls: http://0.0.0.0:2379 + listen-peer-urls: http://172.15.0.22:2380 + units: + - name: etcd2.service + command: start + - name: fleet.service + command: start diff --git a/examples/etcd-large-rkt/cloud/etcd3.yaml b/examples/etcd-large-rkt/cloud/etcd3.yaml new file mode 100644 index 00000000..5a5f2fb6 --- /dev/null +++ b/examples/etcd-large-rkt/cloud/etcd3.yaml @@ -0,0 +1,15 @@ +#cloud-config +coreos: + etcd2: + name: node3 + initial-cluster: node1=http://172.15.0.21:2380,node2=http://172.15.0.22:2380,node3=http://172.15.0.23:2380 + initial-advertise-peer-urls: http://172.15.0.23:2380 + initial-cluster-state: new + advertise-client-urls: http://172.15.0.23:2379 + listen-client-urls: http://0.0.0.0:2379 + listen-peer-urls: http://172.15.0.23:2380 + units: + - name: etcd2.service + command: start + - name: fleet.service + command: start diff --git a/examples/etcd-large-rkt/cloud/worker.yaml b/examples/etcd-large-rkt/cloud/worker.yaml new file mode 100644 index 00000000..c6e9dbbd --- /dev/null +++ b/examples/etcd-large-rkt/cloud/worker.yaml @@ -0,0 +1,14 @@ +#cloud-config +coreos: + etcd2: + initial-cluster: node1=http://172.15.0.21:2380,node2=http://172.15.0.22:2380,node3=http://172.15.0.23:2380 + proxy: on + listen-client-urls: http://localhost:2379 + fleet: + metadata: "role=worker" + etcd_servers: "http://localhost:2379" + units: + - name: etcd2.service + command: start + - name: fleet.service + command: start \ No newline at end of file diff --git a/examples/etcd-large-rkt/config.yaml b/examples/etcd-large-rkt/config.yaml new file mode 100644 index 00000000..899eb602 --- /dev/null +++ b/examples/etcd-large-rkt/config.yaml @@ -0,0 +1,20 @@ +--- +api_version: v1alpha1 +groups: + - name: node1 + spec: etcd1 + require: + uuid: 16e7d8a7-bfa9-428b-9117-363341bb330b + + - name: node2 + spec: etcd2 + require: + uuid: 264cd073-ca62-44b3-98c0-50aad5b5f819 + + - name: node3 + spec: etcd3 + require: + uuid: 39d2e747-2648-4d68-ae92-bbc70b245055 + + - name: default + spec: worker \ No newline at end of file diff --git a/examples/etcd-large-rkt/ignition/etcd1.json b/examples/etcd-large-rkt/ignition/etcd1.json new file mode 100644 index 00000000..c8eb684d --- /dev/null +++ b/examples/etcd-large-rkt/ignition/etcd1.json @@ -0,0 +1,11 @@ +{ + "ignitionVersion": 1, + "networkd": { + "units": [ + { + "name": "00-ens3.network", + "contents": "[Match]\nName=ens3\n\n[Network]\nGateway=172.15.0.1\nAddress=172.15.0.21" + } + ] + } +} \ No newline at end of file diff --git a/examples/etcd-large-rkt/ignition/etcd2.json b/examples/etcd-large-rkt/ignition/etcd2.json new file mode 100644 index 00000000..88c3811f --- /dev/null +++ b/examples/etcd-large-rkt/ignition/etcd2.json @@ -0,0 +1,11 @@ +{ + "ignitionVersion": 1, + "networkd": { + "units": [ + { + "name": "00-ens3.network", + "contents": "[Match]\nName=ens3\n\n[Network]\nGateway=172.15.0.1\nAddress=172.15.0.22" + } + ] + } +} \ No newline at end of file diff --git a/examples/etcd-large-rkt/ignition/etcd3.json b/examples/etcd-large-rkt/ignition/etcd3.json new file mode 100644 index 00000000..35584575 --- /dev/null +++ b/examples/etcd-large-rkt/ignition/etcd3.json @@ -0,0 +1,11 @@ +{ + "ignitionVersion": 1, + "networkd": { + "units": [ + { + "name": "00-ens3.network", + "contents": "[Match]\nName=ens3\n\n[Network]\nGateway=172.15.0.1\nAddress=172.15.0.23" + } + ] + } +} \ No newline at end of file diff --git a/examples/etcd-large-rkt/specs/etcd1/spec.json b/examples/etcd-large-rkt/specs/etcd1/spec.json new file mode 100644 index 00000000..99fc7fa6 --- /dev/null +++ b/examples/etcd-large-rkt/specs/etcd1/spec.json @@ -0,0 +1,15 @@ +{ + "id": "etcd1", + "boot": { + "kernel": "/assets/coreos/835.9.0/coreos_production_pxe.vmlinuz", + "initrd": ["/assets/coreos/835.9.0/coreos_production_pxe_image.cpio.gz"], + "cmdline": { + "cloud-config-url": "http://172.15.0.2:8080/cloud?uuid=${uuid}&mac=${net0/mac:hexhyp}", + "coreos.config.url": "http://bootcfg.example:8080/ignition?uuid=${uuid}", + "coreos.autologin": "", + "coreos.first_boot": "" + } + }, + "cloud_id": "etcd1.yaml", + "ignition_id": "etcd1.json" +} \ No newline at end of file diff --git a/examples/etcd-large-rkt/specs/etcd2/spec.json b/examples/etcd-large-rkt/specs/etcd2/spec.json new file mode 100644 index 00000000..e4eeb48a --- /dev/null +++ b/examples/etcd-large-rkt/specs/etcd2/spec.json @@ -0,0 +1,15 @@ +{ + "id": "etcd2", + "boot": { + "kernel": "/assets/coreos/835.9.0/coreos_production_pxe.vmlinuz", + "initrd": ["/assets/coreos/835.9.0/coreos_production_pxe_image.cpio.gz"], + "cmdline": { + "cloud-config-url": "http://172.15.0.2:8080/cloud?uuid=${uuid}&mac=${net0/mac:hexhyp}", + "coreos.config.url": "http://bootcfg.example:8080/ignition?uuid=${uuid}", + "coreos.autologin": "", + "coreos.first_boot": "" + } + }, + "cloud_id": "etcd2.yaml", + "ignition_id": "etcd2.json" +} \ No newline at end of file diff --git a/examples/etcd-large-rkt/specs/etcd3/spec.json b/examples/etcd-large-rkt/specs/etcd3/spec.json new file mode 100644 index 00000000..6a86d082 --- /dev/null +++ b/examples/etcd-large-rkt/specs/etcd3/spec.json @@ -0,0 +1,15 @@ +{ + "id": "etcd3", + "boot": { + "kernel": "/assets/coreos/835.9.0/coreos_production_pxe.vmlinuz", + "initrd": ["/assets/coreos/835.9.0/coreos_production_pxe_image.cpio.gz"], + "cmdline": { + "cloud-config-url": "http://172.15.0.2:8080/cloud?uuid=${uuid}&mac=${net0/mac:hexhyp}", + "coreos.config.url": "http://bootcfg.example:8080/ignition?uuid=${uuid}", + "coreos.autologin": "", + "coreos.first_boot": "" + } + }, + "cloud_id": "etcd3.yaml", + "ignition_id": "etcd3.json" +} \ No newline at end of file diff --git a/examples/etcd-large-rkt/specs/worker/spec.json b/examples/etcd-large-rkt/specs/worker/spec.json new file mode 100644 index 00000000..a03230d3 --- /dev/null +++ b/examples/etcd-large-rkt/specs/worker/spec.json @@ -0,0 +1,13 @@ +{ + "id": "worker", + "boot": { + "kernel": "/assets/coreos/835.9.0/coreos_production_pxe.vmlinuz", + "initrd": ["/assets/coreos/835.9.0/coreos_production_pxe_image.cpio.gz"], + "cmdline": { + "cloud-config-url": "http://172.15.0.2:8080/cloud?uuid=${uuid}&mac=${net0/mac:hexhyp}", + "coreos.autologin": "" + } + }, + "cloud_id": "worker.yaml", + "ignition_id": "" +} \ No newline at end of file diff --git a/scripts/libvirt b/scripts/libvirt index 93b24012..46ee12f2 100755 --- a/scripts/libvirt +++ b/scripts/libvirt @@ -8,7 +8,8 @@ fi function main { case "$1" in - "create") create;; + "create-docker") create;; + "create-rkt") create_rkt;; "start") start;; "reboot") reboot;; "shutdown") shutdown;; @@ -24,7 +25,8 @@ function main { function usage { echo "USAGE: ${0##*/} " echo "Commands:" - echo -e "\tcreate\t\tcreate 5 libvirt nodes with specific hardware attributes" + echo -e "\tcreate-docker\tcreate 5 libvirt nodes on the docker0 bridge" + echo -e "\tcreate-rkt\tcreate 5 libvirt nodes on a rkt CNI metal0 bridge" echo -e "\tstart\t\tstart the 5 libvirt nodes" echo -e "\treboot\t\treboot the 5 libvirt nodes" echo -e "\tshutdown\tshutdown the 5 libvirt nodes" @@ -40,6 +42,31 @@ function create { virt-install --name node5 -u 53683e94-3273-4a49-9a82-d769b88e3ccf --pxe --network=bridge:docker0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole } +function create_rkt { + metal0 + virt-install --name node1 -u 16e7d8a7-bfa9-428b-9117-363341bb330b --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole + virt-install --name node2 -u 264cd073-ca62-44b3-98c0-50aad5b5f819 --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole + virt-install --name node3 -u 39d2e747-2648-4d68-ae92-bbc70b245055 --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole + virt-install --name node4 -u 4ed46e8e-db69-471e-b874-0990dd65649d --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole + virt-install --name node5 -u 53683e94-3273-4a49-9a82-d769b88e3ccf --pxe --network=bridge:metal0 --memory=1024 --vcpus=1 --os-type=linux --disk none --noautoconsole +} + +function metal0 { + cat > /etc/rkt/net.d/20-metal.conf << EOF +{ + "name": "metal0", + "type": "bridge", + "bridge": "metal0", + "isGateway": true, + "ipam": { + "type": "host-local", + "subnet": "172.15.0.0/16", + "routes" : [ { "dst" : "172.15.0.0/16" } ] + } +} +EOF +} + nodes=(node1 node2 node3 node4 node5) function start { diff --git a/scripts/rkt-gc-force b/scripts/rkt-gc-force new file mode 100755 index 00000000..27fbba6e --- /dev/null +++ b/scripts/rkt-gc-force @@ -0,0 +1,13 @@ +#!/bin/bash -e + +# See https://github.com/coreos/rkt/issues/1572 +rkt gc --grace-period=0s +for p in `mount | grep -oP "/tmp/[^\s,\)]*"`; do umount -l $p; done +for p in `mount | grep -oP "/var/lib/rkt/pods/[^\s,\)]*"`; do umount -l $p; done +rkt gc --grace-period=0s +for i in `rkt image list | grep -oP '^sha512[^ \t]*'`; do rkt image rm $i; done +for ns in `ip netns l | grep -o -E '^[[:alnum:]]+'`; do ip netns del $ns; done +ip l del metal0 +for veth in `ip l show | grep -oE 'veth[^@]+'`; do ip l del $veth; done +rm -Rf /var/lib/cni/networks/* +rm -Rf /var/lib/rkt/* \ No newline at end of file