From 68437b9e2d0ffe4dbb678b94d6af600e5374d3f1 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 3 Feb 2016 21:28:44 -0800 Subject: [PATCH] api: Allow Ignition config templates in YAML * Match machines to their Ignition template file, render the template, parse as JSON or YAML, and serve as Ignition JSON * Existing JSON ignition configs should still work * Remove the Ignition YAML -> JSON transform script which used fuze --- api/groups_test.go | 5 ++- api/ignition.go | 36 ++++++++++++----- api/ignition_test.go | 39 ++++++++++++++++--- api/metadata_test.go | 9 +++-- api/spec_test.go | 4 ++ .../{coreos-disk.yaml => coreos-install.yaml} | 2 +- examples/etcd-docker.yaml | 2 +- examples/ignition/coreos-install.json | 1 - examples/ignition/etcd.json | 1 - examples/ignition/etcd.yaml | 8 ++++ examples/ignition/etcd_proxy.json | 1 - examples/ignition/etcd_proxy.yaml | 8 ++++ examples/ignition/network.json | 1 - examples/ignition/network.yaml | 8 ++++ examples/specs/coreos-install/spec.json | 2 +- examples/specs/etcd/spec.json | 2 +- examples/specs/etcd_proxy/spec.json | 2 +- examples/specs/kubernetes-master/spec.json | 2 +- examples/specs/kubernetes-worker/spec.json | 2 +- scripts/gen-ignition | 12 ------ 20 files changed, 104 insertions(+), 43 deletions(-) rename examples/{coreos-disk.yaml => coreos-install.yaml} (96%) delete mode 100644 examples/ignition/coreos-install.json delete mode 100644 examples/ignition/etcd.json delete mode 100644 examples/ignition/etcd_proxy.json delete mode 100644 examples/ignition/network.json delete mode 100755 scripts/gen-ignition diff --git a/api/groups_test.go b/api/groups_test.go index cdb8b2b6..abd6fc95 100644 --- a/api/groups_test.go +++ b/api/groups_test.go @@ -18,8 +18,9 @@ var ( Name: "test group", Spec: "g1h2i3j4", Metadata: map[string]string{ - "k8s_version": "v1.1.2", - "pod_network": "10.2.0.0/16", + "k8s_version": "v1.1.2", + "pod_network": "10.2.0.0/16", + "service_name": "etcd2", }, Matcher: RequirementSet(map[string]string{"uuid": "a1b2c3d4"}), } diff --git a/api/ignition.go b/api/ignition.go index 6e2f59a7..b3c0e153 100644 --- a/api/ignition.go +++ b/api/ignition.go @@ -2,14 +2,18 @@ package api import ( "bytes" + "gopkg.in/yaml.v2" "net/http" + "strings" ignition "github.com/coreos/ignition/src/config" "golang.org/x/net/context" ) -// ignitionHandler returns a handler that responds with the ignition config -// for the requester. +// ignitionHandler returns a handler that responds with the Ignition config +// for the requester. The Ignition file referenced in the Spec is rendered +// with metadata and parsed and validated as either YAML or JSON based on the +// extension. The Ignition config is served as an HTTP JSON response. func ignitionHandler(store Store) ContextHandler { fn := func(ctx context.Context, w http.ResponseWriter, req *http.Request) { group, err := groupFromContext(ctx) @@ -43,14 +47,28 @@ func ignitionHandler(store Store) ContextHandler { return } - // validate the Ignition JSON - config, err := ignition.Parse(buf.Bytes()) - if err != nil { - log.Errorf("error parsing ignition config: %v", err) - http.NotFound(w, req) - return + // Unmarshal YAML or JSON Ignition config + var cfg ignition.Config + if isYAML(spec.IgnitionConfig) { + if err := yaml.Unmarshal(buf.Bytes(), &cfg); err != nil { + log.Errorf("error parsing YAML Ignition config: %v", err) + http.NotFound(w, req) + return + } + } else { + cfg, err = ignition.Parse(buf.Bytes()) + if err != nil { + log.Errorf("error parsing JSON Ignition config: %v", err) + http.NotFound(w, req) + return + } } - renderJSON(w, config) + // Marshal Ignition config as JSON HTTP response + renderJSON(w, cfg) } return ContextHandlerFunc(fn) } + +func isYAML(filename string) bool { + return strings.HasSuffix(filename, ".yaml") || strings.HasSuffix(filename, ".yml") +} diff --git a/api/ignition_test.go b/api/ignition_test.go index 0662cc93..640670dc 100644 --- a/api/ignition_test.go +++ b/api/ignition_test.go @@ -9,11 +9,13 @@ import ( "golang.org/x/net/context" ) +var expectedIgnition = `{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"etcd2.service","enable":true}]},"networkd":{},"passwd":{}}` + func TestIgnitionHandler(t *testing.T) { - ignitioncfg := `{"ignitionVersion": 1}` + content := `{"ignitionVersion": 1,"systemd":{"units":[{"name":"{{.service_name}}.service","enable":true}]}}` store := &fixedStore{ Specs: map[string]*Spec{testGroup.Spec: testSpec}, - IgnitionConfigs: map[string]string{testSpec.IgnitionConfig: ignitioncfg}, + IgnitionConfigs: map[string]string{testSpec.IgnitionConfig: content}, } h := ignitionHandler(store) ctx := withGroup(context.Background(), &testGroup) @@ -21,11 +23,38 @@ func TestIgnitionHandler(t *testing.T) { req, _ := http.NewRequest("GET", "/", nil) h.ServeHTTP(ctx, w, req) // assert that: - // - the Spec's ignition config is rendered - expectedJSON := `{"ignitionVersion":1,"storage":{},"systemd":{},"networkd":{},"passwd":{}}` + // - Ignition template is rendered with Group metadata + // - Rendered Ignition template is parsed as JSON + // - Ignition Config served as JSON assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, jsonContentType, w.HeaderMap.Get(contentType)) - assert.Equal(t, expectedJSON, w.Body.String()) + assert.Equal(t, expectedIgnition, w.Body.String()) +} + +func TestIgnitionHandler_YAMLIgnition(t *testing.T) { + content := ` +ignition_version: 1 +systemd: + units: + - name: {{.service_name}}.service + enable: true +` + store := &fixedStore{ + Specs: map[string]*Spec{testGroup.Spec: testSpecWithIgnitionYAML}, + IgnitionConfigs: map[string]string{testSpecWithIgnitionYAML.IgnitionConfig: content}, + } + h := ignitionHandler(store) + ctx := withGroup(context.Background(), &testGroup) + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/", nil) + h.ServeHTTP(ctx, w, req) + // assert that: + // - Ignition template is rendered with Group metadata + // - Rendered Ignition template ending in .yaml is parsed as YAML + // - Ignition Config served as JSON + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, jsonContentType, w.HeaderMap.Get(contentType)) + assert.Equal(t, expectedIgnition, w.Body.String()) } func TestIgnitionHandler_MissingCtxSpec(t *testing.T) { diff --git a/api/metadata_test.go b/api/metadata_test.go index 091ec84a..a8c29d0d 100644 --- a/api/metadata_test.go +++ b/api/metadata_test.go @@ -22,10 +22,11 @@ func TestMetadataHandler(t *testing.T) { // - query argument attributes are added to the metadata // - key names are upper case expectedData := map[string]string{ - "K8S_VERSION": "v1.1.2", - "POD_NETWORK": "10.2.0.0/16", - "UUID": "a1b2c3d4", - "MAC": validMACStr, + "K8S_VERSION": "v1.1.2", + "POD_NETWORK": "10.2.0.0/16", + "SERVICE_NAME": "etcd2", + "UUID": "a1b2c3d4", + "MAC": validMACStr, } assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, expectedData, metadataToMap(w.Body.String())) diff --git a/api/spec_test.go b/api/spec_test.go index 18789fb1..0251e83b 100644 --- a/api/spec_test.go +++ b/api/spec_test.go @@ -23,6 +23,10 @@ var ( CloudConfig: "cloud-config.yml", IgnitionConfig: "ignition.json", } + testSpecWithIgnitionYAML = &Spec{ + ID: "g1h2i3j4", + IgnitionConfig: "ignition.yaml", + } emptySpec = &Spec{ ID: "empty", } diff --git a/examples/coreos-disk.yaml b/examples/coreos-install.yaml similarity index 96% rename from examples/coreos-disk.yaml rename to examples/coreos-install.yaml index 65e39951..1776dfd1 100644 --- a/examples/coreos-disk.yaml +++ b/examples/coreos-install.yaml @@ -58,4 +58,4 @@ groups: networkd_name: ens3 networkd_gateway: 172.17.0.1 networkd_dns: 172.17.0.3 - etcd_initial_cluster: "node1=http://172.17.0.21:2380,node2=http://172.17.0.22:2380,node3=http://172.17.0.23:2380" \ No newline at end of file + etcd_initial_cluster: "node1=http://172.17.0.21:2380,node2=http://172.17.0.22:2380,node3=http://172.17.0.23:2380" diff --git a/examples/etcd-docker.yaml b/examples/etcd-docker.yaml index 57d4c9aa..443a301e 100644 --- a/examples/etcd-docker.yaml +++ b/examples/etcd-docker.yaml @@ -46,4 +46,4 @@ groups: networkd_name: ens3 networkd_gateway: 172.17.0.1 networkd_dns: 172.17.0.3 - etcd_initial_cluster: "node1=http://172.17.0.21:2380,node2=http://172.17.0.22:2380,node3=http://172.17.0.23:2380" \ No newline at end of file + etcd_initial_cluster: "node1=http://172.17.0.21:2380,node2=http://172.17.0.22:2380,node3=http://172.17.0.23:2380" diff --git a/examples/ignition/coreos-install.json b/examples/ignition/coreos-install.json deleted file mode 100644 index 4df1cd74..00000000 --- a/examples/ignition/coreos-install.json +++ /dev/null @@ -1 +0,0 @@ -{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"install.service","enable":true,"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/usr/bin/coreos-install -d /dev/sda -C {{.coreos_channel}} -V {{.coreos_version}}\nExecStart=/usr/bin/udevadm settle\nExecStart=/usr/bin/mount /dev/disk/by-label/OEM /mnt\nExecStart=/bin/sh -c 'echo set linux_append=\\\\\"coreos.config.url=\\\"{{.ignition_endpoint}}?{{.query}}\u0026os=installed\\\"\\\\\" \u003e /mnt/grub.cfg'\nExecStart=/usr/bin/systemctl reboot\n[Install]\nWantedBy=multi-user.target\n"}]},"networkd":{},"passwd":{}} \ No newline at end of file diff --git a/examples/ignition/etcd.json b/examples/ignition/etcd.json deleted file mode 100644 index 52eebfee..00000000 --- a/examples/ignition/etcd.json +++ /dev/null @@ -1 +0,0 @@ -{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"metadata.service","enable":true,"contents":"[Unit]\nDescription=Bare Metal Metadata Agent\n[Service]\nType=oneshot\nEnvironment=OUTPUT=/run/metadata/bootcfg\nExecStart=/usr/bin/mkdir --parent /run/metadata\nExecStart=/usr/bin/bash -c 'curl --url \"http://bootcfg.foo:8080/metadata?{{.query}}\" --retry 10 --output ${OUTPUT}'\n[Install]\nWantedBy=multi-user.target\n"},{"name":"etcd2.service","enable":true,"dropins":[{"name":"etcd-metadata.conf","contents":"[Unit]\nRequires=metadata.service\nAfter=metadata.service\n[Service]\nEnvironmentFile=/run/metadata/bootcfg\nExecStart=\nExecStart=/usr/bin/etcd2 \\\n --advertise-client-urls=http://${IPV4_ADDRESS}:2379 \\\n --initial-advertise-peer-urls=http://${IPV4_ADDRESS}:2380 \\\n --listen-client-urls=http://0.0.0.0:2379 \\\n --listen-peer-urls=http://${IPV4_ADDRESS}:2380 \\\n --initial-cluster=${ETCD_INITIAL_CLUSTER}\n"}]}]},"networkd":{"units":[{"name":"00-{{.networkd_name}}.network","contents":"[Match]\nName={{.networkd_name}}\n[Network]\nGateway={{.networkd_gateway}}\nDNS={{.networkd_dns}}\nDNS=8.8.8.8\nAddress={{.networkd_address}}\n"}]},"passwd":{}} \ No newline at end of file diff --git a/examples/ignition/etcd.yaml b/examples/ignition/etcd.yaml index 70ca1b98..b745c9e1 100644 --- a/examples/ignition/etcd.yaml +++ b/examples/ignition/etcd.yaml @@ -42,3 +42,11 @@ networkd: DNS={{.networkd_dns}} DNS=8.8.8.8 Address={{.networkd_address}} + +{{ if .ssh_authorized_key }} +passwd: + users: + - name: core + ssh_authorized_keys: + - {{.ssh_authorized_key}} +{{end}} diff --git a/examples/ignition/etcd_proxy.json b/examples/ignition/etcd_proxy.json deleted file mode 100644 index ba1ed8a1..00000000 --- a/examples/ignition/etcd_proxy.json +++ /dev/null @@ -1 +0,0 @@ -{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"metadata.service","enable":true,"contents":"[Unit]\nDescription=Bare Metal Metadata Agent\n[Service]\nType=oneshot\nEnvironment=OUTPUT=/run/metadata/bootcfg\nExecStart=/usr/bin/mkdir --parent /run/metadata\nExecStart=/usr/bin/bash -c 'curl --url http://bootcfg.foo:8080/metadata --retry 10 --output ${OUTPUT}'\n[Install]\nWantedBy=multi-user.target\n"},{"name":"etcd2.service","enable":true,"dropins":[{"name":"etcd-metadata.conf","contents":"[Unit]\nRequires=metadata.service\nAfter=metadata.service\n[Service]\nEnvironmentFile=/run/metadata/bootcfg\nExecStart=\nExecStart=/usr/bin/etcd2 \\\n --proxy on\n --listen-client-urls=http://localhost:2379 \\\n --initial-cluster=${ETCD_INITIAL_CLUSTER}\n"}]}]},"networkd":{},"passwd":{}} \ No newline at end of file diff --git a/examples/ignition/etcd_proxy.yaml b/examples/ignition/etcd_proxy.yaml index 97761a6e..e5480a05 100644 --- a/examples/ignition/etcd_proxy.yaml +++ b/examples/ignition/etcd_proxy.yaml @@ -29,3 +29,11 @@ systemd: --proxy on --listen-client-urls=http://localhost:2379 \ --initial-cluster=${ETCD_INITIAL_CLUSTER} + +{{ if .ssh_authorized_key }} +passwd: + users: + - name: core + ssh_authorized_keys: + - {{.ssh_authorized_key}} +{{end}} diff --git a/examples/ignition/network.json b/examples/ignition/network.json deleted file mode 100644 index 3b28eee9..00000000 --- a/examples/ignition/network.json +++ /dev/null @@ -1 +0,0 @@ -{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"metadata.service","enable":true,"contents":"[Unit]\nDescription=Bare Metal Metadata Agent\n[Service]\nType=oneshot\nEnvironment=OUTPUT=/run/metadata/bootcfg\nExecStart=/usr/bin/mkdir --parent /run/metadata\nExecStart=/usr/bin/bash -c 'curl --url \"http://bootcfg.foo:8080/metadata?{{.query}}\" --retry 10 --output ${OUTPUT}'\n[Install]\nWantedBy=multi-user.target\n"}]},"networkd":{"units":[{"name":"00-{{.networkd_name}}.network","contents":"[Match]\nName={{.networkd_name}}\n[Network]\nGateway={{.networkd_gateway}}\nDNS={{.networkd_dns}}\nDNS=8.8.8.8\nAddress={{.networkd_address}}\n"}]},"passwd":{}} \ No newline at end of file diff --git a/examples/ignition/network.yaml b/examples/ignition/network.yaml index 8331526f..2a1cb7ac 100644 --- a/examples/ignition/network.yaml +++ b/examples/ignition/network.yaml @@ -25,3 +25,11 @@ networkd: DNS={{.networkd_dns}} DNS=8.8.8.8 Address={{.networkd_address}} + +{{ if .ssh_authorized_key }} +passwd: + users: + - name: core + ssh_authorized_keys: + - {{.ssh_authorized_key}} +{{end}} diff --git a/examples/specs/coreos-install/spec.json b/examples/specs/coreos-install/spec.json index 56f1f4d5..35107b5f 100644 --- a/examples/specs/coreos-install/spec.json +++ b/examples/specs/coreos-install/spec.json @@ -10,5 +10,5 @@ } }, "cloud_id": "", - "ignition_id": "coreos-install.json" + "ignition_id": "coreos-install.yaml" } \ No newline at end of file diff --git a/examples/specs/etcd/spec.json b/examples/specs/etcd/spec.json index 445a59b2..6dea8f25 100644 --- a/examples/specs/etcd/spec.json +++ b/examples/specs/etcd/spec.json @@ -10,5 +10,5 @@ } }, "cloud_id": "", - "ignition_id": "etcd.json" + "ignition_id": "etcd.yaml" } \ No newline at end of file diff --git a/examples/specs/etcd_proxy/spec.json b/examples/specs/etcd_proxy/spec.json index 2ee5babd..e5d10d85 100644 --- a/examples/specs/etcd_proxy/spec.json +++ b/examples/specs/etcd_proxy/spec.json @@ -10,5 +10,5 @@ } }, "cloud_id": "", - "ignition_id": "etcd_proxy.json" + "ignition_id": "etcd_proxy.yaml" } \ No newline at end of file diff --git a/examples/specs/kubernetes-master/spec.json b/examples/specs/kubernetes-master/spec.json index 919eac2d..c791c191 100644 --- a/examples/specs/kubernetes-master/spec.json +++ b/examples/specs/kubernetes-master/spec.json @@ -11,5 +11,5 @@ } }, "cloud_id": "kubernetes-master.sh", - "ignition_id": "network.json" + "ignition_id": "network.yaml" } \ No newline at end of file diff --git a/examples/specs/kubernetes-worker/spec.json b/examples/specs/kubernetes-worker/spec.json index 620464c1..c2b4a407 100644 --- a/examples/specs/kubernetes-worker/spec.json +++ b/examples/specs/kubernetes-worker/spec.json @@ -11,5 +11,5 @@ } }, "cloud_id": "kubernetes-worker.sh", - "ignition_id": "network.json" + "ignition_id": "network.yaml" } \ No newline at end of file diff --git a/scripts/gen-ignition b/scripts/gen-ignition deleted file mode 100755 index fc23c153..00000000 --- a/scripts/gen-ignition +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -e -# USAGE: ./scripts/gen-ignition src -# Example: ./scripts/gen-ignition exampels/dev/ignition -# Use coreos/fuze to transform a human-friendly *.yaml Ignition config to -# machine-friendly JSON - -DEST=$1 - -shopt -s nullglob -for file in $DEST/*.yaml; do - fuze -in-file "$file" -out-file "${file%.yaml}.json" -done \ No newline at end of file