api: Allow Ignition config templates in YAML

* Match machines to their Ignition template file, render the
template, parse as JSON or YAML, and serve as Ignition JSON
* Existing JSON ignition configs should still work
* Remove the Ignition YAML -> JSON transform script which used fuze

api/groups_test.go

@@ -18,8 +18,9 @@ var (
 		Name: "test group",
 		Spec: "g1h2i3j4",
 		Metadata: map[string]string{
-			"k8s_version": "v1.1.2",
-			"pod_network": "10.2.0.0/16",
+			"k8s_version":  "v1.1.2",
+			"pod_network":  "10.2.0.0/16",
+			"service_name": "etcd2",
 		},
 		Matcher: RequirementSet(map[string]string{"uuid": "a1b2c3d4"}),
 	}

api/ignition.go

@@ -2,14 +2,18 @@ package api
 
 import (
 	"bytes"
+	"gopkg.in/yaml.v2"
 	"net/http"
+	"strings"
 
 	ignition "github.com/coreos/ignition/src/config"
 	"golang.org/x/net/context"
 )
 
-// ignitionHandler returns a handler that responds with the ignition config
-// for the requester.
+// ignitionHandler returns a handler that responds with the Ignition config
+// for the requester. The Ignition file referenced in the Spec is rendered
+// with metadata and parsed and validated as either YAML or JSON based on the
+// extension. The Ignition config is served as an HTTP JSON response.
 func ignitionHandler(store Store) ContextHandler {
 	fn := func(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 		group, err := groupFromContext(ctx)
@@ -43,14 +47,28 @@ func ignitionHandler(store Store) ContextHandler {
 			return
 		}
 
-		// validate the Ignition JSON
-		config, err := ignition.Parse(buf.Bytes())
-		if err != nil {
-			log.Errorf("error parsing ignition config: %v", err)
-			http.NotFound(w, req)
-			return
+		// Unmarshal YAML or JSON Ignition config
+		var cfg ignition.Config
+		if isYAML(spec.IgnitionConfig) {
+			if err := yaml.Unmarshal(buf.Bytes(), &cfg); err != nil {
+				log.Errorf("error parsing YAML Ignition config: %v", err)
+				http.NotFound(w, req)
+				return
+			}
+		} else {
+			cfg, err = ignition.Parse(buf.Bytes())
+			if err != nil {
+				log.Errorf("error parsing JSON Ignition config: %v", err)
+				http.NotFound(w, req)
+				return
+			}
 		}
-		renderJSON(w, config)
+		// Marshal Ignition config as JSON HTTP response
+		renderJSON(w, cfg)
 	}
 	return ContextHandlerFunc(fn)
 }
+
+func isYAML(filename string) bool {
+	return strings.HasSuffix(filename, ".yaml") || strings.HasSuffix(filename, ".yml")
+}

api/ignition_test.go

@@ -9,11 +9,13 @@ import (
 	"golang.org/x/net/context"
 )
 
+var expectedIgnition = `{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"etcd2.service","enable":true}]},"networkd":{},"passwd":{}}`
+
 func TestIgnitionHandler(t *testing.T) {
-	ignitioncfg := `{"ignitionVersion": 1}`
+	content := `{"ignitionVersion": 1,"systemd":{"units":[{"name":"{{.service_name}}.service","enable":true}]}}`
 	store := &fixedStore{
 		Specs:           map[string]*Spec{testGroup.Spec: testSpec},
-		IgnitionConfigs: map[string]string{testSpec.IgnitionConfig: ignitioncfg},
+		IgnitionConfigs: map[string]string{testSpec.IgnitionConfig: content},
 	}
 	h := ignitionHandler(store)
 	ctx := withGroup(context.Background(), &testGroup)
@@ -21,11 +23,38 @@ func TestIgnitionHandler(t *testing.T) {
 	req, _ := http.NewRequest("GET", "/", nil)
 	h.ServeHTTP(ctx, w, req)
 	// assert that:
-	// - the Spec's ignition config is rendered
-	expectedJSON := `{"ignitionVersion":1,"storage":{},"systemd":{},"networkd":{},"passwd":{}}`
+	// - Ignition template is rendered with Group metadata
+	// - Rendered Ignition template is parsed as JSON
+	// - Ignition Config served as JSON
 	assert.Equal(t, http.StatusOK, w.Code)
 	assert.Equal(t, jsonContentType, w.HeaderMap.Get(contentType))
-	assert.Equal(t, expectedJSON, w.Body.String())
+	assert.Equal(t, expectedIgnition, w.Body.String())
+}
+
+func TestIgnitionHandler_YAMLIgnition(t *testing.T) {
+	content := `
+ignition_version: 1
+systemd:
+  units:
+    - name: {{.service_name}}.service
+      enable: true
+`
+	store := &fixedStore{
+		Specs:           map[string]*Spec{testGroup.Spec: testSpecWithIgnitionYAML},
+		IgnitionConfigs: map[string]string{testSpecWithIgnitionYAML.IgnitionConfig: content},
+	}
+	h := ignitionHandler(store)
+	ctx := withGroup(context.Background(), &testGroup)
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/", nil)
+	h.ServeHTTP(ctx, w, req)
+	// assert that:
+	// - Ignition template is rendered with Group metadata
+	// - Rendered Ignition template ending in .yaml is parsed as YAML
+	// - Ignition Config served as JSON
+	assert.Equal(t, http.StatusOK, w.Code)
+	assert.Equal(t, jsonContentType, w.HeaderMap.Get(contentType))
+	assert.Equal(t, expectedIgnition, w.Body.String())
 }
 
 func TestIgnitionHandler_MissingCtxSpec(t *testing.T) {

api/metadata_test.go

@@ -22,10 +22,11 @@ func TestMetadataHandler(t *testing.T) {
 	// - query argument attributes are added to the metadata
 	// - key names are upper case
 	expectedData := map[string]string{
-		"K8S_VERSION": "v1.1.2",
-		"POD_NETWORK": "10.2.0.0/16",
-		"UUID":        "a1b2c3d4",
-		"MAC":         validMACStr,
+		"K8S_VERSION":  "v1.1.2",
+		"POD_NETWORK":  "10.2.0.0/16",
+		"SERVICE_NAME": "etcd2",
+		"UUID":         "a1b2c3d4",
+		"MAC":          validMACStr,
 	}
 	assert.Equal(t, http.StatusOK, w.Code)
 	assert.Equal(t, expectedData, metadataToMap(w.Body.String()))

api/spec_test.go

@@ -23,6 +23,10 @@ var (
 		CloudConfig:    "cloud-config.yml",
 		IgnitionConfig: "ignition.json",
 	}
+	testSpecWithIgnitionYAML = &Spec{
+		ID:             "g1h2i3j4",
+		IgnitionConfig: "ignition.yaml",
+	}
 	emptySpec = &Spec{
 		ID: "empty",
 	}

examples/coreos-install.yaml

@@ -58,4 +58,4 @@ groups:
       networkd_name: ens3
       networkd_gateway: 172.17.0.1
       networkd_dns: 172.17.0.3
-      etcd_initial_cluster: "node1=http://172.17.0.21:2380,node2=http://172.17.0.22:2380,node3=http://172.17.0.23:2380"
+      etcd_initial_cluster: "node1=http://172.17.0.21:2380,node2=http://172.17.0.22:2380,node3=http://172.17.0.23:2380"

examples/etcd-docker.yaml

@@ -46,4 +46,4 @@ groups:
       networkd_name: ens3
       networkd_gateway: 172.17.0.1
       networkd_dns: 172.17.0.3
-      etcd_initial_cluster: "node1=http://172.17.0.21:2380,node2=http://172.17.0.22:2380,node3=http://172.17.0.23:2380"
+      etcd_initial_cluster: "node1=http://172.17.0.21:2380,node2=http://172.17.0.22:2380,node3=http://172.17.0.23:2380"

examples/ignition/coreos-install.json

@@ -1 +0,0 @@
-{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"install.service","enable":true,"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/usr/bin/coreos-install -d /dev/sda -C {{.coreos_channel}} -V {{.coreos_version}}\nExecStart=/usr/bin/udevadm settle\nExecStart=/usr/bin/mount /dev/disk/by-label/OEM /mnt\nExecStart=/bin/sh -c 'echo set linux_append=\\\\\"coreos.config.url=\\\"{{.ignition_endpoint}}?{{.query}}\u0026os=installed\\\"\\\\\" \u003e /mnt/grub.cfg'\nExecStart=/usr/bin/systemctl reboot\n[Install]\nWantedBy=multi-user.target\n"}]},"networkd":{},"passwd":{}}

examples/ignition/etcd.json

@@ -1 +0,0 @@
-{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"metadata.service","enable":true,"contents":"[Unit]\nDescription=Bare Metal Metadata Agent\n[Service]\nType=oneshot\nEnvironment=OUTPUT=/run/metadata/bootcfg\nExecStart=/usr/bin/mkdir --parent /run/metadata\nExecStart=/usr/bin/bash -c 'curl --url \"http://bootcfg.foo:8080/metadata?{{.query}}\" --retry 10 --output ${OUTPUT}'\n[Install]\nWantedBy=multi-user.target\n"},{"name":"etcd2.service","enable":true,"dropins":[{"name":"etcd-metadata.conf","contents":"[Unit]\nRequires=metadata.service\nAfter=metadata.service\n[Service]\nEnvironmentFile=/run/metadata/bootcfg\nExecStart=\nExecStart=/usr/bin/etcd2 \\\n  --advertise-client-urls=http://${IPV4_ADDRESS}:2379 \\\n  --initial-advertise-peer-urls=http://${IPV4_ADDRESS}:2380 \\\n  --listen-client-urls=http://0.0.0.0:2379 \\\n  --listen-peer-urls=http://${IPV4_ADDRESS}:2380 \\\n  --initial-cluster=${ETCD_INITIAL_CLUSTER}\n"}]}]},"networkd":{"units":[{"name":"00-{{.networkd_name}}.network","contents":"[Match]\nName={{.networkd_name}}\n[Network]\nGateway={{.networkd_gateway}}\nDNS={{.networkd_dns}}\nDNS=8.8.8.8\nAddress={{.networkd_address}}\n"}]},"passwd":{}}

examples/ignition/etcd.yaml

@@ -42,3 +42,11 @@ networkd:
         DNS={{.networkd_dns}}
         DNS=8.8.8.8
         Address={{.networkd_address}}
+
+{{ if .ssh_authorized_key }}
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - {{.ssh_authorized_key}}
+{{end}}

examples/ignition/etcd_proxy.json

@@ -1 +0,0 @@
-{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"metadata.service","enable":true,"contents":"[Unit]\nDescription=Bare Metal Metadata Agent\n[Service]\nType=oneshot\nEnvironment=OUTPUT=/run/metadata/bootcfg\nExecStart=/usr/bin/mkdir --parent /run/metadata\nExecStart=/usr/bin/bash -c 'curl --url http://bootcfg.foo:8080/metadata --retry 10 --output ${OUTPUT}'\n[Install]\nWantedBy=multi-user.target\n"},{"name":"etcd2.service","enable":true,"dropins":[{"name":"etcd-metadata.conf","contents":"[Unit]\nRequires=metadata.service\nAfter=metadata.service\n[Service]\nEnvironmentFile=/run/metadata/bootcfg\nExecStart=\nExecStart=/usr/bin/etcd2 \\\n  --proxy on\n  --listen-client-urls=http://localhost:2379 \\\n  --initial-cluster=${ETCD_INITIAL_CLUSTER}\n"}]}]},"networkd":{},"passwd":{}}

examples/ignition/etcd_proxy.yaml

@@ -29,3 +29,11 @@ systemd:
               --proxy on
               --listen-client-urls=http://localhost:2379 \
               --initial-cluster=${ETCD_INITIAL_CLUSTER}
+
+{{ if .ssh_authorized_key }}
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - {{.ssh_authorized_key}}
+{{end}}

examples/ignition/network.json

@@ -1 +0,0 @@
-{"ignitionVersion":1,"storage":{},"systemd":{"units":[{"name":"metadata.service","enable":true,"contents":"[Unit]\nDescription=Bare Metal Metadata Agent\n[Service]\nType=oneshot\nEnvironment=OUTPUT=/run/metadata/bootcfg\nExecStart=/usr/bin/mkdir --parent /run/metadata\nExecStart=/usr/bin/bash -c 'curl --url \"http://bootcfg.foo:8080/metadata?{{.query}}\" --retry 10 --output ${OUTPUT}'\n[Install]\nWantedBy=multi-user.target\n"}]},"networkd":{"units":[{"name":"00-{{.networkd_name}}.network","contents":"[Match]\nName={{.networkd_name}}\n[Network]\nGateway={{.networkd_gateway}}\nDNS={{.networkd_dns}}\nDNS=8.8.8.8\nAddress={{.networkd_address}}\n"}]},"passwd":{}}

examples/ignition/network.yaml

@@ -25,3 +25,11 @@ networkd:
         DNS={{.networkd_dns}}
         DNS=8.8.8.8
         Address={{.networkd_address}}
+
+{{ if .ssh_authorized_key }}
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - {{.ssh_authorized_key}}
+{{end}}

examples/specs/coreos-install/spec.json

@@ -10,5 +10,5 @@
     }
   },
   "cloud_id": "",
-  "ignition_id": "coreos-install.json"
+  "ignition_id": "coreos-install.yaml"
 }

examples/specs/etcd/spec.json

@@ -10,5 +10,5 @@
     }
   },
   "cloud_id": "",
-  "ignition_id": "etcd.json"
+  "ignition_id": "etcd.yaml"
 }

examples/specs/etcd_proxy/spec.json

@@ -10,5 +10,5 @@
     }
   },
   "cloud_id": "",
-  "ignition_id": "etcd_proxy.json"
+  "ignition_id": "etcd_proxy.yaml"
 }

examples/specs/kubernetes-master/spec.json

@@ -11,5 +11,5 @@
     }
   },
   "cloud_id": "kubernetes-master.sh",
-  "ignition_id": "network.json"
+  "ignition_id": "network.yaml"
 }

examples/specs/kubernetes-worker/spec.json

@@ -11,5 +11,5 @@
     }
   },
   "cloud_id": "kubernetes-worker.sh",
-  "ignition_id": "network.json"
+  "ignition_id": "network.yaml"
 }

scripts/gen-ignition

@@ -1,12 +0,0 @@
-#!/bin/bash -e
-# USAGE: ./scripts/gen-ignition src
-# Example: ./scripts/gen-ignition exampels/dev/ignition
-# Use coreos/fuze to transform a human-friendly *.yaml Ignition config to
-# machine-friendly JSON
-
-DEST=$1
-
-shopt -s nullglob
-for file in $DEST/*.yaml; do
-  fuze -in-file "$file" -out-file "${file%.yaml}.json"
-done