feat(volumes): Persistent storage for Traefik certs

.gitignore

@@ -29,3 +29,4 @@ override.tf.json
 # example: *tfplan*
 
 .idea
+certs/

QUICKSTART.md

@@ -50,23 +50,16 @@ kubectl apply -f metallb/01-configuration.yml
 
 https://doc.traefik.io/traefik/v2.8/user-guides/crd-acme/
 
-## Create Traefik CRDs
+## Create persistent volume for certs
 
 ```shell
-kubectl apply -f traefik/00-crd-definition.yml
-kubectl apply -f traefik/01-crd-rbac.yml
+kubectl appy -f volumes/volumes.yml
 ```
 
-## Create Service
+## Install using Helm
 
 ```shell
-kubectl apply -f traefik/02-service.yml
-```
-
-## Create Deployment
-
-```shell
-kubectl apply -f traefik/03-deployment.yml
+helm install --values=helm/traefik-values.yaml traefik traefik/traefik
 ```
 
 ## Create test application "whoami" with IngressRoutes

README.md

@@ -82,6 +82,10 @@ Install Cilium
 cilium install
 ```
 
+```shell
+helm template --namespace kube-system cilium cilium/cilium --version 1.12.1 --set cluster.id=0,cluster.name=kubernetes,encryption.nodeEncryption=false,kubeProxyReplacement=disabled,operator.replicas=1,serviceAccounts.cilium.name=cilium,serviceAccounts.operator.name=cilium-operator,tunnel=vxlan
+```
+
 Validate install
 
 ```shell
@@ -89,15 +93,15 @@ cilium status
 ```
 
 ### (Optional) Replace kube-proxy with Cilium [TODO]
+
 https://docs.cilium.io/en/v1.12/gettingstarted/kubeproxy-free/
 
-*NB* Cluster should be initialised with 
+*NB* Cluster should be initialised with
 
 ```shell
 sudo kubeadm init --skip-phases=addon/kube-proxy
 ```
 
-
 ## MetalLB
 
 For load balancing
@@ -115,11 +119,24 @@ Configure IP-pool and advertise as Level 2
 https://metallb.universe.tf/configuration/
 
 ```yaml
-kubectl apply -f metallb/02-configuration
+kubectl apply -f metallb/01-configuration.yml
 ```
 
 # Traefik
 
+## Install using Helm
+
+```shell
+kubectl apply -f volumes/volumes.yml
+```
+
+**NB:** It appears we need the "volume-permissions" init container for Traefik if using `StorageClass` with
+provisioner `kubernetes.io/no-provisioner`
+
+```shell
+helm install --values=helm/traefik-values.yaml traefik traefik/traefik
+```
+
 ## Traefik IngressRoute Custom Resource Definition (CRD)
 
 https://doc.traefik.io/traefik/v2.8/routing/providers/kubernetes-crd/

helm/traefik-values.yaml

@@ -0,0 +1,30 @@
+deployment:
+  initContainers:
+    # The "volume-permissions" init container is required if you run into permission issues.
+    # Related issue: https://github.com/traefik/traefik/issues/6972
+    - name: volume-permissions
+      image: busybox:1.31.1
+      command: [ "sh", "-c", "chmod -Rv 600 /data/*" ]
+      volumeMounts:
+        - name: data
+          mountPath: /data
+
+additionalArguments:
+  - "--log.level=DEBUG"
+  - "--api.insecure"
+
+persistence:
+  enabled: true
+  name: data
+  accessMode: ReadWriteOnce
+  size: 128Mi
+  storageClass: cert-storage
+  path: /data
+
+certResolvers:
+  letsencrypt:
+    email: veghag@gmail.com
+    tlsChallenge: true
+    storage: /data/acme.json
+    # Remove staging server when it's working
+    caServer: https://acme-staging-v02.api.letsencrypt.org/directory

main.tf

@@ -30,40 +30,66 @@ provider "helm" {
 #  version    = "1.11.5"
 #}
 
+## Create namespace for Traefik
 resource "kubernetes_namespace" "traefik" {
   metadata {
     name = "traefik-system"
   }
 }
 
+## Create StorageClass for local volumes
+resource "kubernetes_storage_class" "cert-storage" {
+  metadata {
+    name = "cert-storage"
+  }
+  storage_provisioner = "kubernetes.io/no-provisioner"
+  volume_binding_mode = "WaitForFirstCustomer"
+}
+
+## Create PersistentVolume for Traefik certs
+resource "kubernetes_persistent_volume" "traefik-cert-pv" {
+  metadata {
+    name = "traefik-cert-pv"
+  }
+  spec {
+    capacity                         = {
+      storage = "128Mi"
+    }
+    volume_mode                      = "Filesystem"
+    access_modes                     = ["ReadWriteOnce"]
+    persistent_volume_reclaim_policy = "Retain"
+    storage_class_name               = "cert-storage"
+    persistent_volume_source {
+      local {
+        path = "/mnt/sdb1/terrakube/certs"
+      }
+    }
+    node_affinity {
+      required {
+        node_selector_term {
+          match_expressions {
+            key      = "kubernetes.io/hostname"
+            operator = "In"
+            values   = ["ratatoskr"]
+          }
+        }
+      }
+    }
+  }
+}
+
+## Install Traefik
 resource "helm_release" "traefik" {
   name = "traefik"
 
   repository = "https://helm.traefik.io/traefik"
   chart      = "traefik"
-  namespace  = "traefik"
-  version    = "10.20.0"
+  namespace  = kubernetes_namespace.traefik.metadata.0.name
+  #version    = "10.30.1"
+
+  values = [file("traefik2/custom-values.yaml")]
 }
 
-#resource "kubernetes_service" "traefik" {
-#  metadata {
-#    name      = "traefik"
-#    namespace = kubernetes_namespace.traefik.metadata.0.name
-#  }
-#  spec {
-#    selector = {
-#      # Standard Helm chart label to locate pods
-#      "app.kubernetes.io/name" = helm_release.traefik.name
-#    }
-#
-#    type = "LoadBalancer"
-#    port {
-#      protocol    = "TCP"
-#      port        = 80    # External exposed port to reach container
-#      target_port = 9000  # Internal exposed port of container
-#    }
-#  }
-#}
 
 //resource "kubernetes_namespace" "test" {
 //  metadata {

traefik/custom-values.yaml

@@ -1,9 +0,0 @@
-# File custom-values.yml
-## Install with "helm install --values=traefik/custom-values.yml traefik traefik/traefik
-additionalArguments:
-  - "--log.level=DEBUG"
-  - "--api.insecure"
-  - "--accesslog"
-  - "--certificatesresolvers.myresolver.acme.tlschallenge"
-  - "--certificatesresolvers.myresolver.acme.email=veghag@gmail.com"
-  - "--certificatesresolvers.myresolver.acme.storage=acme.json"

traefik2/02-service.yml

@@ -8,12 +8,14 @@ spec:
   ports:
     - protocol: TCP
       name: web
-      port: 8000
+      port: 80
+      targetPort: 8000
     - protocol: TCP
       name: admin
       port: 8080
     - protocol: TCP
       name: websecure
-      port: 4443
+      port: 443
+      targetPort: 8443
   selector:
     app: traefik

traefik2/03-deployment.yml

@@ -28,17 +28,18 @@ spec:
         - name: traefik
           image: traefik:v2.8
           args:
+            - "--log.level=DEBUG"
             - --api.insecure
             - --accesslog
             - --entrypoints.web.Address=:8000
             - --entrypoints.websecure.Address=:8443
             - --providers.kubernetescrd
-            - --certificatesresolvers.myresolver.acme.tlschallenge
-            - --certificatesresolvers.myresolver.acme.email=veghag@gmail.com
-            - --certificatesresolvers.myresolver.acme.storage=acme.json
+            - --certificatesresolvers.letsencrypt.acme.tlschallenge
+            - --certificatesresolvers.letsencrypt.acme.email=veghag@gmail.com
+            - --certificatesresolvers.letsencrypt.acme.storage=acme.json
             # Please note that this is the staging Let's Encrypt server.
             # Once you get things working, you should remove that whole line altogether.
-            #- --certificatesresolvers.myresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory
+            #- --certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory
           ports:
             - name: web
               containerPort: 8000

traefik2/values.yaml

@@ -0,0 +1,288 @@
+# Default values for Traefik
+image:
+  name: traefik
+  # defaults to appVersion
+  tag: ""
+  pullPolicy: IfNotPresent
+
+#
+# Configure the deployment
+#
+deployment:
+  enabled: true
+  # Can be either Deployment or DaemonSet
+  kind: Deployment
+  # Number of pods of the deployment (only applies when kind == Deployment)
+  replicas: 1
+  # Number of old history to retain to allow rollback (If not set, default Kubernetes value is set to 10)
+  # revisionHistoryLimit: 1
+  # Amount of time (in seconds) before Kubernetes will send the SIGKILL signal if Traefik does not shut down
+  terminationGracePeriodSeconds: 60
+  # The minimum number of seconds Traefik needs to be up and running before the DaemonSet/Deployment controller considers it available
+  minReadySeconds: 0
+  # Additional deployment annotations (e.g. for jaeger-operator sidecar injection)
+  annotations: {}
+  # Additional deployment labels (e.g. for filtering deployment by custom labels)
+  labels: {}
+  # Additional pod annotations (e.g. for mesh injection or prometheus scraping)
+  podAnnotations: {}
+  # Additional Pod labels (e.g. for filtering Pod by custom labels)
+  podLabels: {}
+  # Additional containers (e.g. for metric offloading sidecars)
+  additionalContainers: []
+    # https://docs.datadoghq.com/developers/dogstatsd/unix_socket/?tab=host
+    # - name: socat-proxy
+    # image: alpine/socat:1.0.5
+    # args: ["-s", "-u", "udp-recv:8125", "unix-sendto:/socket/socket"]
+    # volumeMounts:
+    #   - name: dsdsocket
+  #     mountPath: /socket
+  # Additional volumes available for use with initContainers and additionalContainers
+  additionalVolumes: []
+    # - name: dsdsocket
+    #   hostPath:
+  #     path: /var/run/statsd-exporter
+  # Additional initContainers (e.g. for setting file permission as shown below)
+  initContainers: []
+    # The "volume-permissions" init container is required if you run into permission issues.
+    # Related issue: https://github.com/traefik/traefik/issues/6972
+    # - name: volume-permissions
+    #   image: busybox:1.31.1
+    #   command: ["sh", "-c", "chmod -Rv 600 /data/*"]
+    #   volumeMounts:
+    #     - name: data
+  #       mountPath: /data
+  # Use process namespace sharing
+  shareProcessNamespace: false
+  # Custom pod DNS policy. Apply if `hostNetwork: true`
+  # dnsPolicy: ClusterFirstWithHostNet
+  # Additional imagePullSecrets
+  imagePullSecrets: []
+  # - name: myRegistryKeySecretName
+  # Pod lifecycle actions
+  lifecycle: {}
+    # preStop:
+    #   exec:
+    #     command: ["/bin/sh", "-c", "sleep 40"]
+    # postStart:
+    #   httpGet:
+    #     path: /ping
+    #     port: 9000
+    #     host: localhost
+  #     scheme: HTTP
+
+# Pod disruption budget
+podDisruptionBudget:
+  enabled: false
+  # maxUnavailable: 1
+  # maxUnavailable: 33%
+  # minAvailable: 0
+  # minAvailable: 25%
+
+# Use ingressClass. Ignored if Traefik version < 2.3 / kubernetes < 1.18.x
+ingressClass:
+  # true is not unit-testable yet, pending https://github.com/rancher/helm-unittest/pull/12
+  enabled: false
+  isDefaultClass: false
+  # Use to force a networking.k8s.io API Version for certain CI/CD applications. E.g. "v1beta1"
+  fallbackApiVersion: ""
+
+# Activate Pilot integration
+pilot:
+  enabled: false
+  token: ""
+  # Toggle Pilot Dashboard
+  # dashboard: false
+
+# Enable experimental features
+experimental:
+  http3:
+    enabled: false
+  plugins:
+    enabled: false
+  kubernetesGateway:
+    enabled: false
+    gateway:
+      enabled: true
+    # certificate:
+    #   group: "core"
+    #   kind: "Secret"
+    #   name: "mysecret"
+    # By default, Gateway would be created to the Namespace you are deploying Traefik to.
+    # You may create that Gateway in another namespace, setting its name below:
+    # namespace: default
+
+# Create an IngressRoute for the dashboard
+ingressRoute:
+  dashboard:
+    enabled: true
+    # Additional ingressRoute annotations (e.g. for kubernetes.io/ingress.class)
+    annotations: {}
+    # Additional ingressRoute labels (e.g. for filtering IngressRoute by custom labels)
+    labels: {}
+
+rollingUpdate:
+  maxUnavailable: 1
+  maxSurge: 1
+
+# Customize liveness and readiness probe values.
+readinessProbe:
+  failureThreshold: 1
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  successThreshold: 1
+  timeoutSeconds: 2
+
+livenessProbe:
+  failureThreshold: 3
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  successThreshold: 1
+  timeoutSeconds: 2
+
+#
+# Configure providers
+#
+providers:
+  kubernetesCRD:
+    enabled: true
+    allowCrossNamespace: false
+    allowExternalNameServices: false
+    allowEmptyServices: false
+    # ingressClass: traefik-internal
+    # labelSelector: environment=production,method=traefik
+    namespaces: []
+    # - "default"
+
+  kubernetesIngress:
+    enabled: true
+    allowExternalNameServices: false
+    allowEmptyServices: false
+    # ingressClass: traefik-internal
+    # labelSelector: environment=production,method=traefik
+    namespaces: []
+    # - "default"
+    # IP used for Kubernetes Ingress endpoints
+    publishedService:
+      enabled: false
+      # Published Kubernetes Service to copy status from. Format: namespace/servicename
+      # By default this Traefik service
+      # pathOverride: ""
+
+#
+# Add volumes to the traefik pod. The volume name will be passed to tpl.
+# This can be used to mount a cert pair or a configmap that holds a config.toml file.
+# After the volume has been mounted, add the configs into traefik by using the `additionalArguments` list below, eg:
+# additionalArguments:
+# - "--providers.file.filename=/config/dynamic.toml"
+# - "--ping"
+# - "--ping.entrypoint=web"
+volumes: []
+# - name: public-cert
+#   mountPath: "/certs"
+#   type: secret
+# - name: '{{ printf "%s-configs" .Release.Name }}'
+#   mountPath: "/config"
+#   type: configMap
+
+# Additional volumeMounts to add to the Traefik container
+additionalVolumeMounts: []
+  # For instance when using a logshipper for access logs
+  # - name: traefik-logs
+#   mountPath: /var/log/traefik
+
+# Logs
+# https://docs.traefik.io/observability/logs/
+logs:
+  # Traefik logs concern everything that happens to Traefik itself (startup, configuration, events, shutdown, and so on).
+  general:
+    # By default, the logs use a text format (common), but you can
+    # also ask for the json format in the format option
+    # format: json
+    # By default, the level is set to ERROR. Alternative logging levels are DEBUG, PANIC, FATAL, ERROR, WARN, and INFO.
+    level: ERROR
+  access:
+    # To enable access logs
+    enabled: false
+    # By default, logs are written using the Common Log Format (CLF).
+    # To write logs in JSON, use json in the format option.
+    # If the given format is unsupported, the default (CLF) is used instead.
+    # format: json
+    # To write the logs in an asynchronous fashion, specify a bufferingSize option.
+    # This option represents the number of log lines Traefik will keep in memory before writing
+    # them to the selected output. In some cases, this option can greatly help performances.
+    # bufferingSize: 100
+    # Filtering https://docs.traefik.io/observability/access-logs/#filtering
+    filters: {}
+      # statuscodes: "200,300-302"
+      # retryattempts: true
+    # minduration: 10ms
+    # Fields
+    # https://docs.traefik.io/observability/access-logs/#limiting-the-fieldsincluding-headers
+    fields:
+      general:
+        defaultmode: keep
+        names: {}
+          # Examples:
+        # ClientUsername: drop
+      headers:
+        defaultmode: drop
+        names: {}
+          # Examples:
+          # User-Agent: redact
+          # Authorization: drop
+        # Content-Type: keep
+
+metrics:
+  # datadog:
+  #   address: 127.0.0.1:8125
+  # influxdb:
+  #   address: localhost:8089
+  #   protocol: udp
+  prometheus:
+    entryPoint: metrics
+  #  addRoutersLabels: true
+  # statsd:
+  #   address: localhost:8125
+
+tracing: {}
+  # instana:
+  #   localAgentHost: 127.0.0.1
+  #   localAgentPort: 42699
+  #   logLevel: info
+  #   enableAutoProfile: true
+  # datadog:
+  #   localAgentHostPort: 127.0.0.1:8126
+  #   debug: false
+  #   globalTag: ""
+  #   prioritySampling: false
+  # jaeger:
+  #   samplingServerURL: http://localhost:5778/sampling
+  #   samplingType: const
+  #   samplingParam: 1.0
+  #   localAgentHostPort: 127.0.0.1:6831
+  #   gen128Bit: false
+  #   propagation: jaeger
+  #   traceContextHeaderName: uber-trace-id
+  #   disableAttemptReconnecting: true
+  #   collector:
+  #      endpoint: ""
+  #      user: ""
+  #      password: ""
+  # zipkin:
+  #   httpEndpoint: http://localhost:9411/api/v2/spans
+  #   sameSpan: false
+  #   id128Bit: true
+  #   sampleRate: 1.0
+  # haystack:
+  #   localAgentHost: 127.0.0.1
+  #   localAgentPort: 35000
+  #   globalTag: ""
+  #   traceIDHeaderName: ""
+  #   parentIDHeaderName: ""
+  #   spanIDHeaderName: ""
+  #   baggagePrefixHeaderName: ""
+  # elastic:
+  #   serverURL: http://localhost:8200
+  #   secretToken: ""
+#   serviceEnvironment: ""

volumes/volumes.yml

@@ -0,0 +1,29 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: cert-storage
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: traefik-certs-pv
+spec:
+  capacity:
+    storage: 128Mi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: cert-storage
+  local:
+    path: "/mnt/sdb1/terrakube/certs"
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - ratatoskr

whoami/00-whoami.yml

@@ -3,6 +3,7 @@
 apiVersion: v1
 kind: Service
 metadata:
+  namespace: default
   name: whoami
 
 spec:
@@ -52,27 +53,10 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`whoami.ratatoskr.myddns.rocks`)
+    - match: Host(`whoami.stonegarden.dev`)
       kind: Rule
       services:
         - name: whoami
           port: 80
   tls:
-    certResolver: myresolver
-
----
-## IngressRoute for insecure whoami address
-#apiVersion: traefik.containo.us/v1alpha1
-#kind: IngressRoute
-#metadata:
-#  name: simpleingressroute
-#  namespace: default
-#spec:
-#  entryPoints:
-#    - web
-#  routes:
-#    - match: Host(`test.ratatoskr.myddns.rocks`) && PathPrefix(`/notls`)
-#      kind: Rule
-#      services:
-#        - name: whoami
-#          port: 80
+    certResolver: letsencrypt

whoami/01-whoami2.yml

@@ -0,0 +1,68 @@
+# Namespace for whoami
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: whoami
+
+---
+# Service for exposing deployment of whoami
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: whoami
+  name: whoami
+
+spec:
+  type: LoadBalancer
+  ports:
+    - protocol: TCP
+      name: web
+      port: 80
+  selector:
+    app: whoami
+
+---
+# Deployment of whoami
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  namespace: whoami
+  name: whoami
+  labels:
+    app: whoami
+
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: whoami
+  template:
+    metadata:
+      labels:
+        app: whoami
+    spec:
+      containers:
+        - name: whoami
+          image: traefik/whoami
+          ports:
+            - name: web
+              containerPort: 80
+
+---
+# IngressRoute for secure whoami address
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: ingressroutetls
+  namespace: whoami
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`whoami2.stonegarden.dev`)
+      kind: Rule
+      services:
+        - name: whoami
+          port: 80
+  tls:
+    certResolver: letsencrypt