diff --git a/.gitignore b/.gitignore index 16c68e1..a1b4e63 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ override.tf.json # example: *tfplan* .idea +certs/ \ No newline at end of file diff --git a/QUICKSTART.md b/QUICKSTART.md index 8d4c73a..c27a8d8 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -50,23 +50,16 @@ kubectl apply -f metallb/01-configuration.yml https://doc.traefik.io/traefik/v2.8/user-guides/crd-acme/ -## Create Traefik CRDs +## Create persistent volume for certs ```shell -kubectl apply -f traefik/00-crd-definition.yml -kubectl apply -f traefik/01-crd-rbac.yml +kubectl appy -f volumes/volumes.yml ``` -## Create Service +## Install using Helm ```shell -kubectl apply -f traefik/02-service.yml -``` - -## Create Deployment - -```shell -kubectl apply -f traefik/03-deployment.yml +helm install --values=helm/traefik-values.yaml traefik traefik/traefik ``` ## Create test application "whoami" with IngressRoutes diff --git a/README.md b/README.md index 36794ed..b568d09 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,10 @@ Install Cilium cilium install ``` +```shell +helm template --namespace kube-system cilium cilium/cilium --version 1.12.1 --set cluster.id=0,cluster.name=kubernetes,encryption.nodeEncryption=false,kubeProxyReplacement=disabled,operator.replicas=1,serviceAccounts.cilium.name=cilium,serviceAccounts.operator.name=cilium-operator,tunnel=vxlan +``` + Validate install ```shell @@ -89,15 +93,15 @@ cilium status ``` ### (Optional) Replace kube-proxy with Cilium [TODO] + https://docs.cilium.io/en/v1.12/gettingstarted/kubeproxy-free/ -*NB* Cluster should be initialised with +*NB* Cluster should be initialised with ```shell sudo kubeadm init --skip-phases=addon/kube-proxy ``` - ## MetalLB For load balancing @@ -115,11 +119,24 @@ Configure IP-pool and advertise as Level 2 https://metallb.universe.tf/configuration/ ```yaml -kubectl apply -f metallb/02-configuration +kubectl apply -f metallb/01-configuration.yml ``` # Traefik +## Install using Helm + +```shell +kubectl apply -f volumes/volumes.yml +``` + +**NB:** It appears we need the "volume-permissions" init container for Traefik if using `StorageClass` with +provisioner `kubernetes.io/no-provisioner` + +```shell +helm install --values=helm/traefik-values.yaml traefik traefik/traefik +``` + ## Traefik IngressRoute Custom Resource Definition (CRD) https://doc.traefik.io/traefik/v2.8/routing/providers/kubernetes-crd/ diff --git a/helm/traefik-values.yaml b/helm/traefik-values.yaml new file mode 100644 index 0000000..c180503 --- /dev/null +++ b/helm/traefik-values.yaml @@ -0,0 +1,30 @@ +deployment: + initContainers: + # The "volume-permissions" init container is required if you run into permission issues. + # Related issue: https://github.com/traefik/traefik/issues/6972 + - name: volume-permissions + image: busybox:1.31.1 + command: [ "sh", "-c", "chmod -Rv 600 /data/*" ] + volumeMounts: + - name: data + mountPath: /data + +additionalArguments: + - "--log.level=DEBUG" + - "--api.insecure" + +persistence: + enabled: true + name: data + accessMode: ReadWriteOnce + size: 128Mi + storageClass: cert-storage + path: /data + +certResolvers: + letsencrypt: + email: veghag@gmail.com + tlsChallenge: true + storage: /data/acme.json + # Remove staging server when it's working + caServer: https://acme-staging-v02.api.letsencrypt.org/directory \ No newline at end of file diff --git a/main.tf b/main.tf index 62e4b32..358c633 100644 --- a/main.tf +++ b/main.tf @@ -30,40 +30,66 @@ provider "helm" { # version = "1.11.5" #} +## Create namespace for Traefik resource "kubernetes_namespace" "traefik" { metadata { name = "traefik-system" } } +## Create StorageClass for local volumes +resource "kubernetes_storage_class" "cert-storage" { + metadata { + name = "cert-storage" + } + storage_provisioner = "kubernetes.io/no-provisioner" + volume_binding_mode = "WaitForFirstCustomer" +} + +## Create PersistentVolume for Traefik certs +resource "kubernetes_persistent_volume" "traefik-cert-pv" { + metadata { + name = "traefik-cert-pv" + } + spec { + capacity = { + storage = "128Mi" + } + volume_mode = "Filesystem" + access_modes = ["ReadWriteOnce"] + persistent_volume_reclaim_policy = "Retain" + storage_class_name = "cert-storage" + persistent_volume_source { + local { + path = "/mnt/sdb1/terrakube/certs" + } + } + node_affinity { + required { + node_selector_term { + match_expressions { + key = "kubernetes.io/hostname" + operator = "In" + values = ["ratatoskr"] + } + } + } + } + } +} + +## Install Traefik resource "helm_release" "traefik" { name = "traefik" repository = "https://helm.traefik.io/traefik" chart = "traefik" - namespace = "traefik" - version = "10.20.0" + namespace = kubernetes_namespace.traefik.metadata.0.name + #version = "10.30.1" + + values = [file("traefik2/custom-values.yaml")] } -#resource "kubernetes_service" "traefik" { -# metadata { -# name = "traefik" -# namespace = kubernetes_namespace.traefik.metadata.0.name -# } -# spec { -# selector = { -# # Standard Helm chart label to locate pods -# "app.kubernetes.io/name" = helm_release.traefik.name -# } -# -# type = "LoadBalancer" -# port { -# protocol = "TCP" -# port = 80 # External exposed port to reach container -# target_port = 9000 # Internal exposed port of container -# } -# } -#} //resource "kubernetes_namespace" "test" { // metadata { diff --git a/traefik/custom-values.yaml b/traefik/custom-values.yaml deleted file mode 100644 index 0ba55df..0000000 --- a/traefik/custom-values.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# File custom-values.yml -## Install with "helm install --values=traefik/custom-values.yml traefik traefik/traefik -additionalArguments: - - "--log.level=DEBUG" - - "--api.insecure" - - "--accesslog" - - "--certificatesresolvers.myresolver.acme.tlschallenge" - - "--certificatesresolvers.myresolver.acme.email=veghag@gmail.com" - - "--certificatesresolvers.myresolver.acme.storage=acme.json" \ No newline at end of file diff --git a/traefik/00-crd-definition.yml b/traefik2/00-crd-definition.yml similarity index 100% rename from traefik/00-crd-definition.yml rename to traefik2/00-crd-definition.yml diff --git a/traefik/01-crd-rbac.yml b/traefik2/01-crd-rbac.yml similarity index 100% rename from traefik/01-crd-rbac.yml rename to traefik2/01-crd-rbac.yml diff --git a/traefik/02-service.yml b/traefik2/02-service.yml similarity index 76% rename from traefik/02-service.yml rename to traefik2/02-service.yml index 70a7dfe..173c7df 100644 --- a/traefik/02-service.yml +++ b/traefik2/02-service.yml @@ -8,12 +8,14 @@ spec: ports: - protocol: TCP name: web - port: 8000 + port: 80 + targetPort: 8000 - protocol: TCP name: admin port: 8080 - protocol: TCP name: websecure - port: 4443 + port: 443 + targetPort: 8443 selector: app: traefik diff --git a/traefik/03-deployment.yml b/traefik2/03-deployment.yml similarity index 73% rename from traefik/03-deployment.yml rename to traefik2/03-deployment.yml index c048418..a8cda32 100644 --- a/traefik/03-deployment.yml +++ b/traefik2/03-deployment.yml @@ -28,17 +28,18 @@ spec: - name: traefik image: traefik:v2.8 args: + - "--log.level=DEBUG" - --api.insecure - --accesslog - --entrypoints.web.Address=:8000 - --entrypoints.websecure.Address=:8443 - --providers.kubernetescrd - - --certificatesresolvers.myresolver.acme.tlschallenge - - --certificatesresolvers.myresolver.acme.email=veghag@gmail.com - - --certificatesresolvers.myresolver.acme.storage=acme.json + - --certificatesresolvers.letsencrypt.acme.tlschallenge + - --certificatesresolvers.letsencrypt.acme.email=veghag@gmail.com + - --certificatesresolvers.letsencrypt.acme.storage=acme.json # Please note that this is the staging Let's Encrypt server. # Once you get things working, you should remove that whole line altogether. - #- --certificatesresolvers.myresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory + #- --certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory ports: - name: web containerPort: 8000 diff --git a/traefik2/values.yaml b/traefik2/values.yaml new file mode 100644 index 0000000..5faafe6 --- /dev/null +++ b/traefik2/values.yaml @@ -0,0 +1,288 @@ +# Default values for Traefik +image: + name: traefik + # defaults to appVersion + tag: "" + pullPolicy: IfNotPresent + +# +# Configure the deployment +# +deployment: + enabled: true + # Can be either Deployment or DaemonSet + kind: Deployment + # Number of pods of the deployment (only applies when kind == Deployment) + replicas: 1 + # Number of old history to retain to allow rollback (If not set, default Kubernetes value is set to 10) + # revisionHistoryLimit: 1 + # Amount of time (in seconds) before Kubernetes will send the SIGKILL signal if Traefik does not shut down + terminationGracePeriodSeconds: 60 + # The minimum number of seconds Traefik needs to be up and running before the DaemonSet/Deployment controller considers it available + minReadySeconds: 0 + # Additional deployment annotations (e.g. for jaeger-operator sidecar injection) + annotations: {} + # Additional deployment labels (e.g. for filtering deployment by custom labels) + labels: {} + # Additional pod annotations (e.g. for mesh injection or prometheus scraping) + podAnnotations: {} + # Additional Pod labels (e.g. for filtering Pod by custom labels) + podLabels: {} + # Additional containers (e.g. for metric offloading sidecars) + additionalContainers: [] + # https://docs.datadoghq.com/developers/dogstatsd/unix_socket/?tab=host + # - name: socat-proxy + # image: alpine/socat:1.0.5 + # args: ["-s", "-u", "udp-recv:8125", "unix-sendto:/socket/socket"] + # volumeMounts: + # - name: dsdsocket + # mountPath: /socket + # Additional volumes available for use with initContainers and additionalContainers + additionalVolumes: [] + # - name: dsdsocket + # hostPath: + # path: /var/run/statsd-exporter + # Additional initContainers (e.g. for setting file permission as shown below) + initContainers: [] + # The "volume-permissions" init container is required if you run into permission issues. + # Related issue: https://github.com/traefik/traefik/issues/6972 + # - name: volume-permissions + # image: busybox:1.31.1 + # command: ["sh", "-c", "chmod -Rv 600 /data/*"] + # volumeMounts: + # - name: data + # mountPath: /data + # Use process namespace sharing + shareProcessNamespace: false + # Custom pod DNS policy. Apply if `hostNetwork: true` + # dnsPolicy: ClusterFirstWithHostNet + # Additional imagePullSecrets + imagePullSecrets: [] + # - name: myRegistryKeySecretName + # Pod lifecycle actions + lifecycle: {} + # preStop: + # exec: + # command: ["/bin/sh", "-c", "sleep 40"] + # postStart: + # httpGet: + # path: /ping + # port: 9000 + # host: localhost + # scheme: HTTP + +# Pod disruption budget +podDisruptionBudget: + enabled: false + # maxUnavailable: 1 + # maxUnavailable: 33% + # minAvailable: 0 + # minAvailable: 25% + +# Use ingressClass. Ignored if Traefik version < 2.3 / kubernetes < 1.18.x +ingressClass: + # true is not unit-testable yet, pending https://github.com/rancher/helm-unittest/pull/12 + enabled: false + isDefaultClass: false + # Use to force a networking.k8s.io API Version for certain CI/CD applications. E.g. "v1beta1" + fallbackApiVersion: "" + +# Activate Pilot integration +pilot: + enabled: false + token: "" + # Toggle Pilot Dashboard + # dashboard: false + +# Enable experimental features +experimental: + http3: + enabled: false + plugins: + enabled: false + kubernetesGateway: + enabled: false + gateway: + enabled: true + # certificate: + # group: "core" + # kind: "Secret" + # name: "mysecret" + # By default, Gateway would be created to the Namespace you are deploying Traefik to. + # You may create that Gateway in another namespace, setting its name below: + # namespace: default + +# Create an IngressRoute for the dashboard +ingressRoute: + dashboard: + enabled: true + # Additional ingressRoute annotations (e.g. for kubernetes.io/ingress.class) + annotations: {} + # Additional ingressRoute labels (e.g. for filtering IngressRoute by custom labels) + labels: {} + +rollingUpdate: + maxUnavailable: 1 + maxSurge: 1 + +# Customize liveness and readiness probe values. +readinessProbe: + failureThreshold: 1 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + +livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + +# +# Configure providers +# +providers: + kubernetesCRD: + enabled: true + allowCrossNamespace: false + allowExternalNameServices: false + allowEmptyServices: false + # ingressClass: traefik-internal + # labelSelector: environment=production,method=traefik + namespaces: [] + # - "default" + + kubernetesIngress: + enabled: true + allowExternalNameServices: false + allowEmptyServices: false + # ingressClass: traefik-internal + # labelSelector: environment=production,method=traefik + namespaces: [] + # - "default" + # IP used for Kubernetes Ingress endpoints + publishedService: + enabled: false + # Published Kubernetes Service to copy status from. Format: namespace/servicename + # By default this Traefik service + # pathOverride: "" + +# +# Add volumes to the traefik pod. The volume name will be passed to tpl. +# This can be used to mount a cert pair or a configmap that holds a config.toml file. +# After the volume has been mounted, add the configs into traefik by using the `additionalArguments` list below, eg: +# additionalArguments: +# - "--providers.file.filename=/config/dynamic.toml" +# - "--ping" +# - "--ping.entrypoint=web" +volumes: [] +# - name: public-cert +# mountPath: "/certs" +# type: secret +# - name: '{{ printf "%s-configs" .Release.Name }}' +# mountPath: "/config" +# type: configMap + +# Additional volumeMounts to add to the Traefik container +additionalVolumeMounts: [] + # For instance when using a logshipper for access logs + # - name: traefik-logs +# mountPath: /var/log/traefik + +# Logs +# https://docs.traefik.io/observability/logs/ +logs: + # Traefik logs concern everything that happens to Traefik itself (startup, configuration, events, shutdown, and so on). + general: + # By default, the logs use a text format (common), but you can + # also ask for the json format in the format option + # format: json + # By default, the level is set to ERROR. Alternative logging levels are DEBUG, PANIC, FATAL, ERROR, WARN, and INFO. + level: ERROR + access: + # To enable access logs + enabled: false + # By default, logs are written using the Common Log Format (CLF). + # To write logs in JSON, use json in the format option. + # If the given format is unsupported, the default (CLF) is used instead. + # format: json + # To write the logs in an asynchronous fashion, specify a bufferingSize option. + # This option represents the number of log lines Traefik will keep in memory before writing + # them to the selected output. In some cases, this option can greatly help performances. + # bufferingSize: 100 + # Filtering https://docs.traefik.io/observability/access-logs/#filtering + filters: {} + # statuscodes: "200,300-302" + # retryattempts: true + # minduration: 10ms + # Fields + # https://docs.traefik.io/observability/access-logs/#limiting-the-fieldsincluding-headers + fields: + general: + defaultmode: keep + names: {} + # Examples: + # ClientUsername: drop + headers: + defaultmode: drop + names: {} + # Examples: + # User-Agent: redact + # Authorization: drop + # Content-Type: keep + +metrics: + # datadog: + # address: 127.0.0.1:8125 + # influxdb: + # address: localhost:8089 + # protocol: udp + prometheus: + entryPoint: metrics + # addRoutersLabels: true + # statsd: + # address: localhost:8125 + +tracing: {} + # instana: + # localAgentHost: 127.0.0.1 + # localAgentPort: 42699 + # logLevel: info + # enableAutoProfile: true + # datadog: + # localAgentHostPort: 127.0.0.1:8126 + # debug: false + # globalTag: "" + # prioritySampling: false + # jaeger: + # samplingServerURL: http://localhost:5778/sampling + # samplingType: const + # samplingParam: 1.0 + # localAgentHostPort: 127.0.0.1:6831 + # gen128Bit: false + # propagation: jaeger + # traceContextHeaderName: uber-trace-id + # disableAttemptReconnecting: true + # collector: + # endpoint: "" + # user: "" + # password: "" + # zipkin: + # httpEndpoint: http://localhost:9411/api/v2/spans + # sameSpan: false + # id128Bit: true + # sampleRate: 1.0 + # haystack: + # localAgentHost: 127.0.0.1 + # localAgentPort: 35000 + # globalTag: "" + # traceIDHeaderName: "" + # parentIDHeaderName: "" + # spanIDHeaderName: "" + # baggagePrefixHeaderName: "" + # elastic: + # serverURL: http://localhost:8200 + # secretToken: "" +# serviceEnvironment: "" diff --git a/volumes/volumes.yml b/volumes/volumes.yml new file mode 100644 index 0000000..4f16400 --- /dev/null +++ b/volumes/volumes.yml @@ -0,0 +1,29 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: cert-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: traefik-certs-pv +spec: + capacity: + storage: 128Mi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: cert-storage + local: + path: "/mnt/sdb1/terrakube/certs" + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - ratatoskr \ No newline at end of file diff --git a/whoami/00-whoami.yml b/whoami/00-whoami.yml index 8ccd94f..b74732f 100644 --- a/whoami/00-whoami.yml +++ b/whoami/00-whoami.yml @@ -3,6 +3,7 @@ apiVersion: v1 kind: Service metadata: + namespace: default name: whoami spec: @@ -52,27 +53,10 @@ spec: entryPoints: - websecure routes: - - match: Host(`whoami.ratatoskr.myddns.rocks`) + - match: Host(`whoami.stonegarden.dev`) kind: Rule services: - name: whoami port: 80 tls: - certResolver: myresolver - ---- -## IngressRoute for insecure whoami address -#apiVersion: traefik.containo.us/v1alpha1 -#kind: IngressRoute -#metadata: -# name: simpleingressroute -# namespace: default -#spec: -# entryPoints: -# - web -# routes: -# - match: Host(`test.ratatoskr.myddns.rocks`) && PathPrefix(`/notls`) -# kind: Rule -# services: -# - name: whoami -# port: 80 + certResolver: letsencrypt \ No newline at end of file diff --git a/whoami/01-whoami2.yml b/whoami/01-whoami2.yml new file mode 100644 index 0000000..845cfcd --- /dev/null +++ b/whoami/01-whoami2.yml @@ -0,0 +1,68 @@ +# Namespace for whoami +apiVersion: v1 +kind: Namespace +metadata: + name: whoami + +--- +# Service for exposing deployment of whoami +apiVersion: v1 +kind: Service +metadata: + namespace: whoami + name: whoami + +spec: + type: LoadBalancer + ports: + - protocol: TCP + name: web + port: 80 + selector: + app: whoami + +--- +# Deployment of whoami +kind: Deployment +apiVersion: apps/v1 +metadata: + namespace: whoami + name: whoami + labels: + app: whoami + +spec: + replicas: 2 + selector: + matchLabels: + app: whoami + template: + metadata: + labels: + app: whoami + spec: + containers: + - name: whoami + image: traefik/whoami + ports: + - name: web + containerPort: 80 + +--- +# IngressRoute for secure whoami address +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRoute +metadata: + name: ingressroutetls + namespace: whoami +spec: + entryPoints: + - websecure + routes: + - match: Host(`whoami2.stonegarden.dev`) + kind: Rule + services: + - name: whoami + port: 80 + tls: + certResolver: letsencrypt \ No newline at end of file