Migrated fluentd to daemon set

2025-11-04 04:08:16 +00:00 · 2016-12-13 16:14:10 +01:00
parent 59ad9a30ca
commit a52637f09f
14 changed files with 141 additions and 92 deletions
--- a/build/lib/release.sh
+++ b/build/lib/release.sh
@@ -315,7 +315,6 @@ function kube::release::package_kube_manifests_tarball() {

  local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
  cp "${salt_dir}/cluster-autoscaler/cluster-autoscaler.manifest" "${dst_dir}/"
-  cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/"
  cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
  cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/"
  cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/"
--- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml
+++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml
@@ -0,0 +1,44 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: fluentd-es-v1.20
+  namespace: kube-system
+  labels:
+    k8s-app: fluentd-es
+    kubernetes.io/cluster-service: "true"
+    version: v1.20
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: fluentd-es
+        kubernetes.io/cluster-service: "true"
+        version: v1.20
+    spec:
+      containers:
+      - name: fluentd-es
+        image: gcr.io/google_containers/fluentd-elasticsearch:1.20
+        command:
+          - '/bin/sh'
+          - '-c'
+          - '/usr/sbin/td-agent 2>&1 >> /var/log/fluentd.log'
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml
@@ -0,0 +1,55 @@
+# please keep this file synchronized with cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: fluentd-gcp-v1.30
+  namespace: kube-system
+  labels:
+    k8s-app: fluentd-gcp
+    kubernetes.io/cluster-service: "true"
+    version: v1.30
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: fluentd-gcp
+        kubernetes.io/cluster-service: "true"
+        version: v1.30
+    spec:
+      containers:
+      - name: fluentd-gcp
+        image: gcr.io/google_containers/fluentd-gcp:1.30
+        # If fluentd consumes its own logs, the following situation may happen:
+        # fluentd fails to send a chunk to the server => writes it to the log =>
+        # tries to send this message to the server => fails to send a chunk and so on.
+        # Writing to a file, which is not exported to the back-end prevents it.
+        # It also allows to increase the fluentd verbosity by default.
+        command:
+          - '/bin/sh'
+          - '-c'
+          - '/run.sh $FLUENTD_ARGS 2>&1 >>/var/log/fluentd.log'
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: libsystemddir
+          mountPath: /host/lib
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: libsystemddir
+        hostPath:
+          path: /usr/lib64
--- a/cluster/gce/gci/configure-helper.sh
+++ b/cluster/gce/gci/configure-helper.sh
@@ -345,10 +345,13 @@ EOF

 # Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
 # to generate a kubeconfig file for the kubelet to securely connect to the apiserver.
+# Set REGISTER_MASTER_KUBELET to true if kubelet on the master node
+# should register to the apiserver.
 function create-master-kubelet-auth {
  # Only configure the kubelet on the master if the required variables are
  # set in the environment.
  if [[ -n "${KUBELET_APISERVER:-}" && -n "${KUBELET_CERT:-}" && -n "${KUBELET_KEY:-}" ]]; then
+    REGISTER_MASTER_KUBELET="true"
    create-kubelet-kubeconfig
  fi
 }
@@ -504,7 +507,7 @@ function start-kubelet {
  if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
    flags+=" --enable-debugging-handlers=false"
    flags+=" --hairpin-mode=none"
-    if [[ ! -z "${KUBELET_APISERVER:-}" && ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then
+    if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
      flags+=" --api-servers=https://${KUBELET_APISERVER}"
      flags+=" --register-schedulable=false"
      flags+=" --register-with-taints=node.alpha.kubernetes.io/ismaster=:NoSchedule"
@@ -1102,11 +1105,18 @@ function start-kube-addons {
    sed -i -e "s@{{ *pillar\['cluster_registry_disk_size'\] *}}@${CLUSTER_REGISTRY_DISK_SIZE}@g" "${registry_pvc_file}"
    sed -i -e "s@{{ *pillar\['cluster_registry_disk_name'\] *}}@${CLUSTER_REGISTRY_DISK}@g" "${registry_pvc_file}"
  fi
+  # TODO(piosz): figure out how to not run fluentd-es pod from fluentd daemon set on master.
+  #              Running fluentd-es on the master is pointless, as it can't communicate
+  #              with elasticsearch from there in the default configuration.
  if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
     [[ "${LOGGING_DESTINATION:-}" == "elasticsearch" ]] && \
     [[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]]; then
    setup-addon-manifests "addons" "fluentd-elasticsearch"
  fi
+  if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
+     [[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then
+    setup-addon-manifests "addons" "fluentd-gcp"
+  fi
  if [[ "${ENABLE_CLUSTER_UI:-}" == "true" ]]; then
    setup-addon-manifests "addons" "dashboard"
  fi
@@ -1127,17 +1137,14 @@ function start-kube-addons {
  cp "${src_dir}/kube-addon-manager.yaml" /etc/kubernetes/manifests
 }

-# Starts a fluentd static pod for logging.
-function start-fluentd {
+# Starts a fluentd static pod for logging for gcp in case master is not registered.
+function start-fluentd-static-pod {
  echo "Start fluentd pod"
-  if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]]; then
-    if [[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then
+  if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
+     [[ "${LOGGING_DESTINATION:-}" == "gcp" ]] && \
+     [[ "${KUBERNETES_MASTER:-}" == "true" ]] && \
+     [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then
    cp "${KUBE_HOME}/kube-manifests/kubernetes/fluentd-gcp.yaml" /etc/kubernetes/manifests/
-    elif [[ "${LOGGING_DESTINATION:-}" == "elasticsearch" && "${KUBERNETES_MASTER:-}" != "true" ]]; then
-      # Running fluentd-es on the master is pointless, as it can't communicate
-      # with elasticsearch from there in the default configuration.
-      cp "${KUBE_HOME}/kube-manifests/kubernetes/fluentd-es.yaml" /etc/kubernetes/manifests/
-    fi
  fi
 }

@@ -1280,6 +1287,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
  start-cluster-autoscaler
  start-lb-controller
  start-rescheduler
+  start-fluentd-static-pod
 else
  start-kube-proxy
  # Kube-registry-proxy.
@@ -1290,6 +1298,5 @@ else
    start-image-puller
  fi
 fi
-start-fluentd
 reset-motd
 echo "Done for the configuration for kubernetes"
--- a/cluster/gce/trusty/configure-helper.sh
+++ b/cluster/gce/trusty/configure-helper.sh
@@ -152,9 +152,7 @@ assemble_kubelet_flags() {
  fi
  if [ "${KUBERNETES_MASTER:-}" = "true" ]; then
    KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-debugging-handlers=false --hairpin-mode=none"
-    if [ ! -z "${KUBELET_APISERVER:-}" ] && \
-       [ ! -z "${KUBELET_CERT:-}" ] && \
-       [ ! -z "${KUBELET_KEY:-}" ]; then
+    if [ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]; then
      KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER} --register-schedulable=false"
    else
      KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=${MASTER_IP_RANGE}"
@@ -420,10 +418,13 @@ EOF

 # Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
 # to generate a kubeconfig file for the kubelet to securely connect to the apiserver.
+# Set REGISTER_MASTER_KUBELET to true if kubelet on the master node
+# should register to the apiserver.
 create_master_kubelet_auth() {
  # Only configure the kubelet on the master if the required variables are
  # set in the environment.
  if [ -n "${KUBELET_APISERVER:-}" ] && [ -n "${KUBELET_CERT:-}" ] && [ -n "${KUBELET_KEY:-}" ]; then
+    REGISTER_MASTER_KUBELET="true"
    create_kubelet_kubeconfig
  fi
 }
@@ -806,14 +807,13 @@ start-rescheduler() {
  fi
 }

-# Starts a fluentd static pod for logging.
-start_fluentd() {
-  if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then
-    if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then
+# Starts a fluentd static pod for logging for gcp in case master is not registered.
+start_fluentd_static_pod() {
+  if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
+     [[ "${LOGGING_DESTINATION:-}" == "gcp" ]] && \
+     [[ "${KUBERNETES_MASTER:-}" == "true" ]] && \
+     [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then
    cp /home/kubernetes/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/
-    elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then
-      cp /home/kubernetes/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/
-    fi
  fi
 }

@@ -934,6 +934,10 @@ start_kube_addons() {
     [ "${ENABLE_CLUSTER_LOGGING:-}" = "true" ]; then
    setup_addon_manifests "addons" "fluentd-elasticsearch"
  fi
+  if [ "${ENABLE_NODE_LOGGING:-}" = "true" ] && \
+     [ "${LOGGING_DESTINATION:-}" = "gcp" ] ; then
+    setup_addon_manifests "addons" "fluentd-gcp"
+  fi
  if [ "${ENABLE_CLUSTER_UI:-}" = "true" ]; then
    setup_addon_manifests "addons" "dashboard"
  fi
--- a/cluster/gce/trusty/master.yaml
+++ b/cluster/gce/trusty/master.yaml
@@ -187,7 +187,7 @@ script
 	. /etc/kube-configure-helper.sh
 	. /etc/kube-env
 	start_etcd_servers
-	start_fluentd
+	start_fluentd_static_pod
 	compute_master_manifest_variables
 	start_kube_apiserver
 	start_kube_controller_manager
--- a/cluster/gce/trusty/node.yaml
+++ b/cluster/gce/trusty/node.yaml
@@ -242,8 +242,6 @@ script

 	. /etc/kube-configure-helper.sh
 	. /etc/kube-env
-	# Fluentd
-	start_fluentd
 	# Kube-registry-proxy
 	if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then
 		cp /home/kubernetes/kube-manifests/kubernetes/kube-registry-proxy.yaml /etc/kubernetes/manifests/
--- a/cluster/saltbase/salt/README.md
+++ b/cluster/saltbase/salt/README.md
@@ -13,8 +13,6 @@ Config                                              | GCE   | Vagrant | AWS | Az
 [debian-auto-upgrades](debian-auto-upgrades/)       | M n   | M n     | M n | M n
 [docker](docker/)                                   | M n   | M n     | M n | M n
 [etcd](etcd/)                                       | M     | M       | M   | M
-[fluentd-es](fluentd-es/) (pillar conditional)      | M n   | M n     | M n | M n
-[fluentd-gcp](fluentd-gcp/) (pillar conditional)    | M n   | M n     | M n | M n
 [generate-cert](generate-cert/)                     | M     | M       | M   | M
 [kube-addons](kube-addons/)                         | M     | M       | M   | M
 [kube-apiserver](kube-apiserver/)                   | M     | M       | M   | M
--- a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml
+++ b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml
@@ -1,36 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: fluentd-elasticsearch
-  namespace: kube-system
-  labels:
-    k8s-app: fluentd-logging
-spec:
-  containers:
-  - name: fluentd-elasticsearch
-    image: gcr.io/google_containers/fluentd-elasticsearch:1.20
-    command:
-      - '/bin/sh'
-      - '-c'
-      - '/usr/sbin/td-agent 2>&1 >>/var/log/fluentd.log'
-    resources:
-      limits:
-        memory: 200Mi
-      requests:
-        cpu: 100m
-        memory: 200Mi
-    volumeMounts:
-    - name: varlog
-      mountPath: /var/log
-    - name: varlibdockercontainers
-      mountPath: /var/lib/docker/containers
-      readOnly: true
-  terminationGracePeriodSeconds: 30
-  volumes:
-  - name: varlog
-    hostPath:
-      path: /var/log
-  - name: varlibdockercontainers
-    hostPath:
-      path: /var/lib/docker/containers
-
--- a/cluster/saltbase/salt/fluentd-es/init.sls
+++ b/cluster/saltbase/salt/fluentd-es/init.sls
@@ -1,10 +0,0 @@
-{% if grains['roles'][0] != 'kubernetes-master' -%}
-/etc/kubernetes/manifests/fluentd-es.yaml:
-  file.managed:
-    - source: salt://fluentd-es/fluentd-es.yaml
-    - user: root
-    - group: root
-    - mode: 644
-    - makedirs: true
-    - dir_mode: 755
-{% endif %}
--- a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml
+++ b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml
@@ -1,5 +1,4 @@
-# This config should be kept as similar as possible to the one at
-# cluster/saltbase/salt/fluentd-gcp-gci/fluentd-gcp-gci.yaml
+# please keep this file synchronized with cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml
 apiVersion: v1
 kind: Pod
 metadata:
--- a/cluster/saltbase/salt/fluentd-gcp/init.sls
+++ b/cluster/saltbase/salt/fluentd-gcp/init.sls
@@ -1,3 +1,4 @@
+{% if grains.kubelet_api_servers is not defined -%}
 /etc/kubernetes/manifests/fluentd-gcp.yaml:
  file.managed:
    - source: salt://fluentd-gcp/fluentd-gcp.yaml
@@ -6,3 +7,4 @@
    - mode: 644
    - makedirs: true
    - dir_mode: 755
+{% endif %}
--- a/cluster/saltbase/salt/kube-addons/init.sls
+++ b/cluster/saltbase/salt/kube-addons/init.sls
@@ -138,11 +138,11 @@ addon-dir-create:
 {% endif %}

 {% if pillar.get('enable_node_logging', '').lower() == 'true'
-   and pillar.get('logging_destination', '').lower() == 'elasticsearch'
+   and 'logging_destination' in pillar
   and pillar.get('enable_cluster_logging', '').lower() == 'true' %}
-/etc/kubernetes/addons/fluentd-elasticsearch:
+/etc/kubernetes/addons/fluentd-{{ pillar.get('logging_destination') }}:
  file.recurse:
-    - source: salt://kube-addons/fluentd-elasticsearch
+    - source: salt://kube-addons/fluentd-{{ pillar.get('logging_destination') }}
    - include_pat: E@^.+\.yaml$
    - user: root
    - group: root
--- a/cluster/saltbase/salt/top.sls
+++ b/cluster/saltbase/salt/top.sls
@@ -32,13 +32,6 @@ base:
 {% else %}
    - kube-proxy
 {% endif %}
-{% if pillar.get('enable_node_logging', '').lower() == 'true' and pillar['logging_destination'] is defined %}
-  {% if pillar['logging_destination'] == 'elasticsearch' %}
-    - fluentd-es
-  {% elif pillar['logging_destination'] == 'gcp' %}
-    - fluentd-gcp
-  {% endif %}
-{% endif %}
 {% if pillar.get('enable_cluster_registry', '').lower() == 'true' %}
    - kube-registry-proxy
 {% endif %}
@@ -70,12 +63,8 @@ base:
    - kube-client-tools
    - kube-master-addons
    - kube-admission-controls
-{% if pillar.get('enable_node_logging', '').lower() == 'true' and pillar['logging_destination'] is defined %}
-  {% if pillar['logging_destination'] == 'elasticsearch' %}
-    - fluentd-es
-  {% elif pillar['logging_destination'] == 'gcp' %}
+{% if pillar.get('enable_node_logging', '').lower() == 'true' and pillar['logging_destination'] == 'gcp' %}
    - fluentd-gcp
-  {% endif %}
 {% endif %}
 {% if grains['cloud'] is defined and grains['cloud'] != 'vagrant' %}
    - logrotate