Merge pull request #71180 from monotek/fluentd-elasticsearch

fluentd-elasticsearch - update fluentd to 1.3.3 & added filter_parser for json fields in es

cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml

@@ -1,7 +1,7 @@
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: fluentd-es-config-v0.1.6
+  name: fluentd-es-config-v0.2.0
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
@@ -143,6 +143,39 @@ data:
       max_lines 1000
     </match>
 
+    # Concatenate multi-line logs
+    <filter **>
+      @id filter_concat
+      @type concat
+      key message
+      multiline_end_regexp /\n$/
+      separator ""
+    </filter>
+
+    # Enriches records with Kubernetes metadata
+    <filter kubernetes.**>
+      @id filter_kubernetes_metadata
+      @type kubernetes_metadata
+    </filter>
+
+    # Fixes json fields in Elasticsearch
+    <filter kubernetes.**>
+      @id filter_parser
+      @type parser
+      key_name log
+      reserve_data true
+      remove_key_name_field true
+      <parse>
+        @type multi_format
+        <pattern>
+          format json
+        </pattern>
+        <pattern>
+          format none
+        </pattern>
+      </parse>
+    </filter>
+
   system.input.conf: |-
     # Example:
     # 2015-12-21 23:17:22,066 [salt.state       ][INFO    ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
@@ -376,6 +409,7 @@ data:
   forward.input.conf: |-
     # Takes the messages sent over TCP
     <source>
+      @id forward
       @type forward
     </source>
 
@@ -383,15 +417,18 @@ data:
     # Prometheus Exporter Plugin
     # input plugin that exports metrics
     <source>
+      @id prometheus
       @type prometheus
     </source>
 
     <source>
+      @id monitor_agent
       @type monitor_agent
     </source>
 
     # input plugin that collects metrics from MonitorAgent
     <source>
+      @id prometheus_monitor
       @type prometheus_monitor
       <labels>
         host ${hostname}
@@ -400,6 +437,7 @@ data:
 
     # input plugin that collects metrics for output plugin
     <source>
+      @id prometheus_output_monitor
       @type prometheus_output_monitor
       <labels>
         host ${hostname}
@@ -408,6 +446,7 @@ data:
 
     # input plugin that collects metrics for in_tail plugin
     <source>
+      @id prometheus_tail_monitor
       @type prometheus_tail_monitor
       <labels>
         host ${hostname}
@@ -415,24 +454,11 @@ data:
     </source>
 
   output.conf: |-
-    # Enriches records with Kubernetes metadata
-    <filter kubernetes.**>
-      @type kubernetes_metadata
-    </filter>
-
-    # Concatenate multi-line logs
-    <filter **>
-      @type concat
-      key message
-      multiline_end_regexp /\n$/
-      separator ""
-    </filter>
-
     <match **>
       @id elasticsearch
       @type elasticsearch
       @log_level info
-      type_name fluentd
+      type_name _doc
       include_tag_key true
       host elasticsearch-logging
       port 9200

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -48,24 +48,24 @@ roleRef:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v2.2.1
+  name: fluentd-es-v2.4.0
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
-    version: v2.2.1
+    version: v2.4.0
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
       k8s-app: fluentd-es
-      version: v2.2.1
+      version: v2.4.0
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
         kubernetes.io/cluster-service: "true"
-        version: v2.2.1
+        version: v2.4.0
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -77,7 +77,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: k8s.gcr.io/fluentd-elasticsearch:v2.2.0
+        image: k8s.gcr.io/fluentd-elasticsearch:v2.4.0
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q
@@ -107,4 +107,4 @@ spec:
           path: /var/lib/docker/containers
       - name: config-volume
         configMap:
-          name: fluentd-es-config-v0.1.6
+          name: fluentd-es-config-v0.2.0

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile

@@ -1,12 +1,12 @@
 source 'https://rubygems.org'
 
-gem 'fluentd', '<=1.2.4'
-gem 'activesupport', '~>5.2.1'
+gem 'activesupport', '~>5.2.2'
+gem 'fluentd', '<=1.3.3'
 gem 'fluent-plugin-concat', '~>2.3.0'
 gem 'fluent-plugin-detect-exceptions', '~>0.0.11'
-gem 'fluent-plugin-elasticsearch', '~>2.11.5'
-gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0'
+gem 'fluent-plugin-elasticsearch', '~>3.0.1'
+gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.1.6'
 gem 'fluent-plugin-multi-format-parser', '~>1.0.0'
-gem 'fluent-plugin-prometheus', '~>1.0.1'
+gem 'fluent-plugin-prometheus', '~>1.3.0'
 gem 'fluent-plugin-systemd', '~>1.0.1'
-gem 'oj', '~>3.6.5'
+gem 'oj', '~>3.7.6'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = staging-k8s.gcr.io
 IMAGE = fluentd-elasticsearch
-TAG = v2.3.1
+TAG = v2.4.0
 
 build:
 	docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .