WIFI-1172 add JMX to Prometheus PoC (#51)

* add JMX to Prometheus PoC

* add JMX prometheus exporter to all Java services

tip-wlan/charts/opensync-gw-cloud/templates/deployment.yaml

@@ -27,6 +27,7 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
         - name: wait-for-services
           image: opsfleet/depends-on:latest
           args:
@@ -147,7 +148,7 @@ spec:
             - name: tip_wlan_maxHttpThreads
               value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
             - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"
           volumeMounts:
           - mountPath: /opt/tip-wlan/certs/client_keystore.jks
             name: certificates
@@ -169,6 +170,8 @@ spec:
             subPath: logback.xml
           - mountPath: {{ $file_store_path }}
             name: file-store-data
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
           ports:
             - name: {{ .Values.service.name1 }}
               containerPort: {{ .Values.service.port1 }}
@@ -187,6 +190,7 @@ spec:
               containerPort: {{ .Values.service.port5 }}
               protocol: TCP
             {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
@@ -216,3 +220,5 @@ spec:
       {{- else }}
         emptyDir: {}
       {{- end }}
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}

tip-wlan/charts/opensync-gw-cloud/templates/pod-monitor.yaml

@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}

tip-wlan/charts/wlan-port-forwarding-gateway-service/templates/deployment.yaml

@@ -22,6 +22,7 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
         - name: {{ include "common.name" . }}-readiness-opensync-gw
           image: alpine
           imagePullPolicy: {{ .Values.pullPolicy | default .Values.global.pullPolicy }}
@@ -75,6 +76,8 @@ spec:
               value: "{{ include "apDebugPortsStart" . }}"
             - name: EXT_PORT_RANGE_END
               value: "{{ sub (include "apDebugPortsEnd" . | atoi) 1 }}"
+            - name: JVM_MEM_OPTIONS
+              value: "{{ include "jmxPrometheus.jvmOpts" . }}"
           {{- if .Values.probes.enabled }}
           livenessProbe:
             httpGet:
@@ -117,6 +120,8 @@ spec:
           - mountPath: /app/port-forwarding-gateway/logback.xml
             name: configuration
             subPath: logback.xml
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
 
           ports:
             - name: {{ .Values.service.name1 }}
@@ -131,6 +136,7 @@ spec:
               protocol: TCP
             {{- end }}
             {{- include "container.dev.apDebugPorts" . | nindent 10 }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
@@ -154,3 +160,5 @@ spec:
         configMap:
             name: {{ include "common.fullname" . }}-config
             defaultMode: 0755
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}

tip-wlan/charts/wlan-port-forwarding-gateway-service/templates/pod-monitor.yaml

@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}

tip-wlan/charts/wlan-portal-service/templates/pod-monitor.yaml

@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}

tip-wlan/charts/wlan-portal-service/templates/statefulset.yaml

@@ -44,6 +44,8 @@ spec:
       serviceAccountName: {{ include "common.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      initContainers:
+      {{- include "jmxPrometheus.initContainer" . | nindent 6 }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
@@ -71,7 +73,7 @@ spec:
             - name: tip_wlan_maxHttpThreads
               value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
             - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"
           {{- if .Values.probes.enabled }}
           livenessProbe:
             httpGet:
@@ -116,6 +118,8 @@ spec:
             subPath: logback.xml
           - mountPath: {{ $file_store_path }}
             name: file-store-data
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
           ports:
             - name: {{ .Values.service.name1 }}
               containerPort: {{ .Values.service.port1 }}
@@ -128,6 +132,7 @@ spec:
               containerPort: {{ .Values.service.port3 }}
               protocol: TCP
             {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
@@ -154,6 +159,8 @@ spec:
       - name: file-store-data
         emptyDir: {}
       {{- end }}
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}
   {{- if .Values.persistence.enabled }}
   volumeClaimTemplates:
     - metadata:

tip-wlan/charts/wlan-prov-service/templates/deployment.yaml

@@ -23,6 +23,7 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
         - name: wait-for-services
           image: opsfleet/depends-on:latest
           args:
@@ -139,7 +140,7 @@ spec:
             - name: tip_wlan_maxHttpThreads
               value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
             - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"
             - name: singleDataSource_maxTotalConnections
               value: "{{ .Values.scalability.singleDataSource_maxTotalConnections }}"
             - name: singleDataSource_maxIdleConnections
@@ -202,6 +203,8 @@ spec:
           - mountPath: /app/prov/datasource.properties
             name: logback-config
             subPath: datasource.properties
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
           ports:
             - name: {{ .Values.service.name1 }}
               containerPort: {{ .Values.service.port1 }}
@@ -214,6 +217,7 @@ spec:
               containerPort: {{ .Values.service.port3 }}
               protocol: TCP
             {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
@@ -240,3 +244,5 @@ spec:
         emptyDir: {}
       - name: dshm
         emptyDir: {}
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}

tip-wlan/charts/wlan-prov-service/templates/pod-monitor.yaml

@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}

tip-wlan/charts/wlan-spc-service/templates/deployment.yaml

@@ -23,6 +23,7 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
         - name: wait-for-services
           image: opsfleet/depends-on:latest
           args:
@@ -82,7 +83,7 @@ spec:
             - name: tip_wlan_maxHttpThreads
               value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
             - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"
 
           {{- if .Values.probes.enabled }}
           livenessProbe:
@@ -129,6 +130,8 @@ spec:
           - mountPath: /app/spc/logback.xml
             name: logback-config
             subPath: logback.xml
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
           ports:
             - name: {{ .Values.service.name1 }}
               containerPort: {{ .Values.service.port1 }}
@@ -141,6 +144,7 @@ spec:
               containerPort: {{ .Values.service.port3 }}
               protocol: TCP
             {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
@@ -163,3 +167,5 @@ spec:
       - name: logback-config
         configMap:
             name: {{ include "common.fullname" . }}-log-config
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}

tip-wlan/charts/wlan-spc-service/templates/pod-monitor.yaml

@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}

tip-wlan/charts/wlan-ssc-service/templates/deployment.yaml

@@ -24,6 +24,7 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
         - name: wait-for-services
           image: opsfleet/depends-on:latest
           args:
@@ -174,7 +175,7 @@ spec:
             - name: tip_wlan_maxHttpThreads
               value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
             - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"
 
           {{- if .Values.probes.enabled }}
           livenessProbe:
@@ -227,6 +228,8 @@ spec:
           - mountPath: /app/ssc/cassandra-application.conf
             name: ssc-configurations
             subPath: cassandra-application.conf
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
           ports:
             - name: {{ .Values.service.name1 }}
               containerPort: {{ .Values.service.port1 }}
@@ -239,6 +242,7 @@ spec:
               containerPort: {{ .Values.service.port3 }}
               protocol: TCP
             {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}
@@ -261,3 +265,5 @@ spec:
       - name: ssc-configurations
         configMap:
             name: {{ include "common.fullname" . }}-ssc-config
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}

tip-wlan/charts/wlan-ssc-service/templates/pod-monitor.yaml

@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}

tip-wlan/resources/configs/jmx-prometheus-config.yml

@@ -0,0 +1,13 @@
+lowercaseOutputLabelNames: true
+lowercaseOutputName: true
+whitelistObjectNames: ["java.lang:type=OperatingSystem"]
+blacklistObjectNames: []
+rules:
+  - pattern: 'java.lang<type=OperatingSystem><>(committed_virtual_memory|free_physical_memory|free_swap_space|total_physical_memory|total_swap_space)_size:'
+    name: os_$1_bytes
+    type: GAUGE
+    attrNameSnakeCase: true
+  - pattern: 'java.lang<type=OperatingSystem><>((?!process_cpu_time)\w+):'
+    name: os_$1
+    type: GAUGE
+    attrNameSnakeCase: true

tip-wlan/templates/_prometheus-jmx.tpl

@@ -0,0 +1,93 @@
+{{/*
+    This template provides various definitions used for integrating the JMX Prometheus exporter
+  */}}
+
+{{- define "jmxPrometheus.agentDir" -}}
+/jmx-prometheus-exporter-dir
+{{- end -}}
+
+{{- define "jmxPrometheus.configPath" -}}
+/app/jmx-prometheus-config.yml
+{{- end -}}
+
+{{- define "jmxPrometheus.portNumber" -}}
+9404
+{{- end -}}
+
+{{- define "jmxPrometheus.initContainer" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: download-jmx-prometheus-exporter
+  image: alpine:latest
+  command:
+  - wget
+  args:
+  - -P
+  - {{ include "jmxPrometheus.agentDir" . }}
+  - https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.14.0/jmx_prometheus_javaagent-0.14.0.jar
+  volumeMounts:
+{{ include "jmxPrometheus.tmpVolumeMount" . | indent 2 }}
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.port" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: jmx-prometheus
+  containerPort: {{ include "jmxPrometheus.portNumber" . }}
+  protocol: TCP
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.tmpVolume" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: jmx-prometheus-exporter-dir
+  emptyDir: {}
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.tmpVolumeMount" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: jmx-prometheus-exporter-dir
+  mountPath: {{ include "jmxPrometheus.agentDir" . }}
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.configVolume" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: tip-common-jmx-prometheus-config
+  configMap:
+      name: tip-common-jmx-prometheus-config
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.configVolumeMount" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- mountPath: {{ include "jmxPrometheus.configPath" . }}
+  name: tip-common-jmx-prometheus-config
+  subPath: jmx-prometheus-config.yml
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.jvmOpts" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+-javaagent:{{ include "jmxPrometheus.agentDir" . }}/jmx_prometheus_javaagent-0.14.0.jar={{ include "jmxPrometheus.portNumber" . }}:{{ include "jmxPrometheus.configPath" . }}
+{{- end -}}
+{{- end -}}
+
+
+{{- define "jmxPrometheus.podMonitor" -}}
+{{- if and .Values.global.monitoring.enableJmxPrometheusMetrics .Values.global.monitoring.enablePrometheusPodMonitors -}}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "common.fullname" . }}
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    release: prometheus-operator
+spec:
+  selector:
+    matchLabels:
+      {{- include "common.selectorLabels" . | nindent 6 }}
+  podMetricsEndpoints:
+  - port: jmx-prometheus
+{{- end -}}
+{{- end -}}

tip-wlan/templates/prometheus-jmx-configmap.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tip-common-jmx-prometheus-config
+  namespace: {{ .Release.Namespace }}
+data:
+{{ (.Files.Glob "resources/configs/jmx-prometheus-config.yml").AsConfig | indent 2 }}

tip-wlan/values.yaml

@@ -72,6 +72,10 @@ global:
     username: tip-read
     password: tip-read
 
+  monitoring:
+    enableJmxPrometheusMetrics: false
+    enablePrometheusPodMonitors: false
+
 # DockerRegistry Secret
 createDockerRegistrySecret: true