WIFI-1172 add JMX to Prometheus PoC (#51)

* add JMX to Prometheus PoC * add JMX prometheus exporter to all Java services
2025-10-29 17:52:40 +00:00 · 2021-02-19 18:12:43 +01:00
parent 43233798b2
commit de8e8897f1
16 changed files with 168 additions and 6 deletions
--- a/tip-wlan/charts/opensync-gw-cloud/templates/deployment.yaml
+++ b/tip-wlan/charts/opensync-gw-cloud/templates/deployment.yaml
@@ -27,6 +27,7 @@ spec:
      securityContext:
        {{- toYaml .Values.podSecurityContext | nindent 8 }}
      initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
        - name: wait-for-services
          image: opsfleet/depends-on:latest
          args:
@@ -147,7 +148,7 @@ spec:
            - name: tip_wlan_maxHttpThreads
              value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
            - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"
          volumeMounts:
          - mountPath: /opt/tip-wlan/certs/client_keystore.jks
            name: certificates
@@ -169,6 +170,8 @@ spec:
            subPath: logback.xml
          - mountPath: {{ $file_store_path }}
            name: file-store-data
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
          ports:
            - name: {{ .Values.service.name1 }}
              containerPort: {{ .Values.service.port1 }}
@@ -187,6 +190,7 @@ spec:
              containerPort: {{ .Values.service.port5 }}
              protocol: TCP
            {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
@@ -216,3 +220,5 @@ spec:
      {{- else }}
        emptyDir: {}
      {{- end }}
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}
--- a/tip-wlan/charts/opensync-gw-cloud/templates/pod-monitor.yaml
+++ b/tip-wlan/charts/opensync-gw-cloud/templates/pod-monitor.yaml
@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}
--- a/tip-wlan/charts/wlan-port-forwarding-gateway-service/templates/deployment.yaml
+++ b/tip-wlan/charts/wlan-port-forwarding-gateway-service/templates/deployment.yaml
@@ -22,6 +22,7 @@ spec:
      securityContext:
        {{- toYaml .Values.podSecurityContext | nindent 8 }}
      initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
        - name: {{ include "common.name" . }}-readiness-opensync-gw
          image: alpine
          imagePullPolicy: {{ .Values.pullPolicy | default .Values.global.pullPolicy }}
@@ -75,6 +76,8 @@ spec:
              value: "{{ include "apDebugPortsStart" . }}"
            - name: EXT_PORT_RANGE_END
              value: "{{ sub (include "apDebugPortsEnd" . | atoi) 1 }}"
+            - name: JVM_MEM_OPTIONS
+              value: "{{ include "jmxPrometheus.jvmOpts" . }}"
          {{- if .Values.probes.enabled }}
          livenessProbe:
            httpGet:
@@ -117,6 +120,8 @@ spec:
          - mountPath: /app/port-forwarding-gateway/logback.xml
            name: configuration
            subPath: logback.xml
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}

          ports:
            - name: {{ .Values.service.name1 }}
@@ -131,6 +136,7 @@ spec:
              protocol: TCP
            {{- end }}
            {{- include "container.dev.apDebugPorts" . | nindent 10 }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
@@ -153,4 +159,6 @@ spec:
      - name: configuration
        configMap:
            name: {{ include "common.fullname" . }}-config
-            defaultMode: 0755
+            defaultMode: 0755
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}
--- a/tip-wlan/charts/wlan-port-forwarding-gateway-service/templates/pod-monitor.yaml
+++ b/tip-wlan/charts/wlan-port-forwarding-gateway-service/templates/pod-monitor.yaml
@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}
--- a/tip-wlan/charts/wlan-portal-service/templates/pod-monitor.yaml
+++ b/tip-wlan/charts/wlan-portal-service/templates/pod-monitor.yaml
@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}
--- a/tip-wlan/charts/wlan-portal-service/templates/statefulset.yaml
+++ b/tip-wlan/charts/wlan-portal-service/templates/statefulset.yaml
@@ -44,6 +44,8 @@ spec:
      serviceAccountName: {{ include "common.serviceAccountName" . }}
      securityContext:
        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      initContainers:
+      {{- include "jmxPrometheus.initContainer" . | nindent 6 }}
      containers:
        - name: {{ .Chart.Name }}
          securityContext:
@@ -71,7 +73,7 @@ spec:
            - name: tip_wlan_maxHttpThreads
              value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
            - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"
          {{- if .Values.probes.enabled }}
          livenessProbe:
            httpGet:
@@ -116,6 +118,8 @@ spec:
            subPath: logback.xml
          - mountPath: {{ $file_store_path }}
            name: file-store-data
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
          ports:
            - name: {{ .Values.service.name1 }}
              containerPort: {{ .Values.service.port1 }}
@@ -128,6 +132,7 @@ spec:
              containerPort: {{ .Values.service.port3 }}
              protocol: TCP
            {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
@@ -154,6 +159,8 @@ spec:
      - name: file-store-data
        emptyDir: {}
      {{- end }}
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}
  {{- if .Values.persistence.enabled }}
  volumeClaimTemplates:
    - metadata:
--- a/tip-wlan/charts/wlan-prov-service/templates/deployment.yaml
+++ b/tip-wlan/charts/wlan-prov-service/templates/deployment.yaml
@@ -23,6 +23,7 @@ spec:
      securityContext:
        {{- toYaml .Values.podSecurityContext | nindent 8 }}
      initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
        - name: wait-for-services
          image: opsfleet/depends-on:latest
          args:
@@ -139,7 +140,7 @@ spec:
            - name: tip_wlan_maxHttpThreads
              value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
            - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"
            - name: singleDataSource_maxTotalConnections
              value: "{{ .Values.scalability.singleDataSource_maxTotalConnections }}"
            - name: singleDataSource_maxIdleConnections
@@ -202,6 +203,8 @@ spec:
          - mountPath: /app/prov/datasource.properties
            name: logback-config
            subPath: datasource.properties
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
          ports:
            - name: {{ .Values.service.name1 }}
              containerPort: {{ .Values.service.port1 }}
@@ -214,6 +217,7 @@ spec:
              containerPort: {{ .Values.service.port3 }}
              protocol: TCP
            {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
@@ -240,3 +244,5 @@ spec:
        emptyDir: {}
      - name: dshm
        emptyDir: {}
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}
--- a/tip-wlan/charts/wlan-prov-service/templates/pod-monitor.yaml
+++ b/tip-wlan/charts/wlan-prov-service/templates/pod-monitor.yaml
@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}
--- a/tip-wlan/charts/wlan-spc-service/templates/deployment.yaml
+++ b/tip-wlan/charts/wlan-spc-service/templates/deployment.yaml
@@ -23,6 +23,7 @@ spec:
      securityContext:
        {{- toYaml .Values.podSecurityContext | nindent 8 }}
      initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
        - name: wait-for-services
          image: opsfleet/depends-on:latest
          args:
@@ -82,7 +83,7 @@ spec:
            - name: tip_wlan_maxHttpThreads
              value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
            - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"

          {{- if .Values.probes.enabled }}
          livenessProbe:
@@ -129,6 +130,8 @@ spec:
          - mountPath: /app/spc/logback.xml
            name: logback-config
            subPath: logback.xml
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
          ports:
            - name: {{ .Values.service.name1 }}
              containerPort: {{ .Values.service.port1 }}
@@ -141,6 +144,7 @@ spec:
              containerPort: {{ .Values.service.port3 }}
              protocol: TCP
            {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
@@ -163,3 +167,5 @@ spec:
      - name: logback-config
        configMap:
            name: {{ include "common.fullname" . }}-log-config
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}
--- a/tip-wlan/charts/wlan-spc-service/templates/pod-monitor.yaml
+++ b/tip-wlan/charts/wlan-spc-service/templates/pod-monitor.yaml
@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}
--- a/tip-wlan/charts/wlan-ssc-service/templates/deployment.yaml
+++ b/tip-wlan/charts/wlan-ssc-service/templates/deployment.yaml
@@ -24,6 +24,7 @@ spec:
      securityContext:
        {{- toYaml .Values.podSecurityContext | nindent 8 }}
      initContainers:
+        {{- include "jmxPrometheus.initContainer" . | nindent 8 }}
        - name: wait-for-services
          image: opsfleet/depends-on:latest
          args:
@@ -174,7 +175,7 @@ spec:
            - name: tip_wlan_maxHttpThreads
              value: "{{ .Values.scalability.tip_wlan_maxHttpThreads }}"
            - name: JVM_MEM_OPTIONS
-              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }}"
+              value: "{{ .Values.scalability.JVM_MEM_OPTIONS }} {{ include "jmxPrometheus.jvmOpts" . }}"

          {{- if .Values.probes.enabled }}
          livenessProbe:
@@ -227,6 +228,8 @@ spec:
          - mountPath: /app/ssc/cassandra-application.conf
            name: ssc-configurations
            subPath: cassandra-application.conf
+          {{- include "jmxPrometheus.configVolumeMount" . | nindent 10 }}
+          {{- include "jmxPrometheus.tmpVolumeMount" . | nindent 10 }}
          ports:
            - name: {{ .Values.service.name1 }}
              containerPort: {{ .Values.service.port1 }}
@@ -239,6 +242,7 @@ spec:
              containerPort: {{ .Values.service.port3 }}
              protocol: TCP
            {{- end }}
+            {{- include "jmxPrometheus.port" . | nindent 12 }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
@@ -261,3 +265,5 @@ spec:
      - name: ssc-configurations
        configMap:
            name: {{ include "common.fullname" . }}-ssc-config
+      {{- include "jmxPrometheus.configVolume" . | nindent 6 }}
+      {{- include "jmxPrometheus.tmpVolume" . | nindent 6 }}
--- a/tip-wlan/charts/wlan-ssc-service/templates/pod-monitor.yaml
+++ b/tip-wlan/charts/wlan-ssc-service/templates/pod-monitor.yaml
@@ -0,0 +1 @@
+{{ include "jmxPrometheus.podMonitor" . }}
--- a/tip-wlan/resources/configs/jmx-prometheus-config.yml
+++ b/tip-wlan/resources/configs/jmx-prometheus-config.yml
@@ -0,0 +1,13 @@
+lowercaseOutputLabelNames: true
+lowercaseOutputName: true
+whitelistObjectNames: ["java.lang:type=OperatingSystem"]
+blacklistObjectNames: []
+rules:
+  - pattern: 'java.lang<type=OperatingSystem><>(committed_virtual_memory|free_physical_memory|free_swap_space|total_physical_memory|total_swap_space)_size:'
+    name: os_$1_bytes
+    type: GAUGE
+    attrNameSnakeCase: true
+  - pattern: 'java.lang<type=OperatingSystem><>((?!process_cpu_time)\w+):'
+    name: os_$1
+    type: GAUGE
+    attrNameSnakeCase: true
--- a/tip-wlan/templates/_prometheus-jmx.tpl
+++ b/tip-wlan/templates/_prometheus-jmx.tpl
@@ -0,0 +1,93 @@
+{{/*
+    This template provides various definitions used for integrating the JMX Prometheus exporter
+  */}}
+
+{{- define "jmxPrometheus.agentDir" -}}
+/jmx-prometheus-exporter-dir
+{{- end -}}
+
+{{- define "jmxPrometheus.configPath" -}}
+/app/jmx-prometheus-config.yml
+{{- end -}}
+
+{{- define "jmxPrometheus.portNumber" -}}
+9404
+{{- end -}}
+
+{{- define "jmxPrometheus.initContainer" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: download-jmx-prometheus-exporter
+  image: alpine:latest
+  command:
+  - wget
+  args:
+  - -P
+  - {{ include "jmxPrometheus.agentDir" . }}
+  - https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.14.0/jmx_prometheus_javaagent-0.14.0.jar
+  volumeMounts:
+{{ include "jmxPrometheus.tmpVolumeMount" . | indent 2 }}
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.port" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: jmx-prometheus
+  containerPort: {{ include "jmxPrometheus.portNumber" . }}
+  protocol: TCP
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.tmpVolume" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: jmx-prometheus-exporter-dir
+  emptyDir: {}
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.tmpVolumeMount" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: jmx-prometheus-exporter-dir
+  mountPath: {{ include "jmxPrometheus.agentDir" . }}
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.configVolume" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- name: tip-common-jmx-prometheus-config
+  configMap:
+      name: tip-common-jmx-prometheus-config
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.configVolumeMount" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+- mountPath: {{ include "jmxPrometheus.configPath" . }}
+  name: tip-common-jmx-prometheus-config
+  subPath: jmx-prometheus-config.yml
+{{- end -}}
+{{- end -}}
+
+{{- define "jmxPrometheus.jvmOpts" -}}
+{{- if .Values.global.monitoring.enableJmxPrometheusMetrics -}}
+-javaagent:{{ include "jmxPrometheus.agentDir" . }}/jmx_prometheus_javaagent-0.14.0.jar={{ include "jmxPrometheus.portNumber" . }}:{{ include "jmxPrometheus.configPath" . }}
+{{- end -}}
+{{- end -}}
+
+
+{{- define "jmxPrometheus.podMonitor" -}}
+{{- if and .Values.global.monitoring.enableJmxPrometheusMetrics .Values.global.monitoring.enablePrometheusPodMonitors -}}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: {{ include "common.fullname" . }}
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    release: prometheus-operator
+spec:
+  selector:
+    matchLabels:
+      {{- include "common.selectorLabels" . | nindent 6 }}
+  podMetricsEndpoints:
+  - port: jmx-prometheus
+{{- end -}}
+{{- end -}}
--- a/tip-wlan/templates/prometheus-jmx-configmap.yaml
+++ b/tip-wlan/templates/prometheus-jmx-configmap.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tip-common-jmx-prometheus-config
+  namespace: {{ .Release.Namespace }}
+data:
+{{ (.Files.Glob "resources/configs/jmx-prometheus-config.yml").AsConfig | indent 2 }}
--- a/tip-wlan/values.yaml
+++ b/tip-wlan/values.yaml
@@ -72,6 +72,10 @@ global:
    username: tip-read
    password: tip-read

+  monitoring:
+    enableJmxPrometheusMetrics: false
+    enablePrometheusPodMonitors: false
+
 # DockerRegistry Secret
 createDockerRegistrySecret: true