diff --git a/packages/system/linstor/hack/plunger/plunger-drbd-logger.sh b/packages/system/linstor/hack/plunger/plunger-drbd-logger.sh new file mode 100755 index 00000000..297c66b8 --- /dev/null +++ b/packages/system/linstor/hack/plunger/plunger-drbd-logger.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +terminate() { + echo "Caught signal, terminating" + exit 0 +} + +trap terminate SIGINT SIGQUIT SIGTERM + +echo "Started logger of bad DRBD statuses" + +while true; do + + all_devices="$(drbdsetup status --json 2>/dev/null)" + unhealthy_devices="$(echo "${all_devices}" | jq -r ' + map(select( + .suspended != false or + ."force-io-failures" != false or + # Diskless can be legit when allowRemoteVolumeAccess is set to "true" + # TODO how does forced-diskless look? + ([.devices[]."disk-state"] | inside(["UpToDate", "Consistent", "Diskless"]) | not) or + (.connections[]."connection-state" != "Connected") or + # congested is not an alarm but an indicator + (.connections[]."congested" != false) or + (.connections[].peer_devices[]."replication-state" != "Established") + )) + | unique + ')" + if [ "${unhealthy_devices}" != '[]' ]; then + echo -e "Unhealthy devices:\n${unhealthy_devices}" + fi + + sleep 30 & + pid=$! + wait $pid + +done diff --git a/packages/system/linstor/templates/satellites-plunger.yaml b/packages/system/linstor/templates/satellites-plunger.yaml index b3abd152..e3cfa3b1 100644 --- a/packages/system/linstor/templates/satellites-plunger.yaml +++ b/packages/system/linstor/templates/satellites-plunger.yaml @@ -47,6 +47,29 @@ spec: - mountPath: /scripts name: script-volume readOnly: true + - name: drbd-logger + image: {{ include "cozy.linstor.version.satellite" . }} + command: + - "/scripts/plunger-drbd-logger.sh" + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + drop: + - ALL + privileged: true + readOnlyRootFilesystem: false + volumeMounts: + - mountPath: /dev + name: dev + - mountPath: /var/lib/drbd + name: var-lib-drbd + - mountPath: /var/lib/linstor.d + name: var-lib-linstor-d + - mountPath: /scripts + name: script-volume + readOnly: true volumes: - name: script-volume configMap: