From b482b57e6ea85eceef02787cb2e8eb884cceb80c Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Mon, 17 Feb 2020 00:59:58 -0800 Subject: [PATCH] mariadb: avoid state management thread death The mariadb container launches two threads in addition to the mysql daemon, one to mantain a configmap containing the Galera Cluster state, and the other to handle leader elections. These threads die if they suffer any exceptions talking to the kubernetes apiserver. This can happen sometimes, e.g. when a k8s control node reboots. This change logs and ignores the kubernetes.client.rest.ApiException, allowing the threads to retry and hopefully succeed once the k8s api becomes available. Change-Id: I5745a763bb07f719d83a41c1f27be2b76ce998e9 --- mariadb/templates/bin/_start.py.tpl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mariadb/templates/bin/_start.py.tpl b/mariadb/templates/bin/_start.py.tpl index 03654d6f..5640da1d 100644 --- a/mariadb/templates/bin/_start.py.tpl +++ b/mariadb/templates/bin/_start.py.tpl @@ -705,7 +705,10 @@ def check_if_i_lead(): def monitor_cluster(): """Function to kick off grastate configmap updating thread""" while True: - update_grastate_configmap() + try: + update_grastate_configmap() + except kubernetes.client.rest.ApiException as error: + logger.error("Error updating grastate configmap: {0}".format(error)) time.sleep(state_configmap_update_period) @@ -723,7 +726,10 @@ def launch_cluster_monitor(): def leader_election(): """Function to kick off leader election thread""" while True: - deadmans_leader_election() + try: + deadmans_leader_election() + except kubernetes.client.rest.ApiException as error: + logger.error("Error electing leader: {0}".format(error)) time.sleep(cluster_leader_ttl / 2)