scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-11-04 12:18:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Return non-zero code on logexporter failures

Browse Source
This commit is contained in:
Antoni Zawodny
2020-10-14 18:26:06 +02:00
parent 6d01c5a589
commit e42a8dbe38
1 changed files with 18 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
cluster/log-dump/log-dump.sh
View File

@@ -63,6 +63,14 @@ readonly windows_node_otherfiles="C:\\Windows\\MEMORY.dmp"
# file descriptors for large clusters. # file descriptors for large clusters.
readonly max_dump_processes=25 readonly max_dump_processes=25
# Indicator variable whether we experienced a significant failure during
# logexporter creation or execution.
logexporter_failed=0
# Percentage of nodes that must be logexported successfully (otherwise the
# process will exit with a non-zero exit code).
readonly log_dump_expected_success_percentage="${LOG_DUMP_EXPECTED_SUCCESS_PERCENTAGE:-0}"
# TODO: Get rid of all the sourcing of bash dependencies eventually. # TODO: Get rid of all the sourcing of bash dependencies eventually.
function setup() { function setup() {
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
@@ -546,6 +554,7 @@ function dump_nodes_with_logexporter() {
echo 'Failed to create logexporter daemonset.. falling back to logdump through SSH' echo 'Failed to create logexporter daemonset.. falling back to logdump through SSH'
"${KUBECTL}" delete namespace "${logexporter_namespace}" || true "${KUBECTL}" delete namespace "${logexporter_namespace}" || true
dump_nodes "${NODE_NAMES[@]}" dump_nodes "${NODE_NAMES[@]}"
logexporter_failed=1
return return
fi fi
@@ -595,6 +604,7 @@ function dump_nodes_with_logexporter() {
echo 'Final attempt to list marker files failed.. falling back to logdump through SSH' echo 'Final attempt to list marker files failed.. falling back to logdump through SSH'
"${KUBECTL}" delete namespace "${logexporter_namespace}" || true "${KUBECTL}" delete namespace "${logexporter_namespace}" || true
dump_nodes "${NODE_NAMES[@]}" dump_nodes "${NODE_NAMES[@]}"
logexporter_failed=1
return return
fi fi
sleep 2 sleep 2
@@ -611,6 +621,11 @@ function dump_nodes_with_logexporter() {
done done
fi fi
# If less than a certain ratio of the nodes got logexported, report an error.
if [[ $(((${#NODE_NAMES[@]} - ${#failed_nodes[@]}) * 100)) -lt $((${#NODE_NAMES[@]} * log_dump_expected_success_percentage )) ]]; then
logexporter_failed=1
fi
# Delete the logexporter resources and dump logs for the failed nodes (if any) through SSH. # Delete the logexporter resources and dump logs for the failed nodes (if any) through SSH.
"${KUBECTL}" get pods --namespace "${logexporter_namespace}" || true "${KUBECTL}" get pods --namespace "${logexporter_namespace}" || true
"${KUBECTL}" delete namespace "${logexporter_namespace}" || true "${KUBECTL}" delete namespace "${logexporter_namespace}" || true
@@ -675,6 +690,9 @@ function main() {
fi fi
detect_node_failures detect_node_failures
if [[ ${logexporter_failed} -ne 0 && ${log_dump_expected_success_percentage} -gt 0 ]]; then
return 1
fi
} }
main main