Update stats display on dashboard to docker-only situation

2026-01-27 10:19:34 +00:00 · 2022-08-06 15:15:54 +02:00
parent 762ff9bb04
commit 9935b056c3
5 changed files with 235 additions and 49 deletions
--- a/24
+++ b/24
@@ -132,18 +132,18 @@ case $txt in
    svc=$(docker inspect $(docker ps --format "{{.Names}}" | grep -- -control-) | grep -i started | sed -e "s/[^:]*:\(.*\)/\1/" | sed -e "s/.*\"\(.*\)\".*/\1/")
    boulder=$(docker inspect $(docker ps --format "{{.Names}}" | grep -- -boulder-) | grep -i started | grep -v depends_on | sed -e "s/[^:]*:\(.*\)/\1/" | sed -e "s/.*\"\(.*\)\".*/\1/")
    labca=$(docker inspect $(docker ps --format "{{.Names}}" | grep -- -labca-) | grep -i started | grep -v depends_on | sed -e "s/[^:]*:\(.*\)/\1/" | sed -e "s/.*\"\(.*\)\".*/\1/")
-    echo "$nginx|$svc|$boulder|$labca"
+    mysql=$(docker inspect $(docker ps --format "{{.Names}}" | grep -- -bmysql-) | grep -i started | grep -v depends_on | sed -e "s/[^:]*:\(.*\)/\1/" | sed -e "s/.*\"\(.*\)\".*/\1/")
+    echo "$nginx|$svc|$boulder|$labca|$mysql"
+    exit 0
+    ;;
+"log-uptime")
+    timezone=$(cat /etc/timezone)
+    uptime=$(uptime -s)
+    echo "$timezone|$uptime"
    exit 0
    ;;
 "log-stats")
-    timezone=$(cat /etc/timezone)
-    uptime=$(uptime -s)
-    procs=$(ps -ef --no-headers | wc -l)
-    total=$(free -b --si | grep ":" | head -1 | perl -p0e 's/.*?\s+(\d+)\s+.*/$1/')
-    avail=$(free -b --si | grep ":" | head -1 | perl -p0e 's/.*\s+(\d+)$/$1/')
-    let used=$total-$avail
-    echo "$timezone|$uptime|$procs|$used|$avail"
-    exit 0
+    docker stats --no-stream -a | grep " boulder-"
    ;;
 "revoke-cert")
    read serial
@@ -183,6 +183,12 @@ case $txt in
    sleep 15
    wait_up $PS_LABCA &>>$LOGFILE
    ;;
+"mysql-restart")
+    cd /boulder
+    set +e
+    COMPOSE_HTTP_TIMEOUT=120 docker-compose restart bmysql
+    set -e
+    ;;
 "svc-restart")
    cd /boulder
    set +e
--- a/gui/dashboard.go
+++ b/gui/dashboard.go
@@ -166,6 +166,15 @@ func _parseComponents(data string) []Component {

 	parts := strings.Split(data, "|")

+	if len(parts) < 5 {
+		components = append(components, Component{Name: "Boulder (ACME)"})
+		components = append(components, Component{Name: "Controller"})
+		components = append(components, Component{Name: "LabCA Application"})
+		components = append(components, Component{Name: "MySQL Database"})
+		components = append(components, Component{Name: "NGINX Webserver"})
+		return components
+	}
+
 	nginx, err := time.Parse(time.RFC3339Nano, parts[0])
 	nginxReal := ""
 	nginxNice := "stopped"
@@ -206,9 +215,20 @@ func _parseComponents(data string) []Component {
 		labcaClass = ""
 	}

+	mysql, err := time.Parse(time.RFC3339Nano, parts[4])
+	mysqlReal := ""
+	mysqlNice := "stopped"
+	mysqlClass := "error"
+	if err == nil {
+		mysqlReal = mysql.Format("02-Jan-2006 15:04:05 MST")
+		mysqlNice = humanize.RelTime(mysql, time.Now(), "", "")
+		mysqlClass = ""
+	}
+
 	components = append(components, Component{Name: "Boulder (ACME)", Timestamp: boulderReal, TimestampRel: boulderNice, Class: boulderClass})
 	components = append(components, Component{Name: "Controller", Timestamp: svcReal, TimestampRel: svcNice, Class: svcClass})
 	components = append(components, Component{Name: "LabCA Application", Timestamp: labcaReal, TimestampRel: labcaNice, Class: labcaClass})
+	components = append(components, Component{Name: "MySQL Database", Timestamp: mysqlReal, TimestampRel: mysqlNice, Class: mysqlClass})
 	components = append(components, Component{Name: "NGINX Webserver", Timestamp: nginxReal, TimestampRel: nginxNice, Class: nginxClass})

 	return components
@@ -222,7 +242,16 @@ type Stat struct {
 	Class string
 }

-func _parseStats(data string) []Stat {
+// The stats as reported by docker
+type DockerStat struct {
+	Name     string
+	MemUsage uint64
+	MemLimit uint64
+	MemPerc  float64
+	Pids     uint64
+}
+
+func _parseStats(data string, components []Component) []Stat {
 	var stats []Stat

 	if data[len(data)-1:] == "\n" {
@@ -249,24 +278,99 @@ func _parseStats(data string) []Stat {
 	}
 	stats = append(stats, Stat{Name: "System Uptime", Hint: sinceReal, Value: sinceNice})

-	memUsed, err := strconv.ParseUint(parts[3], 10, 64)
-	if err != nil {
-		memUsed = 0
+	if components == nil {
+		return stats
 	}
-	memAvail, err := strconv.ParseUint(parts[4], 10, 64)
-	if err != nil {
-		memAvail = 0
+
+	stats = append(stats, Stat{Name: "Memory Limit", Value: ""})
+	stats = append(stats, Stat{Name: "Memory Used", Value: ""})
+	stats = append(stats, Stat{Name: "Memory Used [%]", Value: ""})
+
+	return stats
+}
+
+// What we return as json
+type AjaxStat struct {
+	Stat
+	MemoryUsed string
+	MemoryPerc string
+	NumPids    int
+}
+
+func parseDockerStats(data string) []AjaxStat {
+	var stats []AjaxStat
+
+	dockerStats := []DockerStat{}
+	rawStats := strings.Split(data, "\n")
+	for _, rawStat := range rawStats {
+		if len(rawStat) > 0 {
+			elms := strings.Fields(rawStat)
+			if len(elms) > 13 {
+				stat := DockerStat{}
+				// CONTAINER ID   NAME                CPU %     MEM USAGE / LIMIT   MEM %     NET I/O           BLOCK I/O   PIDS
+				// 817bdaec6daf   boulder-boulder-1   0.07%     255.3MiB / 1GiB     24.93%    1.18MB / 339kB    0B / 0B     158
+				stat.Name = elms[1]
+				x, err := humanize.ParseBigBytes(elms[3])
+				if err == nil {
+					stat.MemUsage = x.Uint64()
+				}
+				x, err = humanize.ParseBigBytes(elms[5])
+				if err == nil {
+					stat.MemLimit = x.Uint64()
+				}
+				y, err := strconv.ParseFloat(strings.Replace(elms[6], "%", "", -1), 64)
+				if err == nil {
+					stat.MemPerc = y
+				}
+				p, err := strconv.ParseUint(elms[13], 10, 64)
+				if err == nil {
+					stat.Pids = p
+				}
+				dockerStats = append(dockerStats, stat)
+			}
+		}
+	}
+
+	// Update the component stats
+	totalMemUsage := uint64(0)
+	for _, docker := range dockerStats {
+		stat := AjaxStat{}
+		if strings.Contains(docker.Name, "-boulder-") {
+			stat.Name = "Boulder (ACME)"
+		}
+		if strings.Contains(docker.Name, "-control-") {
+			stat.Name = "Controller"
+		}
+		if strings.Contains(docker.Name, "-labca-") {
+			stat.Name = "LabCA Application"
+		}
+		if strings.Contains(docker.Name, "-nginx-") {
+			stat.Name = "NGINX Webserver"
+		}
+		if strings.Contains(docker.Name, "-bmysql-") {
+			stat.Name = "MySQL Database"
+		}
+
+		stat.MemoryUsed = humanize.IBytes(docker.MemUsage)
+		stat.MemoryPerc = fmt.Sprintf("%s%%", humanize.FtoaWithDigits(docker.MemPerc, 1))
+		stat.NumPids = int(docker.Pids)
+
+		stats = append(stats, stat)
+
+		totalMemUsage += docker.MemUsage
 	}

 	percMem := float64(0)
-	if (memUsed + memAvail) > 0 {
-		percMem = float64(100) * float64(memUsed) / float64(memUsed+memAvail)
+	if (dockerStats[0].MemLimit) > 0 {
+		percMem = float64(100) * float64(totalMemUsage) / float64(dockerStats[0].MemLimit)
 	}

-	usedHuman := humanize.IBytes(memUsed)
-	availHuman := humanize.IBytes(memAvail)
-	percHuman := fmt.Sprintf("%s %%", humanize.FtoaWithDigits(percMem, 1))
+	usedHuman := humanize.IBytes(totalMemUsage)
+	limitHuman := humanize.IBytes(dockerStats[0].MemLimit)
+	percHuman := fmt.Sprintf("%s%%", humanize.FtoaWithDigits(percMem, 1))

+	stats = append(stats, AjaxStat{Stat: Stat{Name: "Memory Limit", Value: limitHuman}})
+	stats = append(stats, AjaxStat{Stat: Stat{Name: "Memory Used", Value: usedHuman}})
 	class := ""
 	if percMem > 75 {
 		class = "warning"
@@ -274,16 +378,7 @@ func _parseStats(data string) []Stat {
 	if percMem > 90 {
 		class = "error"
 	}
-	stats = append(stats, Stat{Name: "Memory Usage", Value: percHuman, Class: class})
-	stats = append(stats, Stat{Name: "Memory Used", Value: usedHuman})
-	class = ""
-	if memAvail < 250000000 {
-		class = "warning"
-	}
-	if memAvail < 100000000 {
-		class = "error"
-	}
-	stats = append(stats, Stat{Name: "Memory Available", Value: availHuman, Class: class})
+	stats = append(stats, AjaxStat{Stat: Stat{Name: "Memory Used [%]", Value: percHuman, Class: class}})

 	return stats
 }
@@ -369,11 +464,10 @@ func CollectDashboardData(w http.ResponseWriter, r *http.Request) (map[string]in
 	activity := getLog(w, r, "activity")
 	dashboardData["Activity"] = _parseActivity(activity)

-	components := getLog(w, r, "components")
-	dashboardData["Components"] = _parseComponents(components)
-
-	stats := getLog(w, r, "stats")
-	dashboardData["Stats"] = _parseStats(stats)
+	components := _parseComponents(getLog(w, r, "components"))
+	uptime := getLog(w, r, "uptime")
+	dashboardData["Stats"] = _parseStats(uptime, components)
+	dashboardData["Components"] = components

 	return dashboardData, nil
 }
--- a/gui/main.go
+++ b/gui/main.go
@@ -894,7 +894,8 @@ func (res *Result) ManageComponents(w http.ResponseWriter, r *http.Request, acti
 		if (components[i].Name == "NGINX Webserver" && (action == "nginx-reload" || action == "nginx-restart")) ||
 			(components[i].Name == "Controller" && action == "svc-restart") ||
 			(components[i].Name == "Boulder (ACME)" && (action == "boulder-start" || action == "boulder-stop" || action == "boulder-restart")) ||
-			(components[i].Name == "LabCA Application" && action == "labca-restart") {
+			(components[i].Name == "LabCA Application" && action == "labca-restart") ||
+			(components[i].Name == "MySQL Database" && action == "mysql-restart") {
 			res.Timestamp = components[i].Timestamp
 			res.TimestampRel = components[i].TimestampRel
 			res.Class = components[i].Class
@@ -976,6 +977,7 @@ func _managePost(w http.ResponseWriter, r *http.Request) {
 		"backup-delete",
 		"backup-now",
 		"cert-export",
+		"mysql-restart",
 		"nginx-reload",
 		"nginx-restart",
 		"svc-restart",
@@ -1109,17 +1111,20 @@ func _manageGet(w http.ResponseWriter, r *http.Request) {
 			btn["Label"] = "Restart"
 			components[i].Buttons = append(components[i].Buttons, btn)
 		}
-	}
-	manageData["Components"] = components

-	stats := _parseStats(getLog(w, r, "stats"))
-	for _, stat := range stats {
-		if stat.Name == "System Uptime" {
-			manageData["ServerTimestamp"] = stat.Hint
-			manageData["ServerTimestampRel"] = stat.Value
-			break
+		if components[i].Name == "MySQL Database" {
+			components[i].LogURL = ""
+			components[i].LogTitle = ""
+
+			btn := make(map[string]interface{})
+			btn["Class"] = "btn-warning"
+			btn["Id"] = "mysql-restart"
+			btn["Title"] = "Restart the MySQL database server"
+			btn["Label"] = "Restart"
+			components[i].Buttons = append(components[i].Buttons, btn)
 		}
 	}
+	manageData["Components"] = components

 	backupFiles := strings.Split(getLog(w, r, "backups"), "\n")
 	backupFiles = backupFiles[:len(backupFiles)-1]
@@ -2269,6 +2274,13 @@ func certRevokeHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }

+func statsHandler(w http.ResponseWriter, r *http.Request) {
+	res := parseDockerStats(getLog(w, r, "stats"))
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(res)
+}
+
 type navItem struct {
 	Name     string
 	Icon     string
@@ -2600,6 +2612,7 @@ func main() {

 	r := mux.NewRouter()
 	r.HandleFunc("/", rootHandler).Methods("GET")
+	r.HandleFunc("/stats", statsHandler).Methods("GET")
 	r.HandleFunc("/about", aboutHandler).Methods("GET")
 	r.HandleFunc("/manage", manageHandler).Methods("GET", "POST")
 	r.HandleFunc("/final", finalHandler).Methods("GET")
--- a/gui/templates/views/dashboard.tmpl
+++ b/gui/templates/views/dashboard.tmpl
@@ -136,18 +136,27 @@
                    <div class="clearfix"></div>
                </div>
                <div class="panel-body">
-                    <div class="table-responsive">
+                    <div class="table-responsive" id="dashboard-stats">
                        <table class="table table-bordered mb15">
                            <tbody>
                            {{ range $item := .Components }}
                                <tr>
-                                    <td>{{ $item.Name }}</td>
-                                    <td title="{{ $item.Timestamp }}">
+                                    <td rowspan="3">{{ $item.Name }}</td>
+                                    <td class="pad-low-bottom">Uptime</td>
+                                    <td class="pad-low-bottom" title="{{ $item.Timestamp }}">
                                        <span class="pull-right{{ if $item.Class }} {{ $item.Class }}{{ end }}">{{ $item.TimestampRel }}
                                            {{ if $item.Class }}<i class="fa fa-fw fa-warning"></i>{{ end }}
                                        </span>
                                    </td>
                                </tr>
+                                <tr>
+                                    <td class="pad-low">Memory</td>
+                                    <td class="pad-low"><span class="pull-right"><img height="12px" src="/img/spinner.gif"></span></td>
+                                </tr>
+                                <tr>
+                                    <td class="pad-low-top">Processes</td>
+                                    <td class="pad-low-top"><span class="pull-right"><img height="12px" src="/img/spinner.gif"></span></td>
+                                </tr>
                            {{ end }}
                            </tbody>
                        </table>
@@ -158,7 +167,8 @@
                                <tr>
                                    <td>{{ $item.Name }}</td>
                                    <td{{ if $item.Hint }} title="{{ $item.Hint }}"{{ end }}>
-                                        <span class="pull-right{{ if $item.Class }} {{ $item.Class }}{{ end }}">{{ $item.Value }}
+                                        <span class="pull-right{{ if $item.Class }} {{ $item.Class }}{{ end }}">
+                                            {{ if $item.Value }}{{ $item.Value }}{{ else }}<img height="12px" src="/img/spinner.gif">{{ end }}
                                            {{ if $item.Class }}<i class="fa fa-fw fa-warning"></i>{{ end }}
                                        </span>
                                    </td>
@@ -178,3 +188,53 @@
            </div>

 {{ end }}
+
+{{ define "tail" }}
+    <script>
+        (function() {
+            $.ajax(window.location.href + "/stats", {
+                timeout: 30000
+            })
+            .done(function(data) {
+                for (var i = 0; i < data.length; i++) {
+                    children = $("#dashboard-stats td").filter(function() {
+                        return $(this).text() == data[i].Name;
+                    }).parent().children();
+
+                    if (children.length == 2) {
+                        // a .Stats table row
+                        td = children[1];
+                        $(td).prop('title', data[i].Hint);
+                        span = $(td).children()[0];
+                        $(span).attr("class","pull-right");
+                        val = data[i].Value
+                        if (data[i].Class != "") {
+                            $(span).addClass(data[i].Class);
+                            val += '<i class="fa fa-fw fa-warning"></i>';
+                        }
+                        $(span).html(val);
+                    } else {
+                        // a .Components table row
+                        nextRows = $(children).parent().nextAll("tr");
+
+                        memSpan = $($(nextRows[0]).children()[1]).children()[0];
+                        $(memSpan).text(data[i].MemoryUsed + " (" + data[i].MemoryPerc + ")");
+
+                        procSpan = $($(nextRows[1]).children()[1]).children()[0];
+                        $(procSpan).text(data[i].NumPids);
+                    }
+                }
+
+                // Delete any remaining spinners when one or more dockers is down
+                $('img').each(function() {
+                    if ($(this).attr('height') == '12px' && $(this).attr('src') == '/img/spinner.gif') {
+                        $(this).remove();
+                    }
+                });
+            })
+            .fail(function(xhr, status, err) {
+                console.log("ajax ERROR:", err);
+            });
+        })();
+    </script>
+{{ end }}
--- a/static/css/labca.css
+++ b/static/css/labca.css
@@ -95,6 +95,19 @@ p.caption {
    padding: 4px 8px;
 }

+td.pad-low-bottom {
+    padding-bottom: 4px !important;
+}
+
+td.pad-low {
+    padding-top: 4px !important;
+    padding-bottom: 4px !important;
+}
+
+td.pad-low-top {
+    padding-top: 4px !important;
+}
+
 .btn-reg {
    width: 5em;
 }