Add AP hang detection

BUG=chrome-os-partner:24558
BRANCH=none
TEST=see procedure in bug

Change-Id: I42614a1da5f24c93b6267d81339ff9d721bf0d8f
Signed-off-by: Randall Spangler <rspangler@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/180080
Reviewed-by: Bill Richardson <wfrichar@chromium.org>

board/link/board.h

@@ -9,6 +9,7 @@
 #define __BOARD_H
 
 /* Optional features */
+#define CONFIG_AP_HANG_DETECT
 #define CONFIG_BACKLIGHT_LID
 #define CONFIG_BACKLIGHT_REQ_GPIO GPIO_PCH_BKLTEN
 #define CONFIG_BATTERY_LINK

board/rambi/board.h

@@ -9,6 +9,7 @@
 #define __BOARD_H
 
 /* Optional features */
+#define CONFIG_AP_HANG_DETECT
 #define CONFIG_BACKLIGHT_LID
 #define CONFIG_BATTERY_SMART
 #define CONFIG_BOARD_VERSION

common/ap_hang_detect.c

@@ -0,0 +1,232 @@
+/* Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/* AP hang detect logic */
+
+#include "ap_hang_detect.h"
+#include "chipset.h"
+#include "common.h"
+#include "console.h"
+#include "hooks.h"
+#include "host_command.h"
+#include "lid_switch.h"
+#include "power_button.h"
+#include "timer.h"
+#include "util.h"
+
+/* Console output macros */
+#define CPUTS(outstr) cputs(CC_CHIPSET, outstr)
+#define CPRINTF(format, args...) cprintf(CC_CHIPSET, format, ## args)
+
+static struct ec_params_hang_detect hdparams;
+
+static int active;  /* Is hang detect timer active / counting? */
+static int timeout_will_reboot;  /* Will the deferred call reboot the AP? */
+
+/**
+ * Handle the hang detect timer expiring.
+ */
+static void hang_detect_deferred(void)
+{
+	/* If we're no longer active, nothing to do */
+	if (!active)
+		return;
+
+	/* If we're rebooting the AP, stop hang detection */
+	if (timeout_will_reboot) {
+		CPRINTF("[%T hang detect triggering warm reboot]\n");
+		host_set_single_event(EC_HOST_EVENT_HANG_REBOOT);
+		chipset_reset(0);
+		active = 0;
+		return;
+	}
+
+	/* Otherwise, we're starting with the host event */
+	CPRINTF("[%T hang detect sending host event]\n");
+	host_set_single_event(EC_HOST_EVENT_HANG_DETECT);
+
+	/* If we're also rebooting, defer for the remaining delay */
+	if (hdparams.warm_reboot_timeout_msec) {
+		CPRINTF("[%T hang detect continuing (for reboot)]\n");
+		timeout_will_reboot = 1;
+		hook_call_deferred(hang_detect_deferred,
+				   (hdparams.warm_reboot_timeout_msec -
+				    hdparams.host_event_timeout_msec) * MSEC);
+	} else {
+		/* Not rebooting, so go back to idle */
+		active = 0;
+	}
+}
+DECLARE_DEFERRED(hang_detect_deferred);
+
+/**
+ * Start the hang detect timers.
+ */
+static void hang_detect_start(const char *why)
+{
+	/* If already active, don't restart timer */
+	if (active)
+		return;
+
+	if (hdparams.host_event_timeout_msec) {
+		CPRINTF("[%T hang detect started on %s (for event)]\n", why);
+		timeout_will_reboot = 0;
+		active = 1;
+		hook_call_deferred(hang_detect_deferred,
+				   hdparams.host_event_timeout_msec * MSEC);
+	} else if (hdparams.warm_reboot_timeout_msec) {
+		CPRINTF("[%T hang detect started on %s (for reboot)]\n", why);
+		timeout_will_reboot = 1;
+		active = 1;
+		hook_call_deferred(hang_detect_deferred,
+				   hdparams.warm_reboot_timeout_msec * MSEC);
+	}
+}
+
+/**
+ * Stop the hang detect timers.
+ */
+static void hang_detect_stop(const char *why)
+{
+	if (active)
+		CPRINTF("[%T hang detect stopped on %s]\n", why);
+
+	active = 0;
+}
+
+void hang_detect_stop_on_host_command(void)
+{
+	if (hdparams.flags & EC_HANG_STOP_ON_HOST_COMMAND)
+		hang_detect_stop("host cmd");
+}
+
+/*****************************************************************************/
+/* Hooks */
+
+static void hang_detect_power_button(void)
+{
+	if (power_button_is_pressed()) {
+		if (hdparams.flags & EC_HANG_START_ON_POWER_PRESS)
+			hang_detect_start("power button");
+	} else {
+		if (hdparams.flags & EC_HANG_STOP_ON_POWER_RELEASE)
+			hang_detect_stop("power button");
+	}
+}
+DECLARE_HOOK(HOOK_POWER_BUTTON_CHANGE, hang_detect_power_button,
+	     HOOK_PRIO_DEFAULT);
+
+static void hang_detect_lid(void)
+{
+	if (lid_is_open()) {
+		if (hdparams.flags & EC_HANG_START_ON_LID_OPEN)
+			hang_detect_start("lid open");
+	} else {
+		if (hdparams.flags & EC_HANG_START_ON_LID_CLOSE)
+			hang_detect_start("lid close");
+	}
+}
+DECLARE_HOOK(HOOK_LID_CHANGE, hang_detect_lid, HOOK_PRIO_DEFAULT);
+
+static void hang_detect_resume(void)
+{
+	if (hdparams.flags & EC_HANG_START_ON_RESUME)
+		hang_detect_start("resume");
+}
+DECLARE_HOOK(HOOK_CHIPSET_RESUME, hang_detect_resume, HOOK_PRIO_DEFAULT);
+
+static void hang_detect_suspend(void)
+{
+	if (hdparams.flags & EC_HANG_STOP_ON_SUSPEND)
+		hang_detect_stop("suspend");
+}
+DECLARE_HOOK(HOOK_CHIPSET_SUSPEND, hang_detect_suspend, HOOK_PRIO_DEFAULT);
+
+static void hang_detect_shutdown(void)
+{
+	/* Stop the timers */
+	hang_detect_stop("shutdown");
+
+	/* Disable hang detection; it must be enabled every boot */
+	memset(&hdparams, 0, sizeof(hdparams));
+}
+DECLARE_HOOK(HOOK_CHIPSET_SHUTDOWN, hang_detect_shutdown, HOOK_PRIO_DEFAULT);
+
+/*****************************************************************************/
+/* Host command */
+
+static int hang_detect_host_command(struct host_cmd_handler_args *args)
+{
+	const struct ec_params_hang_detect *p = args->params;
+
+	/* Handle stopping hang timer on request */
+	if (p->flags & EC_HANG_STOP_NOW) {
+		hang_detect_stop("ap request");
+
+		/* Ignore the other params */
+		return EC_RES_SUCCESS;
+	}
+
+	/* Handle starting hang timer on request */
+	if (p->flags & EC_HANG_START_NOW) {
+		hang_detect_start("ap request");
+
+		/* Ignore the other params */
+		return EC_RES_SUCCESS;
+	}
+
+	/* Save new params */
+	hdparams = *p;
+	CPRINTF("[%T hang detect flags=0x%x, event=%d ms, reboot=%d ms]\n",
+		hdparams.flags, hdparams.host_event_timeout_msec,
+		hdparams.warm_reboot_timeout_msec);
+
+	/*
+	 * If warm reboot timeout is shorter than host event timeout, ignore
+	 * the host event timeout because a warm reboot will win.
+	 */
+	if (hdparams.warm_reboot_timeout_msec &&
+	    hdparams.warm_reboot_timeout_msec <=
+	    hdparams.host_event_timeout_msec)
+		hdparams.host_event_timeout_msec = 0;
+
+	return EC_RES_SUCCESS;
+}
+DECLARE_HOST_COMMAND(EC_CMD_HANG_DETECT,
+		     hang_detect_host_command,
+		     EC_VER_MASK(0));
+
+/*****************************************************************************/
+/* Console command */
+
+static int command_hang_detect(int argc, char **argv)
+{
+	ccprintf("flags:  0x%x\n", hdparams.flags);
+
+	ccputs("event:  ");
+	if (hdparams.host_event_timeout_msec)
+		ccprintf("%d ms\n", hdparams.host_event_timeout_msec);
+	else
+		ccputs("disabled\n");
+
+	ccputs("reboot: ");
+	if (hdparams.warm_reboot_timeout_msec)
+		ccprintf("%d ms\n", hdparams.warm_reboot_timeout_msec);
+	else
+		ccputs("disabled\n");
+
+	ccputs("status: ");
+	if (active)
+		ccprintf("active for %s\n",
+			 timeout_will_reboot ? "reboot" : "event");
+	else
+		ccputs("inactive\n");
+
+	return EC_SUCCESS;
+}
+DECLARE_CONSOLE_COMMAND(hangdet, command_hang_detect,
+			NULL,
+			"Print hang detect state",
+			NULL);

common/build.mk

@@ -11,6 +11,7 @@ common-y+=memory_commands.o shared_mem.o system.o hooks.o
 common-y+=gpio.o version.o printf.o queue.o
 
 common-$(CONFIG_ALS)+=als.o
+common-$(CONFIG_AP_HANG_DETECT)+=ap_hang_detect.o
 common-$(CONFIG_BACKLIGHT_LID)+=backlight_lid.o
 # TODO(crosbug.com/p/23821): Why do these include battery_common but
 # the other batteries don't?  Perhaps should use CONFIG_CMD_BATTERY

common/host_command.c

@@ -5,6 +5,7 @@
 
 /* Host command module for Chrome EC */
 
+#include "ap_hang_detect.h"
 #include "common.h"
 #include "console.h"
 #include "host_command.h"
@@ -145,6 +146,11 @@ void host_command_received(struct host_cmd_handler_args *args)
 		args->result = EC_RES_ERROR;
 	}
 
+#ifdef CONFIG_AP_HANG_DETECT
+	/* If hang detection is enabled, check stop on host command */
+	hang_detect_stop_on_host_command();
+#endif
+
 	if (args->result) {
 		; /* driver has signalled an error, respond now */
 #ifdef CONFIG_HOST_COMMAND_STATUS

include/ap_hang_detect.h

@@ -0,0 +1,20 @@
+/* Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/* Power button API for Chrome EC */
+
+#ifndef __CROS_EC_AP_HANG_DETECT_H
+#define __CROS_EC_AP_HANG_DETECT_H
+
+#include "common.h"
+
+/**
+ * If the hang detect timers were started and can be stopped by any host
+ * command, stop them.  This is intended to be called by the the host command
+ * module.
+ */
+void hang_detect_stop_on_host_command(void);
+
+#endif  /* __CROS_EC_AP_HANG_DETECT_H */

include/config.h

@@ -52,6 +52,9 @@
 #undef CONFIG_ALS
 #undef CONFIG_ALS_ISL29035
 
+/* Support AP hang detection host command and state machine */
+#undef CONFIG_AP_HANG_DETECT
+
 /*
  * Support controlling the display backlight based on the state of the lid
  * switch.  The EC will disable the backlight when the lid is closed.

include/ec_commands.h

@@ -249,6 +249,11 @@ enum host_event_code {
 	/* Suggest that the AP resume normal speed */
 	EC_HOST_EVENT_THROTTLE_STOP = 19,
 
+	/* Hang detect logic detected a hang and host event timeout expired */
+	EC_HOST_EVENT_HANG_DETECT = 20,
+	/* Hang detect logic detected a hang and warm rebooted the AP */
+	EC_HOST_EVENT_HANG_REBOOT = 21,
+
 	/*
 	 * The high bit of the event mask is not used as a host event code.  If
 	 * it reads back as set, then the entire event mask should be
@@ -1707,6 +1712,60 @@ struct ec_response_i2c_passthru {
 	uint8_t data[];		/* Data read by messages concatenated here */
 } __packed;
 
+/*****************************************************************************/
+/* Power button hang detect */
+
+#define EC_CMD_HANG_DETECT 0x9f
+
+/* Reasons to start hang detection timer */
+/* Power button pressed */
+#define EC_HANG_START_ON_POWER_PRESS  (1 << 0)
+
+/* Lid closed */
+#define EC_HANG_START_ON_LID_CLOSE    (1 << 1)
+
+ /* Lid opened */
+#define EC_HANG_START_ON_LID_OPEN     (1 << 2)
+
+/* Start of AP S3->S0 transition (booting or resuming from suspend) */
+#define EC_HANG_START_ON_RESUME       (1 << 3)
+
+/* Reasons to cancel hang detection */
+
+/* Power button released */
+#define EC_HANG_STOP_ON_POWER_RELEASE (1 << 8)
+
+/* Any host command from AP received */
+#define EC_HANG_STOP_ON_HOST_COMMAND  (1 << 9)
+
+/* Stop on end of AP S0->S3 transition (suspending or shutting down) */
+#define EC_HANG_STOP_ON_SUSPEND       (1 << 10)
+
+/*
+ * If this flag is set, all the other fields are ignored, and the hang detect
+ * timer is started.  This provides the AP a way to start the hang timer
+ * without reconfiguring any of the other hang detect settings.  Note that
+ * you must previously have configured the timeouts.
+ */
+#define EC_HANG_START_NOW             (1 << 30)
+
+/*
+ * If this flag is set, all the other fields are ignored (including
+ * EC_HANG_START_NOW).  This provides the AP a way to stop the hang timer
+ * without reconfiguring any of the other hang detect settings.
+ */
+#define EC_HANG_STOP_NOW              (1 << 31)
+
+struct ec_params_hang_detect {
+	/* Flags; see EC_HANG_* */
+	uint32_t flags;
+
+	/* Timeout in msec before generating host event, if enabled */
+	uint16_t host_event_timeout_msec;
+
+	/* Timeout in msec before generating warm reboot, if enabled */
+	uint16_t warm_reboot_timeout_msec;
+} __packed;
 
 /*****************************************************************************/
 /* Debug commands for battery charging */

util/ectool.c

@@ -88,6 +88,8 @@ const char help_str[] =
 	"      Get the value of GPIO signal\n"
 	"  gpioset <GPIO name>\n"
 	"      Set the value of GPIO signal\n"
+	"  hangdetect <flags> <event_msec> <reboot_msec> | stop | start\n"
+	"      Configure or start/stop the hang detect timer\n"
 	"  hello\n"
 	"      Checks for basic communication with EC\n"
 	"  kbpress\n"
@@ -3365,6 +3367,59 @@ int cmd_tmp006cal(int argc, char *argv[])
 			  &p, sizeof(p), NULL, 0);
 }
 
+static int cmd_hang_detect(int argc, char *argv[])
+{
+	struct ec_params_hang_detect req;
+	char *e;
+
+	memset(&req, 0, sizeof(req));
+
+	if (argc == 2 && !strcasecmp(argv[1], "stop")) {
+		req.flags = EC_HANG_STOP_NOW;
+		return ec_command(EC_CMD_HANG_DETECT, 0, &req, sizeof(req),
+				  NULL, 0);
+	}
+
+	if (argc == 2 && !strcasecmp(argv[1], "start")) {
+		req.flags = EC_HANG_START_NOW;
+		return ec_command(EC_CMD_HANG_DETECT, 0, &req, sizeof(req),
+				  NULL, 0);
+	}
+
+	if (argc == 4) {
+		req.flags = strtol(argv[1], &e, 0);
+		if (e && *e) {
+			fprintf(stderr, "Bad flags.\n");
+			return -1;
+		}
+
+		req.host_event_timeout_msec = strtol(argv[2], &e, 0);
+		if (e && *e) {
+			fprintf(stderr, "Bad event timeout.\n");
+			return -1;
+		}
+
+		req.warm_reboot_timeout_msec = strtol(argv[3], &e, 0);
+		if (e && *e) {
+			fprintf(stderr, "Bad reboot timeout.\n");
+			return -1;
+		}
+
+		printf("hang flags=0x%x\n"
+		       "event_timeout=%d ms\n"
+		       "reboot_timeout=%d ms\n",
+		       req.flags, req.host_event_timeout_msec,
+		       req.warm_reboot_timeout_msec);
+
+		return ec_command(EC_CMD_HANG_DETECT, 0, &req, sizeof(req),
+				  NULL, 0);
+	}
+
+	fprintf(stderr,
+		"Must specify start/stop or <flags> <event_ms> <reboot_ms>\n");
+	return -1;
+}
+
 struct command {
 	const char *name;
 	int (*handler)(int argc, char *argv[]);
@@ -3403,6 +3458,7 @@ const struct command commands[] = {
 	{"flashinfo", cmd_flash_info},
 	{"gpioget", cmd_gpio_get},
 	{"gpioset", cmd_gpio_set},
+	{"hangdetect", cmd_hang_detect},
 	{"hello", cmd_hello},
 	{"kbpress", cmd_kbpress},
 	{"i2cread", cmd_i2c_read},