Cleanup futility test coverage framework a bit

This cleans up the Makfile and test scripts a bit, and adds a new test for
the builtin commands.

BUG=chromium:224734
BRANCH=ToT
TEST=make runtests

Change-Id: Ibf5aa867d4dcabc0e46daac6633036b035c99ac8
Signed-off-by: Bill Richardson <wfrichar@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/207718

Makefile

@@ -172,7 +172,7 @@ CFLAGS += -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64
 
 # Code coverage
 ifneq (${COV},)
-  COV_FLAGS = -O0 --coverage
+  COV_FLAGS = -O0 --coverage -DCOVERAGE
   CFLAGS += ${COV_FLAGS}
   LDFLAGS += ${COV_FLAGS}
   COV_INFO = ${BUILD}/coverage.info
@@ -758,8 +758,8 @@ ${BUILD}/firmware/linktest/main: ${FWLIB}
 ${BUILD}/firmware/linktest/main: LIBS = ${FWLIB}
 ALL_OBJS += ${BUILD}/firmware/linktest/main.o
 
-.phony: fwlinktest
+.PHONY: fwlinktest
-fwlinktest: ${FWLIB} \
+fwlinktest: \
 	${BUILD}/firmware/linktest/main_vbinit \
 	${BUILD}/firmware/linktest/main_vbsf \
 	${BUILD}/firmware/linktest/main
@@ -1227,9 +1227,9 @@ coverage_html:
 	lcov -a ${COV_INFO}.initial -a ${COV_INFO}.tests -o ${COV_INFO}.total
 	lcov -r ${COV_INFO}.total '/usr/*' '*/linktest/*' -o ${COV_INFO}.local
 	genhtml ${COV_INFO}.local -o ${BUILD}/coverage
-
+# Generate addtional coverage stats just for firmware subdir, because the stats
-# Generate addtional coverage stats just for firmware subdir, because the
+# for the whole project don't include subdirectory summaries. This will print
-# per-directory stats for the whole project don't include their own subdirs.
+# the summary for just the firmware sources.
 	lcov -r ${COV_INFO}.local '*/stub/*' -o ${COV_INFO}.nostub
 	lcov -e ${COV_INFO}.nostub '${SRCDIR}/firmware/*' \
 		-o ${COV_INFO}.firmware

futility/futility.c

@@ -200,6 +200,10 @@ static void log_args(int argc, char *argv[])
 /******************************************************************************/
 /* Here we go */
 
+#ifdef COVERAGE
+void __gcov_flush(void);
+#endif
+
 int main(int argc, char *argv[], char *envp[])
 {
   char *progname;
@@ -278,6 +282,10 @@ int main(int argc, char *argv[], char *envp[])
   }
 
   fflush(0);
+#ifdef COVERAGE
+  /* Write gcov data prior to exec. */
+  __gcov_flush();
+#endif
   execve(oldname, argv, envp);
 
   fprintf(stderr, "%s failed to exec %s: %s\n", MYNAME,

tests/futility/run_test_scripts.sh

@@ -7,7 +7,7 @@
 SCRIPTDIR=$(dirname $(readlink -f "$0"))
 . "$SCRIPTDIR/common.sh"
 
-# Mandatory arg is the path to the futility executable to test.
+# Mandatory arg is the directory where futility is installed.
 [ -z "${1:-}" ] && error "Directory argument is required"
 BINDIR="$1"
 shift
@@ -33,7 +33,10 @@ export SCRIPTDIR
 export OUTDIR
 
 # These are the scripts to run. Binaries are invoked directly by the Makefile.
-TESTS="${SCRIPTDIR}/test_dump_fmap.sh"
+TESTS="
+${SCRIPTDIR}/test_main.sh
+${SCRIPTDIR}/test_dump_fmap.sh
+"
 
 
 # Get ready...
@@ -45,14 +48,14 @@ progs=0
 # everything is built in (chromium:196079).
 
 # Here are the old programs to be wrapped
-# FIXME: dev_debug_vboot isn't tested right now.
+# TODO(crbug.com/224734): dev_debug_vboot isn't tested right now.
 PROGS=${*:-cgpt crossystem dev_sign_file dumpRSAPublicKey
            dump_kernel_config enable_dev_usb_boot gbb_utility
            tpm_init_temp_fix tpmc vbutil_firmware vbutil_kernel vbutil_key
            vbutil_keyblock vbutil_what_keys}
 
 # For now just compare results of invoking each program with no args.
-# FIXME(chromium-os:37062): Create true rigorous tests for every program.
+# TODO: Create true rigorous tests for every program.
 echo "-- old_bins --"
 for i in $PROGS; do
   : $(( progs++ ))
@@ -89,6 +92,12 @@ for i in $PROGS; do
   fi
 done
 
+# How many wrapped executables are left to incorporate? Did we check them all?
+xprogs=$(find ${OLDDIR} -type f -perm /111 | wc -l)
+if [ $xprogs -gt 0 ]; then
+  yellow "${progs}/${xprogs} wrapped executables tested"
+fi
+
 
 ##############################################################################
 # Invoke the scripts that test the builtin functions.
@@ -107,7 +116,7 @@ for i in $TESTS; do
   if [ ! "$rc" ]; then
     green "passed"
     : $(( pass++ ))
-    rm -f ${OUTDIR}/$i.{stdout,stderr,return}
+    rm -f ${OUTDIR}/$j.{stdout,stderr,return}
   else
     red "failed"
   fi

tests/futility/test_dump_fmap.sh

@@ -27,4 +27,10 @@ cd "$OUTDIR"  # TODO: we really need a directory argument for dump_fmap.
 if "$FUTILITY" dump_fmap -x "${SCRIPTDIR}/data_fmap.bin" FMAP ; then
   echo Wait, that was supposed to fail. 1>&2
   exit 1
+else
+  rm -f FMAP
 fi
+
+# cleanup
+rm -f "$TMP"
+exit 0

tests/futility/test_main.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -eux
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+me=${0##*/}
+
+# Work in scratch directory
+cd "$OUTDIR"
+TMP="$me.tmp"
+
+# Built-in do-nothing commands.
+# TODO(crbug.com/224734): Remove these when we have enough built-in commands
+"$FUTILITY" foo hi
+"$FUTILITY" bar there
+"$FUTILITY" hey boys
+
+# No args returns nonzero exit code
+"$FUTILITY" && false
+
+"$FUTILITY" help > "$TMP"
+grep Usage "$TMP"
+# TODO(crbug.com/224734): Make sure all built-in commands have help, too.
+
+# It's weird but okay if the command is a full path.
+"$FUTILITY" /fake/path/to/help  > "$TMP"
+grep Usage "$TMP"
+
+# Make sure logging does something.
+# Note: This will zap any existing log file. Too bad.
+LOG="/tmp/futility.log"
+rm -f "$LOG"
+touch "$LOG"
+"$FUTILITY" help
+grep "$FUTILITY" "$LOG"
+rm "$LOG"
+
+# cleanup
+rm -f "$TMP"
+exit 0