scottk/OpenCellular
1
0
Fork 0
mirror of https://github.com/Telecominfraproject/OpenCellular.git synced 2026-01-08 16:41:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

eCTS: Check order and expectation of test results

Browse Source 
This patch makes the framework verify not only the result but also
the execution order of the tests. It also allows each test to
specify expected return code and strings printed by TH and DUT.
The final test results depends on the return code and the expectation.
Therefore, the output now includes 'RESULT' column showing PASS or FAIL:

   test name       TH_RETURN_CODE DUT_RETURN_CODE TH_STR DUT_STR RESULT
test_task_switch          SUCCESS         SUCCESS      1       1   PASS
test_task_priority        SUCCESS         FAILURE      1       1   FAIL
test_stack_overflow   DID_NOT_END     DID_NOT_END      1       1   PASS

Additionally, this patch:

 * Adds CTS_RC_DID_NOT_START and CTS_RC_DID_NOT_END to indicate whether
   the test did start or end, respectively.
 * Makes stack overflow test check whether stack overflow was detected
   and reboot occurred
 * Removes post_corruption_test and conflict test since now
   the test results are stricly compared against expected results.
 * Fixes gpylint errors.

BUG=none
BRANCH=none
TEST=Run gpio, meta, timer, interrupt, and cts/cts.py -m task

Change-Id: I3b7005236e705dcac0c8f4711b44c85ff9a4f676
Reviewed-on: https://chromium-review.googlesource.com/538878
Commit-Ready: Daisuke Nojiri <dnojiri@chromium.org>
Tested-by: Daisuke Nojiri <dnojiri@chromium.org>
Reviewed-by: Daisuke Nojiri <dnojiri@chromium.org>
This commit is contained in:
Daisuke Nojiri
2017-04-12 09:48:10 -07:00
committed by chrome-bot
parent 60800678ca
commit ff85876719
19 changed files with 313 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
cts/common/board.py
View File

@@ -2,13 +2,13 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from abc import ABCMeta, abstractmethod
from abc import ABCMeta
from abc import abstractmethod
import fcntl
import os
import select
import shutil
import subprocess as sp
import time
OCD_SCRIPT_DIR = '/usr/local/share/openocd/scripts'
@@ -26,7 +26,7 @@ REBOOT_MARKER = 'UART initialized after reboot'
class Board(object):
"""Class representing a single board connected to a host machine
"""Class representing a single board connected to a host machine.
Attributes:
board: String containing actual type of board, i.e. nucleo-f072rb
@@ -40,19 +40,23 @@ class Board(object):
"""
__metaclass__ = ABCMeta # This is an Abstract Base Class (ABC)
def __init__(self, board, module, hla_serial=None):
"""Initializes a board object with given attributes
"""Initializes a board object with given attributes.
Args:
board: String containing board name
module: String of the test module you are building,
i.e. gpio, timer, etc.
hla_serial: Serial number if board's adaptor is an HLA
Raises:
RuntimeError: Board is not supported
"""
if not board in OPENOCD_CONFIGS:
if board not in OPENOCD_CONFIGS:
msg = 'OpenOcd configuration not found for ' + board
raise RuntimeError(msg)
if not board in FLASH_OFFSETS:
if board not in FLASH_OFFSETS:
msg = 'Flash offset not found for ' + board
raise RuntimeError(msg)
self.board = board
@@ -64,14 +68,14 @@ class Board(object):
self.tty = None
def reset_log_dir(self):
"""Reset log directory"""
"""Reset log directory."""
if os.path.isdir(self.log_dir):
shutil.rmtree(self.log_dir)
os.makedirs(self.log_dir)
@staticmethod
def get_stlink_serials():
"""Gets serial numbers of all st-link v2.1 board attached to host
"""Gets serial numbers of all st-link v2.1 board attached to host.
Returns:
List of serials
@@ -81,7 +85,7 @@ class Board(object):
st_link_info = usb_process.communicate()[0]
st_serials = []
for line in st_link_info.split('\n'):
if not 'iSerial' in line:
if 'iSerial' not in line:
continue
words = line.split()
if len(words) <= 2:
@@ -91,14 +95,17 @@ class Board(object):
@abstractmethod
def get_serial(self):
"""Subclass should implement this"""
"""Subclass should implement this."""
pass
def send_open_ocd_commands(self, commands):
"""Send a command to the board via openocd
"""Send a command to the board via openocd.
Args:
commands: A list of commands to send
Returns:
True if execution is successful or False otherwise.
"""
args = ['openocd', '-s', OCD_SCRIPT_DIR,
'-f', self.openocd_config, '-c', 'hla_serial ' + self.hla_serial]
@@ -120,11 +127,14 @@ class Board(object):
with open(self.openocd_log) as log:
print log.read()
def build(self, module, ec_dir):
"""Builds test suite module for board
def build(self, ec_dir):
"""Builds test suite module for board.
Args:
ec_dir: String of the ec directory path
Returns:
True if build is successful or False otherwise.
"""
cmds = ['make',
'--directory=' + ec_dir,
@@ -146,7 +156,7 @@ class Board(object):
print log.read()
def flash(self, image_path):
"""Flashes board with most recent build ec.bin"""
"""Flashes board with most recent build ec.bin."""
cmd = ['reset_config connect_assert_srst',
'init',
'reset init',
@@ -163,11 +173,12 @@ class Board(object):
return s
def reset(self):
"""Reset then halt board """
"""Reset then halt board."""
return self.send_open_ocd_commands(['init', 'reset halt'])
def setup_tty(self):
"""Call this before calling read_tty for the first time.
This is not in the initialization because caller only should call
this function after serial numbers are setup
"""
@@ -185,12 +196,12 @@ class Board(object):
self.tty = tty
def read_tty(self, max_boot_count=1):
"""Read info from a serial port described by a file descriptor
"""Read info from a serial port described by a file descriptor.
Args:
max_boot_count: Stop reading if boot count exceeds this number
Return:
Returns:
result: characters read from tty
boot: boot counts
"""
@@ -219,19 +230,15 @@ class Board(object):
return result, boot
def identify_tty_port(self):
"""Saves this board's serial port"""
"""Saves this board's serial port."""
dev_dir = '/dev'
id_prefix = 'ID_SERIAL_SHORT='
com_devices = [f for f in os.listdir(dev_dir) if f.startswith('ttyACM')]
for device in com_devices:
self.tty_port = os.path.join(dev_dir, device)
properties = sp.check_output(['udevadm',
'info',
'-a',
'-n',
self.tty_port,
'--query=property'])
properties = sp.check_output(
['udevadm', 'info', '-a', '-n', self.tty_port, '--query=property'])
for line in [l.strip() for l in properties.split('\n')]:
if line.startswith(id_prefix):
if self.hla_serial == line[len(id_prefix):]:
@@ -241,7 +248,7 @@ class Board(object):
raise RuntimeError('The device dev path could not be found')
def open_tty(self):
"""Read available bytes from device dev path"""
"""Read available bytes from device dev path."""
fd = os.open(self.tty_port, os.O_RDONLY)
flag = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, flag | os.O_NONBLOCK)
@@ -249,18 +256,19 @@ class Board(object):
class TestHarness(Board):
"""Subclass of Board representing a Test Harness
"""Subclass of Board representing a Test Harness.
Attributes:
serial_path: Path to file containing serial number
"""
def __init__(self, board, module, log_dir, serial_path):
"""Initializes a board object with given attributes
"""Initializes a board object with given attributes.
Args:
board: board name
module: module name
log_dir: Directory where log file is stored
serial_path: Path to file containing serial number
"""
Board.__init__(self, board, module)
@@ -271,9 +279,9 @@ class TestHarness(Board):
self.reset_log_dir()
def get_serial(self):
"""Loads serial number from saved location"""
"""Loads serial number from saved location."""
if self.hla_serial:
return # serial was already loaded
return # serial was already loaded
try:
with open(self.serial_path, mode='r') as ser_f:
self.hla_serial = ser_f.read()
@@ -284,7 +292,7 @@ class TestHarness(Board):
raise RuntimeError(msg)
def save_serial(self):
"""Saves the TH serial number to a file"""
"""Saves the TH serial number to a file."""
serials = Board.get_stlink_serials()
if len(serials) > 1:
msg = ('There are more than one test board connected to the host.'
@@ -296,9 +304,9 @@ class TestHarness(Board):
raise RuntimeError(msg)
serial = serials[0]
dir = os.path.dirname(self.serial_path)
if not os.path.exists(dir):
os.makedirs(dir)
serial_dir = os.path.dirname(self.serial_path)
if not os.path.exists(serial_dir):
os.makedirs(serial_dir)
with open(self.serial_path, mode='w') as ser_f:
ser_f.write(serial)
self.hla_serial = serial
@@ -308,20 +316,21 @@ class TestHarness(Board):
class DeviceUnderTest(Board):
"""Subclass of Board representing a DUT board
"""Subclass of Board representing a DUT board.
Attributes:
th: Reference to test harness board to which this DUT is attached
"""
def __init__(self, board, th, module, log_dir, hla_ser=None):
"""Initializes a Device Under Test object with given attributes
"""Initializes a DUT object.
Args:
board: String containing board name
th: Reference to test harness board to which this DUT is attached
module: module name
hla_serial: Serial number if board uses an HLA adaptor
log_dir: Directory where log file is stored
hla_ser: Serial number if board uses an HLA adaptor
"""
Board.__init__(self, board, module, hla_serial=hla_ser)
self.th = th
@@ -335,8 +344,11 @@ class DeviceUnderTest(Board):
Precondition: The DUT and TH must both be connected, and th.hla_serial
must hold the correct value (the th's serial #)
Raises:
RuntimeError: DUT isn't found or multiple DUTs are found.
"""
if self.hla_serial != None:
if self.hla_serial is not None:
# serial was already set ('' is a valid serial)
return

18
cts/common/cts.rc
View File

@@ -14,11 +14,21 @@
* where <NAME> will be printed on the result screen.
*/
/* Host only codes. Should not be needed by th.c or dut.c. */
CTS_RC_DUPLICATE_RUN = -2,
CTS_RC_NO_RESULT = -1,
/*
* Host only return codes. Should not be needed by th.c or dut.c.
*/
/* Test didn't run */
CTS_RC_DID_NOT_START = -1,
/* Test didn't end */
CTS_RC_DID_NOT_END = -2,
/* Results were reported twice or more */
CTS_RC_DUPLICATE_RUN = -3,
/* Error in parsing return code. Probably it was null or not an integer. */
CTS_RC_INVALID_RC = -4,
/* Regular codes */
/*
* Regular return codes. Used by DUT and TH.
*/
CTS_RC_SUCCESS = 0,
CTS_RC_FAILURE,
CTS_RC_BAD_SYNC,

11
cts/common/cts_testlist.h
View File

@@ -3,18 +3,25 @@
* found in the LICENSE file.
*/
/*
* CTS_TEST macro is used by dut.c, th.c, and cts.py. Currently, the 2nd
* and 3rd arguments are only used by cts.py. They specify the expected
* strings output by TH and DUT, respectively.
*/
struct cts_test {
enum cts_rc (*run)(void);
char *name;
};
#define CTS_TEST(test) {test, STRINGIFY(test)},
#define CTS_TEST(test, th_rc, th_string, dut_rc, dut_string) \
{test, STRINGIFY(test)},
struct cts_test tests[] = {
#include "cts.testlist"
};
#undef CTS_TEST
#define CTS_TEST(test) CTS_TEST_ID_##test,
#define CTS_TEST(test, th_rc, th_string, dut_rc, dut_string) CTS_TEST_ID_##test,
enum {
#include "cts.testlist"
CTS_TEST_ID_COUNT,

254
cts/cts.py
View File

@@ -1,4 +1,5 @@
#!/usr/bin/python2
#!/usr/bin/python
#
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -17,32 +18,33 @@
import argparse
from collections import defaultdict
import common.board as board
import os
import shutil
import time
import common.board as board
# Host only return codes. Make sure they match values in cts.rc
CTS_RC_DUPLICATE_RUN = -2 # The test was run multiple times.
CTS_RC_NO_RESULT = -1 # The test did not run.
CTS_RC_PREFIX = 'CTS_RC_'
DEFAULT_TH = 'stm32l476g-eval'
DEFAULT_DUT = 'nucleo-f072rb'
MAX_SUITE_TIME_SEC = 5
CTS_TEST_RESULT_DIR = '/tmp/ects'
# Host only return codes. Make sure they match values in cts.rc
CTS_RC_DID_NOT_START = -1 # test did not run.
CTS_RC_DID_NOT_END = -2 # test did not run.
CTS_RC_DUPLICATE_RUN = -3 # test was run multiple times.
CTS_RC_INVALID_RETURN_CODE = -4 # failed to parse return code
class Cts(object):
"""Class that represents a CTS testing setup and provides
interface to boards (building, flashing, etc.)
"""Class that represents a eCTS run.
Attributes:
dut: DeviceUnderTest object representing dut
th: TestHarness object representing th
dut: DeviceUnderTest object representing DUT
th: TestHarness object representing a test harness
module: Name of module to build/run tests for
test_names: List of strings of test names contained in given module
testlist: List of strings of test names contained in given module
return_codes: Dict of strings of return codes, with a code's integer
value being the index for the corresponding string representation
"""
@@ -68,22 +70,21 @@ class Cts(object):
self.dut = board.DeviceUnderTest(dut, self.th, module, self.results_dir)
cts_dir = os.path.join(self.ec_dir, 'cts')
testlist_path = os.path.join(cts_dir, self.module, 'cts.testlist')
self.test_names = Cts.get_macro_args(testlist_path, 'CTS_TEST')
return_codes_path = os.path.join(cts_dir, 'common', 'cts.rc')
self.get_return_codes(return_codes_path, 'CTS_RC_')
self.get_return_codes(return_codes_path)
self.testlist = self.get_macro_args(testlist_path, 'CTS_TEST')
def build(self):
"""Build images for DUT and TH"""
"""Build images for DUT and TH."""
print 'Building DUT image...'
if not self.dut.build(self.module, self.ec_dir):
if not self.dut.build(self.ec_dir):
raise RuntimeError('Building module %s for DUT failed' % (self.module))
print 'Building TH image...'
if not self.th.build(self.module, self.ec_dir):
if not self.th.build(self.ec_dir):
raise RuntimeError('Building module %s for TH failed' % (self.module))
def flash_boards(self):
"""Flashes th and dut boards with their most recently build ec.bin"""
"""Flashes TH and DUT with their most recently built ec.bin."""
cts_module = 'cts_' + self.module
image_path = os.path.join('build', self.th.board, cts_module, 'ec.bin')
self.identify_boards()
@@ -96,102 +97,207 @@ class Cts(object):
raise RuntimeError('Flashing DUT failed')
def setup(self):
"""Setup boards"""
"""Setup boards."""
self.th.save_serial()
def identify_boards(self):
"""Updates serials of both th and dut, in that order (order matters)"""
"""Updates serials of TH and DUT in that order (order matters)."""
self.th.get_serial()
self.dut.get_serial()
@staticmethod
def get_macro_args(filepath, macro):
"""Get list of args of a certain macro in a file when macro is used
by itself on a line
def get_macro_args(self, filepath, macro):
"""Get list of args of a macro in a file when macro.
Args:
filepath: String containing absolute path to the file
macro: String containing text of macro to get args of
Returns:
List of dictionaries where each entry is:
'name': Test name,
'th_string': Expected string from TH,
'dut_string': Expected string from DUT,
"""
args = []
tests = []
with open(filepath, 'r') as f:
for l in f.readlines():
lines = f.readlines()
joined = ''.join(lines).replace('\\\n', '').splitlines()
for l in joined:
if not l.strip().startswith(macro):
continue
d = {}
l = l.strip()[len(macro):]
args.append(l.strip('()').replace(',', ''))
return args
l = l.strip('()').split(',')
d['name'] = l[0].strip()
d['th_rc'] = self.get_return_code_value(l[1].strip().strip('"'))
d['th_string'] = l[2].strip().strip('"')
d['dut_rc'] = self.get_return_code_value(l[3].strip().strip('"'))
d['dut_string'] = l[4].strip().strip('"')
tests.append(d)
return tests
def get_return_codes(self, file, prefix):
"""Extract return code names from the definition file (cts.rc)"""
def get_return_codes(self, filepath):
"""Read return code names from the return code definition file."""
self.return_codes = {}
val = 0
with open(file, 'r') as f:
for line in f.readlines():
with open(filepath, 'r') as f:
for line in f:
line = line.strip()
if not line.startswith(prefix):
if not line.startswith(CTS_RC_PREFIX):
continue
line = line[len(prefix):]
line = line.split(',')[0]
if '=' in line:
tokens = line.split('=')
line = tokens[0].strip()
val = int(tokens[1].strip())
self.return_codes[val] = line
self.return_codes[line] = val
val += 1
def parse_output(self, output):
results = defaultdict(lambda: CTS_RC_NO_RESULT)
"""Parse console output from DUT or TH.
Args:
output: String containing consoule output
Returns:
List of dictionaries where each key and value are:
name = 'ects_test_x',
started = True/False,
ended = True/False,
rc = CTS_RC_*,
output = All text between 'ects_test_x start' and 'ects_test_x end'
"""
results = []
i = 0
for test in self.testlist:
results.append({})
results[i]['name'] = test['name']
results[i]['started'] = False
results[i]['rc'] = CTS_RC_DID_NOT_START
results[i]['string'] = False
results[i]['output'] = []
i += 1
i = 0
for ln in [ln.strip() for ln in output.split('\n')]:
if i + 1 > len(results):
break
tokens = ln.split()
if len(tokens) != 2:
continue
test_name = tokens[0].strip()
if test_name not in self.test_names:
continue
try:
return_code = int(tokens[1])
except ValueError: # Second token is not an int
continue
if test_name in results:
results[test_name] = CTS_RC_DUPLICATE_RUN
else:
results[test_name] = return_code
if len(tokens) >= 2:
if tokens[0].strip() == results[i]['name']:
if tokens[1].strip() == 'start':
# start line found
if results[i]['started']: # Already started
results[i]['rc'] = CTS_RC_DUPLICATE_RUN
else:
results[i]['rc'] = CTS_RC_DID_NOT_END
results[i]['started'] = True
continue
elif results[i]['started'] and tokens[1].strip() == 'end':
# end line found
results[i]['rc'] = CTS_RC_INVALID_RETURN_CODE
if len(tokens) == 3:
try:
results[i]['rc'] = int(tokens[2].strip())
except ValueError:
pass
# Since index is incremented when 'end' is encountered, we don't
# need to check duplicate 'end'.
i += 1
continue
if results[i]['started']:
results[i]['output'].append(ln)
return results
def get_return_code_name(self, code):
return self.return_codes.get(code, '%d' % code)
def get_return_code_name(self, code, strip_prefix=False):
name = ''
for k, v in self.return_codes.iteritems():
if v == code:
if strip_prefix:
name = k[len(CTS_RC_PREFIX):]
else:
name = k
return name
def get_return_code_value(self, name):
if name:
return self.return_codes[name]
return 0
def evaluate_run(self, dut_output, th_output):
"""Parse outputs to derive test results
"""Parse outputs to derive test results.
Args;
Args:
dut_output: String output of DUT
th_output: String output of TH
"""
dut_results = self.parse_output(dut_output)
th_results = self.parse_output(th_output)
len_test_name = max(len(s) for s in self.test_names)
len_code_name = max(len(s) for s in self.return_codes.values())
Returns:
th_results: list of test results for TH
dut_results: list of test results for DUT
"""
th_results = self.parse_output(th_output)
dut_results = self.parse_output(dut_output)
# Search for expected string in each output
for i, v in enumerate(self.testlist):
if v['th_string'] in th_results[i]['output'] or not v['th_string']:
th_results[i]['string'] = True
if v['dut_string'] in dut_results[i]['output'] or not v['dut_string']:
dut_results[i]['string'] = True
return th_results, dut_results
def print_result(self, th_results, dut_results):
"""Print results to the screen.
Args:
th_results: list of test results for TH
dut_results: list of test results for DUT
"""
len_test_name = max(len(s['name']) for s in self.testlist)
len_code_name = max(len(self.get_return_code_name(v, True))
for v in self.return_codes.values())
head = '{:^' + str(len_test_name) + '} '
head += '{:^' + str(len_code_name) + '} '
head += '{:^' + str(len_code_name) + '}\n'
head += '{:^' + str(len_code_name) + '}'
head += '{:^' + str(len(' TH_STR')) + '}'
head += '{:^' + str(len(' DUT_STR')) + '}'
head += '{:^' + str(len(' RESULT')) + '}\n'
fmt = '{:' + str(len_test_name) + '} '
fmt += '{:>' + str(len_code_name) + '} '
fmt += '{:>' + str(len_code_name) + '}\n'
fmt += '{:>' + str(len_code_name) + '}'
fmt += '{:>' + str(len(' TH_STR')) + '}'
fmt += '{:>' + str(len(' DUT_STR')) + '}'
fmt += '{:>' + str(len(' RESULT')) + '}\n'
self.formatted_results = head.format('test name', 'TH', 'DUT')
for test_name in self.test_names:
th_cn = self.get_return_code_name(th_results[test_name])
dut_cn = self.get_return_code_name(dut_results[test_name])
self.formatted_results += fmt.format(test_name, th_cn, dut_cn)
self.formatted_results = head.format(
'test name', 'TH_RETURN_CODE', 'DUT_RETURN_CODE',
' TH_STR', ' DUT_STR', ' RESULT')
for i, d in enumerate(dut_results):
th_cn = self.get_return_code_name(th_results[i]['rc'], True)
dut_cn = self.get_return_code_name(dut_results[i]['rc'], True)
th_res = self.evaluate_result(th_results[i],
self.testlist[i]['th_rc'],
self.testlist[i]['th_string'])
dut_res = self.evaluate_result(dut_results[i],
self.testlist[i]['dut_rc'],
self.testlist[i]['dut_string'])
self.formatted_results += fmt.format(
d['name'], th_cn, dut_cn,
th_results[i]['string'], dut_results[i]['string'],
'PASS' if th_res and dut_res else 'FAIL')
def evaluate_result(self, result, expected_rc, expected_string):
if result['rc'] != expected_rc:
return False
if expected_string and expected_string not in result['output']:
return False
return True
def run(self):
"""Resets boards, records test results in results dir"""
"""Resets boards, records test results in results dir."""
print 'Reading serials...'
self.identify_boards()
print 'Opening DUT tty...'
@@ -228,9 +334,9 @@ class Cts(object):
time.sleep(MAX_SUITE_TIME_SEC)
print 'Reading DUT tty...'
dut_output, dut_boot = self.dut.read_tty()
dut_output, _ = self.dut.read_tty()
print 'Reading TH tty...'
th_output, th_boot = self.th.read_tty()
th_output, _ = self.th.read_tty()
print 'Halting TH...'
if not self.th.send_open_ocd_commands(['init', 'reset halt']):
@@ -245,7 +351,10 @@ class Cts(object):
'again.')
print 'Pursing results...'
self.evaluate_run(dut_output, th_output)
th_results, dut_results = self.evaluate_run(dut_output, th_output)
# Print out results
self.print_result(th_results, dut_results)
# Write results
dest = os.path.join(self.results_dir, 'results.log')
@@ -262,11 +371,10 @@ class Cts(object):
print self.formatted_results
# TODO: Should set exit code for the shell
# TODO(chromium:735652): Should set exit code for the shell
def main():
"""Main entry point for CTS script from command line"""
ec_dir = os.path.realpath(os.path.join(
os.path.dirname(os.path.abspath(__file__)), '..'))
os.chdir(ec_dir)
@@ -321,5 +429,5 @@ def main():
cts.flash_boards()
cts.run()
if __name__ == "__main__":
if __name__ == '__main__':
main()

17
cts/gpio/cts.testlist
View File

@@ -6,15 +6,20 @@
/* Currently tests will execute in the order they are listed here */
/* Test whether sync completes successfully */
CTS_TEST(sync_test)
CTS_TEST(sync_test,,,,)
/* Check if the dut can set a line low */
CTS_TEST(set_low_test)
CTS_TEST(set_low_test,,,,)
/* Check if the dut can set a line high */
CTS_TEST(set_high_test)
CTS_TEST(set_high_test,,,,)
/* Check if the dut can read a line that is low */
CTS_TEST(read_high_test)
CTS_TEST(read_high_test,,,,)
/* Check if the dut can read a line that is high */
CTS_TEST(read_low_test)
CTS_TEST(read_low_test,,,,)
/* Check if the dut reads its true pin level (success)
or its register level when configured as a low open drain output pin */
CTS_TEST(od_read_high_test)
CTS_TEST(od_read_high_test,,,,)

3
cts/gpio/dut.c
View File

@@ -81,8 +81,9 @@ void cts_task(void)
uart_flush_output();
for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
sync();
CPRINTF("\n%s start\n", tests[i].name);
result = tests[i].run();
CPRINTF("\n%s %d\n", tests[i].name, result);
CPRINTF("\n%s end %d\n", tests[i].name, result);
uart_flush_output();
}

3
cts/gpio/th.c
View File

@@ -75,8 +75,9 @@ void cts_task(void)
uart_flush_output();
for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
sync();
CPRINTF("\n%s start\n", tests[i].name);
result = tests[i].run();
CPRINTF("\n%s %d\n", tests[i].name, result);
CPRINTF("\n%s end %d\n", tests[i].name, result);
uart_flush_output();
}

12
cts/i2c/cts.testlist
View File

@@ -6,9 +6,9 @@
/* Currently tests will execute in the order they are listed here */
/* Test whether sync completes successfully */
CTS_TEST(write8_test)
CTS_TEST(write16_test)
CTS_TEST(write32_test)
CTS_TEST(read8_test)
CTS_TEST(read16_test)
CTS_TEST(read32_test)
CTS_TEST(write8_test,,,,)
CTS_TEST(write16_test,,,,)
CTS_TEST(write32_test,,,,)
CTS_TEST(read8_test,,,,)
CTS_TEST(read16_test,,,,)
CTS_TEST(read32_test,,,,)

12
cts/interrupt/cts.testlist
View File

@@ -4,14 +4,14 @@
*/
/* Test interrupt_enable/disable */
CTS_TEST(test_interrupt_enable)
CTS_TEST(test_interrupt_disable)
CTS_TEST(test_interrupt_enable,,,,)
CTS_TEST(test_interrupt_disable,,,,)
/* Test task_wait_for_event */
CTS_TEST(test_task_wait_event)
CTS_TEST(test_task_wait_event,,,,)
/* Test task_disable_irq */
CTS_TEST(test_task_disable_irq)
CTS_TEST(test_task_disable_irq,,,,)
/* Test nested interrupt. Lower priority IRQ is fired, followed by
* higher priority IRQ. Handler executions should be nested.
@@ -23,7 +23,7 @@ CTS_TEST(test_task_disable_irq)
* task_cts ----* *----
* A B C D
*/
CTS_TEST(test_nested_interrupt_low_high)
CTS_TEST(test_nested_interrupt_low_high,,,,)
/* Test nested interrupt. Higher priority IRQ is fired, followed by
* lower priority IRQ. Handlers should be executed sequentially.
@@ -35,7 +35,7 @@ CTS_TEST(test_nested_interrupt_low_high)
* task_cts ----* *----
* B C A D
*/
CTS_TEST(test_nested_interrupt_high_low)
CTS_TEST(test_nested_interrupt_high_low,,,,)
/*
* Other ideas

3
cts/interrupt/dut.c
View File

@@ -185,9 +185,10 @@ void cts_task(void)
for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
clear_state();
sync();
CPRINTF("\n%s start\n", tests[i].name);
rc = tests[i].run();
interrupt_enable();
CPRINTF("\n%s %d\n", tests[i].name, rc);
CPRINTF("\n%s end %d\n", tests[i].name, rc);
cflush();
}

3
cts/interrupt/th.c
View File

@@ -74,8 +74,9 @@ void cts_task(void)
gpio_set_level(GPIO_OUTPUT_TEST, 1);
gpio_set_level(GPIO_CTS_IRQ2, 1);
sync();
CPRINTF("\n%s start\n", tests[i].name);
rc = tests[i].run();
CPRINTF("\n%s %d\n", tests[i].name, rc);
CPRINTF("\n%s end %d\n", tests[i].name, rc);
cflush();
}

28
cts/meta/cts.testlist
View File

@@ -12,53 +12,41 @@
* 3
* 4
*/
CTS_TEST(debug_test)
CTS_TEST(debug_test,,,,)
/* Test should succeed if both report success
* (expected result: success)
*/
CTS_TEST(success_test)
CTS_TEST(success_test,,,,)
/* Test should fail if one reports success and
* (one reports failure (expected result: failure)
*/
CTS_TEST(fail_dut_test)
CTS_TEST(fail_dut_test,,, CTS_RC_FAILURE,)
/* Test should fail if one reports success and
* (one reports failure (expected result: failure)
*/
CTS_TEST(fail_th_test)
CTS_TEST(fail_th_test, CTS_RC_FAILURE,,,)
/* Test should fail when both boards report failure
* (expected result: failure)
*/
CTS_TEST(fail_both_test)
CTS_TEST(fail_both_test, CTS_RC_FAILURE,, CTS_RC_FAILURE,)
/* Test should fail with bad sync if one reports bad
* sync and the other reports success (expected result:
* bad_sync)
*/
CTS_TEST(bad_sync_and_success_test)
CTS_TEST(bad_sync_and_success_test, CTS_RC_BAD_SYNC,,,)
/* Test should fail with bad sync if both boards report
* bad sync (expected result: bad_sync)
*/
CTS_TEST(bad_sync_both_test)
/* Test should report conflict if one reports bad sync
* and the other reports failure
* (expected result: conflict)
*/
CTS_TEST(bad_sync_failure_test)
CTS_TEST(bad_sync_both_test, CTS_RC_BAD_SYNC,, CTS_RC_BAD_SYNC,)
/* Test should be listed as corrupted if one test hangs,
* regardless of what the other test outputs
* (expected result: corrupted)
*/
CTS_TEST(hang_test)
/* Test should be corrupted if it follows a corrupted
* test, regardless of what the actual result was
* reported as
*/
CTS_TEST(post_corruption_success)
CTS_TEST(hang_test, CTS_RC_SUCCESS,, CTS_RC_DID_NOT_END,)

17
cts/meta/dut.c
View File

@@ -37,7 +37,7 @@ enum cts_rc fail_both_test(void)
enum cts_rc bad_sync_and_success_test(void)
{
return CTS_RC_BAD_SYNC;
return CTS_RC_SUCCESS;
}
enum cts_rc bad_sync_both_test(void)
@@ -45,11 +45,6 @@ enum cts_rc bad_sync_both_test(void)
return CTS_RC_BAD_SYNC;
}
enum cts_rc bad_sync_failure_test(void)
{
return CTS_RC_BAD_SYNC;
}
enum cts_rc hang_test(void)
{
while (1) {
@@ -60,11 +55,6 @@ enum cts_rc hang_test(void)
return CTS_RC_SUCCESS;
}
enum cts_rc post_corruption_success(void)
{
return CTS_RC_SUCCESS;
}
#include "cts_testlist.h"
void cts_task(void)
@@ -75,12 +65,13 @@ void cts_task(void)
cflush();
for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
sync();
CPRINTF("\n%s start\n", tests[i].name);
result = tests[i].run();
CPRINTF("\n%s %d\n", tests[i].name, result);
CPRINTF("\n%s end %d\n", tests[i].name, result);
cflush();
}
CPRINTS("GPIO test suite finished");
CPRINTS("Meta test finished");
cflush();
while (1) {
watchdog_reload();

16
cts/meta/th.c
View File

@@ -55,23 +55,12 @@ enum cts_rc bad_sync_both_test(void)
return CTS_RC_BAD_SYNC;
}
enum cts_rc bad_sync_failure_test(void)
{
CTS_DEBUG_PRINTF("Expect: Conflict");
return CTS_RC_FAILURE;
}
enum cts_rc hang_test(void)
{
CTS_DEBUG_PRINTF("This and next, expect: Corrupted");
return CTS_RC_SUCCESS;
}
enum cts_rc post_corruption_success(void)
{
return CTS_RC_SUCCESS;
}
#include "cts_testlist.h"
void cts_task(void)
@@ -82,12 +71,13 @@ void cts_task(void)
cflush();
for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
sync();
CPRINTF("\n%s start\n", tests[i].name);
result = tests[i].run();
CPRINTF("\n%s %d\n", tests[i].name, result);
CPRINTF("\n%s end %d\n", tests[i].name, result);
cflush();
}
CPRINTS("GPIO test suite finished");
CPRINTS("Meta test finished");
cflush();
while (1) {
watchdog_reload();

9
cts/task/cts.testlist
View File

@@ -9,18 +9,19 @@
* repeated TEST_COUNT times. It's expected all tasks to run exactly
* TEST_COUNT times. Tick task runs to inject some irregularity.
*/
CTS_TEST(test_task_switch)
CTS_TEST(test_task_switch,,,,)
/*
* Test task priority. CTS task wakes up A and C then goes to sleep. Since C
* has a higher priority, C should run first. This should result in C running
* one more time than A (or B).
*/
CTS_TEST(test_task_priority)
CTS_TEST(test_task_priority,,,,)
/*
* Test stack overflow. CTS task overflows the stack and it should be detected
* when task switch happens. Reboot is expected.
* TODO: Verify stack overflow detection and reboot.
*/
CTS_TEST(test_stack_overflow)
CTS_TEST(test_stack_overflow,\
CTS_RC_DID_NOT_END, "Stack overflow in CTS task!",\
CTS_RC_DID_NOT_END, "Stack overflow in CTS task!")

12
cts/task/dut.c
View File

@@ -125,19 +125,14 @@ static void recurse(int x)
CPRINTS("-%d", x);
}
enum cts_rc test_stack_overflow(void);
#include "cts_testlist.h"
enum cts_rc test_stack_overflow(void)
{
/* recurse() is expected to overflow the stack, which leads to reboot.
* So, we print output proactively. */
CPRINTF("\n%s %d\n", tests[CTS_TEST_ID_COUNT - 1].name, CTS_RC_SUCCESS);
recurse(0);
return CTS_RC_FAILURE;
}
#include "cts_testlist.h"
void cts_task(void)
{
enum cts_rc rc;
@@ -147,8 +142,9 @@ void cts_task(void)
for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
clear_state();
CPRINTF("\n%s start\n", tests[i].name);
rc = tests[i].run();
CPRINTF("\n%s %d\n", tests[i].name, rc);
CPRINTF("\n%s end %d\n", tests[i].name, rc);
cflush();
}

2
cts/timer/cts.testlist
View File

@@ -16,4 +16,4 @@
* - GPIO_OUTPUT connection for sending notification from DUT
* - Calibrated TH timer
*/
CTS_TEST(timer_calibration_test)
CTS_TEST(timer_calibration_test,,,,)

3
cts/timer/dut.c
View File

@@ -29,8 +29,9 @@ void cts_task(void)
for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
sync();
CPRINTF("\n%s start\n", tests[i].name);
rc = tests[i].run();
CPRINTF("\n%s %d\n", tests[i].name, rc);
CPRINTF("\n%s end %d\n", tests[i].name, rc);
cflush();
}

3
cts/timer/th.c
View File

@@ -62,8 +62,9 @@ void cts_task(void)
for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
sync();
CPRINTF("\n%s start\n", tests[i].name);
rc = tests[i].run();
CPRINTF("\n%s %d\n", tests[i].name, rc);
CPRINTF("\n%s end %d\n", tests[i].name, rc);
cflush();
}