Merge pull request #24115 from rmmh/test-history-suites

Automatic merge from submit-queue

Add suite pages to test-history

This changes http://storage.googleapis.com/kubernetes-test-history/static/index.html to look like:

![image](https://cloud.githubusercontent.com/assets/211868/14445317/f263e1da-0000-11e6-9bd9-01150d9ae4ba.png)

And adds sub-pages to show test results for specific suites:

![image](https://cloud.githubusercontent.com/assets/211868/14445339/1bb8555c-0001-11e6-9e5c-41d671af39a1.png)

hack/jenkins/test-history/gen_history

@@ -26,16 +26,8 @@ readonly datestr=$(date +"%Y-%m-%d")
 # Create JSON report
 time python gen_json.py "${jenkins}" kubernetes
 
-# Create static HTML report out of the JSON
-python gen_html.py > static/tests.html
-python gen_html.py kubernetes-e2e > static/tests-e2e.html
-python gen_html.py kubernetes-soak > static/tests-soak.html
-python gen_html.py kubernetes-e2e-gce > static/tests-e2e-gce.html
-python gen_html.py kubernetes-e2e-gke > static/tests-e2e-gke.html
-python gen_html.py kubernetes-upgrade > static/tests-upgrade.html
-
-# Fill in the last updated time into the template.
-cat index_template.html | sed -e "s/TIME/Last updated: ${datestr}/" > static/index.html
+# Create static HTML reports out of the JSON
+python gen_html.py --suites --prefixes ,e2e,soak,e2e-gce,e2e-gke,upgrade --output-dir static --input tests.json
 
 # Upload to GCS
 readonly bucket="kubernetes-test-history"

hack/jenkins/test-history/gen_html.py

@@ -27,20 +27,24 @@ JSON. That would allow custom filtering and stuff like that.
 
 from __future__ import print_function
 
+import argparse
 import json
+import os
 import string
 import sys
+import time
 
-def gen_tests(data, prefix):
+def gen_tests(data, prefix, exact_match):
     """Creates the HTML for all test cases.
 
     Args:
         data: Parsed JSON data that was created by gen_json.py.
         prefix: Considers Jenkins jobs that start with this.
+        exact_match: Only match Jenkins jobs with name equal to prefix.
 
     Returns:
-        The HTML as a list of elements along with the number of passing,
-        unstable, failing, and skipped tests.
+        The HTML as a list of elements along with a tuple of the number of
+        passing, unstable, failing, and skipped tests.
     """
     html = ['<ul class="test">']
     total_okay = 0
@@ -55,6 +59,8 @@ def gen_tests(data, prefix):
         for suite in sorted(data[test]):
             if not suite.startswith(prefix):
                 continue
+            if exact_match and suite != prefix:
+                continue
             has_test = True
             num_failed = 0
             num_builds = 0
@@ -78,10 +84,11 @@ def gen_tests(data, prefix):
             else:
                 status = 'okay'
             test_html.append('<li class="suite">')
-            test_html.append('<span class="{}">{}/{}</span>'.format(status, str(num_builds - num_failed), str(num_builds)))
-            test_html.append('<span class="time">{}</span>'.format(str(int(avg_time)) + unit))
+            test_html.append('<span class="%s">%d/%d</span>' % (status, num_builds - num_failed, num_builds))
+            test_html.append('<span class="time">%.0f%s</span>' % (avg_time, unit))
             test_html.append(suite)
             test_html.append('</li>')
+        test_html.append('</ul>')
         if has_failed:
             status = 'failed'
             total_failed += 1
@@ -94,42 +101,112 @@ def gen_tests(data, prefix):
         else:
             status = 'skipped'
             total_skipped += 1
-        html.append('<li class="test {}">{}'.format(status, test))
-        html.extend(test_html)
-        html.append('</ul>')
+        html.append('<li class="test %s">' % status)
+        if exact_match and len(test_html) > 2:
+            if not (test_html[2].startswith('<span') and test_html[3].startswith('<span')):
+                raise ValueError("couldn't extract suite results for prepending")
+            html.extend(test_html[2:4])
+            html.append(test)
+        else:
+            html.append(test)
+            html.extend(test_html)
         html.append('</li>')
     html.append('</ul>')
-    return html, total_okay, total_unstable, total_failed, total_skipped
+    return '\n'.join(html), (total_okay, total_unstable, total_failed, total_skipped)
 
-def gen_html(data, prefix):
-    """Creates the HTML for the entire page.
-
-    Args: Same as gen_tests.
-    Returns: Just the list of HTML elements.
-    """
-    tests_html, okay, unstable, failed, skipped = gen_tests(data, prefix)
+def html_header():
     html = ['<html>', '<head>']
     html.append('<link rel="stylesheet" type="text/css" href="style.css" />')
     html.append('<script src="script.js"></script>')
     html.append('</head>')
     html.append('<body>')
-    if len(prefix) > 0:
-        html.append('<div id="header">Suites starting with {}:'.format(prefix))
-    else:
-        html.append('<div id="header">All suites:')
-    html.append('<span class="total okay" onclick="toggle(\'okay\');">{}</span>'.format(str(okay)))
-    html.append('<span class="total unstable" onclick="toggle(\'unstable\');">{}</span>'.format(str(unstable)))
-    html.append('<span class="total failed" onclick="toggle(\'failed\');">{}</span>'.format(str(failed)))
-    html.append('<span class="total skipped" onclick="toggle(\'skipped\');">{}</span>'.format(str(skipped)))
-    html.append('</div>')
-    html.extend(tests_html)
-    html.append('</body>')
-    html.append('</html>')
     return html
 
+def gen_html(data, prefix, exact_match=False):
+    """Creates the HTML for the entire page.
+
+    Args: Same as gen_tests.
+    Returns: Same as gen_tests.
+    """
+    tests_html, (okay, unstable, failed, skipped) = gen_tests(data, prefix, exact_match)
+    html = html_header()
+    if exact_match:
+        html.append('<div id="header">Suite %s' % prefix)
+    elif len(prefix) > 0:
+        html.append('<div id="header">Suites starting with %s:' % prefix)
+    else:
+        html.append('<div id="header">All suites:')
+    html.append('<span class="total okay" onclick="toggle(\'okay\');">%s</span>' % okay)
+    html.append('<span class="total unstable" onclick="toggle(\'unstable\');">%d</span>' % unstable)
+    html.append('<span class="total failed" onclick="toggle(\'failed\');">%d</span>' % failed)
+    html.append('<span class="total skipped" onclick="toggle(\'skipped\');">%d</span>' % skipped)
+    html.append('</div>')
+    html.append(tests_html)
+    html.append('</body>')
+    html.append('</html>')
+    return '\n'.join(html), (okay, unstable, failed, skipped)
+
+def gen_metadata_links(suites):
+    html = []
+    for (name, target), (okay, unstable, failed, skipped) in sorted(suites.iteritems()):
+        html.append('<a class="suite-link" href="%s">' % target)
+        html.append('<span class="total okay">%d</span>' % okay)
+        html.append('<span class="total unstable">%d</span>' % unstable)
+        html.append('<span class="total failed">%d</span>' % failed)
+        html.append(name)
+        html.append('</a>')
+    return html
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--suites', action='store_true',
+                        help='output test results for each suite')
+    parser.add_argument('--prefixes',
+                        help='comma-separated list of suite prefixes to create pages for')
+    parser.add_argument('--output-dir', required=True,
+                        help='where to write output pages')
+    parser.add_argument('--input', required=True,
+                        help='JSON test data to read for input')
+    options=parser.parse_args(args)
+
+    with open(options.input) as f:
+        data = json.load(f)
+
+    if options.prefixes:
+        # the empty prefix means "all tests"
+        options.prefixes = options.prefixes.split(',')
+        prefix_metadata = {}
+        for prefix in options.prefixes:
+            if prefix:
+                path = 'tests-%s.html' % prefix
+                prefix = 'kubernetes-%s' % prefix
+            else:
+                path = 'tests.html'
+            html, prefix_metadata[prefix or 'kubernetes', path] = gen_html(data, prefix, False)
+            with open(os.path.join(options.output_dir, path), 'w') as f:
+                f.write(html)
+    if options.suites:
+        suites_set = set()
+        for test, suites in data.iteritems():
+            suites_set.update(suites.keys())
+        suite_metadata = {}
+        for suite in sorted(suites_set):
+            path = 'suite-%s.html' % suite
+            html, suite_metadata[suite, path] = gen_html(data, suite, True)
+            with open(os.path.join(options.output_dir, path), 'w') as f:
+                f.write(html)
+    html = html_header()
+    html.append('<h1>Kubernetes Tests</h1>')
+    html.append('Last updated %s' % time.strftime('%F'))
+    if options.prefixes:
+        html.append('<h2>All suites starting with:</h2>')
+        html.extend(gen_metadata_links(prefix_metadata))
+    if options.suites:
+        html.append('<h2>Specific suites:</h2>')
+        html.extend(gen_metadata_links(suite_metadata))
+    html.extend(['</body>', '</html>'])
+    with open(os.path.join(options.output_dir, 'index.html'), 'w') as f:
+        f.write('\n'.join(html))
+
 if __name__ == '__main__':
-    prefix = ''
-    if len(sys.argv) == 2:
-        prefix = sys.argv[1]
-    with open('tests.json', 'r') as f:
-        print('\n'.join(gen_html(json.load(f), prefix)))
+    main(sys.argv[1:])

hack/jenkins/test-history/gen_html_test.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for gen_html."""
+
+import json
+import os
+import shutil
+import tempfile
+import unittest
+
+import gen_html
+
+TEST_DATA = {
+  "test1":
+      {"kubernetes-release": [{"build": 3, "failed": False, "time": 3.52},
+                              {"build": 4, "failed": True, "time": 63.21}],
+       "kubernetes-debug": [{"build": 5, "failed": False, "time": 7.56},
+                            {"build": 6, "failed": False, "time": 8.43}],
+      },
+  "test2":
+      {"kubernetes-debug": [{"build": 6, "failed": True, "time": 3.53}]},
+}
+
+class GenHtmlTest(unittest.TestCase):
+    def gen_html(self, *args):
+        return gen_html.gen_html(TEST_DATA, *args)[0]
+
+    def testGenHtml(self):
+        html = self.gen_html('')
+        self.assertIn("test1", html)
+        self.assertIn("test2", html)
+        self.assertIn("release", html)
+        self.assertIn("debug", html)
+
+    def testGenHtmlFilter(self):
+        html = self.gen_html('release')
+        self.assertIn("release", html)
+        self.assertIn('skipped">\ntest2', html)
+        self.assertNotIn("debug", html)
+
+    def testGenHtmlFilterExact(self):
+        html = self.gen_html('release', True)
+        self.assertNotIn('debug', html)
+
+    def testMain(self):
+        temp_dir = tempfile.mkdtemp(prefix='kube-test-hist-')
+        try:
+            tests_json = os.path.join(temp_dir, 'tests.json')
+            with open(tests_json, 'w') as f:
+                json.dump(TEST_DATA, f)
+            gen_html.main(['--suites', '--prefixes', ',rel,deb',
+                           '--output-dir', temp_dir, '--input', tests_json])
+            for page in ('index', 'suite-kubernetes-debug', 'tests', 'tests-rel', 'tests-deb'):
+                self.assertTrue(os.path.exists('%s/%s.html' % (temp_dir, page)))
+        finally:
+            shutil.rmtree(temp_dir)
+
+if __name__ == '__main__':
+    unittest.main()

hack/jenkins/test-history/index_template.html

@@ -1,15 +0,0 @@
-<html>
-<body>
-<h2>All tests starting with:</h2>
-<ul>
-<li><a href="tests.html">kubernetes</a></li>
-<li><a href="tests-e2e.html">kubernetes-e2e</a></li>
-<li><a href="tests-e2e-gce.html">kubernetes-e2e-gce</a></li>
-<li><a href="tests-e2e-gke.html">kubernetes-e2e-gke</a></li>
-<li><a href="tests-upgrade.html">kubernetes-upgrade</a></li>
-<li><a href="tests-soak.html">kubernetes-soak</a></li>
-</ul>
-TIME
-</body>
-</html>
-

hack/jenkins/test-history/static/style.css

@@ -11,7 +11,14 @@ body {
     padding: 10px;
 }
 
-#header span{
+a.suite-link {
+    font-weight: bold;
+    font-size: large;
+    display: block;
+    text-decoration: none;
+}
+
+span.total {
     display: inline-block;
     width: 60px;
     padding-left: 30px;
@@ -91,19 +98,24 @@ li.suite span {
     font-weight: bold;
 }
 
-li.suite span.time {
+li.test span.time {
     width: 50px;
     font-weight: normal;
 }
 
-li.suite span.okay {
+li.test span.okay {
     color: green;
 }
 
-li.suite span.unstable {
+li.test span.unstable {
     color: orange;
 }
 
-li.suite span.failed {
+li.test span.failed {
     color: red;
 }
+
+li.test>span {
+    display: inline-block;
+    text-align: right;
+}