Merge pull request #24370 from fejta/crit

Automatic merge from submit-queue

Add blocking suites to daily summary

Add a section for blocking suites, and refactor some of the code.

![blocking](https://cloud.githubusercontent.com/assets/940341/14578274/40415830-0338-11e6-8d8e-9434c87a7288.png)


/cc @rmmh

hack/jenkins/test-history/gen_history

@@ -24,10 +24,12 @@ readonly jenkins="$1"
 readonly datestr=$(date +"%Y-%m-%d")
 
 # Create JSON report
-time python gen_json.py "${jenkins}" kubernetes
+time python gen_json.py \
+  "--server=${jenkins}" \
+  "--match=^kubernetes|kubernetes-build|kubelet-gce-e2e-ci"
 
 # Create static HTML reports out of the JSON
-python gen_html.py --suites --prefixes ,e2e,soak,e2e-gce,e2e-gke,upgrade --output-dir static --input tests.json
+python gen_html.py --output-dir=static --input=tests.json
 
 # Upload to GCS
 readonly bucket="kubernetes-test-history"

hack/jenkins/test-history/gen_html.py

@@ -28,12 +28,23 @@ JSON. That would allow custom filtering and stuff like that.
 from __future__ import print_function
 
 import argparse
+import cgi
+import collections
 import json
 import os
 import string
 import sys
 import time
 
+
+TestMetadata = collections.namedtuple('TestMetadata', [
+    'okay',
+    'unstable',
+    'failed',
+    'skipped',
+])
+
+
 def gen_tests(data, prefix, exact_match):
     """Creates the HTML for all test cases.
 
@@ -43,14 +54,10 @@ def gen_tests(data, prefix, exact_match):
         exact_match: Only match Jenkins jobs with name equal to prefix.
 
     Returns:
-        The HTML as a list of elements along with a tuple of the number of
-        passing, unstable, failing, and skipped tests.
+        (html, TestMetadata) for matching tests
     """
     html = ['<ul class="test">']
-    total_okay = 0
-    total_unstable = 0
-    total_failed = 0
-    total_skipped = 0
+    totals = collections.defaultdict(int)
     for test in sorted(data, key=string.lower):
         test_html = ['<ul class="suite">']
         has_test = False
@@ -84,27 +91,28 @@ def gen_tests(data, prefix, exact_match):
             else:
                 status = 'okay'
             test_html.append('<li class="suite">')
-            test_html.append('<span class="%s">%d/%d</span>' % (status, num_builds - num_failed, num_builds))
-            test_html.append('<span class="time">%.0f%s</span>' % (avg_time, unit))
+            test_html.append('<span class="%s">%d/%d</span>' % (
+                status, num_builds - num_failed, num_builds))
+            test_html.append(
+                '<span class="time">%.0f%s</span>' % (avg_time, unit))
             test_html.append(suite)
             test_html.append('</li>')
         test_html.append('</ul>')
         if has_failed:
             status = 'failed'
-            total_failed += 1
         elif has_unstable:
             status = 'unstable'
-            total_unstable += 1
         elif has_test:
             status = 'okay'
-            total_okay += 1
         else:
             status = 'skipped'
-            total_skipped += 1
+        totals[status] += 1
         html.append('<li class="test %s">' % status)
         if exact_match and len(test_html) > 2:
-            if not (test_html[2].startswith('<span') and test_html[3].startswith('<span')):
-                raise ValueError("couldn't extract suite results for prepending")
+            if not (test_html[2].startswith('<span') and
+                    test_html[3].startswith('<span')):
+                raise ValueError(
+                    'couldn\'t extract suite results for prepending')
             html.extend(test_html[2:4])
             html.append(test)
         else:
@@ -112,101 +120,166 @@ def gen_tests(data, prefix, exact_match):
             html.extend(test_html)
         html.append('</li>')
     html.append('</ul>')
-    return '\n'.join(html), (total_okay, total_unstable, total_failed, total_skipped)
+    return '\n'.join(html), TestMetadata(
+        totals['okay'], totals['unstable'], totals['failed'], totals['skipped'])
 
-def html_header():
+
+def html_header(title, script):
+    """Return html header items."""
     html = ['<html>', '<head>']
     html.append('<link rel="stylesheet" type="text/css" href="style.css" />')
+    if title:
+        html.append('<title>%s</title>' % cgi.escape(title))
+    if script:
         html.append('<script src="script.js"></script>')
     html.append('</head>')
     html.append('<body>')
     return html
 
+
 def gen_html(data, prefix, exact_match=False):
     """Creates the HTML for the entire page.
 
-    Args: Same as gen_tests.
-    Returns: Same as gen_tests.
+    Args:
+        Same as gen_tests.
+    Returns:
+        Same as gen_tests.
     """
-    tests_html, (okay, unstable, failed, skipped) = gen_tests(data, prefix, exact_match)
-    html = html_header()
+    tests_html, meta = gen_tests(data, prefix, exact_match)
     if exact_match:
-        html.append('<div id="header">Suite %s' % prefix)
-    elif len(prefix) > 0:
-        html.append('<div id="header">Suites starting with %s:' % prefix)
+        msg = 'Suite %s' % cgi.escape(prefix)
+    elif prefix:
+        msg = 'Suites starting with %s' % cgi.escape(prefix)
     else:
-        html.append('<div id="header">All suites:')
-    html.append('<span class="total okay" onclick="toggle(\'okay\');">%s</span>' % okay)
-    html.append('<span class="total unstable" onclick="toggle(\'unstable\');">%d</span>' % unstable)
-    html.append('<span class="total failed" onclick="toggle(\'failed\');">%d</span>' % failed)
-    html.append('<span class="total skipped" onclick="toggle(\'skipped\');">%d</span>' % skipped)
+        msg = 'All suites'
+    html = html_header(title=msg, script=True)
+    html.append('<div id="header">%s:' % msg)
+    fmt = '<span class="total %s" onclick="toggle(\'%s\');">%s</span>'
+    html.append(fmt % ('okay', 'okay', meta.okay))
+    html.append(fmt % ('unstable', 'unstable', meta.unstable))
+    html.append(fmt % ('failed', 'failed', meta.failed))
+    html.append(fmt % ('skipped', 'skipped', meta.skipped))
     html.append('</div>')
     html.append(tests_html)
     html.append('</body>')
     html.append('</html>')
-    return '\n'.join(html), (okay, unstable, failed, skipped)
+    return '\n'.join(html), meta
+
 
 def gen_metadata_links(suites):
+    """Write clickable pass, ustabled, failed stats."""
     html = []
-    for (name, target), (okay, unstable, failed, skipped) in sorted(suites.iteritems()):
+    for (name, target), meta in sorted(suites.iteritems()):
         html.append('<a class="suite-link" href="%s">' % target)
-        html.append('<span class="total okay">%d</span>' % okay)
-        html.append('<span class="total unstable">%d</span>' % unstable)
-        html.append('<span class="total failed">%d</span>' % failed)
+        html.append('<span class="total okay">%d</span>' % meta.okay)
+        html.append('<span class="total unstable">%d</span>' % meta.unstable)
+        html.append('<span class="total failed">%d</span>' % meta.failed)
         html.append(name)
         html.append('</a>')
     return html
 
-def main(args):
+
+def write_html(outdir, path, html):
+    """Write html to outdir/path."""
+    with open(os.path.join(outdir, path), 'w') as buf:
+        buf.write(html)
+
+
+def write_metadata(infile, outdir):
+    """Writes tests-*.html and suite-*.html files.
+
+    Args:
+      infile: the json file created by gen_json.py
+      outdir: a path to write the html files.
+    """
+    with open(infile) as buf:
+        data = json.load(buf)
+
+    prefix_metadata = {}
+    prefixes = [
+        'kubernetes',
+        'kubernetes-e2e',
+        'kubernetes-soak',
+        'kubernetes-e2e-gce',
+        'kubernetes-e2e-gke',
+        'kubernetes-upgrade',
+    ]
+    for prefix in prefixes:
+        path = 'tests-%s.html' % prefix
+        html, metadata = gen_html(data, prefix, False)
+        write_html(outdir, path, html)
+        prefix_metadata[prefix or 'kubernetes', path] = metadata
+
+    suite_metadata = {}
+    suites = set()
+    for suite_names in data.values():
+        suites.update(suite_names.keys())
+    for suite in sorted(suites):
+        path = 'suite-%s.html' % suite
+        html, metadata = gen_html(data, suite, True)
+        write_html(outdir, path, html)
+        suite_metadata[suite, path] = metadata
+
+    blocking = {
+        'kubelet-gce-e2e-ci',
+        'kubernetes-build',
+        'kubernetes-e2e-gce',
+        'kubernetes-e2e-gce-scalability',
+        'kubernetes-e2e-gce-slow',
+        'kubernetes-e2e-gke',
+        'kubernetes-e2e-gke-slow',
+        'kubernetes-kubemark-5-gce',
+        'kubernetes-kubemark-500-gce',
+        'kubernetes-test-go',
+    }
+    blocking_suite_metadata = {
+        k: v for (k, v) in suite_metadata.items() if k[0] in blocking}
+
+    return prefix_metadata, suite_metadata, blocking_suite_metadata
+
+
+def write_index(outdir, prefixes, suites, blockers):
+    """Write the index.html with links to each view, including stat summaries.
+
+    Args:
+      outdir: the path to write the index.html file
+      prefixes: the {(prefix, path): TestMetadata} map
+      suites: the {(suite, path): TestMetadata} map
+      blockers: the {(suite, path): TestMetadata} map of blocking suites
+    """
+    html = html_header(title='Kubernetes Test Summary', script=False)
+    html.append('<h1>Kubernetes Tests</h1>')
+    html.append('Last updated %s' % time.strftime('%F'))
+
+    html.append('<h2>Tests from suites starting with:</h2>')
+    html.extend(gen_metadata_links(prefixes))
+
+    html.append('<h2>Blocking suites:</h2>')
+    html.extend(gen_metadata_links(blockers))
+
+    html.append('<h2>All suites:</h2>')
+    html.extend(gen_metadata_links(suites))
+
+    html.extend(['</body>', '</html>'])
+    write_html(outdir, 'index.html', '\n'.join(html))
+
+
+def main(infile, outdir):
+    """Use infile to write test, suite and index html files to outdir."""
+    prefixes, suites, blockers = write_metadata(infile, outdir)
+    write_index(outdir, prefixes, suites, blockers)
+
+
+def get_options(argv):
+    """Process command line arguments."""
     parser = argparse.ArgumentParser()
-    parser.add_argument('--suites', action='store_true',
-                        help='output test results for each suite')
-    parser.add_argument('--prefixes',
-                        help='comma-separated list of suite prefixes to create pages for')
     parser.add_argument('--output-dir', required=True,
                         help='where to write output pages')
     parser.add_argument('--input', required=True,
                         help='JSON test data to read for input')
-    options=parser.parse_args(args)
+    return parser.parse_args(argv)
 
-    with open(options.input) as f:
-        data = json.load(f)
-
-    if options.prefixes:
-        # the empty prefix means "all tests"
-        options.prefixes = options.prefixes.split(',')
-        prefix_metadata = {}
-        for prefix in options.prefixes:
-            if prefix:
-                path = 'tests-%s.html' % prefix
-                prefix = 'kubernetes-%s' % prefix
-            else:
-                path = 'tests.html'
-            html, prefix_metadata[prefix or 'kubernetes', path] = gen_html(data, prefix, False)
-            with open(os.path.join(options.output_dir, path), 'w') as f:
-                f.write(html)
-    if options.suites:
-        suites_set = set()
-        for test, suites in data.iteritems():
-            suites_set.update(suites.keys())
-        suite_metadata = {}
-        for suite in sorted(suites_set):
-            path = 'suite-%s.html' % suite
-            html, suite_metadata[suite, path] = gen_html(data, suite, True)
-            with open(os.path.join(options.output_dir, path), 'w') as f:
-                f.write(html)
-    html = html_header()
-    html.append('<h1>Kubernetes Tests</h1>')
-    html.append('Last updated %s' % time.strftime('%F'))
-    if options.prefixes:
-        html.append('<h2>All suites starting with:</h2>')
-        html.extend(gen_metadata_links(prefix_metadata))
-    if options.suites:
-        html.append('<h2>Specific suites:</h2>')
-        html.extend(gen_metadata_links(suite_metadata))
-    html.extend(['</body>', '</html>'])
-    with open(os.path.join(options.output_dir, 'index.html'), 'w') as f:
-        f.write('\n'.join(html))
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
+    OPTIONS = get_options(sys.argv[1:])
+    main(OPTIONS.input, OPTIONS.output_dir)

hack/jenkins/test-history/gen_html_test.py

@@ -25,46 +25,112 @@ import unittest
 import gen_html
 
 TEST_DATA = {
-  "test1":
-      {"kubernetes-release": [{"build": 3, "failed": False, "time": 3.52},
-                              {"build": 4, "failed": True, "time": 63.21}],
-       "kubernetes-debug": [{"build": 5, "failed": False, "time": 7.56},
-                            {"build": 6, "failed": False, "time": 8.43}],
+  'test1':
+      {'kubernetes-release': [{'build': 3, 'failed': False, 'time': 3.52},
+                              {'build': 4, 'failed': True, 'time': 63.21}],
+       'kubernetes-debug': [{'build': 5, 'failed': False, 'time': 7.56},
+                            {'build': 6, 'failed': False, 'time': 8.43}],
       },
-  "test2":
-      {"kubernetes-debug": [{"build": 6, "failed": True, "time": 3.53}]},
+  'test2':
+      {'kubernetes-debug': [{'build': 6, 'failed': True, 'time': 3.53}]},
 }
 
 class GenHtmlTest(unittest.TestCase):
-    def gen_html(self, *args):
+    """Unit tests for gen_html.py."""
+    # pylint: disable=invalid-name
+
+    def testHtmlHeader_NoScript(self):
+        result = '\n'.join(gen_html.html_header('', False))
+        self.assertNotIn('<script', result)
+
+    def testHtmlHeader_NoTitle(self):
+        def Test(title):
+            result = '\n'.join(gen_html.html_header(title, False))
+            self.assertNotIn('<title', result)
+        Test('')
+        Test(None)
+
+    def testHtmlHeader_Title(self):
+        lines = gen_html.html_header('foo', False)
+        for item in lines:
+          if '<title' in item:
+            self.assertIn('foo', item)
+            break
+        else:
+          self.fail('No foo in: %s' % '\n'.join(lines))
+
+    def testHtmlHeader_Script(self):
+        lines = gen_html.html_header('', True)
+        for item in lines:
+          if '<script' in item:
+            break
+        else:
+          self.fail('No script in: %s' % '\n'.join(lines))
+
+    @staticmethod
+    def gen_html(*args):
+        """Call gen_html with TEST_DATA."""
         return gen_html.gen_html(TEST_DATA, *args)[0]
 
     def testGenHtml(self):
+        """Test that the expected tests and jobs are in the results."""
         html = self.gen_html('')
-        self.assertIn("test1", html)
-        self.assertIn("test2", html)
-        self.assertIn("release", html)
-        self.assertIn("debug", html)
+        self.assertIn('test1', html)
+        self.assertIn('test2', html)
+        self.assertIn('release', html)
+        self.assertIn('debug', html)
 
     def testGenHtmlFilter(self):
+        """Test that filtering to just the release jobs works."""
         html = self.gen_html('release')
-        self.assertIn("release", html)
+        self.assertIn('release', html)
         self.assertIn('skipped">\ntest2', html)
-        self.assertNotIn("debug", html)
-
-    def testGenHtmlFilterExact(self):
-        html = self.gen_html('release', True)
         self.assertNotIn('debug', html)
 
+    def testGenHtmlFilterExact(self):
+        """Test that filtering to an exact name works."""
+        html = self.gen_html('release', True)
+        self.assertIn('release', html)
+        self.assertNotIn('debug', html)
+
+    def testGetOptions(self):
+        """Test argument parsing works correctly."""
+
+        def check(args, expected_output_dir, expected_input):
+            """Check that args is parsed correctly."""
+            options = gen_html.get_options(args)
+            self.assertEquals(expected_output_dir, options.output_dir)
+            self.assertEquals(expected_input, options.input)
+
+
+        check(['--output-dir=foo', '--input=bar'], 'foo', 'bar')
+        check(['--output-dir', 'foo', '--input', 'bar'], 'foo', 'bar')
+        check(['--input=bar', '--output-dir=foo'], 'foo', 'bar')
+
+    def testGetOptions_Missing(self):
+        """Test missing arguments raise an exception."""
+        def check(args):
+            """Check that args raise an exception."""
+            with self.assertRaises(SystemExit):
+                gen_html.get_options(args)
+
+        check([])
+        check(['--output-dir=foo'])
+        check(['--input=bar'])
+
     def testMain(self):
+        """Test main() creates pages."""
         temp_dir = tempfile.mkdtemp(prefix='kube-test-hist-')
         try:
             tests_json = os.path.join(temp_dir, 'tests.json')
-            with open(tests_json, 'w') as f:
-                json.dump(TEST_DATA, f)
-            gen_html.main(['--suites', '--prefixes', ',rel,deb',
-                           '--output-dir', temp_dir, '--input', tests_json])
-            for page in ('index', 'suite-kubernetes-debug', 'tests', 'tests-rel', 'tests-deb'):
+            with open(tests_json, 'w') as buf:
+                json.dump(TEST_DATA, buf)
+            gen_html.main(tests_json, temp_dir)
+            for page in (
+                    'index',
+                    'tests-kubernetes',
+                    'suite-kubernetes-release',
+                    'suite-kubernetes-debug'):
                 self.assertTrue(os.path.exists('%s/%s.html' % (temp_dir, page)))
         finally:
             shutil.rmtree(temp_dir)

hack/jenkins/test-history/gen_json.py

@@ -21,6 +21,7 @@ Writes the JSON out to tests.json.
 
 from __future__ import print_function
 
+import argparse
 import json
 import os
 import re
@@ -28,7 +29,7 @@ import subprocess
 import sys
 import time
 import urllib2
-import xml.etree.ElementTree as ET
+from xml.etree import ElementTree
 import zlib
 
 
@@ -40,6 +41,7 @@ def get_json(url):
     except urllib2.HTTPError:
         return None
 
+
 def get_jobs(server):
     """Generates all job names running on the server."""
     jenkins_json = get_json('{}/api/json'.format(server))
@@ -56,14 +58,16 @@ def get_builds(server, job):
     for build in job_json['builds']:
         yield build['number']
 
+
 def get_build_info(server, job, build):
     """Returns building status along with timestamp for a given build."""
     path = '{}/job/{}/{}/api/json'.format(server, job, str(build))
     build_json = get_json(path)
     if not build_json:
-        return
+        return True, 0
     return build_json['building'], build_json['timestamp']
 
+
 def gcs_ls(path):
     """Lists objects under a path on gcs."""
     try:
@@ -81,6 +85,7 @@ def gcs_ls_build(job, build):
     for path in gcs_ls(url):
         yield path
 
+
 def gcs_ls_artifacts(job, build):
     """Lists all artifacts for a build."""
     for path in gcs_ls_build(job, build):
@@ -88,12 +93,14 @@ def gcs_ls_artifacts(job, build):
             for artifact in gcs_ls(path):
                 yield artifact
 
+
 def gcs_ls_junit_paths(job, build):
     """Lists the paths of JUnit XML files for a build."""
     for path in gcs_ls_artifacts(job, build):
-        if re.match('.*/junit.*\.xml$', path):
+        if re.match(r'.*/junit.*\.xml$', path):
             yield path
 
+
 def gcs_get_tests(path):
     """Generates test data out of the provided JUnit path.
 
@@ -113,13 +120,13 @@ def gcs_get_tests(path):
         pass
 
     try:
-        root = ET.fromstring(data)
-    except ET.ParseError:
+        root = ElementTree.fromstring(data)
+    except ElementTree.ParseError:
         return
 
     for child in root:
         name = child.attrib['name']
-        time = float(child.attrib['time'])
+        ctime = float(child.attrib['time'])
         failed = False
         skipped = False
         for param in child:
@@ -127,7 +134,8 @@ def gcs_get_tests(path):
                 skipped = True
             elif param.tag == 'failure':
                 failed = True
-        yield name, time, failed, skipped
+        yield name, ctime, failed, skipped
+
 
 def get_tests_from_junit_path(path):
     """Generates all tests in a JUnit GCS path."""
@@ -136,17 +144,19 @@ def get_tests_from_junit_path(path):
             continue
         yield test
 
+
 def get_tests_from_build(job, build):
     """Generates all tests for a build."""
     for junit_path in gcs_ls_junit_paths(job, build):
         for test in get_tests_from_junit_path(junit_path):
             yield test
 
-def get_daily_builds(server, prefix):
+
+def get_daily_builds(server, matcher):
     """Generates all (job, build) pairs for the last day."""
     now = time.time()
     for job in get_jobs(server):
-        if not job.startswith(prefix):
+        if not matcher(job):
             continue
         for build in reversed(sorted(get_builds(server, job))):
             building, timestamp = get_build_info(server, job, build)
@@ -158,10 +168,11 @@ def get_daily_builds(server, prefix):
                 break
             yield job, build
 
-def get_tests(server, prefix):
+
+def get_tests(server, matcher):
     """Returns a dictionary of tests to be JSON encoded."""
     tests = {}
-    for job, build in get_daily_builds(server, prefix):
+    for job, build in get_daily_builds(server, matcher):
         print('{}/{}'.format(job, str(build)))
         for name, duration, failed, skipped in get_tests_from_build(job, build):
             if name not in tests:
@@ -177,12 +188,33 @@ def get_tests(server, prefix):
             })
     return tests
 
+
+def main(server, match):
+    """Collect test info in matching jobs."""
+    print('Finding tests in jobs matching {} at server {}'.format(
+        match, server))
+    matcher = re.compile(match)
+    tests = get_tests(server, matcher)
+    with open('tests.json', 'w') as buf:
+        json.dump(tests, buf, sort_keys=True)
+
+
+def get_options(argv):
+    """Process command line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--server',
+        help='hostname of jenkins server',
+        required=True,
+    )
+    parser.add_argument(
+        '--match',
+        help='filter to job names matching this re',
+        required=True,
+    )
+    return parser.parse_args(argv)
+
+
 if __name__ == '__main__':
-    if len(sys.argv) != 3:
-        print('Usage: {} <server> <prefix>'.format(sys.argv[0]))
-        sys.exit(1)
-    server, prefix = sys.argv[1:]
-    print('Finding tests prefixed with {} at server {}'.format(prefix, server))
-    tests = get_tests(server, prefix)
-    with open('tests.json', 'w') as f:
-        json.dump(tests, f, sort_keys=True)
+    OPTIONS = get_options(sys.argv[1:])
+    main(OPTIONS.server, OPTIONS.match)

hack/jenkins/test-history/gen_json_test.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for gen_json."""
+
+import unittest
+
+import gen_json
+
+
+class GenJsonTest(unittest.TestCase):
+    """Unit tests for gen_json.py."""
+    # pylint: disable=invalid-name
+
+    def testGetOptions(self):
+        """Test argument parsing works correctly."""
+        def check(args, expected_server, expected_match):
+            """Check that all args are parsed as expected."""
+            options = gen_json.get_options(args)
+            self.assertEquals(expected_server, options.server)
+            self.assertEquals(expected_match, options.match)
+
+
+        check(['--server=foo', '--match=bar'], 'foo', 'bar')
+        check(['--server', 'foo', '--match', 'bar'], 'foo', 'bar')
+        check(['--match=bar', '--server=foo'], 'foo', 'bar')
+
+    def testGetOptions_Missing(self):
+        """Test missing arguments raise an exception."""
+        def check(args):
+            """Check that missing args raise an exception."""
+            with self.assertRaises(SystemExit):
+                gen_json.get_options(args)
+
+        check([])
+        check(['--server=foo'])
+        check(['--match=bar'])
+
+
+
+if __name__ == '__main__':
+    unittest.main()