code.vuplus.com Git - vuplus_webkit/blob - Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 """Unit tests for json_results_generator.py."""
  30
  31 import unittest
  32 import optparse
  33 import random
  34
  35 from webkitpy.common.system import filesystem_mock
  36 from webkitpy.layout_tests.layout_package import json_results_generator
  37 from webkitpy.layout_tests.models import test_expectations
  38 from webkitpy.layout_tests.port import test
  39 from webkitpy.thirdparty.mock import Mock
  40 from webkitpy.thirdparty import simplejson
  41
  42 class JSONGeneratorTest(unittest.TestCase):
  43     def setUp(self):
  44         self.builder_name = 'DUMMY_BUILDER_NAME'
  45         self.build_name = 'DUMMY_BUILD_NAME'
  46         self.build_number = 'DUMMY_BUILDER_NUMBER'
  47
  48         # For archived results.
  49         self._json = None
  50         self._num_runs = 0
  51         self._tests_set = set([])
  52         self._test_timings = {}
  53         self._failed_count_map = {}
  54
  55         self._PASS_count = 0
  56         self._DISABLED_count = 0
  57         self._FLAKY_count = 0
  58         self._FAILS_count = 0
  59         self._fixable_count = 0
  60
  61     def _test_json_generation(self, passed_tests_list, failed_tests_list):
  62         tests_set = set(passed_tests_list) | set(failed_tests_list)
  63
  64         DISABLED_tests = set([t for t in tests_set
  65                              if t.startswith('DISABLED_')])
  66         FLAKY_tests = set([t for t in tests_set
  67                            if t.startswith('FLAKY_')])
  68         FAILS_tests = set([t for t in tests_set
  69                            if t.startswith('FAILS_')])
  70         PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests)
  71
  72         failed_tests = set(failed_tests_list) - DISABLED_tests
  73         failed_count_map = dict([(t, 1) for t in failed_tests])
  74
  75         test_timings = {}
  76         i = 0
  77         for test in tests_set:
  78             test_timings[test] = float(self._num_runs * 100 + i)
  79             i += 1
  80
  81         test_results_map = dict()
  82         for test in tests_set:
  83             test_results_map[test] = json_results_generator.TestResult(test,
  84                 failed=(test in failed_tests),
  85                 elapsed_time=test_timings[test])
  86
  87         port = Mock()
  88         port._filesystem = filesystem_mock.MockFileSystem()
  89         generator = json_results_generator.JSONResultsGeneratorBase(port,
  90             self.builder_name, self.build_name, self.build_number,
  91             '',
  92             None,   # don't fetch past json results archive
  93             test_results_map)
  94
  95         failed_count_map = dict([(t, 1) for t in failed_tests])
  96
  97         # Test incremental json results
  98         incremental_json = generator.get_json()
  99         self._verify_json_results(
 100             tests_set,
 101             test_timings,
 102             failed_count_map,
 103             len(PASS_tests),
 104             len(DISABLED_tests),
 105             len(FLAKY_tests),
 106             len(DISABLED_tests | failed_tests),
 107             incremental_json,
 108             1)
 109
 110         # We don't verify the results here, but at least we make sure the code runs without errors.
 111         generator.generate_json_output()
 112         generator.generate_times_ms_file()
 113
 114     def _verify_json_results(self, tests_set, test_timings, failed_count_map,
 115                              PASS_count, DISABLED_count, FLAKY_count,
 116                              fixable_count,
 117                              json, num_runs):
 118         # Aliasing to a short name for better access to its constants.
 119         JRG = json_results_generator.JSONResultsGeneratorBase
 120
 121         self.assertTrue(JRG.VERSION_KEY in json)
 122         self.assertTrue(self.builder_name in json)
 123
 124         buildinfo = json[self.builder_name]
 125         self.assertTrue(JRG.FIXABLE in buildinfo)
 126         self.assertTrue(JRG.TESTS in buildinfo)
 127         self.assertEqual(len(buildinfo[JRG.BUILD_NUMBERS]), num_runs)
 128         self.assertEqual(buildinfo[JRG.BUILD_NUMBERS][0], self.build_number)
 129
 130         if tests_set or DISABLED_count:
 131             fixable = {}
 132             for fixable_items in buildinfo[JRG.FIXABLE]:
 133                 for (type, count) in fixable_items.iteritems():
 134                     if type in fixable:
 135                         fixable[type] = fixable[type] + count
 136                     else:
 137                         fixable[type] = count
 138
 139             if PASS_count:
 140                 self.assertEqual(fixable[JRG.PASS_RESULT], PASS_count)
 141             else:
 142                 self.assertTrue(JRG.PASS_RESULT not in fixable or
 143                                 fixable[JRG.PASS_RESULT] == 0)
 144             if DISABLED_count:
 145                 self.assertEqual(fixable[JRG.SKIP_RESULT], DISABLED_count)
 146             else:
 147                 self.assertTrue(JRG.SKIP_RESULT not in fixable or
 148                                 fixable[JRG.SKIP_RESULT] == 0)
 149             if FLAKY_count:
 150                 self.assertEqual(fixable[JRG.FLAKY_RESULT], FLAKY_count)
 151             else:
 152                 self.assertTrue(JRG.FLAKY_RESULT not in fixable or
 153                                 fixable[JRG.FLAKY_RESULT] == 0)
 154
 155         if failed_count_map:
 156             tests = buildinfo[JRG.TESTS]
 157             for test_name in failed_count_map.iterkeys():
 158                 test = self._find_test_in_trie(test_name, tests)
 159
 160                 failed = 0
 161                 for result in test[JRG.RESULTS]:
 162                     if result[1] == JRG.FAIL_RESULT:
 163                         failed += result[0]
 164                 self.assertEqual(failed_count_map[test_name], failed)
 165
 166                 timing_count = 0
 167                 for timings in test[JRG.TIMES]:
 168                     if timings[1] == test_timings[test_name]:
 169                         timing_count = timings[0]
 170                 self.assertEqual(1, timing_count)
 171
 172         if fixable_count:
 173             self.assertEqual(sum(buildinfo[JRG.FIXABLE_COUNT]), fixable_count)
 174
 175     def _find_test_in_trie(self, path, trie):
 176         nodes = path.split("/")
 177         sub_trie = trie
 178         for node in nodes:
 179             self.assertTrue(node in sub_trie)
 180             sub_trie = sub_trie[node]
 181         return sub_trie
 182
 183     def test_json_generation(self):
 184         self._test_json_generation([], [])
 185         self._test_json_generation(['A1', 'B1'], [])
 186         self._test_json_generation([], ['FAILS_A2', 'FAILS_B2'])
 187         self._test_json_generation(['DISABLED_A3', 'DISABLED_B3'], [])
 188         self._test_json_generation(['A4'], ['B4', 'FAILS_C4'])
 189         self._test_json_generation(['DISABLED_C5', 'DISABLED_D5'], ['A5', 'B5'])
 190         self._test_json_generation(
 191             ['A6', 'B6', 'FAILS_C6', 'DISABLED_E6', 'DISABLED_F6'],
 192             ['FAILS_D6'])
 193
 194         # Generate JSON with the same test sets. (Both incremental results and
 195         # archived results must be updated appropriately.)
 196         self._test_json_generation(
 197             ['A', 'FLAKY_B', 'DISABLED_C'],
 198             ['FAILS_D', 'FLAKY_E'])
 199         self._test_json_generation(
 200             ['A', 'DISABLED_C', 'FLAKY_E'],
 201             ['FLAKY_B', 'FAILS_D'])
 202         self._test_json_generation(
 203             ['FLAKY_B', 'DISABLED_C', 'FAILS_D'],
 204             ['A', 'FLAKY_E'])
 205
 206     def test_hierarchical_json_generation(self):
 207         # FIXME: Re-work tests to be more comprehensible and comprehensive.
 208         self._test_json_generation(['foo/A'], ['foo/B', 'bar/C'])
 209
 210     def test_test_timings_trie(self):
 211         test_port = test.TestPort()
 212         individual_test_timings = []
 213         individual_test_timings.append(json_results_generator.TestResult('foo/bar/baz.html', elapsed_time=1.2))
 214         individual_test_timings.append(json_results_generator.TestResult('bar.html', elapsed_time=0.0001))
 215         trie = json_results_generator.test_timings_trie(test_port, individual_test_timings)
 216
 217         expected_trie = {
 218           'bar.html': 0,
 219           'foo': {
 220               'bar': {
 221                   'baz.html': 1200,
 222               }
 223           }
 224         }
 225
 226         self.assertEqual(simplejson.dumps(trie), simplejson.dumps(expected_trie))
 227
 228
 229
 230 if __name__ == '__main__':
 231     unittest.main()