code.vuplus.com Git - vuplus_webkit/blob - Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Unit tests for printing.py."""
  31
  32 import logging
  33 import optparse
  34 import time
  35 import unittest
  36
  37 from webkitpy.common import array_stream
  38 from webkitpy.common.system import logtesting
  39 from webkitpy.layout_tests import port
  40 from webkitpy.layout_tests.controllers import manager
  41 from webkitpy.layout_tests.models import result_summary
  42 from webkitpy.layout_tests.models import test_expectations
  43 from webkitpy.layout_tests.models import test_failures
  44 from webkitpy.layout_tests.models import test_results
  45 from webkitpy.layout_tests.views import printing
  46
  47
  48 def get_options(args):
  49     print_options = printing.print_options()
  50     option_parser = optparse.OptionParser(option_list=print_options)
  51     return option_parser.parse_args(args)
  52
  53
  54 class TestUtilityFunctions(unittest.TestCase):
  55     def test_configure_logging(self):
  56         options, args = get_options([])
  57         stream = array_stream.ArrayStream()
  58         handler = printing._configure_logging(stream, options.verbose)
  59         logging.info("this should be logged")
  60         self.assertFalse(stream.empty())
  61
  62         stream.reset()
  63         logging.debug("this should not be logged")
  64         self.assertTrue(stream.empty())
  65
  66         printing._restore_logging(handler)
  67
  68         stream.reset()
  69         options, args = get_options(['--verbose'])
  70         handler = printing._configure_logging(stream, options.verbose)
  71         logging.debug("this should be logged")
  72         self.assertFalse(stream.empty())
  73         printing._restore_logging(handler)
  74
  75     def test_print_options(self):
  76         options, args = get_options([])
  77         self.assertTrue(options is not None)
  78
  79     def test_parse_print_options(self):
  80         def test_switches(args, expected_switches_str, verbose=False):
  81             options, args = get_options(args)
  82             if expected_switches_str:
  83                 expected_switches = set(expected_switches_str.split(','))
  84             else:
  85                 expected_switches = set()
  86             switches = printing.parse_print_options(options.print_options,
  87                                                     verbose)
  88             self.assertEqual(expected_switches, switches)
  89
  90         # test that we default to the default set of switches
  91         test_switches([], printing.PRINT_DEFAULT)
  92
  93         # test that verbose defaults to everything
  94         test_switches([], printing.PRINT_EVERYTHING, verbose=True)
  95
  96         # test that --print default does what it's supposed to
  97         test_switches(['--print', 'default'], printing.PRINT_DEFAULT)
  98
  99         # test that --print nothing does what it's supposed to
 100         test_switches(['--print', 'nothing'], None)
 101
 102         # test that --print everything does what it's supposed to
 103         test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)
 104
 105         # this tests that '--print X' overrides '--verbose'
 106         test_switches(['--print', 'actual'], 'actual', verbose=True)
 107
 108
 109
 110 class  Testprinter(unittest.TestCase):
 111     def get_printer(self, args=None, tty=False):
 112         args = args or []
 113         printing_options = printing.print_options()
 114         option_parser = optparse.OptionParser(option_list=printing_options)
 115         options, args = option_parser.parse_args(args)
 116         self._port = port.get('test', options)
 117         nproc = 2
 118
 119         regular_output = array_stream.ArrayStream(tty=tty)
 120         buildbot_output = array_stream.ArrayStream()
 121         printer = printing.Printer(self._port, options, regular_output,
 122                                    buildbot_output, configure_logging=True)
 123         return printer, regular_output, buildbot_output
 124
 125     def get_result(self, test_name, result_type=test_expectations.PASS, run_time=0):
 126         failures = []
 127         if result_type == test_expectations.TIMEOUT:
 128             failures = [test_failures.FailureTimeout()]
 129         elif result_type == test_expectations.CRASH:
 130             failures = [test_failures.FailureCrash()]
 131         return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
 132
 133     def get_result_summary(self, test_names, expectations_str):
 134         expectations = test_expectations.TestExpectations(
 135             self._port, test_names, expectations_str,
 136             self._port.test_configuration(),
 137             is_lint_mode=False)
 138
 139         rs = result_summary.ResultSummary(expectations, test_names)
 140         return test_names, rs, expectations
 141
 142     def test_help_printer(self):
 143         # Here and below we'll call the "regular" printer err and the
 144         # buildbot printer out; this corresponds to how things run on the
 145         # bots with stderr and stdout.
 146         printer, err, out = self.get_printer()
 147
 148         # This routine should print something to stdout. testing what it is
 149         # is kind of pointless.
 150         printer.help_printing()
 151         self.assertFalse(err.empty())
 152         self.assertTrue(out.empty())
 153
 154     def do_switch_tests(self, method_name, switch, to_buildbot,
 155                         message='hello', exp_err=None, exp_bot=None):
 156         def do_helper(method_name, switch, message, exp_err, exp_bot):
 157             printer, err, bot = self.get_printer(['--print', switch], tty=True)
 158             getattr(printer, method_name)(message)
 159             self.assertEqual(err.get(), exp_err)
 160             self.assertEqual(bot.get(), exp_bot)
 161
 162         if to_buildbot:
 163             if exp_err is None:
 164                 exp_err = []
 165             if exp_bot is None:
 166                 exp_bot = [message + "\n"]
 167         else:
 168             if exp_err is None:
 169                 exp_err = [message + "\n"]
 170             if exp_bot is None:
 171                 exp_bot = []
 172         do_helper(method_name, 'nothing', 'hello', [], [])
 173         do_helper(method_name, switch, 'hello', exp_err, exp_bot)
 174         do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
 175
 176     def test_configure_and_cleanup(self):
 177         # This test verifies that calling cleanup repeatedly and deleting
 178         # the object is safe.
 179         printer, err, out = self.get_printer(['--print', 'everything'])
 180         printer.cleanup()
 181         printer.cleanup()
 182         printer = None
 183
 184     def test_print_actual(self):
 185         # Actual results need to be logged to the buildbot's stream.
 186         self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
 187
 188     def test_print_actual_buildbot(self):
 189         # FIXME: Test that the format of the actual results matches what the
 190         # buildbot is expecting.
 191         pass
 192
 193     def test_print_config(self):
 194         self.do_switch_tests('print_config', 'config', to_buildbot=False)
 195
 196     def test_print_expected(self):
 197         self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
 198
 199     def test_print_timing(self):
 200         self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
 201
 202     def test_print_update(self):
 203         # Note that there shouldn't be a carriage return here; updates()
 204         # are meant to be overwritten.
 205         self.do_switch_tests('print_update', 'updates', to_buildbot=False,
 206                              message='hello', exp_err=['hello'])
 207
 208     def test_print_one_line_summary(self):
 209         printer, err, out = self.get_printer(['--print', 'nothing'])
 210         printer.print_one_line_summary(1, 1, 0)
 211         self.assertTrue(err.empty())
 212
 213         printer, err, out = self.get_printer(['--print', 'one-line-summary'])
 214         printer.print_one_line_summary(1, 1, 0)
 215         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
 216
 217         printer, err, out = self.get_printer(['--print', 'everything'])
 218         printer.print_one_line_summary(1, 1, 0)
 219         self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
 220
 221         err.reset()
 222         printer.print_one_line_summary(2, 1, 1)
 223         self.assertEquals(err.get(),
 224                           ["1 test ran as expected, 1 didn't:\n", "\n"])
 225
 226         err.reset()
 227         printer.print_one_line_summary(3, 2, 1)
 228         self.assertEquals(err.get(),
 229                           ["2 tests ran as expected, 1 didn't:\n", "\n"])
 230
 231         err.reset()
 232         printer.print_one_line_summary(3, 2, 0)
 233         self.assertEquals(err.get(),
 234                           ['\n', "2 tests ran as expected (1 didn't run).\n",
 235                            '\n'])
 236
 237
 238     def test_print_test_result(self):
 239         # Note here that we don't use meaningful exp_str and got_str values;
 240         # the actual contents of the string are treated opaquely by
 241         # print_test_result() when tracing, and usually we don't want
 242         # to test what exactly is printed, just that something
 243         # was printed (or that nothing was printed).
 244         #
 245         # FIXME: this is actually some goofy layering; it would be nice
 246         # we could refactor it so that the args weren't redundant. Maybe
 247         # the TestResult should contain what was expected, and the
 248         # strings could be derived from the TestResult?
 249         printer, err, out = self.get_printer(['--print', 'nothing'])
 250         result = self.get_result('passes/image.html')
 251         printer.print_test_result(result, expected=False, exp_str='',
 252                                   got_str='')
 253         self.assertTrue(err.empty())
 254
 255         printer, err, out = self.get_printer(['--print', 'unexpected'])
 256         printer.print_test_result(result, expected=True, exp_str='',
 257                                   got_str='')
 258         self.assertTrue(err.empty())
 259         printer.print_test_result(result, expected=False, exp_str='',
 260                                   got_str='')
 261         self.assertEquals(err.get(),
 262                           ['  passes/image.html -> unexpected pass\n'])
 263
 264         printer, err, out = self.get_printer(['--print', 'everything'])
 265         printer.print_test_result(result, expected=True, exp_str='',
 266                                   got_str='')
 267         self.assertTrue(err.empty())
 268
 269         printer.print_test_result(result, expected=False, exp_str='',
 270                                   got_str='')
 271         self.assertEquals(err.get(),
 272                           ['  passes/image.html -> unexpected pass\n'])
 273
 274         printer, err, out = self.get_printer(['--print', 'nothing'])
 275         printer.print_test_result(result, expected=False, exp_str='',
 276                                   got_str='')
 277         self.assertTrue(err.empty())
 278
 279         printer, err, out = self.get_printer(['--print',
 280                                               'trace-unexpected'])
 281         printer.print_test_result(result, expected=True, exp_str='',
 282                                   got_str='')
 283         self.assertTrue(err.empty())
 284
 285         printer, err, out = self.get_printer(['--print',
 286                                               'trace-unexpected'])
 287         printer.print_test_result(result, expected=False, exp_str='',
 288                                   got_str='')
 289         self.assertFalse(err.empty())
 290
 291         printer, err, out = self.get_printer(['--print',
 292                                               'trace-unexpected'])
 293         result = self.get_result("passes/text.html")
 294         printer.print_test_result(result, expected=False, exp_str='',
 295                                   got_str='')
 296         self.assertFalse(err.empty())
 297
 298         err.reset()
 299         printer.print_test_result(result, expected=False, exp_str='',
 300                                   got_str='')
 301         self.assertFalse(err.empty())
 302
 303         printer, err, out = self.get_printer(['--print', 'trace-everything'])
 304         result = self.get_result('passes/image.html')
 305         printer.print_test_result(result, expected=True, exp_str='',
 306                                   got_str='')
 307         result = self.get_result('failures/expected/missing_text.html')
 308         printer.print_test_result(result, expected=True, exp_str='',
 309                                   got_str='')
 310         result = self.get_result('failures/expected/missing_check.html')
 311         printer.print_test_result(result, expected=True, exp_str='',
 312                                   got_str='')
 313         result = self.get_result('failures/expected/missing_image.html')
 314         printer.print_test_result(result, expected=True, exp_str='',
 315                                   got_str='')
 316         self.assertFalse(err.empty())
 317
 318         err.reset()
 319         printer.print_test_result(result, expected=False, exp_str='',
 320                                   got_str='')
 321
 322     def test_print_progress(self):
 323         expectations = ''
 324
 325         printer, err, out = self.get_printer(['--print', 'nothing'])
 326         tests = ['passes/text.html', 'failures/expected/timeout.html',
 327                  'failures/expected/crash.html']
 328         paths, rs, exp = self.get_result_summary(tests, expectations)
 329
 330         # First, test that we print nothing.
 331         printer.print_progress(rs, False, paths)
 332         self.assertTrue(out.empty())
 333         self.assertTrue(err.empty())
 334
 335         printer.print_progress(rs, True, paths)
 336         self.assertTrue(out.empty())
 337         self.assertTrue(err.empty())
 338
 339         self.times = [1, 2, 12, 13, 14, 23, 33]
 340
 341         def mock_time():
 342             return self.times.pop(0)
 343
 344         orig_time = time.time
 345         try:
 346             time.time = mock_time
 347
 348             # Test printing progress updates to a file.
 349             printer, err, out = self.get_printer(['--print', 'one-line-progress'])
 350             printer.print_progress(rs, False, paths)
 351             printer.print_progress(rs, False, paths)
 352             self.assertTrue(out.empty())
 353             self.assertTrue(err.empty())
 354
 355             printer.print_progress(rs, False, paths)
 356             self.assertTrue(out.empty())
 357             self.assertFalse(err.empty())
 358
 359             err.reset()
 360             out.reset()
 361             printer.print_progress(rs, True, paths)
 362             self.assertTrue(out.empty())
 363             self.assertTrue(err.empty())
 364
 365             printer.print_progress(rs, True, paths)
 366             self.assertTrue(out.empty())
 367             self.assertFalse(err.empty())
 368
 369             # Now reconfigure the printer to test printing to a TTY instead of a file.
 370             self.times = [1, 1.01, 2, 3]
 371             printer, err, out = self.get_printer(['--print', 'one-line-progress'], tty=True)
 372             printer.print_progress(rs, False, paths)
 373             printer.print_progress(rs, False, paths)
 374             self.assertTrue(out.empty())
 375             self.assertTrue(err.empty())
 376
 377             printer.print_progress(rs, False, paths)
 378             self.assertTrue(out.empty())
 379             self.assertFalse(err.empty())
 380
 381             err.reset()
 382             out.reset()
 383             printer.print_progress(rs, True, paths)
 384             self.assertTrue(out.empty())
 385             self.assertFalse(err.empty())
 386         finally:
 387             time.time = orig_time
 388
 389     def test_write_nothing(self):
 390         printer, err, out = self.get_printer(['--print', 'nothing'])
 391         printer.write("foo")
 392         self.assertTrue(err.empty())
 393
 394     def test_write_misc(self):
 395         printer, err, out = self.get_printer(['--print', 'misc'])
 396         printer.write("foo")
 397         self.assertFalse(err.empty())
 398         err.reset()
 399         printer.write("foo", "config")
 400         self.assertTrue(err.empty())
 401
 402     def test_write_everything(self):
 403         printer, err, out = self.get_printer(['--print', 'everything'])
 404         printer.write("foo")
 405         self.assertFalse(err.empty())
 406         err.reset()
 407         printer.write("foo", "config")
 408         self.assertFalse(err.empty())
 409
 410     def test_write_verbose(self):
 411         printer, err, out = self.get_printer(['--verbose'])
 412         printer.write("foo")
 413         self.assertTrue(not err.empty() and "foo" in err.get()[0])
 414         self.assertTrue(out.empty())
 415
 416     def test_print_unexpected_results(self):
 417         # This routine is the only one that prints stuff that the bots
 418         # care about.
 419         #
 420         # FIXME: there's some weird layering going on here. It seems
 421         # like we shouldn't be both using an expectations string and
 422         # having to specify whether or not the result was expected.
 423         # This whole set of tests should probably be rewritten.
 424         #
 425         # FIXME: Plus, the fact that we're having to call into
 426         # run_webkit_tests is clearly a layering inversion.
 427         def get_unexpected_results(expected, passing, flaky):
 428             """Return an unexpected results summary matching the input description.
 429
 430             There are a lot of different combinations of test results that
 431             can be tested; this routine produces various combinations based
 432             on the values of the input flags.
 433
 434             Args
 435                 expected: whether the tests ran as expected
 436                 passing: whether the tests should all pass
 437                 flaky: whether the tests should be flaky (if False, they
 438                     produce the same results on both runs; if True, they
 439                     all pass on the second run).
 440
 441             """
 442             paths, rs, exp = self.get_result_summary(tests, expectations)
 443             if expected:
 444                 rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected)
 445                 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected)
 446                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected)
 447             elif passing:
 448                 rs.add(self.get_result('passes/text.html'), expected)
 449                 rs.add(self.get_result('failures/expected/timeout.html'), expected)
 450                 rs.add(self.get_result('failures/expected/crash.html'), expected)
 451             else:
 452                 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected)
 453                 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected)
 454                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected)
 455             retry = rs
 456             if flaky:
 457                 paths, retry, exp = self.get_result_summary(tests, expectations)
 458                 retry.add(self.get_result('passes/text.html'), True)
 459                 retry.add(self.get_result('failures/expected/timeout.html'), True)
 460                 retry.add(self.get_result('failures/expected/crash.html'), True)
 461             unexpected_results = manager.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False)
 462             return unexpected_results
 463
 464         tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html']
 465         expectations = ''
 466
 467         printer, err, out = self.get_printer(['--print', 'nothing'])
 468         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 469         printer.print_unexpected_results(ur)
 470         self.assertTrue(err.empty())
 471         self.assertTrue(out.empty())
 472
 473         printer, err, out = self.get_printer(['--print', 'unexpected-results'])
 474
 475         # test everything running as expected
 476         ur = get_unexpected_results(expected=True, passing=False, flaky=False)
 477         printer.print_unexpected_results(ur)
 478         self.assertTrue(err.empty())
 479         self.assertTrue(out.empty())
 480
 481         # test failures
 482         err.reset()
 483         out.reset()
 484         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 485         printer.print_unexpected_results(ur)
 486         self.assertTrue(err.empty())
 487         self.assertFalse(out.empty())
 488
 489         # test unexpected flaky
 490         err.reset()
 491         out.reset()
 492         ur = get_unexpected_results(expected=False, passing=False, flaky=True)
 493         printer.print_unexpected_results(ur)
 494         self.assertTrue(err.empty())
 495         self.assertFalse(out.empty())
 496
 497         err.reset()
 498         out.reset()
 499         printer, err, out = self.get_printer(['--print', 'everything'])
 500         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 501         printer.print_unexpected_results(ur)
 502         self.assertTrue(err.empty())
 503         self.assertFalse(out.empty())
 504
 505         expectations = """
 506 BUGX : failures/expected/crash.html = CRASH
 507 BUGX : failures/expected/timeout.html = TIMEOUT
 508 """
 509         err.reset()
 510         out.reset()
 511         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 512         printer.print_unexpected_results(ur)
 513         self.assertTrue(err.empty())
 514         self.assertFalse(out.empty())
 515
 516         err.reset()
 517         out.reset()
 518         ur = get_unexpected_results(expected=False, passing=True, flaky=False)
 519         printer.print_unexpected_results(ur)
 520         self.assertTrue(err.empty())
 521         self.assertFalse(out.empty())
 522
 523         # Test handling of --verbose as well.
 524         err.reset()
 525         out.reset()
 526         printer, err, out = self.get_printer(['--verbose'])
 527         ur = get_unexpected_results(expected=False, passing=False, flaky=False)
 528         printer.print_unexpected_results(ur)
 529         # FIXME: debug output from the port and scm objects may or may not go
 530         # to stderr, so there's no point in testing its contents here.
 531         self.assertFalse(out.empty())
 532
 533     def test_print_unexpected_results_buildbot(self):
 534         # FIXME: Test that print_unexpected_results() produces the printer the
 535         # buildbot is expecting.
 536         pass
 537
 538 if __name__ == '__main__':
 539     unittest.main()