code.vuplus.com Git - vuplus_webkit/blob - Tools/Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Unit tests for rebaseline_chromium_webkit_tests.py."""
  31
  32 import unittest
  33
  34 from webkitpy.tool import mocktool
  35 from webkitpy.common.system import urlfetcher_mock
  36 from webkitpy.common.system import filesystem_mock
  37 from webkitpy.common.system import zipfileset_mock
  38 from webkitpy.common.system import outputcapture
  39 from webkitpy.common.system.executive import Executive, ScriptError
  40
  41 from webkitpy.layout_tests import port
  42 from webkitpy.layout_tests.port.test import unit_test_filesystem
  43 from webkitpy.to_be_moved import rebaseline_chromium_webkit_tests
  44
  45
  46 class MockPort(object):
  47     def __init__(self, image_diff_exists):
  48         self.image_diff_exists = image_diff_exists
  49
  50     def check_image_diff(self, override_step, logging):
  51         return self.image_diff_exists
  52
  53
  54 def get_mock_get(config_expectations):
  55     def mock_get(port_name, options):
  56         return MockPort(config_expectations[options.configuration])
  57     return mock_get
  58
  59
  60 ARCHIVE_URL = 'http://localhost/layout_test_results'
  61
  62
  63 def test_options():
  64     return mocktool.MockOptions(configuration=None,
  65                                 backup=False,
  66                                 html_directory='/tmp',
  67                                 archive_url=ARCHIVE_URL,
  68                                 force_archive_url=None,
  69                                 verbose=False,
  70                                 quiet=False,
  71                                 platforms=None)
  72
  73
  74 def test_host_port_and_filesystem(options, expectations):
  75     filesystem = unit_test_filesystem()
  76     host_port_obj = port.get('test', options, filesystem=filesystem, user=mocktool.MockUser())
  77
  78     expectations_path = host_port_obj.path_to_test_expectations_file()
  79     filesystem.write_text_file(expectations_path, expectations)
  80     return (host_port_obj, filesystem)
  81
  82
  83 def test_url_fetcher(filesystem):
  84     urls = {
  85         ARCHIVE_URL + '/Webkit_Mac10_6/': '<a href="4/">',
  86         ARCHIVE_URL + '/Webkit_Mac10_5/': '<a href="1/"><a href="2/">',
  87         ARCHIVE_URL + '/Webkit_Mac10_6__CG_/': '<a href="4/">',
  88         ARCHIVE_URL + '/Webkit_Mac10_5__CG_/': '<a href="1/"><a href="2/">',
  89         ARCHIVE_URL + '/Webkit_Win7/': '<a href="1/">',
  90         ARCHIVE_URL + '/Webkit_Vista/': '<a href="1/">',
  91         ARCHIVE_URL + '/Webkit_Win/': '<a href="1/">',
  92         ARCHIVE_URL + '/Webkit_Linux/': '<a href="1/">',
  93     }
  94     return urlfetcher_mock.make_fetcher_cls(urls)(filesystem)
  95
  96
  97 def test_zip_factory():
  98     ziphashes = {
  99         ARCHIVE_URL + '/Webkit_Mac10_5/2/layout-test-results.zip': {
 100             'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
 101             'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
 102             'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
 103             'layout-test-results/failures/expected/image_checksum-actual.txt': 'png-comment-txt',
 104             'layout-test-results/failures/expected/image_checksum-actual.checksum': '0123456789',
 105             'layout-test-results/failures/expected/image_checksum-actual.png': 'tEXtchecksum\x000123456789',
 106         },
 107         ARCHIVE_URL + '/Webkit_Mac10_6/4/layout-test-results.zip': {
 108             'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
 109             'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
 110             'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
 111         },
 112         ARCHIVE_URL + '/Webkit_Mac10_5__CG_/2/layout-test-results.zip': {
 113             'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
 114             'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
 115             'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
 116             'layout-test-results/failures/expected/image_checksum-actual.txt': 'png-comment-txt',
 117             'layout-test-results/failures/expected/image_checksum-actual.checksum': '0123456789',
 118             'layout-test-results/failures/expected/image_checksum-actual.png': 'tEXtchecksum\x000123456789',
 119         },
 120         ARCHIVE_URL + '/Webkit_Mac10_6__CG_/4/layout-test-results.zip': {
 121             'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
 122             'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
 123             'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
 124         },
 125         ARCHIVE_URL + '/Webkit_Vista/1/layout-test-results.zip': {
 126             'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
 127             'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
 128             'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
 129         },
 130         ARCHIVE_URL + '/Webkit_Win7/1/layout-test-results.zip': {
 131             'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
 132             'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
 133             'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
 134         },
 135         ARCHIVE_URL + '/Webkit_Win/1/layout-test-results.zip': {
 136             'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
 137             'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
 138             'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
 139         },
 140         ARCHIVE_URL + '/Webkit_Linux/1/layout-test-results.zip': {
 141             'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
 142             'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
 143             'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
 144         },
 145     }
 146     return zipfileset_mock.make_factory(ziphashes)
 147
 148
 149 def test_archive(orig_archive_dict):
 150     new_archive_dict = {}
 151     for platform, dirname in orig_archive_dict.iteritems():
 152         # This is a giant hack. :(
 153         platform = platform.replace('chromium', 'test')
 154         platform = platform.replace('test-cg', 'test')
 155         new_archive_dict[platform] = dirname
 156     return new_archive_dict
 157
 158
 159 class TestGetHostPortObject(unittest.TestCase):
 160     def assert_result(self, release_present, debug_present, valid_port_obj):
 161         # Tests whether we get a valid port object returned when we claim
 162         # that Image diff is (or isn't) present in the two configs.
 163         port.get = get_mock_get({'Release': release_present,
 164                                  'Debug': debug_present})
 165         options = mocktool.MockOptions(configuration=None,
 166                                        html_directory='/tmp')
 167         port_obj = rebaseline_chromium_webkit_tests.get_host_port_object(options)
 168         if valid_port_obj:
 169             self.assertNotEqual(port_obj, None)
 170         else:
 171             self.assertEqual(port_obj, None)
 172
 173     def test_get_host_port_object(self):
 174         # Save the normal port.get() function for future testing.
 175         old_get = port.get
 176
 177         # Test whether we get a valid port object back for the four
 178         # possible cases of having ImageDiffs built. It should work when
 179         # there is at least one binary present.
 180         self.assert_result(False, False, False)
 181         self.assert_result(True, False, True)
 182         self.assert_result(False, True, True)
 183         self.assert_result(True, True, True)
 184
 185         # Restore the normal port.get() function.
 186         port.get = old_get
 187
 188
 189 class TestOptions(unittest.TestCase):
 190     def test_parse_options(self):
 191         (options, target_options) = rebaseline_chromium_webkit_tests.parse_options([])
 192         self.assertTrue(target_options.chromium)
 193         self.assertEqual(options.tolerance, 0)
 194
 195         (options, target_options) = rebaseline_chromium_webkit_tests.parse_options(['--target-platform', 'qt'])
 196         self.assertFalse(hasattr(target_options, 'chromium'))
 197         self.assertEqual(options.tolerance, 0)
 198
 199
 200 class TestRebaseliner(unittest.TestCase):
 201     def setUp(self):
 202         if not hasattr(self, '_orig_archive'):
 203             self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT
 204             rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive)
 205
 206     def tearDown(self):
 207         rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive
 208
 209     def make_rebaseliner(self, expectations):
 210         options = test_options()
 211         host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations)
 212
 213         target_options = options
 214         target_port_obj = port.get('test', target_options,
 215                                    filesystem=filesystem)
 216         target_port_obj._expectations = expectations
 217         platform = target_port_obj.name()
 218
 219         url_fetcher = test_url_fetcher(filesystem)
 220         zip_factory = test_zip_factory()
 221         mock_scm = mocktool.MockSCM(filesystem)
 222         filesystem.maybe_make_directory(mock_scm.checkout_root)
 223
 224         # FIXME: SCM module doesn't handle paths that aren't relative to the checkout_root consistently.
 225         filesystem.chdir("/test.checkout")
 226
 227         rebaseliner = rebaseline_chromium_webkit_tests.Rebaseliner(host_port_obj,
 228             target_port_obj, platform, options, url_fetcher, zip_factory, mock_scm)
 229         return rebaseliner, filesystem
 230
 231     def test_noop(self):
 232         # this method tests that was can at least instantiate an object, even
 233         # if there is nothing to do.
 234         rebaseliner, filesystem = self.make_rebaseliner("")
 235         rebaseliner.run()
 236         self.assertEqual(len(filesystem.written_files), 1)
 237
 238     def test_rebaselining_tests(self):
 239         rebaseliner, filesystem = self.make_rebaseliner(
 240             "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE")
 241         compile_success = rebaseliner._compile_rebaselining_tests()
 242         self.assertTrue(compile_success)
 243         self.assertEqual(set(['failures/expected/image.html']), rebaseliner._rebaselining_tests)
 244
 245     def test_rebaselining_tests_should_ignore_reftests(self):
 246         rebaseliner, filesystem = self.make_rebaseliner(
 247             "BUGX REBASELINE : failures/expected/reftest.html = IMAGE")
 248         compile_success = rebaseliner._compile_rebaselining_tests()
 249         self.assertFalse(compile_success)
 250         self.assertFalse(rebaseliner._rebaselining_tests)
 251
 252     def test_one_platform(self):
 253         rebaseliner, filesystem = self.make_rebaseliner(
 254             "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE")
 255
 256         rebaseliner.run()
 257         # We expect to have written 13 files over the course of this rebaseline:
 258         # *) 3 files in /__im_tmp for the extracted archive members
 259         # *) 3 new baselines under '/test.checkout/LayoutTests'
 260         # *) 4 files in /tmp for the new and old baselines in the result file
 261         #    (-{old,new}.{txt,png}
 262         # *) 1 text diff in /tmp for the result file (-diff.txt).
 263         # *) 1 image diff in /tmp for the result file (-diff.png).
 264         # *) 1 updated test_expectations file
 265         self.assertEqual(len(filesystem.written_files), 13)
 266         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum')
 267         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png')
 268         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt')
 269
 270     def test_all_platforms(self):
 271         rebaseliner, filesystem = self.make_rebaseliner(
 272             "BUGX REBASELINE : failures/expected/image.html = IMAGE")
 273         rebaseliner.run()
 274         # See comment in test_one_platform for an explanation of the 13 written tests.
 275         # Note that even though the rebaseline is marked for all platforms, each
 276         # rebaseliner only ever does one.
 277         self.assertEqual(len(filesystem.written_files), 13)
 278         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum')
 279         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png')
 280         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt')
 281
 282     def test_png_file_with_comment(self):
 283         rebaseliner, filesystem = self.make_rebaseliner(
 284             "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
 285         compile_success = rebaseliner._compile_rebaselining_tests()
 286         self.assertTrue(compile_success)
 287         self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
 288         rebaseliner.run()
 289         # There is one less file written than |test_one_platform| because we only
 290         # write 2 expectations (the png and the txt file).
 291         self.assertEqual(len(filesystem.written_files), 12)
 292         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789')
 293         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt')
 294         self.assertFalse(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None))
 295
 296     def test_png_file_with_comment_remove_old_checksum(self):
 297         rebaseliner, filesystem = self.make_rebaseliner(
 298             "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
 299         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'] = 'old'
 300         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum'] = 'old'
 301         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'] = 'old'
 302
 303         compile_success = rebaseliner._compile_rebaselining_tests()
 304         self.assertTrue(compile_success)
 305         self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
 306         rebaseliner.run()
 307         # There is one more file written than |test_png_file_with_comment_remove_old_checksum|
 308         # because we also delete the old checksum.
 309         self.assertEqual(len(filesystem.written_files), 13)
 310         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789')
 311         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt')
 312         self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None)
 313
 314     def test_png_file_with_comment_as_duplicate(self):
 315         rebaseliner, filesystem = self.make_rebaseliner(
 316             "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
 317         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.png'] = 'tEXtchecksum\x000123456789'
 318         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.txt'] = 'png-comment-txt'
 319
 320         compile_success = rebaseliner._compile_rebaselining_tests()
 321         self.assertTrue(compile_success)
 322         self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
 323         rebaseliner.run()
 324         self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png', None), None)
 325         self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt', None), None)
 326         self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None)
 327
 328     def test_diff_baselines_txt(self):
 329         rebaseliner, filesystem = self.make_rebaseliner("")
 330         port = rebaseliner._port
 331         output = port.expected_text('passes/text.html')
 332         self.assertFalse(rebaseliner._diff_baselines(output, output,
 333                                                      is_image=False))
 334
 335     def test_diff_baselines_png(self):
 336         rebaseliner, filesystem = self.make_rebaseliner('')
 337         port = rebaseliner._port
 338         image = port.expected_image('passes/image.html')
 339         self.assertFalse(rebaseliner._diff_baselines(image, image,
 340                                                      is_image=True))
 341
 342
 343 class TestRealMain(unittest.TestCase):
 344     def setUp(self):
 345         if not hasattr(self, '_orig_archive'):
 346             self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT
 347             rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive)
 348
 349     def tearDown(self):
 350         rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive
 351
 352     def test_all_platforms(self):
 353         expectations = "BUGX REBASELINE : failures/expected/image.html = IMAGE"
 354
 355         options = test_options()
 356         host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations)
 357         url_fetcher = test_url_fetcher(filesystem)
 358         zip_factory = test_zip_factory()
 359         mock_scm = mocktool.MockSCM()
 360         filesystem.maybe_make_directory(mock_scm.checkout_root)
 361
 362         # FIXME: SCM module doesn't handle paths that aren't relative to the checkout_root consistently.
 363         filesystem.chdir("/test.checkout")
 364
 365         oc = outputcapture.OutputCapture()
 366         oc.capture_output()
 367         res = rebaseline_chromium_webkit_tests.real_main(options, options,
 368             host_port_obj, host_port_obj, url_fetcher, zip_factory, mock_scm)
 369         oc.restore_output()
 370
 371         # We expect to have written 38 files over the course of this rebaseline:
 372         # *) 6*3 files in /__im_tmp/ for the archived members of the 6 ports
 373         # *) 2*3 files in /test.checkout for actually differing baselines
 374         # *) 1 file in /test.checkout for the updated test_expectations file
 375         # *) 2*4 files in /tmp for the old/new baselines for the two actual ports
 376         # *) 2 files in /tmp for the text diffs for the two ports
 377         # *) 2 files in /tmp for the image diffs for the two ports
 378         # *) 1 file in /tmp for the rebaseline results html file
 379         # FIXME: These numbers depend on MockSCM.exists() returning True for all files.
 380         self.assertEqual(res, 0)
 381         self.assertEqual(len(filesystem.written_files), 38)
 382
 383
 384 class TestHtmlGenerator(unittest.TestCase):
 385     def make_generator(self, files, tests):
 386         options = mocktool.MockOptions(configuration=None, html_directory='/tmp')
 387         host_port = port.get('test', options, filesystem=unit_test_filesystem(files))
 388         generator = rebaseline_chromium_webkit_tests.HtmlGenerator(host_port,
 389             target_port=None, options=options, platforms=['test-mac-leopard'], rebaselining_tests=tests)
 390         return generator, host_port
 391
 392     def test_generate_baseline_links(self):
 393         files = {
 394             "/tmp/foo-expected-mac-old.txt": "",
 395             "/tmp/foo-expected-mac-new.txt": "",
 396             "/tmp/foo-expected-mac-diff.txt": "",
 397         }
 398         tests = ["foo.txt"]
 399         generator, host_port = self.make_generator(files, tests)
 400         links = generator._generate_baseline_links("foo", ".txt", "mac")
 401         expected_links = '<td align=center><a href="file:///tmp/foo-expected-mac-old.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-new.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-diff.txt">Diff</a></td>'
 402         self.assertEqual(links, expected_links)
 403
 404
 405 if __name__ == '__main__':
 406     unittest.main()