code.vuplus.com Git - vuplus_webkit/blob - Tools/Scripts/webkitpy/tool/servers/rebaselineserver_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import unittest
  30
  31 try:
  32     import json
  33 except ImportError:
  34     # python 2.5 compatibility
  35     import webkitpy.thirdparty.simplejson as json
  36
  37 from webkitpy.common.net import resultsjsonparser_unittest
  38 from webkitpy.common.system import filesystem_mock
  39 from webkitpy.layout_tests.layout_package.json_results_generator import strip_json_wrapper
  40 from webkitpy.layout_tests.port.webkit import WebKitPort
  41 from webkitpy.tool.commands.rebaselineserver import TestConfig, RebaselineServer
  42 from webkitpy.tool.mocktool import MockSCM
  43 from webkitpy.tool.servers import rebaselineserver
  44
  45
  46 class RebaselineTestTest(unittest.TestCase):
  47     def test_text_rebaseline_update(self):
  48         self._assertRebaseline(
  49             test_files=(
  50                 'fast/text-expected.txt',
  51                 'platform/mac/fast/text-expected.txt',
  52             ),
  53             results_files=(
  54                 'fast/text-actual.txt',
  55             ),
  56             test_name='fast/text.html',
  57             baseline_target='mac',
  58             baseline_move_to='none',
  59             expected_success=True,
  60             expected_log=[
  61                 'Rebaselining fast/text...',
  62                 '  Updating baselines for mac',
  63                 '    Updated text-expected.txt',
  64             ])
  65
  66     def test_text_rebaseline_new(self):
  67         self._assertRebaseline(
  68             test_files=(
  69                 'fast/text-expected.txt',
  70             ),
  71             results_files=(
  72                 'fast/text-actual.txt',
  73             ),
  74             test_name='fast/text.html',
  75             baseline_target='mac',
  76             baseline_move_to='none',
  77             expected_success=True,
  78             expected_log=[
  79                 'Rebaselining fast/text...',
  80                 '  Updating baselines for mac',
  81                 '    Updated text-expected.txt',
  82             ])
  83
  84     def test_text_rebaseline_move_no_op_1(self):
  85         self._assertRebaseline(
  86             test_files=(
  87                 'fast/text-expected.txt',
  88                 'platform/win/fast/text-expected.txt',
  89             ),
  90             results_files=(
  91                 'fast/text-actual.txt',
  92             ),
  93             test_name='fast/text.html',
  94             baseline_target='mac',
  95             baseline_move_to='mac-leopard',
  96             expected_success=True,
  97             expected_log=[
  98                 'Rebaselining fast/text...',
  99                 '  Updating baselines for mac',
 100                 '    Updated text-expected.txt',
 101             ])
 102
 103     def test_text_rebaseline_move_no_op_2(self):
 104         self._assertRebaseline(
 105             test_files=(
 106                 'fast/text-expected.txt',
 107                 'platform/mac/fast/text-expected.checksum',
 108             ),
 109             results_files=(
 110                 'fast/text-actual.txt',
 111             ),
 112             test_name='fast/text.html',
 113             baseline_target='mac',
 114             baseline_move_to='mac-leopard',
 115             expected_success=True,
 116             expected_log=[
 117                 'Rebaselining fast/text...',
 118                 '  Moving current mac baselines to mac-leopard',
 119                 '    No current baselines to move',
 120                 '  Updating baselines for mac',
 121                 '    Updated text-expected.txt',
 122             ])
 123
 124     def test_text_rebaseline_move(self):
 125         self._assertRebaseline(
 126             test_files=(
 127                 'fast/text-expected.txt',
 128                 'platform/mac/fast/text-expected.txt',
 129             ),
 130             results_files=(
 131                 'fast/text-actual.txt',
 132             ),
 133             test_name='fast/text.html',
 134             baseline_target='mac',
 135             baseline_move_to='mac-leopard',
 136             expected_success=True,
 137             expected_log=[
 138                 'Rebaselining fast/text...',
 139                 '  Moving current mac baselines to mac-leopard',
 140                 '    Moved text-expected.txt',
 141                 '  Updating baselines for mac',
 142                 '    Updated text-expected.txt',
 143             ])
 144
 145     def test_text_rebaseline_move_only_images(self):
 146         self._assertRebaseline(
 147             test_files=(
 148                 'fast/image-expected.txt',
 149                 'platform/mac/fast/image-expected.txt',
 150                 'platform/mac/fast/image-expected.png',
 151                 'platform/mac/fast/image-expected.checksum',
 152             ),
 153             results_files=(
 154                 'fast/image-actual.png',
 155                 'fast/image-actual.checksum',
 156             ),
 157             test_name='fast/image.html',
 158             baseline_target='mac',
 159             baseline_move_to='mac-leopard',
 160             expected_success=True,
 161             expected_log=[
 162                 'Rebaselining fast/image...',
 163                 '  Moving current mac baselines to mac-leopard',
 164                 '    Moved image-expected.checksum',
 165                 '    Moved image-expected.png',
 166                 '  Updating baselines for mac',
 167                 '    Updated image-expected.checksum',
 168                 '    Updated image-expected.png',
 169             ])
 170
 171     def test_text_rebaseline_move_already_exist(self):
 172         self._assertRebaseline(
 173             test_files=(
 174                 'fast/text-expected.txt',
 175                 'platform/mac-leopard/fast/text-expected.txt',
 176                 'platform/mac/fast/text-expected.txt',
 177             ),
 178             results_files=(
 179                 'fast/text-actual.txt',
 180             ),
 181             test_name='fast/text.html',
 182             baseline_target='mac',
 183             baseline_move_to='mac-leopard',
 184             expected_success=False,
 185             expected_log=[
 186                 'Rebaselining fast/text...',
 187                 '  Moving current mac baselines to mac-leopard',
 188                 '    Already had baselines in mac-leopard, could not move existing mac ones',
 189             ])
 190
 191     def test_image_rebaseline(self):
 192         self._assertRebaseline(
 193             test_files=(
 194                 'fast/image-expected.txt',
 195                 'platform/mac/fast/image-expected.png',
 196                 'platform/mac/fast/image-expected.checksum',
 197             ),
 198             results_files=(
 199                 'fast/image-actual.png',
 200                 'fast/image-actual.checksum',
 201             ),
 202             test_name='fast/image.html',
 203             baseline_target='mac',
 204             baseline_move_to='none',
 205             expected_success=True,
 206             expected_log=[
 207                 'Rebaselining fast/image...',
 208                 '  Updating baselines for mac',
 209                 '    Updated image-expected.checksum',
 210                 '    Updated image-expected.png',
 211             ])
 212
 213     def test_gather_baselines(self):
 214         example_json = resultsjsonparser_unittest.ResultsJSONParserTest._example_full_results_json
 215         results_json = json.loads(strip_json_wrapper(example_json))
 216         server = RebaselineServer()
 217         server._test_config = get_test_config()
 218         server._gather_baselines(results_json)
 219         self.assertEqual(results_json['tests']['svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html']['state'], 'needs_rebaseline')
 220         self.assertFalse('prototype-chocolate.html' in results_json['tests'])
 221
 222     def _assertRebaseline(self, test_files, results_files, test_name, baseline_target, baseline_move_to, expected_success, expected_log):
 223         log = []
 224         test_config = get_test_config(test_files, results_files)
 225         success = rebaselineserver._rebaseline_test(
 226             test_name,
 227             baseline_target,
 228             baseline_move_to,
 229             test_config,
 230             log=lambda l: log.append(l))
 231         self.assertEqual(expected_log, log)
 232         self.assertEqual(expected_success, success)
 233
 234
 235 class GetActualResultFilesTest(unittest.TestCase):
 236     def test(self):
 237         test_config = get_test_config(result_files=(
 238             'fast/text-actual.txt',
 239             'fast2/text-actual.txt',
 240             'fast/text2-actual.txt',
 241             'fast/text-notactual.txt',
 242         ))
 243         self.assertEqual(
 244             ('text-actual.txt',),
 245             rebaselineserver._get_actual_result_files(
 246                 'fast/text.html', test_config))
 247
 248
 249 class GetBaselinesTest(unittest.TestCase):
 250     def test_no_baselines(self):
 251         self._assertBaselines(
 252             test_files=(),
 253             test_name='fast/missing.html',
 254             expected_baselines={})
 255
 256     def test_text_baselines(self):
 257         self._assertBaselines(
 258             test_files=(
 259                 'fast/text-expected.txt',
 260                 'platform/mac/fast/text-expected.txt',
 261             ),
 262             test_name='fast/text.html',
 263             expected_baselines={
 264                 'mac': {'.txt': True},
 265                 'base': {'.txt': False},
 266             })
 267
 268     def test_image_and_text_baselines(self):
 269         self._assertBaselines(
 270             test_files=(
 271                 'fast/image-expected.txt',
 272                 'platform/mac/fast/image-expected.png',
 273                 'platform/mac/fast/image-expected.checksum',
 274                 'platform/win/fast/image-expected.png',
 275                 'platform/win/fast/image-expected.checksum',
 276             ),
 277             test_name='fast/image.html',
 278             expected_baselines={
 279                 'base': {'.txt': True},
 280                 'mac': {'.checksum': True, '.png': True},
 281                 'win': {'.checksum': False, '.png': False},
 282             })
 283
 284     def test_extra_baselines(self):
 285         self._assertBaselines(
 286             test_files=(
 287                 'fast/text-expected.txt',
 288                 'platform/nosuchplatform/fast/text-expected.txt',
 289             ),
 290             test_name='fast/text.html',
 291             expected_baselines={'base': {'.txt': True}})
 292
 293     def _assertBaselines(self, test_files, test_name, expected_baselines):
 294         actual_baselines = rebaselineserver.get_test_baselines(test_name, get_test_config(test_files))
 295         self.assertEqual(expected_baselines, actual_baselines)
 296
 297
 298 def get_test_config(test_files=[], result_files=[]):
 299     # We could grab this from port.layout_tests_dir(), but instantiating a fully mocked port is a pain.
 300     layout_tests_directory = "/mock-checkout/LayoutTests"
 301     results_directory = '/WebKitBuild/Debug/layout-test-results'
 302     mock_filesystem = filesystem_mock.MockFileSystem()
 303     for file in test_files:
 304         file_path = mock_filesystem.join(layout_tests_directory, file)
 305         mock_filesystem.files[file_path] = ''
 306     for file in result_files:
 307         file_path = mock_filesystem.join(results_directory, file)
 308         mock_filesystem.files[file_path] = ''
 309
 310     class TestMacPort(WebKitPort):
 311         port_name = "mac"
 312         def __init__(self):
 313             # FIXME: This should use MockExecutive and MockUser as well.
 314             WebKitPort.__init__(self, filesystem=mock_filesystem)
 315
 316     return TestConfig(
 317         TestMacPort(),
 318         layout_tests_directory,
 319         results_directory,
 320         ('mac', 'mac-leopard', 'win', 'linux'),
 321         mock_filesystem,
 322         MockSCM())