code.vuplus.com Git - vuplus_webkit/blob - Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py

   1 # Copyright (C) 2011 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29
  30 import logging
  31 import os
  32
  33 from webkitpy.common.system.crashlogs import CrashLogs
  34 from webkitpy.layout_tests.models import test_failures
  35
  36
  37 _log = logging.getLogger(__name__)
  38
  39
  40 def write_test_result(port, test_name, driver_output,
  41                       expected_driver_output, failures):
  42     """Write the test result to the result output directory."""
  43     root_output_dir = port.results_directory()
  44     writer = TestResultWriter(port, root_output_dir, test_name)
  45     if driver_output.error:
  46         writer.write_stderr(driver_output.error)
  47
  48     for failure in failures:
  49         # FIXME: Instead of this long 'if' block, each failure class might
  50         # have a responsibility for writing a test result.
  51         if isinstance(failure, (test_failures.FailureMissingResult,
  52                                 test_failures.FailureTextMismatch)):
  53             writer.write_text_files(driver_output.text, expected_driver_output.text)
  54             writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text)
  55         elif isinstance(failure, test_failures.FailureMissingImage):
  56             writer.write_image_files(driver_output.image, expected_image=None)
  57         elif isinstance(failure, test_failures.FailureMissingImageHash):
  58             writer.write_image_files(driver_output.image, expected_driver_output.image)
  59         elif isinstance(failure, test_failures.FailureImageHashMismatch):
  60             writer.write_image_files(driver_output.image, expected_driver_output.image)
  61             writer.write_image_diff_files(driver_output.image_diff)
  62         elif isinstance(failure, (test_failures.FailureAudioMismatch,
  63                                   test_failures.FailureMissingAudio)):
  64             writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
  65         elif isinstance(failure, test_failures.FailureCrash):
  66             if failure.is_reftest:
  67                 writer.write_crash_report(expected_driver_output.error)
  68             else:
  69                 writer.write_crash_report(driver_output.error)
  70         elif isinstance(failure, test_failures.FailureReftestMismatch):
  71             writer.write_image_files(driver_output.image, expected_driver_output.image)
  72             # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
  73             image_diff = port.diff_image(driver_output.image, expected_driver_output.image)
  74             if image_diff:
  75                 writer.write_image_diff_files(image_diff)
  76             writer.copy_file(port.reftest_expected_filename(test_name), '-expected.html')
  77         elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
  78             writer.write_image_files(driver_output.image, expected_image=None)
  79             writer.copy_file(port.reftest_expected_mismatch_filename(test_name), '-expected-mismatch.html')
  80         else:
  81             assert isinstance(failure, (test_failures.FailureTimeout,))
  82
  83
  84 class TestResultWriter(object):
  85     """A class which handles all writing operations to the result directory."""
  86
  87     # Filename pieces when writing failures to the test results directory.
  88     FILENAME_SUFFIX_ACTUAL = "-actual"
  89     FILENAME_SUFFIX_EXPECTED = "-expected"
  90     FILENAME_SUFFIX_DIFF = "-diff"
  91     FILENAME_SUFFIX_STDERR = "-stderr"
  92     FILENAME_SUFFIX_CRASH_LOG = "-crash-log"
  93     FILENAME_SUFFIX_WDIFF = "-wdiff.html"
  94     FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html"
  95     FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png"
  96     FILENAME_SUFFIX_IMAGE_DIFFS_HTML = "-diffs.html"
  97
  98     def __init__(self, port, root_output_dir, test_name):
  99         self._port = port
 100         self._root_output_dir = root_output_dir
 101         self._test_name = test_name
 102
 103     def _make_output_directory(self):
 104         """Creates the output directory (if needed) for a given test filename."""
 105         fs = self._port._filesystem
 106         output_filename = fs.join(self._root_output_dir, self._test_name)
 107         self._port.maybe_make_directory(fs.dirname(output_filename))
 108
 109     def output_filename(self, modifier):
 110         """Returns a filename inside the output dir that contains modifier.
 111
 112         For example, if test name is "fast/dom/foo.html" and modifier is "-expected.txt",
 113         the return value is "/<path-to-root-output-dir>/fast/dom/foo-expected.txt".
 114
 115         Args:
 116           modifier: a string to replace the extension of filename with
 117
 118         Return:
 119           The absolute path to the output filename
 120         """
 121         fs = self._port._filesystem
 122         output_filename = fs.join(self._root_output_dir, self._test_name)
 123         return fs.splitext(output_filename)[0] + modifier
 124
 125     def _output_testname(self, modifier):
 126         fs = self._port._filesystem
 127         return fs.splitext(fs.basename(self._test_name))[0] + modifier
 128
 129     def write_output_files(self, file_type, output, expected):
 130         """Writes the test output, the expected output in the results directory.
 131
 132         The full output filename of the actual, for example, will be
 133           <filename>-actual<file_type>
 134         For instance,
 135           my_test-actual.txt
 136
 137         Args:
 138           file_type: A string describing the test output file type, e.g. ".txt"
 139           output: A string containing the test output
 140           expected: A string containing the expected test output
 141         """
 142         self._make_output_directory()
 143         actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type)
 144         expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
 145
 146         fs = self._port._filesystem
 147         if output is not None:
 148             fs.write_binary_file(actual_filename, output)
 149         if expected is not None:
 150             fs.write_binary_file(expected_filename, expected)
 151
 152     def write_stderr(self, error):
 153         fs = self._port._filesystem
 154         filename = self.output_filename(self.FILENAME_SUFFIX_STDERR + ".txt")
 155         fs.maybe_make_directory(fs.dirname(filename))
 156         fs.write_binary_file(filename, error)
 157
 158     def write_crash_report(self, error):
 159         fs = self._port._filesystem
 160         filename = self.output_filename(self.FILENAME_SUFFIX_CRASH_LOG + ".txt")
 161         fs.maybe_make_directory(fs.dirname(filename))
 162         # FIXME: We shouldn't be grabbing private members of port.
 163         crash_logs = CrashLogs(fs)
 164         log = crash_logs.find_newest_log(self._port.driver_name())
 165         # CrashLogs doesn't support every platform, so we fall back to
 166         # including the stderr output, which is admittedly somewhat redundant.
 167         fs.write_text_file(filename, log if log else error)
 168
 169     def write_text_files(self, actual_text, expected_text):
 170         self.write_output_files(".txt", actual_text, expected_text)
 171
 172     def create_text_diff_and_write_result(self, actual_text, expected_text):
 173         # FIXME: This function is actually doing the diffs as well as writing results.
 174         # It might be better to extract code which does 'diff' and make it a separate function.
 175         if not actual_text or not expected_text:
 176             return
 177
 178         self._make_output_directory()
 179         file_type = '.txt'
 180         actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type)
 181         expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
 182         fs = self._port._filesystem
 183         # We treat diff output as binary. Diff output may contain multiple files
 184         # in conflicting encodings.
 185         diff = self._port.diff_text(expected_text, actual_text, expected_filename, actual_filename)
 186         diff_filename = self.output_filename(self.FILENAME_SUFFIX_DIFF + file_type)
 187         fs.write_binary_file(diff_filename, diff)
 188
 189         # Shell out to wdiff to get colored inline diffs.
 190         wdiff = self._port.wdiff_text(expected_filename, actual_filename)
 191         wdiff_filename = self.output_filename(self.FILENAME_SUFFIX_WDIFF)
 192         fs.write_binary_file(wdiff_filename, wdiff)
 193
 194         # Use WebKit's PrettyPatch.rb to get an HTML diff.
 195         pretty_patch = self._port.pretty_patch_text(diff_filename)
 196         pretty_patch_filename = self.output_filename(self.FILENAME_SUFFIX_PRETTY_PATCH)
 197         fs.write_binary_file(pretty_patch_filename, pretty_patch)
 198
 199     def write_audio_files(self, actual_audio, expected_audio):
 200         self.write_output_files('.wav', actual_audio, expected_audio)
 201
 202     def write_image_files(self, actual_image, expected_image):
 203         self.write_output_files('.png', actual_image, expected_image)
 204
 205     def write_image_diff_files(self, image_diff):
 206         diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF)
 207         fs = self._port._filesystem
 208         fs.write_binary_file(diff_filename, image_diff)
 209
 210         diffs_html_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFFS_HTML)
 211         # FIXME: old-run-webkit-tests shows the diff percentage as the text contents of the "diff" link.
 212         # FIXME: old-run-webkit-tests include a link to the test file.
 213         html = """<!DOCTYPE HTML>
 214 <html>
 215 <head>
 216 <title>%(title)s</title>
 217 <style>.label{font-weight:bold}</style>
 218 </head>
 219 <body>
 220 Difference between images: <a href="%(diff_filename)s">diff</a><br>
 221 <div class=imageText></div>
 222 <div class=imageContainer data-prefix="%(prefix)s">Loading...</div>
 223 <script>
 224 (function() {
 225     var preloadedImageCount = 0;
 226     function preloadComplete() {
 227         ++preloadedImageCount;
 228         if (preloadedImageCount < 2)
 229             return;
 230         toggleImages();
 231         setInterval(toggleImages, 2000)
 232     }
 233
 234     function preloadImage(url) {
 235         image = new Image();
 236         image.addEventListener('load', preloadComplete);
 237         image.src = url;
 238         return image;
 239     }
 240
 241     function toggleImages() {
 242         if (text.textContent == 'Expected Image') {
 243             text.textContent = 'Actual Image';
 244             container.replaceChild(actualImage, container.firstChild);
 245         } else {
 246             text.textContent = 'Expected Image';
 247             container.replaceChild(expectedImage, container.firstChild);
 248         }
 249     }
 250
 251     var text = document.querySelector('.imageText');
 252     var container = document.querySelector('.imageContainer');
 253     var actualImage = preloadImage(container.getAttribute('data-prefix') + '-actual.png');
 254     var expectedImage = preloadImage(container.getAttribute('data-prefix') + '-expected.png');
 255 })();
 256 </script>
 257 </body>
 258 </html>
 259 """ % {
 260             'title': self._test_name,
 261             'diff_filename': self._output_testname(self.FILENAME_SUFFIX_IMAGE_DIFF),
 262             'prefix': self._output_testname(''),
 263         }
 264         # FIXME: This seems like a text file, not a binary file.
 265         self._port._filesystem.write_binary_file(diffs_html_filename, html)
 266
 267     def copy_file(self, src_filepath, dst_extension):
 268         fs = self._port._filesystem
 269         assert fs.exists(src_filepath), 'src_filepath: %s' % src_filepath
 270         dst_filename = self.output_filename(dst_extension)
 271         self._make_output_directory()
 272         fs.copyfile(src_filepath, dst_filename)