code.vuplus.com Git - vuplus_webkit/blob - Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js

   1 /*
   2  * Copyright (C) 2011 Google Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  23  * THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 (function () {
  27
  28 module("results");
  29
  30 var kExampleResultsJSON = {
  31     "tests": {
  32         "scrollbars": {
  33             "custom-scrollbar-with-incomplete-style.html": {
  34                 "expected": "IMAGE",
  35                 "actual": "IMAGE"
  36             },
  37             "flaky-scrollbar.html": {
  38                 "expected": "PASS",
  39                 "actual": "PASS TEXT"
  40             },
  41             "unexpected-pass.html": {
  42                 "expected": "FAIL",
  43                 "actual": "PASS"
  44             }
  45         },
  46         "userscripts": {
  47             "user-script-video-document.html": {
  48                 "expected": "FAIL",
  49                 "actual": "TEXT"
  50             },
  51             "another-test.html": {
  52                 "expected": "PASS",
  53                 "actual": "TEXT"
  54             }
  55         },
  56     },
  57     "skipped": 339,
  58     "num_regressions": 14,
  59     "interrupted": false,
  60     "layout_tests_dir": "\/mnt\/data\/b\/build\/slave\/Webkit_Linux\/build\/src\/third_party\/WebKit\/LayoutTests",
  61     "version": 3,
  62     "num_passes": 15566,
  63     "has_pretty_patch": false,
  64     "fixable": 1233,
  65     "num_flaky":1,
  66     "uses_expectations_file": true,
  67     "has_wdiff": true,
  68     "revision": "90430"
  69 };
  70
  71 test("unexpectedFailures", 1, function() {
  72     var unexpectedFailures = results.unexpectedFailures(kExampleResultsJSON);
  73     deepEqual(unexpectedFailures, {
  74         "userscripts/another-test.html": {
  75             "expected": "PASS",
  76             "actual": "TEXT"
  77         }
  78     });
  79 });
  80
  81 test("unexpectedFailuresByTest", 1, function() {
  82     var unexpectedFailuresByTest = results.unexpectedFailuresByTest({
  83         "Mock Builder": kExampleResultsJSON
  84     });
  85     deepEqual(unexpectedFailuresByTest, {
  86         "userscripts/another-test.html": {
  87             "Mock Builder": {
  88                 "expected": "PASS",
  89                 "actual": "TEXT"
  90             }
  91         }
  92     });
  93 });
  94
  95 test("unexpectedSuccessesByTest", 1, function() {
  96     var unexpectedFailuresByTest = results.unexpectedSuccessesByTest({
  97         "Mock Builder": kExampleResultsJSON
  98     });
  99     deepEqual(unexpectedFailuresByTest, {
 100         "scrollbars/unexpected-pass.html": {
 101             "Mock Builder": {
 102                 "expected": "FAIL",
 103                 "actual": "PASS"
 104             }
 105         }
 106     });
 107 });
 108
 109 test("failureInfoForTestAndBuilder", 1, function() {
 110     var unexpectedFailuresByTest = results.unexpectedFailuresByTest({
 111         "Mock Builder": kExampleResultsJSON
 112     });
 113     var failureInfo = results.failureInfoForTestAndBuilder(unexpectedFailuresByTest, "userscripts/another-test.html", "Mock Builder");
 114     deepEqual(failureInfo, {
 115         "testName": "userscripts/another-test.html",
 116         "builderName": "Mock Builder",
 117         "failureTypeList": ["TEXT"],
 118     });
 119 });
 120
 121 test("resultKind", 12, function() {
 122     equals(results.resultKind("http://example.com/foo-actual.txt"), "actual");
 123     equals(results.resultKind("http://example.com/foo-expected.txt"), "expected");
 124     equals(results.resultKind("http://example.com/foo-diff.txt"), "diff");
 125     equals(results.resultKind("http://example.com/foo.bar-actual.txt"), "actual");
 126     equals(results.resultKind("http://example.com/foo.bar-expected.txt"), "expected");
 127     equals(results.resultKind("http://example.com/foo.bar-diff.txt"), "diff");
 128     equals(results.resultKind("http://example.com/foo-actual.png"), "actual");
 129     equals(results.resultKind("http://example.com/foo-expected.png"), "expected");
 130     equals(results.resultKind("http://example.com/foo-diff.png"), "diff");
 131     equals(results.resultKind("http://example.com/foo-pretty-diff.html"), "diff");
 132     equals(results.resultKind("http://example.com/foo-wdiff.html"), "diff");
 133     equals(results.resultKind("http://example.com/foo-xyz.html"), "unknown");
 134 });
 135
 136 test("resultType", 12, function() {
 137     equals(results.resultType("http://example.com/foo-actual.txt"), "text");
 138     equals(results.resultType("http://example.com/foo-expected.txt"), "text");
 139     equals(results.resultType("http://example.com/foo-diff.txt"), "text");
 140     equals(results.resultType("http://example.com/foo.bar-actual.txt"), "text");
 141     equals(results.resultType("http://example.com/foo.bar-expected.txt"), "text");
 142     equals(results.resultType("http://example.com/foo.bar-diff.txt"), "text");
 143     equals(results.resultType("http://example.com/foo-actual.png"), "image");
 144     equals(results.resultType("http://example.com/foo-expected.png"), "image");
 145     equals(results.resultType("http://example.com/foo-diff.png"), "image");
 146     equals(results.resultType("http://example.com/foo-pretty-diff.html"), "text");
 147     equals(results.resultType("http://example.com/foo-wdiff.html"), "text");
 148     equals(results.resultType("http://example.com/foo.xyz"), "text");
 149 });
 150
 151 test("resultNodeForTest", 4, function() {
 152     deepEqual(results.resultNodeForTest(kExampleResultsJSON, "userscripts/another-test.html"), {
 153         "expected": "PASS",
 154         "actual": "TEXT"
 155     });
 156     equals(results.resultNodeForTest(kExampleResultsJSON, "foo.html"), null);
 157     equals(results.resultNodeForTest(kExampleResultsJSON, "userscripts/foo.html"), null);
 158     equals(results.resultNodeForTest(kExampleResultsJSON, "userscripts/foo/bar.html"), null);
 159 });
 160
 161 test("walkHistory", 6, function() {
 162     var simulator = new NetworkSimulator();
 163
 164     var keyMap = {
 165         "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw": {
 166             "tests": {
 167                 "userscripts": {
 168                     "another-test.html": {
 169                         "expected": "PASS",
 170                         "actual": "TEXT"
 171                     }
 172                 },
 173             },
 174             "revision": "90430"
 175         },
 176         "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw":{
 177             "tests": {
 178                 "userscripts": {
 179                     "user-script-video-document.html": {
 180                         "expected": "FAIL",
 181                         "actual": "TEXT"
 182                     },
 183                     "another-test.html": {
 184                         "expected": "PASS",
 185                         "actual": "TEXT"
 186                     }
 187                 },
 188             },
 189             "revision": "90429"
 190         },
 191         "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw":{
 192             "tests": {
 193                 "userscripts": {
 194                     "another-test.html": {
 195                         "expected": "PASS",
 196                         "actual": "TEXT"
 197                     }
 198                 },
 199             },
 200             "revision": "90426"
 201         },
 202         "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw":{
 203             "tests": {
 204                 "userscripts": {
 205                     "user-script-video-document.html": {
 206                         "expected": "FAIL",
 207                         "actual": "TEXT"
 208                     },
 209                 },
 210             },
 211             "revision": "90424"
 212         },
 213         "abc":{
 214             "tests": {
 215                 "userscripts": {
 216                     "another-test.html": {
 217                         "expected": "PASS",
 218                         "actual": "TEXT"
 219                     }
 220                 },
 221             },
 222             "revision": "90426"
 223         },
 224         "xyz":{
 225             "tests": {
 226             },
 227             "revision": "90425"
 228         }
 229     };
 230
 231     simulator.jsonp = function(url, callback) {
 232         simulator.scheduleCallback(function() {
 233             if (/dir=1/.test(url)) {
 234                 if (/builder=Mock/.test(url)) {
 235                     callback([
 236                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw" },
 237                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw" },
 238                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw" },
 239                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw" },
 240                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGOj5UAw" },
 241                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGP-AUQw" },
 242                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGPL3UAw" },
 243                         { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNHJQAw" },
 244                     ]);
 245                 } else if (/builder=Another/.test(url)) {
 246                     callback([
 247                         { "key": "abc" },
 248                         { "key": "xyz" },
 249                     ]);
 250                 } else {
 251                     ok(false, 'Unexpected URL: ' + url);
 252                 }
 253             } else {
 254                 var key = url.match(/key=([^&]+)/)[1];
 255                 callback(keyMap[key]);
 256             }
 257         });
 258     };
 259     simulator.runTest(function() {
 260         results.regressionRangeForFailure("Mock Builder", "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
 261             equals(oldestFailingRevision, 90426);
 262             equals(newestPassingRevision, 90424);
 263         });
 264
 265         results.unifyRegressionRanges(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
 266             equals(oldestFailingRevision, 90426);
 267             equals(newestPassingRevision, 90425);
 268         });
 269
 270         results.countFailureOccurances(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(failureCount) {
 271             equals(failureCount, 4);
 272         });
 273     });
 274 });
 275
 276 test("walkHistory (no revision)", 3, function() {
 277     var simulator = new NetworkSimulator();
 278
 279     var keyMap = {
 280         "vsfdsfdsafsdafasd": {
 281             "tests": {
 282                 "userscripts": {
 283                     "another-test.html": {
 284                         "expected": "PASS",
 285                         "actual": "TEXT"
 286                     }
 287                 },
 288             },
 289             "revision": ""
 290         },
 291         "gavsavsrfgwaevwefawvae":{
 292             "tests": {
 293             },
 294             "revision": ""
 295         },
 296     };
 297
 298     simulator.jsonp = function(url, callback) {
 299         simulator.scheduleCallback(function() {
 300             if (/dir=1/.test(url)) {
 301                 callback([
 302                     { "key": "vsfdsfdsafsdafasd" },
 303                     { "key": "gavsavsrfgwaevwefawvae" },
 304                 ]);
 305             } else {
 306                 var key = url.match(/key=([^&]+)/)[1];
 307                 callback(keyMap[key]);
 308             }
 309         });
 310     };
 311
 312     simulator.runTest(function() {
 313         results.regressionRangeForFailure("Mock Builder", "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
 314             equals(oldestFailingRevision, 0);
 315             equals(newestPassingRevision, 0);
 316         });
 317     });
 318 });
 319
 320 test("collectUnexpectedResults", 1, function() {
 321     var dictionaryOfResultNodes = {
 322         "foo": {
 323             "expected": "IMAGE",
 324             "actual": "IMAGE"
 325         },
 326         "bar": {
 327             "expected": "PASS",
 328             "actual": "PASS TEXT"
 329         },
 330         "baz": {
 331             "expected": "TEXT",
 332             "actual": "IMAGE"
 333         },
 334         "qux": {
 335             "expected": "PASS",
 336             "actual": "TEXT"
 337         },
 338         "taco": {
 339             "expected": "PASS",
 340             "actual": "TEXT"
 341         },
 342     };
 343
 344     var collectedResults = results.collectUnexpectedResults(dictionaryOfResultNodes);
 345     deepEqual(collectedResults, ["TEXT", "IMAGE"]);
 346 });
 347
 348 test("failureTypeToExtensionList", 5, function() {
 349     deepEqual(results.failureTypeToExtensionList('TEXT'), ['txt']);
 350     deepEqual(results.failureTypeToExtensionList('IMAGE+TEXT'), ['txt', 'png']);
 351     deepEqual(results.failureTypeToExtensionList('IMAGE'), ['png']);
 352     deepEqual(results.failureTypeToExtensionList('CRASH'), []);
 353     deepEqual(results.failureTypeToExtensionList('TIMEOUT'), []);
 354 });
 355
 356 test("canRebaseline", 6, function() {
 357     ok(results.canRebaseline(['TEXT']));
 358     ok(results.canRebaseline(['IMAGE+TEXT', 'CRASH']));
 359     ok(results.canRebaseline(['IMAGE']));
 360     ok(!results.canRebaseline(['CRASH']));
 361     ok(!results.canRebaseline(['TIMEOUT']));
 362     ok(!results.canRebaseline([]));
 363 });
 364
 365 test("fetchResultsURLs", 5, function() {
 366     var simulator = new NetworkSimulator();
 367
 368     var probedURLs = [];
 369     simulator.probe = function(url, options)
 370     {
 371         simulator.scheduleCallback(function() {
 372             probedURLs.push(url);
 373             if (base.endsWith(url, '.txt'))
 374                 options.success.call();
 375             else if (/taco.+png$/.test(url))
 376                 options.success.call();
 377             else
 378                 options.error.call();
 379         });
 380     };
 381
 382     simulator.runTest(function() {
 383         results.fetchResultsURLs({
 384             'builderName': "Mock Builder",
 385             'testName': "userscripts/another-test.html",
 386             'failureTypeList': ['IMAGE', 'CRASH'],
 387         }, function(resultURLs) {
 388             deepEqual(resultURLs, [
 389                 "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-crash-log.txt"
 390             ]);
 391         });
 392         results.fetchResultsURLs({
 393             'builderName': "Mock Builder",
 394             'testName': "userscripts/another-test.html",
 395             'failureTypeList': ['TIMEOUT'],
 396         }, function(resultURLs) {
 397             deepEqual(resultURLs, []);
 398         });
 399         results.fetchResultsURLs({
 400             'builderName': "Mock Builder",
 401             'testName': "userscripts/taco.html",
 402             'failureTypeList': ['IMAGE', 'IMAGE+TEXT'],
 403         }, function(resultURLs) {
 404             deepEqual(resultURLs, [
 405                 "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-expected.png",
 406                 "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-actual.png",
 407                 "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-diff.png",
 408                 "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-expected.txt",
 409                 "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-actual.txt",
 410                 "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-diff.txt",
 411             ]);
 412         });
 413     });
 414
 415     deepEqual(probedURLs, [
 416         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.png",
 417         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.png",
 418         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.png",
 419         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-crash-log.txt",
 420         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-expected.png",
 421         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-actual.png",
 422         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-diff.png",
 423         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-actual.txt",
 424         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-expected.txt",
 425         "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/taco-diff.txt",
 426     ]);
 427 });
 428
 429 })();