2 # Copyright (C) 2010 Google Inc. All rights reserved.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 """A helper class for reading in and dealing with tests expectations
41 # python 2.5 compatibility
42 import webkitpy.thirdparty.simplejson as json
44 from webkitpy.layout_tests.models.test_configuration import TestConfiguration, TestConfigurationConverter
46 _log = logging.getLogger(__name__)
49 # Test expectation and modifier constants.
50 # FIXME: range() starts with 0 which makes if expectation checks harder
52 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX,
53 SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(16)
56 def result_was_expected(result, expected_results, test_needs_rebaselining, test_is_skipped):
57 """Returns whether we got a result we were expecting.
59 result: actual result of a test execution
60 expected_results: set of results listed in test_expectations
61 test_needs_rebaselining: whether test was marked as REBASELINE
62 test_is_skipped: whether test was marked as SKIP"""
63 if result in expected_results:
65 if result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and FAIL in expected_results:
67 if result == MISSING and test_needs_rebaselining:
69 if result == SKIP and test_is_skipped:
74 def remove_pixel_failures(expected_results):
75 """Returns a copy of the expected results for a test, except that we
76 drop any pixel failures and return the remaining expectations. For example,
77 if we're not running pixel tests, then tests expected to fail as IMAGE
79 expected_results = expected_results.copy()
80 if IMAGE in expected_results:
81 expected_results.remove(IMAGE)
82 expected_results.add(PASS)
83 if IMAGE_PLUS_TEXT in expected_results:
84 expected_results.remove(IMAGE_PLUS_TEXT)
85 expected_results.add(TEXT)
86 return expected_results
89 def has_pixel_failures(actual_results):
90 return IMAGE in actual_results or IMAGE_PLUS_TEXT in actual_results
93 # FIXME: This method is no longer used here in this module. Remove remaining callsite in manager.py and this method.
94 def strip_comments(line):
95 """Strips comments from a line and return None if the line is empty
96 or else the contents of line with leading and trailing spaces removed
97 and all other whitespace collapsed"""
99 commentIndex = line.find('//')
100 if commentIndex is -1:
101 commentIndex = len(line)
103 line = re.sub(r'\s+', ' ', line[:commentIndex].strip())
110 class ParseError(Exception):
111 def __init__(self, fatal, errors):
116 return '\n'.join(map(str, self.errors))
119 return 'ParseError(fatal=%s, errors=%s)' % (self.fatal, self.errors)
122 class TestExpectationSerializer(object):
123 """Provides means of serializing TestExpectationLine instances."""
124 def __init__(self, test_configuration_converter=None):
125 self._test_configuration_converter = test_configuration_converter
126 self._parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
128 def to_string(self, expectation_line):
129 if expectation_line.is_malformed():
130 return expectation_line.original_string or ''
132 if expectation_line.name is None:
133 return '' if expectation_line.comment is None else "//%s" % expectation_line.comment
135 if self._test_configuration_converter and expectation_line.parsed_bug_modifiers:
136 specifiers_list = self._test_configuration_converter.to_specifiers_list(expectation_line.matching_configurations)
138 for specifiers in specifiers_list:
139 modifiers = self._parsed_modifier_string(expectation_line, specifiers)
140 expectations = self._parsed_expectations_string(expectation_line)
141 result.append(self._format_result(modifiers, expectation_line.name, expectations, expectation_line.comment))
142 return "\n".join(result) if result else None
144 return self._format_result(" ".join(expectation_line.modifiers), expectation_line.name, " ".join(expectation_line.expectations), expectation_line.comment)
146 def _parsed_expectations_string(self, expectation_line):
148 for index in TestExpectations.EXPECTATION_ORDER:
149 if index in expectation_line.parsed_expectations:
150 result.append(self._parsed_expectation_to_string[index])
151 return ' '.join(result)
153 def _parsed_modifier_string(self, expectation_line, specifiers):
154 assert(self._test_configuration_converter)
156 if expectation_line.parsed_bug_modifiers:
157 result.extend(sorted(expectation_line.parsed_bug_modifiers))
158 result.extend(sorted(expectation_line.parsed_modifiers))
159 result.extend(self._test_configuration_converter.specifier_sorter().sort_specifiers(specifiers))
160 return ' '.join(result)
163 def _format_result(cls, modifiers, name, expectations, comment):
164 result = "%s : %s = %s" % (modifiers.upper(), name, expectations.upper())
165 if comment is not None:
166 result += " //%s" % comment
170 def list_to_string(cls, expectation_lines, test_configuration_converter=None, reconstitute_only_these=None):
171 serializer = cls(test_configuration_converter)
173 def serialize(expectation_line):
174 if not reconstitute_only_these or expectation_line in reconstitute_only_these:
175 return serializer.to_string(expectation_line)
176 return expectation_line.original_string
178 def nones_out(expectation_line):
179 return expectation_line is not None
181 return "\n".join(filter(nones_out, map(serialize, expectation_lines)))
184 class TestExpectationParser(object):
185 """Provides parsing facilities for lines in the test_expectation.txt file."""
187 BUG_MODIFIER_PREFIX = 'bug'
188 BUG_MODIFIER_REGEX = 'bug\d+'
189 REBASELINE_MODIFIER = 'rebaseline'
190 SKIP_MODIFIER = 'skip'
191 SLOW_MODIFIER = 'slow'
192 WONTFIX_MODIFIER = 'wontfix'
194 TIMEOUT_EXPECTATION = 'timeout'
196 def __init__(self, port, full_test_list, allow_rebaseline_modifier):
198 self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
199 self._full_test_list = full_test_list
200 self._allow_rebaseline_modifier = allow_rebaseline_modifier
202 def parse(self, expectation_line):
203 if not expectation_line.name:
206 self._check_modifiers_against_expectations(expectation_line)
207 if self._check_path_does_not_exist(expectation_line):
210 expectation_line.path = self._port.normalize_test_name(expectation_line.name)
211 self._collect_matching_tests(expectation_line)
213 self._parse_modifiers(expectation_line)
214 self._parse_expectations(expectation_line)
216 def _parse_modifiers(self, expectation_line):
218 parsed_specifiers = set()
219 for modifier in expectation_line.modifiers:
220 if modifier in TestExpectations.MODIFIERS:
221 expectation_line.parsed_modifiers.append(modifier)
222 if modifier == self.WONTFIX_MODIFIER:
224 elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
225 if re.match(self.BUG_MODIFIER_REGEX, modifier):
226 expectation_line.errors.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.')
228 expectation_line.parsed_bug_modifiers.append(modifier)
230 parsed_specifiers.add(modifier)
232 if not expectation_line.parsed_bug_modifiers and not has_wontfix:
233 expectation_line.warnings.append('Test lacks BUG modifier.')
235 if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in expectation_line.modifiers:
236 expectation_line.errors.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
238 expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.errors)
240 def _parse_expectations(self, expectation_line):
242 for part in expectation_line.expectations:
243 expectation = TestExpectations.expectation_from_string(part)
244 if expectation is None: # Careful, PASS is currently 0.
245 expectation_line.errors.append('Unsupported expectation: %s' % part)
247 result.add(expectation)
248 expectation_line.parsed_expectations = result
250 def _check_modifiers_against_expectations(self, expectation_line):
251 if self.SLOW_MODIFIER in expectation_line.modifiers and self.TIMEOUT_EXPECTATION in expectation_line.expectations:
252 expectation_line.errors.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
254 def _check_path_does_not_exist(self, expectation_line):
255 # WebKit's way of skipping tests is to add a -disabled suffix.
256 # So we should consider the path existing if the path or the
257 # -disabled version exists.
258 if (not self._port.test_exists(expectation_line.name)
259 and not self._port.test_exists(expectation_line.name + '-disabled')):
260 # Log a warning here since you hit this case any
261 # time you update test_expectations.txt without syncing
262 # the LayoutTests directory
263 expectation_line.warnings.append('Path does not exist.')
267 def _collect_matching_tests(self, expectation_line):
268 """Convert the test specification to an absolute, normalized
269 path and make sure directories end with the OS path separator."""
270 # FIXME: full_test_list can quickly contain a big amount of
271 # elements. We should consider at some point to use a more
272 # efficient structure instead of a list. Maybe a dictionary of
273 # lists to represent the tree of tests, leaves being test
274 # files and nodes being categories.
276 if not self._full_test_list:
277 expectation_line.matching_tests = [expectation_line.path]
280 if self._port.test_isdir(expectation_line.path):
281 # this is a test category, return all the tests of the category.
282 expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
285 # this is a test file, do a quick check if it's in the
287 if expectation_line.path in self._full_test_list:
288 expectation_line.matching_tests.append(expectation_line.path)
291 def tokenize(cls, expectation_string, line_number=None):
292 """Tokenizes a line from test_expectations.txt and returns an unparsed TestExpectationLine instance.
294 The format of a test expectation line is:
296 [[<modifiers>] : <name> = <expectations>][ //<comment>]
298 Any errant whitespace is not preserved.
301 expectation_line = TestExpectationLine()
302 expectation_line.original_string = expectation_string
303 expectation_line.line_number = line_number
304 comment_index = expectation_string.find("//")
305 if comment_index == -1:
306 comment_index = len(expectation_string)
308 expectation_line.comment = expectation_string[comment_index + 2:]
310 remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
311 if len(remaining_string) == 0:
312 return expectation_line
314 parts = remaining_string.split(':')
316 expectation_line.errors.append("Missing a ':'" if len(parts) < 2 else "Extraneous ':'")
318 test_and_expectation = parts[1].split('=')
319 if len(test_and_expectation) != 2:
320 expectation_line.errors.append("Missing expectations" if len(test_and_expectation) < 2 else "Extraneous '='")
322 if not expectation_line.is_malformed():
323 expectation_line.modifiers = cls._split_space_separated(parts[0])
324 expectation_line.name = test_and_expectation[0].strip()
325 expectation_line.expectations = cls._split_space_separated(test_and_expectation[1])
327 return expectation_line
330 def tokenize_list(cls, expectations_string):
331 """Returns a list of TestExpectationLines, one for each line in expectations_string."""
332 expectation_lines = []
334 for line in expectations_string.split("\n"):
336 expectation_lines.append(cls.tokenize(line, line_number))
337 return expectation_lines
340 def _split_space_separated(cls, space_separated_string):
341 """Splits a space-separated string into an array."""
342 # FIXME: Lower-casing is necessary to support legacy code. Need to eliminate.
343 return [part.strip().lower() for part in space_separated_string.strip().split(' ')]
346 class TestExpectationLine:
347 """Represents a line in test expectations file."""
350 """Initializes a blank-line equivalent of an expectation."""
351 self.original_string = None
352 self.line_number = None
356 self.parsed_modifiers = []
357 self.parsed_bug_modifiers = []
358 self.matching_configurations = set()
359 self.expectations = []
360 self.parsed_expectations = set()
362 self.matching_tests = []
366 def is_malformed(self):
367 return len(self.errors) > 0
369 def is_invalid(self):
370 return self.is_malformed() or len(self.warnings) > 0
373 return len(self.parsed_expectations) > 1
376 def create_passing_expectation(cls, test):
377 expectation_line = TestExpectationLine()
378 expectation_line.name = test
379 expectation_line.path = test
380 expectation_line.parsed_expectations = set([PASS])
381 expectation_line.matching_tests = [test]
382 return expectation_line
385 # FIXME: Refactor API to be a proper CRUD.
386 class TestExpectationsModel(object):
387 """Represents relational store of all expectations and provides CRUD semantics to manage it."""
390 # Maps a test to its list of expectations.
391 self._test_to_expectations = {}
393 # Maps a test to list of its modifiers (string values)
394 self._test_to_modifiers = {}
396 # Maps a test to a TestExpectationLine instance.
397 self._test_to_expectation_line = {}
399 # List of tests that are in the overrides file (used for checking for
400 # duplicates inside the overrides file itself). Note that just because
401 # a test is in this set doesn't mean it's necessarily overridding a
402 # expectation in the regular expectations; the test might not be
403 # mentioned in the regular expectations file at all.
404 self._overridding_tests = set()
406 self._modifier_to_tests = self._dict_of_sets(TestExpectations.MODIFIERS)
407 self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS)
408 self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES)
409 self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES)
411 def _dict_of_sets(self, strings_to_constants):
412 """Takes a dict of strings->constants and returns a dict mapping
413 each constant to an empty set."""
415 for c in strings_to_constants.values():
419 def get_test_set(self, modifier, expectation=None, include_skips=True):
420 if expectation is None:
421 tests = self._modifier_to_tests[modifier]
423 tests = (self._expectation_to_tests[expectation] &
424 self._modifier_to_tests[modifier])
426 if not include_skips:
427 tests = tests - self.get_test_set(SKIP, expectation)
431 def get_tests_with_result_type(self, result_type):
432 return self._result_type_to_tests[result_type]
434 def get_tests_with_timeline(self, timeline):
435 return self._timeline_to_tests[timeline]
437 def get_modifiers(self, test):
438 """This returns modifiers for the given test (the modifiers plus the BUGXXXX identifier). This is used by the LTTF dashboard."""
439 return self._test_to_modifiers[test]
441 def has_modifier(self, test, modifier):
442 return test in self._modifier_to_tests[modifier]
444 def has_test(self, test):
445 return test in self._test_to_expectation_line
447 def get_expectation_line(self, test):
448 return self._test_to_expectation_line.get(test)
450 def get_expectations(self, test):
451 return self._test_to_expectations[test]
453 def add_expectation_line(self, expectation_line, overrides_allowed):
454 """Returns a list of errors, encountered while matching modifiers."""
456 if expectation_line.is_invalid():
459 for test in expectation_line.matching_tests:
460 if self._already_seen_better_match(test, expectation_line, overrides_allowed):
463 self._clear_expectations_for_test(test, expectation_line)
464 self._test_to_expectation_line[test] = expectation_line
465 self._add_test(test, expectation_line, overrides_allowed)
467 def _add_test(self, test, expectation_line, overrides_allowed):
468 """Sets the expected state for a given test.
470 This routine assumes the test has not been added before. If it has,
471 use _clear_expectations_for_test() to reset the state prior to
476 expectation_line: expectation to add
477 overrides_allowed: whether we're parsing the regular expectations
478 or the overridding expectations"""
479 self._test_to_expectations[test] = expectation_line.parsed_expectations
480 for expectation in expectation_line.parsed_expectations:
481 self._expectation_to_tests[expectation].add(test)
483 self._test_to_modifiers[test] = expectation_line.modifiers
484 for modifier in expectation_line.parsed_modifiers:
485 mod_value = TestExpectations.MODIFIERS[modifier]
486 self._modifier_to_tests[mod_value].add(test)
488 if TestExpectationParser.WONTFIX_MODIFIER in expectation_line.parsed_modifiers:
489 self._timeline_to_tests[WONTFIX].add(test)
491 self._timeline_to_tests[NOW].add(test)
493 if TestExpectationParser.SKIP_MODIFIER in expectation_line.parsed_modifiers:
494 self._result_type_to_tests[SKIP].add(test)
495 elif expectation_line.parsed_expectations == set([PASS]):
496 self._result_type_to_tests[PASS].add(test)
497 elif expectation_line.is_flaky():
498 self._result_type_to_tests[FLAKY].add(test)
500 self._result_type_to_tests[FAIL].add(test)
502 if overrides_allowed:
503 self._overridding_tests.add(test)
505 def _clear_expectations_for_test(self, test, expectation_line):
506 """Remove prexisting expectations for this test.
507 This happens if we are seeing a more precise path
508 than a previous listing.
510 if self.has_test(test):
511 self._test_to_expectations.pop(test, '')
512 self._remove_from_sets(test, self._expectation_to_tests)
513 self._remove_from_sets(test, self._modifier_to_tests)
514 self._remove_from_sets(test, self._timeline_to_tests)
515 self._remove_from_sets(test, self._result_type_to_tests)
517 self._test_to_expectation_line[test] = expectation_line
519 def _remove_from_sets(self, test, dict):
520 """Removes the given test from the sets in the dictionary.
523 test: test to look for
524 dict: dict of sets of files"""
525 for set_of_tests in dict.itervalues():
526 if test in set_of_tests:
527 set_of_tests.remove(test)
529 def _already_seen_better_match(self, test, expectation_line, overrides_allowed):
530 """Returns whether we've seen a better match already in the file.
532 Returns True if we've already seen a expectation_line.name that matches more of the test
535 # FIXME: See comment below about matching test configs and specificity.
536 if not self.has_test(test):
537 # We've never seen this test before.
540 prev_expectation_line = self._test_to_expectation_line[test]
542 if len(prev_expectation_line.path) > len(expectation_line.path):
543 # The previous path matched more of the test.
546 if len(prev_expectation_line.path) < len(expectation_line.path):
547 # This path matches more of the test.
550 if overrides_allowed and test not in self._overridding_tests:
551 # We have seen this path, but that's okay because it is
552 # in the overrides and the earlier path was in the
553 # expectations (not the overrides).
556 # At this point we know we have seen a previous exact match on this
557 # base path, so we need to check the two sets of modifiers.
559 if overrides_allowed:
560 expectation_source = "override"
562 expectation_source = "expectation"
564 # FIXME: This code was originally designed to allow lines that matched
565 # more modifiers to override lines that matched fewer modifiers.
566 # However, we currently view these as errors.
568 # To use the "more modifiers wins" policy, change the errors for overrides
569 # to be warnings and return False".
571 if prev_expectation_line.matching_configurations == expectation_line.matching_configurations:
572 expectation_line.errors.append('Duplicate or ambiguous %s.' % expectation_source)
575 if prev_expectation_line.matching_configurations >= expectation_line.matching_configurations:
576 expectation_line.errors.append('More specific entry on line %d overrides line %d' % (expectation_line.line_number, prev_expectation_line.line_number))
577 # FIXME: return False if we want more specific to win.
580 if prev_expectation_line.matching_configurations <= expectation_line.matching_configurations:
581 expectation_line.errors.append('More specific entry on line %d overrides line %d' % (prev_expectation_line.line_number, expectation_line.line_number))
584 if prev_expectation_line.matching_configurations & expectation_line.matching_configurations:
585 expectation_line.errors.append('Entries on line %d and line %d match overlapping sets of configurations' % (prev_expectation_line.line_number, expectation_line.line_number))
588 # Configuration sets are disjoint, then.
592 class BugManager(object):
593 """A simple interface for managing bugs from TestExpectationsEditor."""
594 def close_bug(self, bug_ids, reference_bug_ids=None):
595 raise NotImplementedError("BugManager.close_bug")
597 def create_bug(self):
598 """Should return a newly created bug id in the form of r"BUG[^\d].*"."""
599 raise NotImplementedError("BugManager.create_bug")
602 class TestExpectationsEditor(object):
604 The editor assumes that the expectation data is error-free.
607 def __init__(self, expectation_lines, bug_manager):
608 self._bug_manager = bug_manager
609 self._expectation_lines = expectation_lines
610 self._tests_with_directory_paths = set()
611 # FIXME: Unify this with TestExpectationsModel.
612 self._test_to_expectation_lines = {}
613 for expectation_line in expectation_lines:
614 for test in expectation_line.matching_tests:
615 if test == expectation_line.path:
616 self._test_to_expectation_lines.setdefault(test, []).append(expectation_line)
618 self._tests_with_directory_paths.add(test)
620 def remove_expectation(self, test, test_config_set, remove_flakes=False):
621 """Removes existing expectations for {test} in the of test configurations {test_config_set}.
622 If the test is flaky, the expectation is not removed, unless remove_flakes is True.
624 In this context, removing expectations does not imply that the test is passing -- we are merely removing
625 any information about this test from the expectations.
627 We do not remove the actual expectation lines here. Instead, we adjust TestExpectationLine.matching_configurations.
628 The serializer will figure out what to do:
629 * An empty matching_configurations set means that the this line matches nothing and will serialize as None.
630 * A matching_configurations set that can't be expressed as one line will be serialized as multiple lines.
632 Also, we do only adjust matching_configurations for lines that match tests exactly, because expectation lines with
633 better path matches are valid and always win.
635 For example, the expectation with the path "fast/events/shadow/" will
636 be ignored when removing expectations for the test "fast/event/shadow/awesome-crash.html", since we can just
637 add a new expectation line for "fast/event/shadow/awesome-crash.html" to influence expected results.
639 expectation_lines = self._test_to_expectation_lines.get(test, [])
640 for expectation_line in expectation_lines:
641 if (not expectation_line.is_flaky() or remove_flakes) and expectation_line.matching_configurations & test_config_set:
642 expectation_line.matching_configurations = expectation_line.matching_configurations - test_config_set
643 if not expectation_line.matching_configurations:
644 self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers)
647 def update_expectation(self, test, test_config_set, expectation_set, parsed_bug_modifiers=None):
648 """Updates expectations for {test} in the set of test configuration {test_config_set} to the values of {expectation_set}.
649 If {parsed_bug_modifiers} is supplied, it is used for updated expectations. Otherwise, a new bug is created.
651 Here, we treat updating expectations to PASS as special: if possible, the corresponding lines are completely removed.
653 # FIXME: Allow specifying modifiers (SLOW, SKIP, WONTFIX).
654 updated_expectations = []
655 expectation_lines = self._test_to_expectation_lines.get(test, [])
656 remaining_configurations = test_config_set.copy()
657 bug_ids = self._get_valid_bug_ids(parsed_bug_modifiers)
658 new_expectation_line_insertion_point = len(self._expectation_lines)
659 remove_expectations = expectation_set == set([PASS]) and test not in self._tests_with_directory_paths
661 for expectation_line in expectation_lines:
662 if expectation_line.matching_configurations == remaining_configurations:
663 # Tweak expectations on existing line.
664 if expectation_line.parsed_expectations == expectation_set:
665 return updated_expectations
666 self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers, bug_ids)
667 updated_expectations.append(expectation_line)
668 if remove_expectations:
669 expectation_line.matching_configurations = set()
671 expectation_line.parsed_expectations = expectation_set
672 expectation_line.parsed_bug_modifiers = bug_ids
673 return updated_expectations
674 elif expectation_line.matching_configurations >= remaining_configurations:
675 # 1) Split up into two expectation lines:
676 # * one with old expectations (existing expectation_line)
677 # * one with new expectations (new expectation_line)
678 # 2) Finish looking, since there will be no more remaining configs to test for.
679 expectation_line.matching_configurations -= remaining_configurations
680 updated_expectations.append(expectation_line)
681 new_expectation_line_insertion_point = self._expectation_lines.index(expectation_line) + 1
683 elif expectation_line.matching_configurations <= remaining_configurations:
684 # Remove existing expectation line.
685 self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers, bug_ids)
686 expectation_line.matching_configurations = set()
687 updated_expectations.append(expectation_line)
689 intersection = expectation_line.matching_configurations & remaining_configurations
691 expectation_line.matching_configurations -= intersection
692 updated_expectations.append(expectation_line)
693 new_expectation_line_insertion_point = self._expectation_lines.index(expectation_line) + 1
695 if not remove_expectations:
696 new_expectation_line = self._create_new_line(test, bug_ids, remaining_configurations, expectation_set)
697 updated_expectations.append(new_expectation_line)
698 self._expectation_lines.insert(new_expectation_line_insertion_point, new_expectation_line)
700 return updated_expectations
702 def _get_valid_bug_ids(self, suggested_bug_ids):
703 # FIXME: Flesh out creating a bug properly (title, etc.)
704 return suggested_bug_ids or [self._bug_manager.create_bug()]
706 def _create_new_line(self, name, bug_ids, config_set, expectation_set):
707 new_line = TestExpectationLine()
709 new_line.parsed_bug_modifiers = bug_ids
710 new_line.matching_configurations = config_set
711 new_line.parsed_expectations = expectation_set
712 # Ensure index integrity for multiple operations.
713 self._test_to_expectation_lines.setdefault(name, []).append(new_line)
717 class TestExpectations(object):
718 """Test expectations consist of lines with specifications of what
719 to expect from layout test cases. The test cases can be directories
720 in which case the expectations apply to all test cases in that
721 directory and any subdirectory. The format is along the lines of:
723 LayoutTests/fast/js/fixme.js = FAIL
724 LayoutTests/fast/js/flaky.js = FAIL PASS
725 LayoutTests/fast/js/crash.js = CRASH TIMEOUT FAIL PASS
729 SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
730 DEBUG : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
731 DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
732 LINUX DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
733 LINUX WIN : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
735 SKIP: Doesn't run the test.
736 SLOW: The test takes a long time to run, but does not timeout indefinitely.
737 WONTFIX: For tests that we never intend to pass on a given platform.
740 -A test cannot be both SLOW and TIMEOUT
741 -A test should only be one of IMAGE, TEXT, IMAGE+TEXT, AUDIO, or FAIL.
742 FAIL is a legacy value that currently means either IMAGE,
743 TEXT, or IMAGE+TEXT. Once we have finished migrating the expectations,
744 we should change FAIL to have the meaning of IMAGE+TEXT and remove the
745 IMAGE+TEXT identifier.
746 -A test can be included twice, but not via the same path.
747 -If a test is included twice, then the more precise path wins.
748 -CRASH tests cannot be WONTFIX
751 TEST_LIST = "test_expectations.txt"
753 EXPECTATIONS = {'pass': PASS,
757 'image+text': IMAGE_PLUS_TEXT,
763 EXPECTATION_DESCRIPTIONS = {SKIP: ('skipped', 'skipped'),
764 PASS: ('pass', 'passes'),
765 FAIL: ('failure', 'failures'),
766 TEXT: ('text diff mismatch',
767 'text diff mismatch'),
768 IMAGE: ('image mismatch', 'image mismatch'),
769 IMAGE_PLUS_TEXT: ('image and text mismatch',
770 'image and text mismatch'),
771 AUDIO: ('audio mismatch', 'audio mismatch'),
772 CRASH: ('DumpRenderTree crash',
773 'DumpRenderTree crashes'),
774 TIMEOUT: ('test timed out', 'tests timed out'),
775 MISSING: ('no expected result found',
776 'no expected results found')}
778 EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, IMAGE_PLUS_TEXT, TEXT, IMAGE, AUDIO, FAIL, SKIP)
780 BUILD_TYPES = ('debug', 'release')
782 MODIFIERS = {TestExpectationParser.SKIP_MODIFIER: SKIP,
783 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
784 TestExpectationParser.SLOW_MODIFIER: SLOW,
785 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE,
788 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
791 RESULT_TYPES = {'skip': SKIP,
797 def expectation_from_string(cls, string):
798 assert(' ' not in string) # This only handles one expectation at a time.
799 return cls.EXPECTATIONS.get(string.lower())
801 def __init__(self, port, tests, expectations,
802 test_config, is_lint_mode=False, overrides=None):
803 """Loads and parses the test expectations given in the string.
805 port: handle to object containing platform-specific functionality
806 tests: list of all of the test files
807 expectations: test expectations as a string
808 test_config: specific values to check against when
809 parsing the file (usually port.test_config(),
810 but may be different when linting or doing other things).
811 is_lint_mode: If True, just parse the expectations string
813 overrides: test expectations that are allowed to override any
814 entries in |expectations|. This is used by callers
815 that need to manage two sets of expectations (e.g., upstream
816 and downstream expectations).
818 self._full_test_list = tests
819 self._test_config = test_config
820 self._is_lint_mode = is_lint_mode
821 self._model = TestExpectationsModel()
822 self._parser = TestExpectationParser(port, tests, is_lint_mode)
823 self._test_configuration_converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
825 self._expectations = TestExpectationParser.tokenize_list(expectations)
826 self._add_expectations(self._expectations, overrides_allowed=False)
829 overrides_expectations = TestExpectationParser.tokenize_list(overrides)
830 self._add_expectations(overrides_expectations, overrides_allowed=True)
831 self._expectations += overrides_expectations
833 self._has_warnings = False
834 self._report_errors()
835 self._process_tests_without_expectations()
837 # TODO(ojan): Allow for removing skipped tests when getting the list of
838 # tests to run, but not when getting metrics.
840 def get_rebaselining_failures(self):
841 return (self._model.get_test_set(REBASELINE, FAIL) |
842 self._model.get_test_set(REBASELINE, IMAGE) |
843 self._model.get_test_set(REBASELINE, TEXT) |
844 self._model.get_test_set(REBASELINE, IMAGE_PLUS_TEXT) |
845 self._model.get_test_set(REBASELINE, AUDIO))
847 # FIXME: Change the callsites to use TestExpectationsModel and remove.
848 def get_expectations(self, test):
849 return self._model.get_expectations(test)
851 # FIXME: Change the callsites to use TestExpectationsModel and remove.
852 def has_modifier(self, test, modifier):
853 return self._model.has_modifier(test, modifier)
855 # FIXME: Change the callsites to use TestExpectationsModel and remove.
856 def get_tests_with_result_type(self, result_type):
857 return self._model.get_tests_with_result_type(result_type)
859 # FIXME: Change the callsites to use TestExpectationsModel and remove.
860 def get_test_set(self, modifier, expectation=None, include_skips=True):
861 return self._model.get_test_set(modifier, expectation, include_skips)
863 # FIXME: Change the callsites to use TestExpectationsModel and remove.
864 def get_modifiers(self, test):
865 return self._model.get_modifiers(test)
867 # FIXME: Change the callsites to use TestExpectationsModel and remove.
868 def get_tests_with_timeline(self, timeline):
869 return self._model.get_tests_with_timeline(timeline)
871 def get_expectations_string(self, test):
872 """Returns the expectatons for the given test as an uppercase string.
873 If there are no expectations for the test, then "PASS" is returned."""
874 expectations = self._model.get_expectations(test)
877 for expectation in expectations:
878 retval.append(self.expectation_to_string(expectation))
880 return " ".join(retval)
882 def expectation_to_string(self, expectation):
883 """Return the uppercased string equivalent of a given expectation."""
884 for item in self.EXPECTATIONS.items():
885 if item[1] == expectation:
886 return item[0].upper()
887 raise ValueError(expectation)
889 def matches_an_expected_result(self, test, result, pixel_tests_are_enabled):
890 expected_results = self._model.get_expectations(test)
891 if not pixel_tests_are_enabled:
892 expected_results = remove_pixel_failures(expected_results)
893 return result_was_expected(result,
895 self.is_rebaselining(test),
896 self._model.has_modifier(test, SKIP))
898 def is_rebaselining(self, test):
899 return self._model.has_modifier(test, REBASELINE)
901 def _report_errors(self):
904 for expectation in self._expectations:
905 for error in expectation.errors:
906 errors.append("Line:%s %s %s" % (expectation.line_number, error, expectation.name if expectation.expectations else expectation.original_string))
907 for warning in expectation.warnings:
908 warnings.append("Line:%s %s %s" % (expectation.line_number, warning, expectation.name if expectation.expectations else expectation.original_string))
910 if len(errors) or len(warnings):
911 _log.error("FAILURES FOR %s" % str(self._test_config))
915 for warning in warnings:
919 raise ParseError(fatal=True, errors=errors)
921 self._has_warnings = True
922 if self._is_lint_mode:
923 raise ParseError(fatal=False, errors=warnings)
925 def _process_tests_without_expectations(self):
926 if self._full_test_list:
927 for test in self._full_test_list:
928 if not self._model.has_test(test):
929 self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test), overrides_allowed=False)
931 def has_warnings(self):
932 return self._has_warnings
934 def remove_rebaselined_tests(self, except_these_tests):
935 """Returns a copy of the expectations with the tests removed."""
936 def without_rebaseline_modifier(expectation):
937 return not (not expectation.is_malformed() and expectation.name in except_these_tests and "rebaseline" in expectation.modifiers)
939 return TestExpectationSerializer.list_to_string(filter(without_rebaseline_modifier, self._expectations))
941 def _add_expectations(self, expectation_list, overrides_allowed):
942 for expectation_line in expectation_list:
943 if not expectation_line.expectations:
946 self._parser.parse(expectation_line)
947 if self._test_config in expectation_line.matching_configurations:
948 self._model.add_expectation_line(expectation_line, overrides_allowed)