Tools/ChangeLog

 12018-01-12 Carlos Garcia Campos <cgarcia@igalia.com>
 2
 3 [GTK][WPE] Add support for unit test expectations
 4 https://bugs.webkit.org/show_bug.cgi?id=181589
 5
 6 Reviewed by NOBODY (OOPS!).
 7
 8 We currently have a way to skip tests by annotating them in the api test runner script. The main problem of this
 9 approach is that we skip test when they fail in the bots and we never notice if they stop failing, keeping the
 10 tests skipped forever. This is indeed the case of several WebKit2 C API tests. Annotating skipped tests in the
 11 script itself is not a good idea either.
 12
 13 This patch adds a generic TestExpectations class for simple tests based on tests with subtests, like our unit
 14 tests, but also WebDriver tests. It parses a json file with the tests and subtests expectations and provides
 15 convenient methods to query them.
 16
 17 * Scripts/run-gtk-tests:
 18 (GtkTestRunner): Remove all Skipped and Slow tests marked here.
 19 * Scripts/run-wpe-tests:
 20 (WPETestRunner): Ditto.
 21 * Scripts/webkitpy/common/test_expectations.py: Added.
 22 (TestExpectations):
 23 (TestExpectations.__init__):
 24 (TestExpectations._port_name_for_expected):
 25 (TestExpectations._expected_value):
 26 (TestExpectations.skipped_tests):
 27 (TestExpectations.skipped_subtests):
 28 (TestExpectations._expectation_value):
 29 (TestExpectations.is_slow):
 30 (TestExpectations.get_expectation):
 31 * Scripts/webkitpy/common/test_expectations_unittest.py: Added.
 32 (MockTestExpectations):
 33 (MockTestExpectations.__init__):
 34 (MockTestExpectations.is_skip):
 35 (ExpectationsTest):
 36 (assert_exp):
 37 (assert_not_exp):
 38 (assert_bad_exp):
 39 (assert_skip):
 40 (test_basic):
 41 (test_skip):
 42 (test_flaky):
 43 (test_build_type):
 44 * TestWebKitAPI/glib/expectations.json: Added.
 45 * glib/api_test_runner.py:
 46 (TestRunner): Remove SkippedTest implementation.
 47 (TestRunner.__init__): Create a TestExpectations.
 48 (TestRunner._test_cases_to_skip): Use TestExpectations to check skipped tests.
 49 (TestRunner._should_run_test_program): Ditto.
 50 (TestRunner._run_test_glib): Use TestExpectations to check if test suite is slow.
 51 (TestRunner._run_test_glib.parse_line.set_test_result): Register also tests passing.
 52 (TestRunner._run_google_test): Use TestExpectations to check if test cases is slow and register tests passing.
 53 (TestRunner.run_tests): Check if actual result is the expected one and register also unexpected passes.
 54 (TestRunner.run_tests.report): Helper to write report to stdout.
 55
1562018-01-11 Jiewen Tan <jiewen_tan@apple.com>
257
358 [WebAuthN] Import a CBOR coder from Chromium

Tools/Scripts/run-gtk-tests

@@sys.path.insert(0, os.path.join(top_level_directory, "Tools", "jhbuild"))
2929sys.path.insert(0, os.path.join(top_level_directory, "Tools", "glib"))
3030import common
3131import jhbuildutils
32 from api_test_runner import TestRunner, SkippedTest, add_options
 32from api_test_runner import TestRunner, add_options
3333
3434class GtkTestRunner(TestRunner):
3535 TestRunner.TEST_DIRS = [ "WebKit2Gtk", "WebKit", "JavaScriptCore", "WTF", "WebCore" ]
3636
37  TestRunner.SKIPPED = [
38  SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/mouse-target", "Test times out after r150890", 117689),
39  SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/usermedia-permission-requests", "Test times out", 158257),
40  SkippedTest("WebKit2Gtk/TestUIClient", "/webkit2/WebKitWebView/audio-usermedia-permission-request", "Test times out", 158257),
41  SkippedTest("WebKit2Gtk/TestCookieManager", "/webkit2/WebKitCookieManager/persistent-storage", "Test is flaky", 134580),
42  SkippedTest("WebKit2Gtk/TestPrinting", "/webkit2/WebKitPrintOperation/custom-widget", "Test is flaky", 168196),
43  SkippedTest("WebKit2Gtk/TestWebViewEditor", "/webkit2/WebKitWebView/editable/editable", "Test hits an assertion in Debug builds", 151654, "Debug"),
44  SkippedTest("WebKit2Gtk/TestWebExtensions", "/webkit2/WebKitWebView/install-missing-plugins-permission-request", "Test times out", 147822),
45  SkippedTest("WebKit2Gtk/TestWebsiteData", "/webkit2/WebKitWebsiteData/databases", "Test fails to clear database data", 181251),
46  SkippedTest("WebKit/TestWebKit", "WebKit.MouseMoveAfterCrash", "Test is flaky", 85066),
47  SkippedTest("WebKit/TestWebKit", "WebKit.NewFirstVisuallyNonEmptyLayoutForImages", "Test is flaky", 85066),
48  SkippedTest("WebKit/TestWebKit", "WebKit.NewFirstVisuallyNonEmptyLayoutFrames", "Test fails", 85037),
49  SkippedTest("WebKit/TestWebKit", "WebKit.SpacebarScrolling", "Test fails", 84961),
50  SkippedTest("WebKit/TestWebKit", "WebKit.WKConnection", "Tests fail and time out out", 84959),
51  SkippedTest("WebKit/TestWebKit", "WebKit.ForceRepaint", "Test times out", 105532),
52  SkippedTest("WebKit/TestWebKit", "WebKit.ReloadPageAfterCrash", "Test flakily times out", 110129),
53  SkippedTest("WebKit/TestWebKit", "WebKit.DidAssociateFormControls", "Test times out", 120302),
54  SkippedTest("WebKit/TestWebKit", "WebKit.InjectedBundleFrameHitTest", "Test times out", 120303),
55  SkippedTest("WebKit/TestWebKit", "WebKit.TerminateTwice", "Test causes crash on the next test", 121970),
56  SkippedTest("WebKit/TestWebKit", "WebKit.GeolocationTransitionToHighAccuracy", "Test causes crash on the next test", 125068),
57  SkippedTest("WebKit/TestWebKit", "WebKit.GeolocationTransitionToLowAccuracy", "Test causes crash on the next test", 125068),
58  ]
59 
60  TestRunner.SLOW = [
61  "WTF_Lock.ContendedShortSection",
62  "WTF_Lock.ContendedLongSection",
63  "WTF_WordLock.ContendedShortSection",
64  "WTF_WordLock.ContendedLongSection",
65  "WebKit2Gtk/TestInspectorServer",
66  ]
67 
6837 def __init__(self, options, tests=[]):
6938 super(GtkTestRunner, self).__init__("gtk", options, tests)
7039

Tools/Scripts/run-wpe-tests

@@top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "
2626sys.path.insert(0, os.path.join(top_level_directory, "Tools", "jhbuild"))
2727sys.path.insert(0, os.path.join(top_level_directory, "Tools", "glib"))
2828import jhbuildutils
29 from api_test_runner import TestRunner, SkippedTest, add_options
 29from api_test_runner import TestRunner, add_options
3030
3131class WPETestRunner(TestRunner):
3232 TestRunner.TEST_DIRS = [ "WPE", "WebKit", "JavaScriptCore", "WTF", "WebCore" ]
3333
34  TestRunner.SLOW = [
35  "WTF_Lock.ContendedShortSection",
36  "WTF_Lock.ContendedLongSection",
37  "WTF_WordLock.ContendedShortSection",
38  "WTF_WordLock.ContendedLongSection",
39  ]
40 
4134 def __init__(self, options, tests=[]):
4235 super(WPETestRunner, self).__init__("wpe", options, tests)
4336

Tools/Scripts/webkitpy/common/test_expectations.py

 1# Copyright (C) 2018 Igalia S.L.
 2#
 3# Redistribution and use in source and binary forms, with or without
 4# modification, are permitted provided that the following conditions
 5# are met:
 6# 1. Redistributions of source code must retain the above copyright
 7# notice, this list of conditions and the following disclaimer.
 8# 2. Redistributions in binary form must reproduce the above copyright
 9# notice, this list of conditions and the following disclaimer in the
 10# documentation and/or other materials provided with the distribution.
 11#
 12# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
 13# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 14# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 15# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
 16# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 17# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 18# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 19# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 20# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 21# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 22
 23import json
 24import os
 25
 26
 27class TestExpectations(object):
 28
 29 def __init__(self, port_name, expectations_file, build_type='Release'):
 30 self._port_name = port_name
 31 self._build_type = build_type
 32 if os.path.isfile(expectations_file):
 33 with open(expectations_file, 'r') as fd:
 34 self._expectations = json.load(fd)
 35 else:
 36 self._expectations = {}
 37
 38 def _port_name_for_expected(self, expected):
 39 if self._port_name in expected:
 40 return self._port_name
 41
 42 name_with_build = self._port_name + '@' + self._build_type
 43 if name_with_build in expected:
 44 return name_with_build
 45
 46 if 'all' in expected:
 47 return 'all'
 48
 49 name_with_build = 'all@' + self._build_type
 50 if name_with_build in expected:
 51 return name_with_build
 52
 53 return None
 54
 55 def _expected_value(self, expected, value, default):
 56 port_name = self._port_name_for_expected(expected)
 57 if port_name is None:
 58 return default
 59
 60 port_expected = expected[port_name]
 61 if value in port_expected:
 62 return port_expected[value]
 63
 64 return default
 65
 66 def skipped_tests(self):
 67 skipped = []
 68 for test in self._expectations:
 69 if 'expected' not in self._expectations[test]:
 70 continue
 71
 72 expected = self._expectations[test]['expected']
 73 if 'SKIP' in self._expected_value(expected, 'status', []):
 74 skipped.append(test)
 75 return skipped
 76
 77 def skipped_subtests(self, test):
 78 skipped = []
 79 if test not in self._expectations:
 80 return skipped
 81
 82 test_expectation = self._expectations[test]
 83 if 'subtests' not in test_expectation:
 84 return skipped
 85
 86 subtests = test_expectation['subtests']
 87 for subtest in subtests:
 88 if 'SKIP' in self._expected_value(subtests[subtest]['expected'], 'status', []):
 89 skipped.append(subtest)
 90 return skipped
 91
 92 def _expectation_value(self, test, subtest, value, default):
 93 retval = default
 94 if test not in self._expectations:
 95 return retval
 96
 97 test_expectation = self._expectations[test]
 98 if 'expected' in test_expectation:
 99 retval = self._expected_value(test_expectation['expected'], value, retval)
 100
 101 if subtest is None or 'subtests' not in test_expectation:
 102 return retval
 103
 104 subtests = test_expectation['subtests']
 105 if subtest not in subtests:
 106 return retval
 107
 108 return self._expected_value(subtests[subtest]['expected'], value, retval)
 109
 110 def is_slow(self, test, subtest=None):
 111 return self._expectation_value(test, subtest, 'slow', False)
 112
 113 def get_expectation(self, test, subtest=None):
 114 return self._expectation_value(test, subtest, 'status', ['PASS'])

Tools/Scripts/webkitpy/common/test_expectations_unittest.py

 1# Copyright (C) 2018 Igalia S.L.
 2#
 3# Redistribution and use in source and binary forms, with or without
 4# modification, are permitted provided that the following conditions
 5# are met:
 6# 1. Redistributions of source code must retain the above copyright
 7# notice, this list of conditions and the following disclaimer.
 8# 2. Redistributions in binary form must reproduce the above copyright
 9# notice, this list of conditions and the following disclaimer in the
 10# documentation and/or other materials provided with the distribution.
 11#
 12# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
 13# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 14# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 15# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
 16# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 17# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 18# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 19# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 20# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 21# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 22
 23import unittest
 24
 25import json
 26import os
 27
 28from webkitpy.common.host_mock import MockHost
 29from webkitpy.common.test_expectations import TestExpectations
 30from webkitpy.common.webkit_finder import WebKitFinder
 31
 32
 33class MockTestExpectations(TestExpectations):
 34
 35 def __init__(self, port, expectations, build_type='Release'):
 36 host = MockHost()
 37 port = host.port_factory.get(port)
 38 self._port_name = port.name()
 39 self._build_type = build_type
 40 self._expectations = json.loads(expectations)
 41
 42 def is_skip(self, test, subtest):
 43 if test in self.skipped_tests():
 44 return True
 45 return subtest in self.skipped_subtests(test)
 46
 47
 48class ExpectationsTest(unittest.TestCase):
 49
 50 BASIC = """
 51{
 52 "imported/w3c/webdriver/tests/test1.py": {
 53 "subtests": {
 54 "test1_one": {
 55 "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
 56 },
 57 "test1_two": {
 58 "expected": {
 59 "all": {"status": ["FAIL"], "bug": "1234"},
 60 "gtk": {"status": ["PASS"]}
 61 }
 62 }
 63 }
 64 },
 65 "imported/w3c/webdriver/tests/test2.py": {
 66 "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
 67 },
 68 "imported/w3c/webdriver/tests/test3.py": {
 69 "expected": {"all": {"status": ["FAIL"], "bug": "1234"}},
 70 "subtests": {
 71 "test3_one": {
 72 "expected": {"all": {"status": ["PASS"]}}
 73 }
 74 }
 75 }
 76}"""
 77
 78 SKIP = """
 79{
 80 "imported/w3c/webdriver/tests/test1.py": {
 81 "expected": {"all": {"status": ["SKIP"], "bug": "1234"}}
 82 },
 83 "imported/w3c/webdriver/tests/test2.py": {
 84 "expected": {"gtk": {"status": ["SKIP"], "bug": "1234"}}
 85 },
 86 "imported/w3c/webdriver/tests/test3.py": {
 87 "expected": {"all": {"status": ["SKIP"], "bug": "1234"}},
 88 "subtests": {
 89 "test3_one": {
 90 "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
 91 }
 92 }
 93 },
 94 "imported/w3c/webdriver/tests/test4.py": {
 95 "subtests": {
 96 "test4_one": {
 97 "expected": {"all": {"status": ["SKIP"], "bug": "1234"}}
 98 }
 99 }
 100 }
 101}"""
 102
 103 FLAKY = """
 104{
 105 "TestCookieManager": {
 106 "subtests": {
 107 "/webkit2/WebKitCookieManager/persistent-storage": {
 108 "expected": {"gtk": {"status": ["FAIL", "PASS"], "bug": "1234"}}
 109 }
 110 }
 111 },
 112 "TestWebKit": {
 113 "subtests": {
 114 "WebKit.MouseMoveAfterCrash": {
 115 "expected": {"all": {"status": ["CRASH", "PASS"], "bug": "1234"}}
 116 },
 117 "WebKit.WKConnection": {
 118 "expected": {"wpe": {"status": ["FAIL", "TIMEOUT"], "bug": "1234"}}
 119 }
 120 }
 121 }
 122}"""
 123
 124 BUILD_TYPE = """
 125{
 126 "TestCookieManager": {
 127 "subtests": {
 128 "/webkit2/WebKitCookieManager/persistent-storage": {
 129 "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
 130 }
 131 }
 132 },
 133 "TestWebKit": {
 134 "subtests": {
 135 "WebKit.MouseMoveAfterCrash": {
 136 "expected": {"all@Debug": {"status": ["CRASH"], "bug": "1234"}}
 137 },
 138 "WebKit.WKConnection": {
 139 "expected": {"gtk@Release": {"status": ["FAIL"], "bug": "1234"},
 140 "gtk@Debug": {"status": ["CRASH"], "bug": "1234"}}
 141 }
 142 }
 143 },
 144 "TestWebCore": {
 145 "expected": {"all@Debug": {"status": ["CRASH"]}},
 146 "subtests": {
 147 "ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL": {
 148 "expected": {"all": {"status": ["PASS"], "bug": "1234"}}
 149 },
 150 "FileMonitorTest.DetectChange": {
 151 "expected": {"all@Release": {"status": ["FAIL"], "bug": "1234"}}
 152 }
 153 }
 154 },
 155 "TestWebViewEditor": {
 156 "expected": {"all@Release": {"status": ["SKIP"]},
 157 "wpe@Debug": {"status": ["SKIP"]}},
 158 "subtests": {
 159 "/webkit2/WebKitWebView/editable/editable": {
 160 "expected": {"gtk": {"status": ["FAIL"], "bug": "1234"}}
 161 }
 162 }
 163 }
 164}"""
 165
 166 def assert_exp(self, test, subtest, result):
 167 self.assertIn(result, self.expectations.get_expectation(test, subtest))
 168
 169 def assert_not_exp(self, test, subtest, result):
 170 self.assertNotIn(result, self.expectations.get_expectation(test, subtest))
 171
 172 def assert_bad_exp(self, test):
 173 self.assertRaises(AssertionError, self.expectations.get_expectation(test))
 174
 175 def assert_skip(self, test, subtest, result):
 176 self.assertEqual(self.expectations.is_skip(test, subtest), result)
 177
 178 def test_basic(self):
 179 self.expectations = MockTestExpectations('gtk', self.BASIC)
 180 self.assert_exp('imported/w3c/webdriver/tests/test5.py', 'test5_two', 'PASS')
 181
 182 self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_five', 'PASS')
 183 self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_one', 'FAIL')
 184 self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'PASS')
 185
 186 self.assert_exp('imported/w3c/webdriver/tests/test2.py', 'test2_one', 'FAIL')
 187
 188 self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_two', 'FAIL')
 189 self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_one', 'PASS')
 190
 191 self.expectations = MockTestExpectations('wpe', self.BASIC)
 192 self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'FAIL')
 193
 194 def test_skip(self):
 195 self.expectations = MockTestExpectations('gtk', self.SKIP)
 196 self.assert_skip('imported/w3c/webdriver/tests/test5.py', None, False)
 197 self.assert_skip('imported/w3c/webdriver/tests/test1.py', None, True)
 198 self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, True)
 199 self.assert_skip('imported/w3c/webdriver/tests/test3.py', None, True)
 200 self.assert_skip('imported/w3c/webdriver/tests/test3.py', 'test3_one', True)
 201 self.assert_skip('imported/w3c/webdriver/tests/test4.py', None, False)
 202 self.assert_skip('imported/w3c/webdriver/tests/test4.py', 'test4_one', True)
 203 self.assert_exp('imported/w3c/webdriver/tests/test4.py', 'test4_one', 'SKIP')
 204
 205 self.expectations = MockTestExpectations('wpe', self.SKIP)
 206 self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, False)
 207
 208 def test_flaky(self):
 209 self.expectations = MockTestExpectations('gtk', self.FLAKY)
 210 self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS')
 211 self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
 212 self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
 213 self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
 214 self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'PASS')
 215 self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
 216 self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT')
 217
 218 self.expectations = MockTestExpectations('wpe', self.FLAKY)
 219 self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS')
 220 self.assert_not_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
 221 self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
 222 self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
 223 self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
 224 self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT')
 225
 226 def test_build_type(self):
 227 self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Debug')
 228 self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
 229 self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
 230 self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'CRASH')
 231 self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS')
 232 self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'CRASH')
 233 self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'CRASH')
 234 self.assert_skip('TestWebViewEditor', None, False)
 235 self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', False)
 236 self.assert_exp('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', 'FAIL')
 237
 238 self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Release')
 239 self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
 240 self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
 241 self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
 242 self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS')
 243 self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'FAIL')
 244 self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'PASS')
 245 self.assert_skip('TestWebViewEditor', None, True)
 246 self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)
 247
 248 self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Release')
 249 self.assert_skip('TestWebViewEditor', None, True)
 250 self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)
 251
 252 self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Debug')
 253 self.assert_skip('TestWebViewEditor', None, True)
 254 self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)

Tools/TestWebKitAPI/glib/expectations.json

 1{
 2 "TestUIClient": {
 3 "subtests": {
 4 "/webkit2/WebKitWebView/mouse-target": {
 5 "expected": {"gtk": {"status": ["TIMEOUT"], "bug": "webkit.org/b/117689"}}
 6 },
 7 "/webkit2/WebKitWebView/usermedia-permission-requests": {
 8 "expected": {"gtk": {"status": ["TIMEOUT"], "bug": "webkit.org/b/158257"}}
 9 },
 10 "/webkit2/WebKitWebView/audio-usermedia-permission-request": {
 11 "expected": {"gtk": {"status": ["TIMEOUT"], "bug": "webkit.org/b/158257"}}
 12 }
 13 }
 14 },
 15 "TestCookieManager": {
 16 "subtests": {
 17 "/webkit2/WebKitCookieManager/persistent-storage": {
 18 "expected": {"gtk": {"status": ["FAIL", "PASS"], "bug": "webkit.org/b/134580"}}
 19 }
 20 }
 21 },
 22 "TestPrinting": {
 23 "subtests": {
 24 "/webkit2/WebKitPrintOperation/custom-widget": {
 25 "expected": {"gtk": {"status": ["FAIL", "PASS"], "bug": "webkit.org/b/168196"}}
 26 }
 27 }
 28 },
 29 "TestWebViewEditor": {
 30 "subtests": {
 31 "/webkit2/WebKitWebView/editable/editable": {
 32 "expected": {"gtk@Debug": {"status": ["FAIL"], "bug": "webkit.org/b/134580"}}
 33 }
 34 }
 35 },
 36 "TestWebExtensions": {
 37 "subtests": {
 38 "/webkit2/WebKitWebView/install-missing-plugins-permission-request": {
 39 "expected": {"gtk": {"status": ["TIMEOUT"], "bug": "webkit.org/b/147822"}}
 40 }
 41 }
 42 },
 43 "TestWebsiteData": {
 44 "subtests": {
 45 "/webkit2/WebKitWebsiteData/databases": {
 46 "expected": {"gtk": {"status": ["FAIL"], "bug": "webkit.org/b/181251"}}
 47 }
 48 }
 49 },
 50 "TestInspectorServer": {
 51 "expected": {"all": {"slow": true}}
 52 },
 53 "TestWebKit": {
 54 "subtests": {
 55 "WebKit.MouseMoveAfterCrash": {
 56 "expected": {"gtk": {"status": ["CRASH", "PASS"], "bug": "webkit.org/b/85066"}}
 57 },
 58 "WebKit.NewFirstVisuallyNonEmptyLayoutForImages": {
 59 "expected": {"gtk": {"status": ["FAIL", "PASS"], "bug": "webkit.org/b/85066"}}
 60 },
 61 "WebKit.NewFirstVisuallyNonEmptyLayoutFrames": {
 62 "expected": {"gtk": {"status": ["FAIL"], "bug": "webkit.org/b/85037"}}
 63 },
 64 "WebKit.SpacebarScrolling": {
 65 "expected": {"gtk": {"status": ["FAIL"], "bug": "webkit.org/b/84961"}}
 66 },
 67 "WebKit.WKConnection": {
 68 "expected": {"gtk": {"status": ["FAIL", "TIMEOUT"], "bug": "webkit.org/b/84959"}}
 69 },
 70 "WebKit.ForceRepaint": {
 71 "expected": {"gtk": {"status": ["TIMEOUT"], "bug": "webkit.org/b/105532"}}
 72 },
 73 "WebKit.ReloadPageAfterCrash": {
 74 "expected": {"gtk": {"status": ["TIMEOUT", "PASS"], "bug": "webkit.org/b/110129"}}
 75 },
 76 "WebKit.DidAssociateFormControls": {
 77 "expected": {"gtk": {"status": ["TIMEOUT"], "bug": "webkit.org/b/120302"}}
 78 },
 79 "WebKit.InjectedBundleFrameHitTest": {
 80 "expected": {"gtk": {"status": ["TIMEOUT"], "bug": "webkit.org/b/120303"}}
 81 },
 82 "WebKit.TerminateTwice": {
 83 "expected": {"gtk": {"status": ["SKIP"], "bug": "webkit.org/b/121970"}}
 84 },
 85 "WebKit.GeolocationTransitionToHighAccuracy": {
 86 "expected": {"gtk": {"status": ["CRASH"], "bug": "webkit.org/b/125068"}}
 87 },
 88 "WebKit.GeolocationTransitionToLowAccuracy": {
 89 "expected": {"gtk": {"status": ["CRASH"], "bug": "webkit.org/b/125068"}}
 90 }
 91 }
 92 },
 93 "TestWTF": {
 94 "subtests": {
 95 "WTF_Lock.ContendedShortSection": {
 96 "expected": {"all": {"slow": true}}
 97 },
 98 "WTF_Lock.ContendedLongSection": {
 99 "expected": {"all": {"slow": true}}
 100 },
 101 "WTF_WordLock.ContendedShortSection": {
 102 "expected": {"all": {"slow": true}}
 103 },
 104 "WTF_WordLock.ContendedLongSection": {
 105 "expected": {"all": {"slow": true}}
 106 }
 107 }
 108 }
 109}

Tools/glib/api_test_runner.py

@@top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), "
2828sys.path.insert(0, os.path.join(top_level_directory, "Tools", "glib"))
2929import common
3030from webkitpy.common.host import Host
31 
32 
33 class SkippedTest:
34  ENTIRE_SUITE = None
35 
36  def __init__(self, test, test_case, reason, bug, build_type=None):
37  self.test = test
38  self.test_case = test_case
39  self.reason = reason
40  self.bug = bug
41  self.build_type = build_type
42 
43  def __str__(self):
44  skipped_test_str = "%s" % self.test
45 
46  if not(self.skip_entire_suite()):
47  skipped_test_str += " [%s]" % self.test_case
48 
49  skipped_test_str += ": %s (https://bugs.webkit.org/show_bug.cgi?id=%d)" % (self.reason, self.bug)
50  return skipped_test_str
51 
52  def skip_entire_suite(self):
53  return self.test_case == SkippedTest.ENTIRE_SUITE
54 
55  def skip_for_build_type(self, build_type):
56  if self.build_type is None:
57  return True
58 
59  return self.build_type == build_type
 31from webkitpy.common.test_expectations import TestExpectations
6032
6133
6234class TestTimeout(Exception):

@@class TestTimeout(Exception):
6537
6638class TestRunner(object):
6739 TEST_DIRS = []
68  SKIPPED = []
69  SLOW = []
7040
7141 def __init__(self, port, options, tests=[]):
7242 self._options = options

@@class TestRunner(object):
7747 self._driver = self._create_driver()
7848
7949 self._programs_path = common.binary_build_path()
 50 expectations_file = os.path.join(common.top_level_path(), "Tools", "TestWebKitAPI", "glib", "expectations.json")
 51 self._expectations = TestExpectations(self._port.name(), expectations_file, self._build_type)
8052 self._tests = self._get_tests(tests)
81  self._skipped_tests = [skipped for skipped in TestRunner.SKIPPED if skipped.skip_for_build_type(self._build_type)]
8253 self._disabled_tests = []
8354
8455 def _test_programs_base_dir(self):

@@class TestRunner(object):
136107 if self._options.skipped_action != 'skip':
137108 return []
138109
139  test_cases = []
140  for skipped in self._skipped_tests:
141  if test_program.endswith(skipped.test) and not skipped.skip_entire_suite():
142  test_cases.append(skipped.test_case)
143  return test_cases
 110 return self._expectations.skipped_subtests(os.path.basename(test_program))
144111
145112 def _should_run_test_program(self, test_program):
146113 for disabled_test in self._disabled_tests:

@@class TestRunner(object):
150117 if self._options.skipped_action != 'skip':
151118 return True
152119
153  for skipped in self._skipped_tests:
154  if test_program.endswith(skipped.test) and skipped.skip_entire_suite():
155  return False
156  return True
 120 return os.path.basename(test_program) not in self._expectations.skipped_tests()
157121
158122 def _kill_process(self, pid):
159123 try:

@@class TestRunner(object):
211175
212176 timeout = self._options.timeout
213177 test = os.path.join(os.path.basename(os.path.dirname(test_program)), os.path.basename(test_program))
214  if test in TestRunner.SLOW:
 178 if self._expectations.is_slow(os.path.basename(test_program)):
215179 timeout *= 5
216180
217181 test_context = {"child-pid": -1, "did-timeout": False, "current_test": None}

@@class TestRunner(object):
232196 test_context[test] = "TIMEOUT"
233197 else:
234198 test_context[test] = result
 199 else:
 200 test_context[test] = 'PASS'
235201 test_context["did-timeout"] = False
236202 test_context["current_test"] = None
237203 self._stop_timeout(timeout)

@@class TestRunner(object):
302268 def _run_google_test(self, test_program, subtest):
303269 command = [test_program, '--gtest_filter=%s' % (subtest)]
304270 timeout = self._options.timeout
305  if subtest in TestRunner.SLOW:
 271 if self._expectations.is_slow(os.path.basename(test_program), subtest):
306272 timeout *= 5
307273
308274 pid, fd = os.forkpty()

@@class TestRunner(object):
328294 if status != 0:
329295 return {subtest: "FAIL"}
330296
331  return {}
 297 return {subtest: "PASS"}
332298
333299 def _run_google_test_suite(self, test_program):
334300 result = {}

@@class TestRunner(object):
367333 crashed_tests = {}
368334 failed_tests = {}
369335 timed_out_tests = {}
 336 passed_tests = {}
370337 try:
371338 for test in self._tests:
372339 results = self._run_test(test)
373340 for test_case, result in results.iteritems():
 341 if result in self._expectations.get_expectation(os.path.basename(test), test_case):
 342 continue
 343
374344 if result == "FAIL":
375345 failed_tests.setdefault(test, []).append(test_case)
376346 elif result == "TIMEOUT":
377347 timed_out_tests.setdefault(test, []).append(test_case)
378348 elif result == "CRASH":
379349 crashed_tests.setdefault(test, []).append(test_case)
 350 elif result == "PASS":
 351 passed_tests.setdefault(test, []).append(test_case)
380352 finally:
381353 self._tear_down_testing_environment()
382354
383  if failed_tests:
384  sys.stdout.write("\nUnexpected failures (%d)\n" % (sum(len(value) for value in failed_tests.itervalues())))
385  for test in failed_tests:
386  sys.stdout.write(" %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
387  for test_case in failed_tests[test]:
388  sys.stdout.write(" %s\n" % (test_case))
389  sys.stdout.flush()
390 
391  if crashed_tests:
392  sys.stdout.write("\nUnexpected crashes (%d)\n" % (sum(len(value) for value in crashed_tests.itervalues())))
393  for test in crashed_tests:
394  sys.stdout.write(" %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
395  for test_case in crashed_tests[test]:
 355 def report(tests, title, base_dir):
 356 if not tests:
 357 return
 358 sys.stdout.write("\nUnexpected %s (%d)\n" % (title, sum(len(value) for value in tests.itervalues())))
 359 for test in tests:
 360 sys.stdout.write(" %s\n" % (test.replace(base_dir, '', 1)))
 361 for test_case in tests[test]:
396362 sys.stdout.write(" %s\n" % (test_case))
397363 sys.stdout.flush()
398364
399  if timed_out_tests:
400  sys.stdout.write("\nUnexpected timeouts (%d)\n" % (sum(len(value) for value in timed_out_tests.itervalues())))
401  for test in timed_out_tests:
402  sys.stdout.write(" %s\n" % (test.replace(self._test_programs_base_dir(), '', 1)))
403  for test_case in timed_out_tests[test]:
404  sys.stdout.write(" %s\n" % (test_case))
405  sys.stdout.flush()
 365 report(failed_tests, "failures", self._test_programs_base_dir())
 366 report(crashed_tests, "crashes", self._test_programs_base_dir())
 367 report(timed_out_tests, "timeouts", self._test_programs_base_dir())
 368 report(passed_tests, "passes", self._test_programs_base_dir())
406369
407370 return len(failed_tests) + len(timed_out_tests)
408371