Attachment 342067 Details for Bug 186359 – Patch

[patch] Patch

bug-186359-20180606144537.patch (text/plain), 13.77 KB, created by Brendan McLoughlin on 2018-06-06 11:45:38 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Brendan McLoughlin

Created: 2018-06-06 11:45:38 PDT

Size: 13.77 KB

patch

obsolete

>Subversion Revision: 232551
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index 357f65e9fb45266d3292a74b659553ae90fd1ca9..3a07762785fd345b47cb48017790baa28d33b1b8 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,42 @@
>+2018-06-06  Brendan McLoughlin  <brendan@bocoup.com>
>+
>+        WIP Add script to update web-platform-tests expectations after import
>+        https://bugs.webkit.org/show_bug.cgi?id=186359
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        * Scripts/update-w3c-tests: Added.
>+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
>+        (Manager._upload_json_files):
>+        * Scripts/webkitpy/w3c/test_updater.py: Added.
>+        (configure_logging):
>+        (configure_logging.LogHandler):
>+        (configure_logging.LogHandler.format):
>+        (render_expectations):
>+        (remove_test_expectation):
>+        (flatten_path):
>+        (pre_process_tests):
>+        (main):
>+        (TestExpecationUpdater):
>+        (TestExpecationUpdater.__init__):
>+        (TestExpecationUpdater.do_update):
>+        (TestExpecationUpdater.update_expectation):
>+        (TestExpecationUpdater.missing_test):
>+        (TestExpecationUpdater.failing_ref_test):
>+        (TestExpecationUpdater.reset_testharness_test):
>+        (TestExpecationUpdater.failing_testharness_test):
>+        (TestExpecationUpdater.crash_test):
>+        (TestExpecationUpdater.timeout_test):
>+        (TestExpecationUpdater.resolve_flaky_test):
>+        (TestExpecationUpdater.unexpected_pass_test):
>+        (TestExpecationUpdater.run_webkit_tests):
>+        (TestExpecationUpdater.extract_test_result):
>+        (TestExpecationUpdater.update_test_expectation):
>+        (TestExpecationUpdater.remove_test_expectation):
>+        (TestExpecationUpdater.test_expectations_path):
>+        (TestExpecationUpdater.results_file):
>+        (results_match_expectation):
>+
> 2018-06-06  Brent Fulgham  <bfulgham@apple.com>
> 
>         Adjust compile and runtime flags to match shippable state of features (Part 2)
>diff --git a/Tools/Scripts/update-w3c-tests b/Tools/Scripts/update-w3c-tests
>new file mode 100755
>index 0000000000000000000000000000000000000000..96fc9dd9dd24a92a97c685308e941704a147cd0a
>--- /dev/null
>+++ b/Tools/Scripts/update-w3c-tests
>@@ -0,0 +1,35 @@
>+#!/usr/bin/env python
>+
>+# Copyright (C) 2018 Bocoup LLC. All rights reserved.
>+#
>+# Redistribution and use in source and binary forms, with or without
>+# modification, are permitted provided that the following conditions
>+# are met:
>+#
>+# 1. Redistributions of source code must retain the above
>+#    copyright notice, this list of conditions and the following
>+#    disclaimer.
>+# 2. Redistributions in binary form must reproduce the above
>+#    copyright notice, this list of conditions and the following
>+#    disclaimer in the documentation and/or other materials
>+#    provided with the distribution.
>+#
>+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
>+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
>+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
>+# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
>+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
>+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
>+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
>+# SUCH DAMAGE.
>+
>+import sys
>+
>+from webkitpy.w3c import test_updater
>+
>+
>+sys.exit(test_updater.main(sys.argv[1:], sys.stdout, sys.stderr))
>diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
>index cb10dba4c259e7d3f29b8c40d4c3246b086bac17..a94832b118c9434cec79c26acee00e42dab0e46f 100644
>--- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
>+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
>@@ -423,8 +423,10 @@ class Manager(object):
>         self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
> 
>         full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
>+        full_results_path_1 = self._filesystem.join(self._results_directory, "full-results.json")
>         # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
>         json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
>+        json_results_generator.write_json(self._filesystem, summarized_results, full_results_path_1)
> 
>         results_json_path = self._filesystem.join(self._results_directory, "results_including_passes.json")
>         if results_including_passes:
>diff --git a/Tools/Scripts/webkitpy/w3c/test_updater.py b/Tools/Scripts/webkitpy/w3c/test_updater.py
>new file mode 100644
>index 0000000000000000000000000000000000000000..ecff612983786dd4216431a4b29288502f4a872d
>--- /dev/null
>+++ b/Tools/Scripts/webkitpy/w3c/test_updater.py
>@@ -0,0 +1,243 @@
>+import argparse
>+import json
>+from subprocess import Popen
>+import io
>+from sets import Set
>+from webkitpy.layout_tests.run_webkit_tests import parse_args
>+import logging
>+
>+from webkitpy.common.host import Host
>+
>+_log = logging.getLogger(__name__)
>+
>+
>+def configure_logging():
>+    class LogHandler(logging.StreamHandler):
>+
>+        def format(self, record):
>+            if record.levelno > logging.INFO:
>+                return "%s: %s" % (record.levelname, record.getMessage())
>+            return record.getMessage()
>+
>+    logger = logging.getLogger()
>+    logger.setLevel(logging.INFO)
>+    handler = LogHandler()
>+    handler.setLevel(logging.INFO)
>+    logger.addHandler(handler)
>+    return handler
>+
>+
>+# TODO
>+# remote missing tests from expectations
>+# Add documentation of argorithm for updating test expectations
>+# cleanup code to follow webkitpy standards
>+# add unittests
>+
>+# Think about flaky tests some more
>+def render_expectations(failures):
>+    expectation_map = {
>+        'TIMEOUT': 'Timeout',
>+        'FAIL': 'Failure',
>+        'TEXT': 'Failure',
>+        'IMAGE+TEXT': 'Failure',
>+        'MISSING': '',
>+        'PASS': 'Pass',
>+        'IMAGE': 'ImageOnlyFailure',
>+    }
>+    return ' '.join([expectation_map[failure] for failure in failures])
>+
>+
>+def remove_test_expectation(expectation_file, test_name):
>+    with io.open(expectation_file, 'r', encoding="utf-8") as fd:
>+        lines = fd.readlines()
>+
>+    with io.open(expectation_file, 'w', encoding="utf-8") as fd:
>+        for line in lines:
>+            if test_name not in line:
>+                fd.write(line)
>+
>+
>+def flatten_path(obj):
>+    to_return = {}
>+    for k, v in obj.items():
>+        if 'expected' in v:
>+            # terminary node
>+            to_return[k] = v
>+            pass
>+        else:
>+            flat_object = flatten_path(v)
>+            for k2, v2 in flat_object.items():
>+                to_return[k + '/' + k2] = v2
>+    return to_return
>+
>+
>+def pre_process_tests(self, test_dict):
>+    tests = flatten_path(test_dict)
>+    processed_tests = []
>+    for file_name, results in tests.items():
>+        results['name'] = file_name
>+        processed_tests.append(results)
>+
>+    processed_tests = [test for test in processed_tests if not results_match_expectation(test)]
>+
>+    return processed_tests
>+
>+
>+def main(_argv, _stdout, _stderr):
>+    configure_logging()
>+
>+    test_updater = TestExpecationUpdater(Host(), _argv)
>+    test_updater.do_update()
>+
>+
>+class TestExpecationUpdater(object):
>+    def __init__(self, host, args):
>+        self._host = host
>+        options, path_args = parse_args(args)
>+        self._options = options
>+        option_args = list(Set(args).difference(Set(path_args)))
>+        # preserve original order of arguments
>+        option_args = [arg for arg in args if arg in option_args]
>+
>+        self._option_args = option_args
>+        self._base_test = path_args
>+        self._port = host.port_factory.get(options.platform, options)
>+
>+    def do_update(self):
>+        # Run tests once to get a baseline
>+        self.run_webkit_tests(self._base_test)
>+        with open(self.results_file()) as f:
>+            data = json.load(f)
>+        tests = pre_process_tests(self, data['tests'])
>+
>+        for test in tests:
>+            _log.info('%s/%s Processing test %s' % (tests.index(test), len(tests), test['name']))
>+            self.update_expectation(test)
>+
>+    def update_expectation(self, test, post_reset_result=False, previous_result=None):
>+        _log.debug(test)
>+        if test.get('report') == 'FLAKY' or previous_result:
>+            return self.resolve_flaky_test(test, previous_result)
>+        if test.get('report') == 'MISSING':
>+            return self.missing_test(test)
>+        if test.get('report') == 'REGRESSION' and test.get('expected') == 'CRASH':
>+            self.unexpected_pass_test(test)
>+        if test.get('actual') == 'CRASH':
>+            return self.crash_test(test)
>+        if test.get('actual') == 'TIMEOUT':
>+            return self.timeout_test(test)
>+        if test.get('actual') == 'PASS':
>+            return self.unexpected_pass_test(test)
>+        if test.get('actual') == 'IMAGE':
>+            return self.failing_ref_test(test)
>+        if test.get('actual') == 'TEXT MISSING':
>+            return self.missing_test(test)
>+        if test.get('actual') == 'TEXT' and post_reset_result:
>+            return self.failing_testharness_test(test)
>+        if test.get('actual') == 'TEXT IMAGE+TEXT':
>+            return self.failing_testharness_test(test)
>+        if test.get('actual') == 'TEXT':
>+            return self.reset_testharness_test(test)
>+        raise NotImplementedError('The test updater decision engine could not figure out how to handle test: %s' % json.dumps(test))
>+
>+    def missing_test(self, test):
>+        self.run_webkit_tests([test['name']])
>+        result = self.extract_test_result(test)
>+        if result:
>+            # Test is still failing, attempt to re-classify
>+            self.update_expectation(result)
>+
>+    def failing_ref_test(self, test):
>+        self.update_test_expectation(test['name'], 'ImageOnlyFailure')
>+        self.run_webkit_tests([test['name']])
>+        result = self.extract_test_result(test)
>+        if result:
>+            # Test is still failing, attempt to re-classify
>+            self.update_expectation(result, previous_result=test)
>+
>+    def reset_testharness_test(self, test):
>+        self.run_webkit_tests([test['name']], reset_results=True)
>+        self.run_webkit_tests([test['name']])
>+        result = self.extract_test_result(test)
>+        if result:
>+            self.update_expectation(result, post_reset_result=True)
>+
>+    def failing_testharness_test(self, test):
>+        self.update_test_expectation(test['name'], 'Failure')
>+        self.run_webkit_tests([test['name']])
>+        result = self.extract_test_result(test)
>+        if result:
>+            self.update_expectation(result, previous_result=test)
>+
>+    def crash_test(self, test):
>+        self.update_test_expectation(test['name'], 'Skip')
>+
>+    def timeout_test(self, test):
>+        self.update_test_expectation(test['name'], 'Skip')
>+
>+    def resolve_flaky_test(self, test, previous_result=None):
>+        expectations = test['actual'].split(' ')
>+        if previous_result:
>+            expectations = expectations + previous_result['actual'].split(' ')
>+            expectations = list(Set(expectations + ['FAIL']))
>+        self.update_test_expectation(test['name'], render_expectations(expectations))
>+        self.run_webkit_tests([test['name']])
>+        result = self.extract_test_result(test)
>+        if result:
>+            self.update_expectation(result, previous_result=test)
>+
>+    def unexpected_pass_test(self, test):
>+        self.remove_test_expectation(test['name'])
>+
>+        self.run_webkit_tests([test['name']])
>+        result = self.extract_test_result(test)
>+        if result:
>+            self.update_expectation(result, previous_result=test)
>+
>+    def run_webkit_tests(self, test_files, reset_results=False):
>+        args = ['Tools/Scripts/run-webkit-tests'] + self._option_args
>+        if reset_results:
>+            args.append('--reset-results')
>+
>+        args = args + test_files
>+
>+        _log.info('Running webkit tests for: %s' % test_files)
>+        p = Popen(args)
>+        return p.wait()
>+
>+    def extract_test_result(self, test):
>+        with open(self.results_file()) as f:
>+            data = json.load(f)
>+        tests = pre_process_tests(self, data['tests'])
>+        matching_tests = [t for t in tests if t['name'] == test['name']]
>+
>+        if len(matching_tests) and not results_match_expectation(matching_tests[0]):
>+            return matching_tests[0]
>+        else:
>+            return None
>+
>+    def update_test_expectation(self, test, expectation):
>+        self.remove_test_expectation(test)
>+        with open(self.test_expectations_path(), 'a') as myfile:
>+            _log.info('Updating TestExpectations %s [ %s ]' % (test, expectation))
>+            myfile.write('\n%s [ %s ]\n' % (test, expectation))
>+
>+    def remove_test_expectation(self, test_name):
>+        for path in self._port.expectations_files():
>+            remove_test_expectation(path, test_name)
>+
>+    def test_expectations_path(self):
>+        # expectations_files
>+        return self._port.path_to_generic_test_expectations_file()
>+
>+    def results_file(self):
>+        options = self._options
>+        return self._host.filesystem.join(options.build_directory, options.configuration, 'layout-test-results/full-results.json')
>+
>+
>+def results_match_expectation(result):
>+    if 'FAIL' in result['expected'] and result['actual'] == 'TEXT':
>+        return True
>+    if result['actual'] in result['expected']:
>+        return True
>+    return False

Actions: View | Formatted Diff | Diff

Attachments on bug 186359: 342067 | 342081 | 342161 | 342162 | 342430 | 342464 | 342509