The test runner doesn't seem to handle platform specific ref test results just right, it complains about a missing directory: If one creates that directory the tests pass just fine, likely the tools just needs to be updated to properly handle this case. [eae@eae]:[15:35:22]:[~/projects/webkit3] $ ./Tools/Scripts/new-run-webkit-tests --chromium --debug --pixel fast/regions/overflow-in-uniform-regions.html Exception raised, exiting Traceback (most recent call last): File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py", line 452, in <module> sys.exit(main()) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py", line 447, in main return run(port, options, args) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py", line 120, in run unexpected_result_count = manager.run() File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py", line 894, in run interrupted, keyboard_interrupted, thread_timings, test_timings, individual_test_timings = self._run_tests(self._test_files_list, result_summary, int(self._options.child_processes)) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py", line 787, in _run_tests manager_connection.run_message_loop(delay_secs=1.0) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py", line 350, in run_message_loop self._inline_worker.run() File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py", line 404, in run self._client.run() File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 132, in run super(Worker, self).run() File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py", line 299, in run self._worker_connection.raise_exception(sys.exc_info()) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py", line 290, in run self._worker_connection.run_message_loop() File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py", line 243, in run_message_loop self._broker.run_message_loop(self._run_topic, self._client, delay_secs) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py", line 175, in run_message_loop self._run_loop(topic_name, client, block=True, delay_secs=delay_secs) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py", line 189, in _run_loop self._dispatch_message(msg, client) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py", line 198, in _dispatch_message message_handler(message.src, *optargs) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 144, in handle_test_list self._run_test(test_input) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 171, in _run_test result = self.run_test_with_timeout(test_input, test_timeout_sec) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 213, in run_test_with_timeout return self._run_test_in_this_thread(test_input) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 298, in _run_test_in_this_thread return self.run_single_test(self._driver, test_input) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 302, in run_single_test test_input, driver, self._name) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 46, in run_single_test return runner.run() File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 101, in run return self._run_reftest() File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 299, in _run_reftest test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, test_output, reference_output, test_result.failures) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 77, in write_test_result writer.copy_file(failure.reference_filename) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 269, in copy_file fs.copyfile(src_filepath, dst_filepath) File "/usr/local/google/projects/webkit3/Tools/Scripts/webkitpy/common/system/filesystem.py", line 76, in copyfile shutil.copyfile(source, destination) File "/usr/lib/python2.6/shutil.py", line 53, in copyfile fdst = open(dst, 'wb') IOError: [Errno 2] No such file or directory: '/usr/local/google/projects/webkit3/Source/WebKit/chromium/webkit/Debug/layout-test-results/platform/chromium/fast/regions/overflow-in-uniform-regions-expected.html' [eae@eae]:[15:38:46]:[~/projects/webkit3] $ mkdir -p /usr/local/google/projects/webkit3/Source/WebKit/chromium/webkit/Debug/layout-test-results/platform/chromium/fast/regions/ [eae@eae]:[15:38:48]:[~/projects/webkit3] $ ./Tools/Scripts/new-run-webkit-tests --chromium --debug --pixel fast/regions/overflow-in-uniform-regions.html All 1 tests ran as expected.
Is this something that needs to be supported? This seems like the whole idea of reftests. Perhaps one case where this is needed is when a feature is not implemented or disabled on some platforms, yet we don't want to disable the test for some reason.
(In reply to comment #1) > Is this something that needs to be supported? This seems like the whole idea of reftests. > > Perhaps one case where this is needed is when a feature is not implemented or disabled on some platforms, yet we don't want to disable the test for some reason. This is precisely the case. We ran into this issue when enabling sub-pixel layout on Chromium. That feature can lead to contents of a page shifting by several pixels. Since the actual layout of the reference vs the test often differ, it can be extremely difficult to tweak the expectations to be identical to the test with both the feature enabled and disabled.
I believe this is now supported, for instance: http://trac.webkit.org/browser/trunk/LayoutTests/fast/media/mq-transform-02-expected.html http://trac.webkit.org/browser/trunk/LayoutTests/platform/gtk/fast/media/mq-transform-02-expected.html Works as expected, the GTK-specific baseline overriding the generic one.