91181 2012-07-12 18:46:51 -0700 nrwt: don't choke when printing invalid utf-8 to stderr 2012-07-13 01:23:36 -0700 1 1 1 Unclassified WebKit Tools / Tests 528+ (Nightly build) Unspecified Unspecified RESOLVED FIXED P2 Normal --- 1 dgrogan dgrogan abarth alecflett dpranke jsbell ojan webkit.review.bot oldest_to_newest 668492 0 dgrogan 2012-07-12 18:46:51 -0700 This seems to only reproduce on one of my branches, so I've probably introduced an error that interacts badly with the recent nrwt changes. Though I have no changes in Tools, just Source. Dirk, if you could try to glean some hints about a possible nrwt bug from this stacktrace, I'd appreciate it. $ new-run-webkit-tests -f --debug --no-new-test-results --no-retry-failures storage/indexeddb File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/common/message_pool.py", line 242, in run worker.handle(message.name, message.src, *message.args) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 81, in handle self._run_test(test_input) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 104, in _run_test result = self._run_test_with_timeout(test_input, test_timeout_sec) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 146, in _run_test_with_timeout return self._run_test_in_this_thread(test_input) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 231, in _run_test_in_this_thread return self._run_single_test(self._driver, test_input) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 235, in _run_single_test test_input, driver, self._name) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 46, in run_single_test return runner.run() File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 105, in run return self._run_compare_test() File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 118, in _run_compare_test test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, driver_output, expected_driver_output, test_result.failures) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 45, in write_test_result writer.write_stderr(driver_output.error) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 163, in write_stderr self._write_text_file(filename, error) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 136, in _write_text_file self._filesystem.write_text_file(path, contents) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/common/system/filesystem.py", line 226, in write_text_file f.write(contents) File "/usr/lib/python2.6/codecs.py", line 686, in write return self.writer.write(data) File "/usr/lib/python2.6/codecs.py", line 351, in write data, consumed = self.encode(object, self.errors) worker/12 raised UnicodeDecodeError(''ascii' codec can't decode byte 0xef in position 914: ordinal not in range(128)'): Exception raised, exiting 668499 1 dpranke 2012-07-12 18:57:02 -0700 the stack trace is saying that you printed something to stderr from DRT/WebKit that isn't a valid utf-8 codepoint, specifically a byte with value 0xef. Does that help? 668519 2 dgrogan 2012-07-12 19:24:19 -0700 Ah to stderr. Yes, there is a printf that prints invalid characters when running storage/indexeddb/odd-strings.html. Indeed, running odd-strings.html alone causes an exception. Though this just started happening today, at 122505. Before then, running new-run-webkit-tests --debug storage/indexeddb/odd-strings.html succeeds. After 122505, I get the stack trace below (practically the same as the first). Any chance of making nrwt tolerate the unprintable characters? $ new-run-webkit-tests --debug storage/indexeddb/odd-strings.html Exception raised, exiting File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py", line 129, in run unexpected_result_count = manager.run() File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py", line 866, in run interrupted, keyboard_interrupted, thread_timings, test_timings, individual_test_timings = self._run_tests(self._test_files_list, result_summary, int(self._options.child_processes)) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py", line 777, in _run_tests pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/common/message_pool.py", line 97, in run self.wait() File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/common/message_pool.py", line 117, in wait self._workers[0].run() File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/common/message_pool.py", line 242, in run worker.handle(message.name, message.src, *message.args) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 81, in handle self._run_test(test_input) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 104, in _run_test result = self._run_test_with_timeout(test_input, test_timeout_sec) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 146, in _run_test_with_timeout return self._run_test_in_this_thread(test_input) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 231, in _run_test_in_this_thread return self._run_single_test(self._driver, test_input) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py", line 235, in _run_single_test test_input, driver, self._name) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 46, in run_single_test return runner.run() File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 105, in run return self._run_compare_test() File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py", line 118, in _run_compare_test test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, driver_output, expected_driver_output, test_result.failures) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 45, in write_test_result writer.write_stderr(driver_output.error) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 163, in write_stderr self._write_text_file(filename, error) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 136, in _write_text_file self._filesystem.write_text_file(path, contents) File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/common/system/filesystem.py", line 226, in write_text_file f.write(contents) File "/usr/lib/python2.6/codecs.py", line 686, in write return self.writer.write(data) File "/usr/lib/python2.6/codecs.py", line 351, in write data, consumed = self.encode(object, self.errors) 668526 3 dpranke 2012-07-12 19:38:25 -0700 As a workaround, all you need to do is change: File "/sdb1/chrome/1/src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py", line 163, in write_stderr self._write_text_file(filename, error) to self._write_binary_file(filename, error) this would probably be a reasonable change to make, period. I don't think we have a strong requirement that stderr is utf-8 :). 668539 4 152137 dgrogan 2012-07-12 19:56:45 -0700 Created attachment 152137 Patch 668543 5 dgrogan 2012-07-12 19:57:38 -0700 Dirk, could you review this? 668547 6 152137 dpranke 2012-07-12 20:09:37 -0700 Comment on attachment 152137 Patch yes, I can :). 668717 7 152137 webkit.review.bot 2012-07-13 01:23:31 -0700 Comment on attachment 152137 Patch Clearing flags on attachment: 152137 Committed r122551: <http://trac.webkit.org/changeset/122551> 668718 8 webkit.review.bot 2012-07-13 01:23:36 -0700 All reviewed patches have been landed. Closing bug. 152137 2012-07-12 19:56:45 -0700 2012-07-13 01:23:31 -0700 Patch bug-91181-20120712195644.patch text/plain 1477 dgrogan U3VidmVyc2lvbiBSZXZpc2lvbjogMTIyNTE1CmRpZmYgLS1naXQgYS9Ub29scy9DaGFuZ2VMb2cg Yi9Ub29scy9DaGFuZ2VMb2cKaW5kZXggZDc5ZDA1MWMxZTQ5M2JkZDlhOTc2Y2FjMWZkY2RiMmRh ZmQ2OTg1Ni4uMmFjODhhOTY0MjZlYTU3YWQzY2JiMDNhMmE5ZTFmZmY0MDI2NjRjYSAxMDA2NDQK LS0tIGEvVG9vbHMvQ2hhbmdlTG9nCisrKyBiL1Rvb2xzL0NoYW5nZUxvZwpAQCAtMSwzICsxLDEz IEBACisyMDEyLTA3LTEyICBEYXZpZCBHcm9nYW4gIDxkZ3JvZ2FuQGNocm9taXVtLm9yZz4KKwor ICAgICAgICBucnd0OiBkb24ndCBjaG9rZSB3aGVuIHByaW50aW5nIGludmFsaWQgdXRmLTggdG8g c3RkZXJyCisgICAgICAgIGh0dHBzOi8vYnVncy53ZWJraXQub3JnL3Nob3dfYnVnLmNnaT9pZD05 MTE4MQorCisgICAgICAgIFJldmlld2VkIGJ5IE5PQk9EWSAoT09QUyEpLgorCisgICAgICAgICog U2NyaXB0cy93ZWJraXRweS9sYXlvdXRfdGVzdHMvY29udHJvbGxlcnMvdGVzdF9yZXN1bHRfd3Jp dGVyLnB5OgorICAgICAgICAoVGVzdFJlc3VsdFdyaXRlci53cml0ZV9zdGRlcnIpOgorCiAyMDEy LTA3LTEyICBEaXJrIFByYW5rZSAgPGRwcmFua2VAY2hyb21pdW0ub3JnPgogCiAgICAgICAgIHdl YmtpdHB5OiByZW5hbWUgbWFuYWdlcl93b3JrZXJfYnJva2VyIHRvIG1lc3NhZ2VfcG9vbApkaWZm IC0tZ2l0IGEvVG9vbHMvU2NyaXB0cy93ZWJraXRweS9sYXlvdXRfdGVzdHMvY29udHJvbGxlcnMv dGVzdF9yZXN1bHRfd3JpdGVyLnB5IGIvVG9vbHMvU2NyaXB0cy93ZWJraXRweS9sYXlvdXRfdGVz dHMvY29udHJvbGxlcnMvdGVzdF9yZXN1bHRfd3JpdGVyLnB5CmluZGV4IGUxMjAxYWNmNjI1NTRi ZGNjNDYwNzM1NWQ2ZGViYTA4MGQ3NjY4OWQuLjI0M2ExMWQ4ZDA5OTg2ZmMzMTRkYTgyNzMzMmRm MGUwNTJmNWI4M2UgMTAwNjQ0Ci0tLSBhL1Rvb2xzL1NjcmlwdHMvd2Via2l0cHkvbGF5b3V0X3Rl c3RzL2NvbnRyb2xsZXJzL3Rlc3RfcmVzdWx0X3dyaXRlci5weQorKysgYi9Ub29scy9TY3JpcHRz L3dlYmtpdHB5L2xheW91dF90ZXN0cy9jb250cm9sbGVycy90ZXN0X3Jlc3VsdF93cml0ZXIucHkK QEAgLTE2MCw3ICsxNjAsNyBAQCBjbGFzcyBUZXN0UmVzdWx0V3JpdGVyKG9iamVjdCk6CiAKICAg ICBkZWYgd3JpdGVfc3RkZXJyKHNlbGYsIGVycm9yKToKICAgICAgICAgZmlsZW5hbWUgPSBzZWxm Lm91dHB1dF9maWxlbmFtZShzZWxmLkZJTEVOQU1FX1NVRkZJWF9TVERFUlIgKyAiLnR4dCIpCi0g ICAgICAgIHNlbGYuX3dyaXRlX3RleHRfZmlsZShmaWxlbmFtZSwgZXJyb3IpCisgICAgICAgIHNl bGYuX3dyaXRlX2JpbmFyeV9maWxlKGZpbGVuYW1lLCBlcnJvcikKIAogICAgIGRlZiB3cml0ZV9j cmFzaF9sb2coc2VsZiwgY3Jhc2hfbG9nKToKICAgICAgICAgZmlsZW5hbWUgPSBzZWxmLm91dHB1 dF9maWxlbmFtZShzZWxmLkZJTEVOQU1FX1NVRkZJWF9DUkFTSF9MT0cgKyAiLnR4dCIpCg==