Tools/ChangeLog

 12017-05-31 Jonathan Bedard <jbedard@apple.com>
 2
 3 webkitpy: Process crash-logs for iOS devices
 4 https://bugs.webkit.org/show_bug.cgi?id=171976
 5 <rdar://problem/32134551>
 6
 7 Reviewed by NOBODY (OOPS!).
 8
 9 When running layout tests on an iOS device, crash logs should be processed.
 10 Implement crash log searching and parsing for iOS devices.
 11
 12 * Scripts/webkitpy/common/system/crashlogs.py:
 13 (CrashLogs.find_newest_log): Add iOS as a potential platform.
 14 (CrashLogs.find_all_logs): Ditto.
 15 (CrashLogs._find_newest_log_darwin): Support iOS device crash logs. Check
 16 cached_crashlogs for efficiency and symbolicate the crashlog before parsing it. Continue searching
 17 for process ID until it is found in log.
 18 (CrashLogs._find_all_logs_darwin):Ditto.
 19 * Scripts/webkitpy/common/system/crashlogs_unittest.py:
 20 (make_mock_crash_report_darwin): Crashlogs may not have their process on the first line.
 21 * Scripts/webkitpy/common/system/platforminfo.py:
 22 (PlatformInfo.is_ios): Added.
 23 * Scripts/webkitpy/common/system/platforminfo_mock.py:
 24 (MockPlatformInfo.is_ios): Added.
 25 * Scripts/webkitpy/common/system/systemhost.py:
 26 (SystemHost.__init__): Initialize cached_crash_logs array.
 27 (SystemHost.symbolicate_crash_log_if_needed): The symbolicated crash log for most systems
 28 is just the crashlog, use this behavior by default.
 29 * Scripts/webkitpy/common/system/systemhost_mock.py: Ditto.
 30 * Scripts/webkitpy/layout_tests/controllers/manager.py:
 31 (Manager._set_up_run): Cache crash logs currently on the target hosts before running tests.
 32 * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
 33 (RunTest.test_crash_log): Initialize cached_crash_logs before running so that the crash
 34 logs we're testing are not ignored.
 35 (RunTest.test_web_process_crash_log): Ditto.
 36 * Scripts/webkitpy/layout_tests/controllers/manager.py:
 37 (Manager._set_up_run): Cache crash logs already on the host so we don't have to read them.
 38 * Scripts/webkitpy/port/base.py:
 39 (Port._get_crash_log): Pass optional target host when getting crashlogs.
 40 * Scripts/webkitpy/port/darwin.py:
 41 (DarwinPort._get_crash_log): Pass optional target host when getting crashlogs.
 42 * Scripts/webkitpy/port/device.py:
 43 (Device.__init__): Initialize cached_crash_logs array.
 44 (Device.symbolicate_crash_log_if_needed): If the platform device has a function with this
 45 name, call it. Otherwise, assume the crash log is already symbolicated.
 46 * Scripts/webkitpy/port/driver.py:
 47 (Driver._get_crash_log): Pass optional target host when getting crashlogs.
 48 * Scripts/webkitpy/port/gtk.py:
 49 (GtkPort._get_crash_log): Pass optional target host when getting crashlogs.
 50 * Scripts/webkitpy/port/ios_device.py:
 51 (IOSDevicePort.path_to_crash_logs): Consult apple_additions for the path to crash logs.
 52 (IOSDevicePort._look_for_all_crash_logs_in_log_dir): Search every connected device for
 53 crash logs.
 54 (IOSDevicePort._get_crash_log): Search the specified target host for a crash log
 55 if a target host is specified. Else, search all connected devices for the specified
 56 crash-log.
 57 (IOSDevicePort.look_for_new_crash_logs): Deleted, use Darwin implementation.
 58 * Scripts/webkitpy/port/ios_device_unittest.py:
 59 (IOSDeviceTest.test_crashlog_path): Without apple_additions, path_to_crash_logs should raise
 60 an exception.
 61 * Scripts/webkitpy/port/win.py:
 62 (WinPort._get_crash_log): Pass optional target host when getting crashlogs.
 63 * Scripts/webkitpy/port/wpe.py:
 64 (WPEPort._get_crash_log): Pass optional target host when getting crashlogs.
 65
1662017-06-01 Carlos Garcia Campos <cgarcia@igalia.com>
267
368 [GTK] Bump GTK+ and its dependencies in jhbuild
217687

Tools/Scripts/webkitpy/common/system/crashlogs.py

2727# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
2828# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2929
30 import codecs
3130import datetime
3231import re
3332

@@class CrashLogs(object):
4241 self._crash_log_directory = crash_log_directory
4342
4443 def find_newest_log(self, process_name, pid=None, include_errors=False, newer_than=None):
45  if self._host.platform.is_mac():
 44 if self._host.platform.is_mac() or self._host.platform.is_ios():
4645 return self._find_newest_log_darwin(process_name, pid, include_errors, newer_than)
4746 elif self._host.platform.is_win():
4847 return self._find_newest_log_win(process_name, pid, include_errors, newer_than)
4948 return None
5049
5150 def find_all_logs(self, include_errors=False, newer_than=None):
52  if self._host.platform.is_mac():
 51 if self._host.platform.is_mac() or self._host.platform.is_ios():
5352 return self._find_all_logs_darwin(include_errors, newer_than)
5453 return None
5554
5655 def _find_newest_log_darwin(self, process_name, pid, include_errors, newer_than):
5756 def is_crash_log(fs, dirpath, basename):
58  return basename.startswith(process_name + "_") and basename.endswith(".crash")
 57 if fs.join(dirpath, basename) in self._host.cached_crash_logs:
 58 return False
 59 return (basename.startswith(process_name + '_') and (basename.endswith('.crash')) or (process_name in basename and basename.endswith('.ips')))
5960
6061 logs = self._host.filesystem.files_under(self._crash_log_directory, file_filter=is_crash_log)
61  first_line_regex = re.compile(r'^Process:\s+(?P<process_name>.*) \[(?P<pid>\d+)\]$')
 62 process_regex = re.compile(r'^Process:\s+(?P<process_name>.*) \[(?P<pid>\d+)\]$')
6263 errors = ''
6364 for path in reversed(sorted(logs)):
6465 try:
6566 if not newer_than or self._host.filesystem.mtime(path) > newer_than:
66  f = self._host.filesystem.read_text_file(path)
67  match = first_line_regex.match(f[0:f.find('\n')])
 67 f = self._host.symbolicate_crash_log_if_needed(path)
 68 for line in f.splitlines():
 69 match = process_regex.match(line)
 70 if match:
 71 break
6872 if match and match.group('process_name') == process_name and (pid is None or int(match.group('pid')) == pid):
6973 return errors + f
7074 except IOError, e:

@@class CrashLogs(object):
8084
8185 def _find_newest_log_win(self, process_name, pid, include_errors, newer_than):
8286 def is_crash_log(fs, dirpath, basename):
 87 if fs.join(dirpath, basename) in self._host.cached_crash_logs:
 88 return False
8389 return basename.startswith("CrashLog")
8490
8591 logs = self._host.filesystem.files_under(self._crash_log_directory, file_filter=is_crash_log)

@@class CrashLogs(object):
117123
118124 def _find_all_logs_darwin(self, include_errors, newer_than):
119125 def is_crash_log(fs, dirpath, basename):
120  return basename.endswith(".crash")
 126 if fs.join(dirpath, basename) in self._host.cached_crash_logs:
 127 return False
 128 return basename.endswith('.crash') or basename.endswith('.ips')
121129
122130 logs = self._host.filesystem.files_under(self._crash_log_directory, file_filter=is_crash_log)
123  first_line_regex = re.compile(r'^Process:\s+(?P<process_name>.*) \[(?P<pid>\d+)\]$')
 131 process_regex = re.compile(r'^Process:\s+(?P<process_name>.*) \[(?P<pid>\d+)\]$')
124132 errors = ''
125133 crash_logs = {}
126134 for path in reversed(sorted(logs)):

@@class CrashLogs(object):
128136 if not newer_than or self._host.filesystem.mtime(path) > newer_than:
129137 result_name = "Unknown"
130138 pid = 0
131  log_contents = self._host.filesystem.read_text_file(path)
 139 log_contents = self._host.symbolicate_crash_log_if_needed(path)
132140 # Verify timestamp from log contents
133141 crash_time = self.get_timestamp_from_log(log_contents)
134142 if crash_time is not None and newer_than is not None:

@@class CrashLogs(object):
136144 if crash_time < start_time:
137145 continue
138146
139  match = first_line_regex.match(log_contents[0:log_contents.find('\n')])
 147 for line in log_contents.splitlines():
 148 match = process_regex.match(line)
 149 if match:
 150 break
 151
140152 if match:
141153 process_name = match.group('process_name')
142154 pid = str(match.group('pid'))
217680

Tools/Scripts/webkitpy/common/system/crashlogs_unittest.py

@@from webkitpy.port.win import WinPort
3434
3535
3636def make_mock_crash_report_darwin(process_name, pid):
37  return """Process: {process_name} [{pid}]
 37 return """May have garbage at the beginning
 38Process: {process_name} [{pid}]
3839Path: /Volumes/Data/slave/snowleopard-intel-release-tests/build/WebKitBuild/Release/{process_name}
3940Identifier: {process_name}
4041Version: ??? (???)
217680

Tools/Scripts/webkitpy/common/system/platforminfo.py

@@class PlatformInfo(object):
5252 self.os_name = self._determine_os_name(sys_module.platform)
5353 if self.os_name == 'linux':
5454 self.os_version = self._determine_linux_version()
55  if self.os_name == 'freebsd' or self.os_name == 'openbsd' or self.os_name == 'netbsd':
 55 if self.os_name == 'freebsd' or self.os_name == 'openbsd' or self.os_name == 'netbsd' or self.os_name == 'ios':
5656 self.os_version = platform_module.release()
5757 if self.os_name.startswith('mac'):
5858 self.os_version = self._determine_mac_version(platform_module.mac_ver()[0])

@@class PlatformInfo(object):
6363 def is_mac(self):
6464 return self.os_name == 'mac'
6565
 66 def is_ios(self):
 67 return self.os_name == 'ios'
 68
6669 def is_win(self):
6770 return self.os_name == 'win'
6871

@@class PlatformInfo(object):
141144 def _determine_os_name(self, sys_platform):
142145 if sys_platform == 'darwin':
143146 return 'mac'
 147 if sys_platform == 'ios':
 148 return 'ios'
144149 if sys_platform.startswith('linux'):
145150 return 'linux'
146151 if sys_platform.startswith('win') or sys_platform == 'cygwin':
217680

Tools/Scripts/webkitpy/common/system/platforminfo_mock.py

@@class MockPlatformInfo(object):
3636 def is_mac(self):
3737 return self.os_name == 'mac'
3838
 39 def is_ios(self):
 40 return self.os_name == 'ios'
 41
3942 def is_linux(self):
4043 return self.os_name == 'linux'
4144
217680

Tools/Scripts/webkitpy/common/system/systemhost.py

@@class SystemHost(object):
4040 self.user = user.User()
4141 self.platform = platforminfo.PlatformInfo(sys, platform, self.executive)
4242 self.workspace = workspace.Workspace(self.filesystem, self.executive)
 43 self.cached_crash_logs = []
4344
4445 def copy_current_environment(self):
4546 return environment.Environment(os.environ.copy())
4647
4748 def make_file_lock(self, path):
4849 return file_lock.FileLock(path)
 50
 51 def symbolicate_crash_log_if_needed(self, path):
 52 return self.filesystem.read_text_file(path)
217680

Tools/Scripts/webkitpy/common/system/systemhost_mock.py

@@class MockSystemHost(object):
4848
4949 # FIXME: Should this take pointers to the filesystem and the executive?
5050 self.workspace = MockWorkspace()
 51 self.cached_crash_logs = []
5152
5253 def copy_current_environment(self):
5354 return Environment({"MOCK_ENVIRON_COPY": '1'})
5455
5556 def make_file_lock(self, path):
5657 return MockFileLock(path)
 58
 59 def symbolicate_crash_log_if_needed(self, path):
 60 return self.filesystem.read_text_file(path)
217680

Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@class RunTest(unittest.TestCase, StreamT
537537 return
538538 mock_crash_report = make_mock_crash_report_darwin('DumpRenderTree', 12345)
539539 host = MockHost()
 540 host.cached_crash_logs = ['file-to-ignore']
540541 host.filesystem.write_text_file('/tmp/layout-test-results/DumpRenderTree_2011-06-13-150719_quadzen.crash', mock_crash_report)
541542 _, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html', '--dump-render-tree'], tests_included=True, host=host)
542543 expected_crash_log = mock_crash_report

@@class RunTest(unittest.TestCase, StreamT
549550 return
550551 mock_crash_report = make_mock_crash_report_darwin('WebProcess', 12345)
551552 host = MockHost()
 553 host.cached_crash_logs = ['file-to-ignore']
552554 host.filesystem.write_text_file('/tmp/layout-test-results/WebProcess_2011-06-13-150719_quadzen.crash', mock_crash_report)
553555 logging_run(['failures/unexpected/web-process-crash-with-stderr.html'], tests_included=True, host=host)
554556 self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/web-process-crash-with-stderr-crash-log.txt'), mock_crash_report)
217680

Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

@@class Manager(object):
190190 self._port.host.filesystem.maybe_make_directory(self._results_directory)
191191
192192 self._port.setup_test_run(self._options.device_class)
 193
 194 # Cache crash logs already on host.
 195 try:
 196 crash_log_directory = self._port.path_to_crash_logs()
 197 except NotImplementedError:
 198 return True
 199 for i in xrange(self._options.child_processes):
 200 target_host = self._port.target_host(i)
 201 if not target_host.cached_crash_logs:
 202 target_host.cached_crash_logs = target_host.filesystem.files_under(crash_log_directory)
 203
193204 return True
194205
195206 def run(self, args):
217680

Tools/Scripts/webkitpy/port/base.py

@@class Port(object):
13641364 def path_to_crash_logs(self):
13651365 raise NotImplementedError
13661366
1367  def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
 1367 def _get_crash_log(self, name, pid, stdout, stderr, newer_than, target_host=None):
13681368 name_str = name or '<unknown process name>'
13691369 pid_str = str(pid or '<unknown>')
13701370 stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines()
217680

Tools/Scripts/webkitpy/port/darwin.py

@@class DarwinPort(ApplePort):
110110 crash_log = CrashLogs(self.host, self.path_to_crash_logs())
111111 return crash_log.find_all_logs(include_errors=True, newer_than=newer_than)
112112
113  def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=None, sleep_fn=None, wait_for_log=True):
 113 def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=None, sleep_fn=None, wait_for_log=True, target_host=None):
114114 # Note that we do slow-spin here and wait, since it appears the time
115115 # ReportCrash takes to actually write and flush the file varies when there are
116116 # lots of simultaneous crashes going on.
117117 time_fn = time_fn or time.time
118118 sleep_fn = sleep_fn or time.sleep
119119 crash_log = ''
120  crash_logs = CrashLogs(self.host, self.path_to_crash_logs())
 120 crash_logs = CrashLogs(target_host or self.host, self.path_to_crash_logs())
121121 now = time_fn()
122122 deadline = now + 5 * int(self.get_option('child_processes', 1))
123123 while not crash_log and now <= deadline:
217680

Tools/Scripts/webkitpy/port/darwin_testcase.py

@@class DarwinTest(port_testcase.PortTestC
137137 OutputCapture().assert_outputs(self, port.sample_process, args=['test', 42])
138138
139139 def test_get_crash_log(self):
140  # Darwin crash logs are tested elsewhere, so here we just make sure we don't crash.
141  def fake_time_cb():
142  times = [0, 20, 40]
143  return lambda: times.pop(0)
144140 port = self.make_port(port_name=self.port_name)
145141 port._get_crash_log('DumpRenderTree', 1234, None, None, time.time(), wait_for_log=False)
217680

Tools/Scripts/webkitpy/port/device.py

@@class Device(object):
2727 def __init__(self, platform_device):
2828 self.platform_device = platform_device
2929 self.listening_socket = None
 30 self.cached_crash_logs = []
3031
3132 def listening_port(self):
3233 if not self.listening_socket:

@@class Device(object):
6364
6465 self.listening_socket = None
6566
 67 def symbolicate_crash_log_if_needed(self, path):
 68 if hasattr(self.platform_device, 'symbolicate_crash_log_if_needed'):
 69 return self.platform_device.symbolicate_crash_log_if_needed(path)
 70 return path
 71
6672 @property
6773 def executive(self):
6874 return self.platform_device.executive
217680

Tools/Scripts/webkitpy/port/driver.py

@@class Driver(object):
241241 crashed_pid=self._crashed_pid, crash_log=crash_log, pid=pid)
242242
243243 def _get_crash_log(self, stdout, stderr, newer_than):
244  return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than)
 244 return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than, target_host=self._target_host)
245245
246246 def _command_wrapper(self):
247247 # Hook for injecting valgrind or other runtime instrumentation, used by e.g. tools/valgrind/valgrind_tests.py.
217680

Tools/Scripts/webkitpy/port/gtk.py

@@class GtkPort(Port):
220220 def check_sys_deps(self, needs_http):
221221 return super(GtkPort, self).check_sys_deps(needs_http) and self._driver_class().check_driver(self)
222222
223  def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
 223 def _get_crash_log(self, name, pid, stdout, stderr, newer_than, target_host=None):
224224 name = "WebKitWebProcess" if name == "WebProcess" else name
225225 return GDBCrashLogGenerator(name, pid, newer_than, self._filesystem, self._path_to_driver).generate_crash_log(stdout, stderr)
226226
217680

Tools/Scripts/webkitpy/port/ios_device.py

2323import logging
2424
2525from webkitpy.common.memoized import memoized
 26from webkitpy.common.system.crashlogs import CrashLogs
2627from webkitpy.port.config import apple_additions
2728from webkitpy.port.ios import IOSPort
2829

@@class IOSDevicePort(IOSPort):
6768 port_name = port_name + '-' + major_version_number
6869 return port_name
6970
70  # FIXME: These need device implementations <rdar://problem/30497991>.
7171 def path_to_crash_logs(self):
72  raise NotImplementedError
 72 if not apple_additions():
 73 raise NotImplementedError
 74 return apple_additions().ios_crash_log_path()
7375
 76 def _look_for_all_crash_logs_in_log_dir(self, newer_than):
 77 log_list = {}
 78 for device in self._device_for_worker_number_map():
 79 crash_log = CrashLogs(device, self.path_to_crash_logs())
 80 log_list.update(crash_log.find_all_logs(include_errors=True, newer_than=newer_than))
 81 return log_list
 82
 83 def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=None, sleep_fn=None, wait_for_log=True, target_host=None):
 84 if target_host:
 85 return super(IOSDevicePort, self)._get_crash_log(name, pid, stdout, stderr, newer_than, time_fn=time_fn, sleep_fn=sleep_fn, wait_for_log=wait_for_log, target_host=target_host)
 86
 87 # We need to search every device since one was not specified.
 88 for device in self._device_for_worker_number_map():
 89 stderr_out, crashlog = super(IOSDevicePort, self)._get_crash_log(name, pid, stdout, stderr, newer_than, time_fn=time_fn, sleep_fn=sleep_fn, wait_for_log=False, target_host=device)
 90 if crashlog:
 91 return (stderr, crashlog)
 92 return (stderr, None)
 93
 94 # FIXME: These need device implementations <rdar://problem/30497991>.
7495 def check_for_leaks(self, process_name, process_pid):
7596 pass
7697
77  def look_for_new_crash_logs(self, crashed_processes, start_time):
78  return {}
79 
8098 def look_for_new_samples(self, unresponsive_processes, start_time):
8199 return {}
82100

@@class IOSDevicePort(IOSPort):
90108 def operating_system(self):
91109 return 'ios-device'
92110
93  def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=None, sleep_fn=None, wait_for_log=True):
94  return (stderr, None)
95 
96111 def _create_devices(self, device_class):
97112 if not apple_additions():
98113 raise RuntimeError(self.NO_ON_DEVICE_TESTING)
217680

Tools/Scripts/webkitpy/port/ios_device_unittest.py

2020# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
2121# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2222
 23import time
 24
2325from webkitpy.port.ios_device import IOSDevicePort
2426from webkitpy.port import ios_testcase
2527

@@class IOSDeviceTest(ios_testcase.IOSTest
3739 def test_operating_system(self):
3840 self.assertEqual('ios-device', self.make_port().operating_system())
3941
40  # FIXME: Update tests when <rdar://problem/30497991> is completed.
4142 def test_crashlog_path(self):
42  pass
 43 port = self.make_port()
 44 with self.assertRaises(NotImplementedError):
 45 port.path_to_crash_logs()
 46
 47 def test_get_crash_log(self):
 48 port = self.make_port(port_name=self.port_name)
 49 with self.assertRaises(RuntimeError):
 50 port._get_crash_log('DumpRenderTree', 1234, None, None, time.time(), wait_for_log=False)
4351
 52 # FIXME: Update tests when <rdar://problem/30497991> is completed.
4453 def test_spindump(self):
4554 pass
4655
217680

Tools/Scripts/webkitpy/port/win.py

@@class WinPort(ApplePort):
372372 def path_to_crash_logs(self):
373373 return self.results_directory()
374374
375  def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=None, sleep_fn=None, wait_for_log=True):
 375 def _get_crash_log(self, name, pid, stdout, stderr, newer_than, time_fn=None, sleep_fn=None, wait_for_log=True, target_host=None):
376376 # Note that we do slow-spin here and wait, since it appears the time
377377 # ReportCrash takes to actually write and flush the file varies when there are
378378 # lots of simultaneous crashes going on.

@@class WinPort(ApplePort):
380380 time_fn = time_fn or time.time
381381 sleep_fn = sleep_fn or time.sleep
382382 crash_log = ''
383  crash_logs = CrashLogs(self.host, self.path_to_crash_logs())
 383 crash_logs = CrashLogs(target_host or self.host, self.path_to_crash_logs())
384384 now = time_fn()
385385 # FIXME: delete this after we're sure this code is working ...
386386 _log.debug('looking for crash log for %s:%s' % (name, str(pid)))
217680

Tools/Scripts/webkitpy/port/wpe.py

@@class WPEPort(Port):
7474 def _path_to_image_diff(self):
7575 return self._built_executables_path('ImageDiff')
7676
77  def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
 77 def _get_crash_log(self, name, pid, stdout, stderr, newer_than, target_host=None):
7878 name = "WPEWebProcess" if name == "WebProcess" else name
7979 return GDBCrashLogGenerator(name, pid, newer_than, self._filesystem, self._path_to_driver).generate_crash_log(stdout, stderr)
217680