WebKitTools/ChangeLog

 12010-10-11 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>
 2
 3 Reviewed by NOBODY (OOPS!).
 4
 5 Implement http locking in NRWT.
 6 https://bugs.webkit.org/show_bug.cgi?id=47072
 7
 8 * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
 9 * Scripts/webkitpy/layout_tests/port/base.py:
 10 * Scripts/webkitpy/layout_tests/port/http_lock.py: Added.
 11 * Scripts/webkitpy/layout_tests/port/http_lock_unittest.py: Added.
 12 * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
 13
1142010-10-11 Adam Barth <abarth@webkit.org>
215
316 Unreviewed.
69480

WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@class TestRunner:
704704 """Returns whether the test runner needs an HTTP server."""
705705 return self._contains_tests(self.HTTP_SUBDIR)
706706
 707 def needs_websocket(self):
 708 """Returns whether the test runner needs a WEBSOCKET server."""
 709 return self._contains_tests(self.WEBSOCKET_SUBDIR)
 710
707711 def set_up_run(self):
708712 """Configures the system to be ready to run tests.
709713

@@class TestRunner:
736740 if not result_summary:
737741 return None
738742
739  if self.needs_http():
740  self._printer.print_update('Starting HTTP server ...')
741  self._port.start_http_server()
742 
743  if self._contains_tests(self.WEBSOCKET_SUBDIR):
744  self._printer.print_update('Starting WebSocket server ...')
745  self._port.start_websocket_server()
746  # self._websocket_secure_server.Start()
 743 # Do not start when http locking is enabled.
 744 if not self._options.wait_for_httpd:
 745 if self.needs_http():
 746 self._printer.print_update('Starting HTTP server ...')
 747 self._port.start_http_server()
 748
 749 if self.needs_websocket():
 750 self._printer.print_update('Starting WebSocket server ...')
 751 self._port.start_websocket_server()
 752 # self._websocket_secure_server.Start()
747753
748754 return result_summary
749755

@@class TestRunner:
834840 sys.stdout.flush()
835841 _log.debug("flushing stderr")
836842 sys.stderr.flush()
837  _log.debug("stopping http server")
838  self._port.stop_http_server()
839  _log.debug("stopping websocket server")
840  self._port.stop_websocket_server()
 843 if not self._options.wait_for_httpd:
 844 _log.debug("stopping http server")
 845 self._port.stop_http_server()
 846 _log.debug("stopping websocket server")
 847 self._port.stop_websocket_server()
841848 _log.debug("stopping helper")
842849 self._port.stop_helper()
843850

@@def parse_args(args=None):
15931600 optparse.make_option("--no-record-results", action="store_false",
15941601 default=True, dest="record_results",
15951602 help="Don't record the results."),
 1603 optparse.make_option("--wait-for-httpd", action="store_true",
 1604 default=False, dest="wait_for_httpd",
 1605 help="Wait for http locks."),
15961606 # old-run-webkit-tests also has HTTP toggle options:
15971607 # --[no-]http Run (or do not run) http tests
15981608 # (default: run)
1599  # --[no-]wait-for-httpd Wait for httpd if some other test
1600  # session is using it already (same
1601  # as WEBKIT_WAIT_FOR_HTTPD=1).
1602  # (default: 0)
16031609 ]
16041610
16051611 test_options = [
69479

WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py

11#!/usr/bin/env python
22# Copyright (C) 2010 Google Inc. All rights reserved.
 3# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
34#
45# Redistribution and use in source and binary forms, with or without
56# modification, are permitted provided that the following conditions are

@@class TestShellThread(WatchableThread):
276277 self._num_tests = 0
277278 self._start_time = 0
278279 self._stop_time = 0
 280 self._have_http_lock = False
 281 self._http_lock_wait_begin = 0
 282 self._http_lock_wait_end = 0
279283
280284 # Current group of tests we're running.
281285 self._current_group = None

@@class TestShellThread(WatchableThread):
298302 return self._test_results
299303
300304 def get_total_time(self):
301  return max(self._stop_time - self._start_time, 0.0)
 305 return max(self._stop_time - self._start_time -
 306 self._http_lock_wait_time(), 0.0)
302307
303308 def get_num_tests(self):
304309 return self._num_tests

@@class TestShellThread(WatchableThread):
337342 do multi-threaded debugging."""
338343 self._run(test_runner, result_summary)
339344
 345 def cancel(self):
 346 """Clean up http lock and set a flag telling this thread to quit."""
 347 self._stop_http_lock()
 348 WatchableThread.cancel(self)
 349
 350 def next_timeout(self):
 351 """Return the time the test is supposed to finish by."""
 352 if self._next_timeout:
 353 return self._next_timeout + self._http_lock_wait_time()
 354 return self._next_timeout
 355
 356 def _http_lock_wait_time(self):
 357 """Return the time what http locking takes."""
 358 if self._http_lock_wait_begin == 0:
 359 return 0
 360 if self._http_lock_wait_end == 0:
 361 return time.time() - self._http_lock_wait_begin
 362 return self._http_lock_wait_end - self._http_lock_wait_begin
 363
340364 def _run(self, test_runner, result_summary):
341365 """Main work entry point of the thread. Basically we pull urls from the
342366 filename queue and run the tests until we run out of urls.

@@class TestShellThread(WatchableThread):
368392 self._current_group, self._filename_list = \
369393 self._filename_list_queue.get_nowait()
370394 except Queue.Empty:
 395 self._stop_http_lock()
371396 self._kill_dump_render_tree()
372397 tests_run_file.close()
373398 return
374399
 400 if self._options.wait_for_httpd:
 401 if self._current_group == "tests_to_http_lock":
 402 self._http_lock_wait_begin = time.time()
 403 self._port.acquire_http_lock()
 404
 405 self._port.start_http_server()
 406 self._port.start_websocket_server()
 407
 408 self._have_http_lock = True
 409 self._http_lock_wait_end = time.time()
 410 elif self._have_http_lock:
 411 self._stop_http_lock()
 412
375413 self._num_tests_in_current_group = len(self._filename_list)
376414 self._current_group_start_time = time.time()
377415

@@class TestShellThread(WatchableThread):
517555 self._options)
518556 self._driver.start()
519557
 558 def _stop_http_lock(self):
 559 """Stop the servers and release http lock."""
 560 if self._have_http_lock:
 561 self._port.stop_http_server()
 562 self._port.stop_websocket_server()
 563 self._port.release_http_lock()
 564 self._have_http_lock = False
 565
520566 def _kill_dump_render_tree(self):
521567 """Kill the DumpRenderTree process if it's running."""
522568 if self._driver:
69479

WebKitTools/Scripts/webkitpy/layout_tests/port/base.py

@@import time
4343
4444import apache_http_server
4545import test_files
 46import http_lock
4647import http_server
4748import websocket_server
4849

@@class Port(object):
9293 self._http_server = None
9394 self._webkit_base_dir = None
9495 self._websocket_server = None
 96 self._http_lock = None
9597
9698 def default_child_processes(self):
9799 """Return the number of DumpRenderTree instances to use for this

@@class Port(object):
500502 self._options.results_directory)
501503 self._websocket_server.start()
502504
 505 def acquire_http_lock(self):
 506 self._http_lock = http_lock.HttpLock(None)
 507 self._http_lock.wait_for_httpd_lock()
 508
503509 def stop_helper(self):
504510 """Shut down the test helper if it is running. Do nothing if
505511 it isn't, or it isn't available. If a port overrides start_helper()

@@class Port(object):
518524 if self._websocket_server:
519525 self._websocket_server.stop()
520526
 527 def release_http_lock(self):
 528 if self._http_lock:
 529 self._http_lock.cleanup_http_lock()
 530
521531 def test_expectations(self):
522532 """Returns the test expectations for this port.
523533
69479

WebKitTools/Scripts/webkitpy/layout_tests/port/http_lock.py

 1#!/usr/bin/env python
 2# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 3# Copyright (C) 2010 Andras Becsi (abecsi@inf.u-szeged.hu), University of Szeged
 4#
 5# All rights reserved.
 6#
 7# Redistribution and use in source and binary forms, with or without
 8# modification, are permitted provided that the following conditions
 9# are met:
 10# 1. Redistributions of source code must retain the above copyright
 11# notice, this list of conditions and the following disclaimer.
 12# 2. Redistributions in binary form must reproduce the above copyright
 13# notice, this list of conditions and the following disclaimer in the
 14# documentation and/or other materials provided with the distribution.
 15#
 16# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
 17# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 19# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
 20# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 21# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 22# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 23# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 24# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 27
 28"""This class helps to block NRWT threads when more NRWTs run
 29http and websocket tests in a same time."""
 30
 31import fcntl
 32import glob
 33import os
 34import signal
 35import sys
 36import tempfile
 37import time
 38
 39
 40class HttpLock(object):
 41
 42 def __init__(self, lock_path, lock_file_prefix="WebKitHttpd.lock.",
 43 guard_lock="WebKit.lock"):
 44 if not lock_path:
 45 self._lock_path = tempfile.gettempdir()
 46 self._lock_file_prefix = lock_file_prefix
 47 self._lock_file_path_prefix = os.path.join(self._lock_path,
 48 self._lock_file_prefix)
 49 self._guard_lock_file = os.path.join(self._lock_path, guard_lock)
 50 self._process_lock_file_name = ""
 51
 52 def cleanup_http_lock(self):
 53 """Delete the lock file if exists."""
 54 if os.path.exists(self._process_lock_file_name):
 55 os.unlink(self._process_lock_file_name)
 56
 57 def _extract_lock_number(self, lock_file_name):
 58 """Return the lock number from lock file."""
 59 prefix_length = len(self._lock_file_path_prefix)
 60 return int(lock_file_name[prefix_length:])
 61
 62 def _lock_file_list(self):
 63 """Return the list of lock files sequentially."""
 64 lock_list = glob.glob(self._lock_file_path_prefix + '*')
 65 lock_list.sort(key=self._extract_lock_number)
 66 return lock_list
 67
 68 def _next_lock_number(self):
 69 """Return the next available lock number."""
 70 lock_list = self._lock_file_list()
 71 if not lock_list:
 72 return 0
 73 return self._extract_lock_number(lock_list[-1]) + 1
 74
 75 def _check_pid(self, current_pid):
 76 """Return True if pid is alive, otherwise return False."""
 77 try:
 78 os.kill(current_pid, 0)
 79 except OSError:
 80 return False
 81 else:
 82 return True
 83
 84 def _curent_lock_pid(self):
 85 """Return with the current lock pid. If the lock is not valid
 86 it deletes the lock file."""
 87 lock_list = self._lock_file_list()
 88 if not lock_list:
 89 return
 90 try:
 91 current_lock_file = open(lock_list[0], 'r')
 92 current_pid = current_lock_file.readline()
 93 current_lock_file.close()
 94 if not (current_pid and
 95 sys.platform in ('darwin', 'linux2') and
 96 self._check_pid(int(current_pid))):
 97 os.unlink(lock_list[0])
 98 return
 99 except IOError, OSError:
 100 return
 101 return int(current_pid)
 102
 103 def _create_lock_file(self):
 104 """The lock files are used to schedule the running test sessions in first
 105 come first served order. The sequential guard lock ensures that the lock
 106 numbers are sequential."""
 107 while(True):
 108 try:
 109 sequential_guard_lock = os.open(self._guard_lock_file,
 110 os.O_CREAT | os.O_NONBLOCK | os.O_EXCL)
 111
 112 self._process_lock_file_name = (self._lock_file_path_prefix +
 113 str(self._next_lock_number()))
 114 lock_file = open(self._process_lock_file_name, 'w')
 115 lock_file.write(str(os.getpid()))
 116 lock_file.close()
 117 os.close(sequential_guard_lock)
 118 os.unlink(self._guard_lock_file)
 119 break
 120 except OSError:
 121 pass
 122
 123 def wait_for_httpd_lock(self):
 124 """Create a lock file and wait until it's turn comes."""
 125 self._create_lock_file()
 126 while self._curent_lock_pid() != os.getpid():
 127 time.sleep(1)
0

WebKitTools/Scripts/webkitpy/layout_tests/port/http_lock_unittest.py

 1#!/usr/bin/env python
 2# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 3#
 4# All rights reserved.
 5#
 6# Redistribution and use in source and binary forms, with or without
 7# modification, are permitted provided that the following conditions
 8# are met:
 9# 1. Redistributions of source code must retain the above copyright
 10# notice, this list of conditions and the following disclaimer.
 11# 2. Redistributions in binary form must reproduce the above copyright
 12# notice, this list of conditions and the following disclaimer in the
 13# documentation and/or other materials provided with the distribution.
 14#
 15# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
 16# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 18# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
 19# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 20# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 21# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 22# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 23# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 26
 27import glob
 28import http_lock
 29import os
 30import unittest
 31
 32
 33class HttpLockTest(unittest.TestCase):
 34
 35 def __init__(self, testFunc):
 36 self.http_lock_obj = http_lock.HttpLock(None, "WebKitTestHttpd.lock.", "WebKitTest.lock")
 37 self.lock_file_path_prefix = os.path.join(self.http_lock_obj._lock_path,
 38 self.http_lock_obj._lock_file_prefix)
 39 self.lock_file_name = self.lock_file_path_prefix + "0"
 40 self.guard_lock_file = self.http_lock_obj._guard_lock_file
 41 self.clean_all_lockfile()
 42 unittest.TestCase.__init__(self, testFunc)
 43
 44 def clean_all_lockfile(self):
 45 if os.path.exists(self.guard_lock_file):
 46 os.unlink(guard_lock_file)
 47 lock_list = glob.glob(self.lock_file_path_prefix + '*')
 48 for file_name in lock_list:
 49 os.unlink(file_name)
 50
 51 def assertEqual(self, first, second):
 52 if first != second:
 53 self.clean_all_lockfile()
 54 unittest.TestCase.assertEqual(self, first, second)
 55
 56 def _check_lock_file(self):
 57 if os.path.exists(self.lock_file_name):
 58 pid = os.getpid()
 59 lock_file = open(self.lock_file_name, 'r')
 60 lock_file_pid = lock_file.readline()
 61 lock_file.close()
 62 self.assertEqual(pid, int(lock_file_pid))
 63 return True
 64 return False
 65
 66 def test_lock_lifecycle(self):
 67 self.http_lock_obj._create_lock_file()
 68
 69 self.assertEqual(True, self._check_lock_file())
 70 self.assertEqual(1, self.http_lock_obj._next_lock_number())
 71
 72 self.http_lock_obj.cleanup_http_lock()
 73
 74 self.assertEqual(False, self._check_lock_file())
 75 self.assertEqual(0, self.http_lock_obj._next_lock_number())
 76
 77 def test_extract_lock_number(self,):
 78 lock_file_list = [
 79 self.lock_file_path_prefix + "00",
 80 self.lock_file_path_prefix + "9",
 81 self.lock_file_path_prefix + "001",
 82 self.lock_file_path_prefix + "021",
 83 ]
 84
 85 expected_number_list = ["0", "9", "1", "21"]
 86
 87 for i in range(len(lock_file_list)):
 88 self.assertEqual(self.http_lock_obj._extract_lock_number(lock_file_list[i]),
 89 int(expected_number_list[i]))
 90
 91 def test_lock_file_list(self):
 92 lock_file_list = [
 93 self.lock_file_path_prefix + "6",
 94 self.lock_file_path_prefix + "1",
 95 self.lock_file_path_prefix + "4",
 96 self.lock_file_path_prefix + "3",
 97 ]
 98
 99 expected_file_list = [
 100 self.lock_file_path_prefix + "1",
 101 self.lock_file_path_prefix + "3",
 102 self.lock_file_path_prefix + "4",
 103 self.lock_file_path_prefix + "6",
 104 ]
 105
 106 for file_name in lock_file_list:
 107 open(file_name, 'w')
 108
 109 self.assertEqual(self.http_lock_obj._lock_file_list(), expected_file_list)
 110
 111 for file_name in lock_file_list:
 112 os.unlink(file_name)
0