Tools/ChangeLog

2012-06-18  Dirk Pranke  <dpranke@chromium.org>

        NRWT should not take memory used as disk cache into account when deciding how many processes to launch

        https://bugs.webkit.org/show_bug.cgi?id=81379

        Reviewed by NOBODY (OOPS!).

        The 'free memory' calculation we were running on the mac seemed

        to underestimate how many children we can run in parallel, and

        it was complex. This patch replaces that calculation with a

        simpler one that reserves 2GB for overhead and assumes 256MB per

        DRT/WTR, so if we had 4GB of RAM we can run up to 8 DRTs.

        Also, there was a bug where we were truncating the memory

        installed on the machine to 4GB by casting to an int instead of

        a long; this was probably the source of some of the earlier

        problems when using total memory.

        This patch also removes the no-longer-needed restrictions on the

        number of workers on beefy Lion boxes for both Apple Mac and Chromium

        Mac; we should now use all of the cores by default.

        The memory calculations have only been implemented on the mac;

        having the calculation in base.default_child_processes() was IMO

        misleading, and so this patch also moves the computation into

        the MacPort. I have not heard of the # of workers being an issue

        on any other ports, so this should be fine.

        * Scripts/webkitpy/common/system/platforminfo.py:

        (PlatformInfo.display_name):

        (PlatformInfo.total_bytes_memory):

        (PlatformInfo._win_version_tuple_from_cmd):

        * Scripts/webkitpy/common/system/platforminfo_unittest.py:

        (TestPlatformInfo.test_total_bytes_memory):

        * Scripts/webkitpy/layout_tests/port/base.py:

        (Port.default_child_processes):

        * Scripts/webkitpy/layout_tests/port/chromium_mac.py:

        (ChromiumMacPort.operating_system):

        * Scripts/webkitpy/layout_tests/port/mac.py:

        (MacPort.default_child_processes):

2012-06-18  Csaba Osztrogonác  <ossy@webkit.org>

        REGRESSION(r100558): NRWT should work without SVN or GIT

Tools/Scripts/webkitpy/common/system/platforminfo.py

@@class PlatformInfo(object):

        # Windows-2008ServerR2-6.1.7600

        return self._platform_module.platform()

    def free_bytes_memory(self):

        if self.is_mac():

            vm_stat_output = self._executive.run_command(["vm_stat"])

            free_bytes = self._compute_bytes_from_vm_stat_output("Pages free", vm_stat_output)

            # Per https://bugs.webkit.org/show_bug.cgi?id=74650 include inactive memory since the OS is lazy about freeing memory.

            free_bytes += self._compute_bytes_from_vm_stat_output("Pages inactive", vm_stat_output)

            return free_bytes

        return None

9384 def total_bytes_memory(self):

9485 if self.is_mac():

95 return int(self._executive.run_command(["sysctl", "-n", "hw.memsize"]))

86 return long(self._executive.run_command(["sysctl", "-n", "hw.memsize"]))

        return None

    def _determine_os_name(self, sys_platform):

@@class PlatformInfo(object):

        match_object = re.search(r'(?P<major>\d)\.(?P<minor>\d)\.(?P<build>\d+)', ver_output)

        assert match_object, 'cmd returned an unexpected version string: ' + ver_output

        return tuple(map(int, match_object.groups()))

    def _compute_bytes_from_vm_stat_output(self, label_text, vm_stat_output):

        page_size_match = re.search(r"page size of (\d+) bytes", vm_stat_output)

        free_pages_match = re.search(r"%s:\s+(\d+)." % label_text, vm_stat_output)

        # Fail hard if vmstat's output isn't what we expect.

        assert(page_size_match and free_pages_match)

        free_page_count = int(free_pages_match.group(1))

        page_size = int(page_size_match.group(1))

        return free_page_count * page_size

Tools/Scripts/webkitpy/common/system/platforminfo_unittest.py

@@class TestPlatformInfo(unittest.TestCase):

        info = self.make_info(fake_sys('freebsd9'))

        self.assertEquals(info.total_bytes_memory(), None)

    def test_free_bytes_memory(self):

        vmstat_output = ("Mach Virtual Memory Statistics: (page size of 4096 bytes)\n"

                         "Pages free:                        1.\n"

                         "Pages inactive:                    1.\n")

        info = self.make_info(fake_sys('darwin'), fake_platform('10.6.3'), fake_executive(vmstat_output))

        self.assertEquals(info.free_bytes_memory(), 8192)

        info = self.make_info(fake_sys('win32', tuple([6, 1, 7600])))

        self.assertEquals(info.free_bytes_memory(), None)

        info = self.make_info(fake_sys('linux2'))

        self.assertEquals(info.free_bytes_memory(), None)

        info = self.make_info(fake_sys('freebsd9'))

        self.assertEquals(info.free_bytes_memory(), None)

if __name__ == '__main__':

    unittest.main()

Tools/Scripts/webkitpy/layout_tests/port/base.py

@@class Port(object):

    def default_child_processes(self):

        """Return the number of DumpRenderTree instances to use for this port."""

        cpu_count = self._executive.cpu_count()

        # Make sure we have enough ram to support that many instances:

        free_memory = self.host.platform.free_bytes_memory()

        if free_memory:

            bytes_per_drt = 200 * 1024 * 1024  # Assume each DRT needs 200MB to run.

            supportable_instances = max(free_memory / bytes_per_drt, 1)  # Always use one process, even if we don't have space for it.

            if supportable_instances < cpu_count:

                # FIXME: The Printer isn't initialized when this is called, so using _log would just show an unitialized logger error.

                print "This machine could support %s child processes, but only has enough memory for %s." % (cpu_count, supportable_instances)

            return min(supportable_instances, cpu_count)

        return cpu_count

157 return self._executive.cpu_count()

    def worker_startup_delay_secs(self):

        # FIXME: If we start workers up too quickly, DumpRenderTree appears

Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py

@@class ChromiumMacPort(chromium.ChromiumPort):

    def operating_system(self):

        return 'mac'

    def default_child_processes(self):

        # FIXME: As a temporary workaround while we figure out what's going

        # on with https://bugs.webkit.org/show_bug.cgi?id=83076, reduce by

        # half the # of workers we run by default on bigger machines.

        default_count = super(ChromiumMacPort, self).default_child_processes()

        if default_count >= 8:

            cpu_count = self._executive.cpu_count()

            return max(1, min(default_count, int(cpu_count / 2)))

        return default_count

    #

    # PROTECTED METHODS

    #

Tools/Scripts/webkitpy/layout_tests/port/mac.py

@@class MacPort(ApplePort):

        return self._version == "lion"

    def default_child_processes(self):

120 # FIXME: The Printer isn't initialized when this is called, so using _log would just show an unitialized logger error.

121

120122 if self.is_snowleopard():

121 ~~_log.warn(~~"Cannot run tests in parallel on Snow Leopard due to rdar://problem/10621525.")

123 print >> sys.stderr, "Cannot run tests in parallel on Snow Leopard due to rdar://problem/10621525."

122124 return 1

123125

        # FIXME: As a temporary workaround while we figure out what's going

        # on with https://bugs.webkit.org/show_bug.cgi?id=83076, reduce by

        # half the # of workers we run by default on bigger machines.

127126 default_count = super(MacPort, self).default_child_processes()

        if default_count >= 8:

            cpu_count = self._executive.cpu_count()

            return max(1, min(default_count, int(cpu_count / 2)))

        return default_count

        # Make sure we have enough ram to support that many instances:

        total_memory = self.host.platform.total_bytes_memory()

        bytes_per_drt = 256 * 1024 * 1024  # Assume each DRT needs 256MB to run.

        overhead = 2048 * 1024 * 1024  # Assume we need 2GB free for the O/S

        supportable_instances = max((total_memory - overhead) / bytes_per_drt, 1)  # Always use one process, even if we don't have space for it.

        if supportable_instances < default_count:

            print >> sys.stderr, "This machine could support %s child processes, but only has enough memory for %s." % (default_count, supportable_instances)

        return min(supportable_instances, default_count)

    def _build_java_test_support(self):

        java_tests_path = self._filesystem.join(self.layout_tests_dir(), "java")