Attachment 342318 Details for Bug 186443 – Patch

[patch] Patch

bug-186443-20180608165021.patch (text/plain), 6.16 KB, created by Leo Balter on 2018-06-08 13:50:22 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Leo Balter

Created: 2018-06-08 13:50:22 PDT

Size: 6.16 KB

patch

obsolete

>Subversion Revision: 232619
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index b785bb2d42e07ff8f59faf994fed5bcfd0a33a6f..bc4b33e75e263c4c21ed2d14d88b8e1e8f747df0 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,15 @@
>+2018-06-08  Leo Balter  <leonardo.balter@gmail.com>
>+
>+        Test262-Runner: Improve files queue to optimize CPU usage/balancing
>+        https://bugs.webkit.org/show_bug.cgi?id=186443
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        This patch creates an option to set a max number of files per child process in each run. The default value is 96. This allows optimizations for each running target, setting custom amount of processes and how many files received by each. This allows some load balancing to avoid a single process to be stacked up for too long. While running a single test file is not worth enough of a single child process, this new option allow the user to define chunks of tests by their size.
>+        * Scripts/test262/Runner.pm:
>+        (processCLI):
>+        (main):
>+
> 2018-06-07  Yusuke Suzuki  <utatane.tea@gmail.com>
> 
>         [WTF] Add WorkerPool
>diff --git a/Tools/Scripts/test262/Runner.pm b/Tools/Scripts/test262/Runner.pm
>index dab4b1fdb9223f6311fc15094ecd5258166096b3..24bcbd1258497c17cc5faa0938eeb799f358bd31 100755
>--- a/Tools/Scripts/test262/Runner.pm
>+++ b/Tools/Scripts/test262/Runner.pm
>@@ -37,6 +37,7 @@ use File::Temp qw(tempfile tempdir);
> use File::Spec::Functions qw(abs2rel);
> use File::Basename qw(dirname);
> use File::Path qw(mkpath);
>+use List::Util qw(min max);
> use Cwd qw(abs_path);
> use FindBin;
> use Env qw(DYLD_FRAMEWORK_PATH);
>@@ -74,7 +75,7 @@ if (eval {require Pod::Usage; 1;}) {
> }
> 
> # Commandline settings
>-my $max_process;
>+my $maxProcesses;
> my @cliTestDirs;
> my $verbose;
> my $JSC;
>@@ -90,6 +91,7 @@ my $failingOnly;
> my $latestImport;
> my $runningAllTests;
> my $timeout;
>+my $maxFiles;
> 
> my $expectationsFile = abs_path("$Bin/../../../JSTests/test262/expectations.yaml");
> my $configFile = abs_path("$Bin/../../../JSTests/test262/config.yaml");
>@@ -128,7 +130,8 @@ sub processCLI {
>         'j|jsc=s' => \$JSC,
>         't|t262=s' => \$test262Dir,
>         'o|test-only=s@' => \@cliTestDirs,
>-        'p|child-processes=i' => \$max_process,
>+        'p|child-processes=i' => \$maxProcesses,
>+        'm|max-files=i' => \$maxFiles,
>         'h|help' => \$help,
>         'release' => \$release,
>         'v|verbose' => \$verbose,
>@@ -234,12 +237,14 @@ sub processCLI {
>         %filterFeatures = map { $_ => 1 } @features;
>     }
> 
>-    $max_process ||= getProcesses();
>+    $maxProcesses ||= getProcesses();
>+    $maxFiles ||= 96;
> 
>     print "\n-------------------------Settings------------------------\n"
>         . "Test262 Dir: $test262Dir\n"
>         . "JSC: $JSC\n"
>-        . "Child Processes: $max_process\n";
>+        . "Child Processes: $maxProcesses\n"
>+        . "Max files per queue: $maxFiles\n";
> 
>     print "Test timeout: $timeout\n" if $timeout;
>     print "DYLD_FRAMEWORK_PATH: $DYLD_FRAMEWORK_PATH\n" if $DYLD_FRAMEWORK_PATH;
>@@ -289,52 +294,49 @@ sub main {
>         }
>     }
> 
>-    # If we are processing many files, fork process
>-    if (scalar @files > $max_process * 5) {
>+    my $pm = Parallel::ForkManager->new($maxProcesses);
> 
>-        # Make temporary files to record results
>-        my @resultsfhs;
>-        for (my $i = 0; $i <= $max_process-1; $i++) {
>-            my ($fh, $filename) = getTempFile();
>-            $resultsfhs[$i] = $fh;
>-        }
>-
>-        my $pm = Parallel::ForkManager->new($max_process);
>-        my $filesperprocess = int(scalar @files / $max_process);
>+    # How many processes are required
>+    my $reqProcesses = ceil(scalar @files / $maxFiles);
> 
>-        FILES:
>-        for (my $i = 0; $i <= $max_process-1; $i++) {
>-            $pm->start and next FILES; # do the fork
>-            srand(time ^ $$); # Creates a new seed for each fork
>+    # Make temporary files to record results
>+    my @resultsfhs;
>+    for (my $i = 0; $i < $reqProcesses; $i++) {
>+        my ($fh, $filename) = getTempFile();
>+        $resultsfhs[$i] = $fh;
>+    }
> 
>-            my $first = $filesperprocess * $i;
>-            my $last = $i == $max_process-1 ? scalar @files : $filesperprocess * ($i+1);
>+    FILES:
>+    for (my $i = 0; $i < $reqProcesses; $i++) {
>+        $pm->start and next FILES; # do the fork
>+        srand(time ^ $$); # Creates a new seed for each fork
> 
>-            for (my $j = $first; $j < $last; $j++) {
>-                processFile($files[$j], $resultsfhs[$i]);
>-            };
>+        my $first = $maxFiles * $i;
>+        my $last = min($maxFiles * ($i+1), scalar @files);
> 
>-            $pm->finish; # do the exit in the child process
>-        };
>+        # End loop if it's not necessary to create more processes
>+        if ($first > $last) {
>+            $pm->finish;
>+            last;
>+        }
> 
>-        $pm->wait_all_children;
>+        print '.';
>+        # printf("%-5s %-5s %-3s %s\n", $first, $last, $i, scalar @files);
> 
>-        # Read results from file into @results and close
>-        for (my $i = 0; $i <= $max_process-1; $i++) {
>-            seek($resultsfhs[$i], 0, 0);
>-            push @results, LoadFile($resultsfhs[$i]);
>-            close $resultsfhs[$i];
>+        for (my $j = $first; $j < $last; $j++) {
>+            processFile($files[$j], $resultsfhs[$i]);
>         }
>-    }
>-    # Otherwising, running sequentially is fine
>-    else {
>-        my ($resfh, $resfilename) = getTempFile();
>-        foreach my $file (@files) {
>-            processFile($file, $resfh);
>-        };
>-        seek($resfh, 0, 0);
>-        @results = LoadFile($resfh);
>-        close $resfh;
>+
>+        $pm->finish; # do the exit in the child process
>+    };
>+
>+    $pm->wait_all_children;
>+
>+    # Read results from file into @results and close
>+    for (my $i = 0; $i < $reqProcesses; $i++) {
>+        seek($resultsfhs[$i], 0, 0);
>+        push @results, LoadFile($resultsfhs[$i]);
>+        close $resultsfhs[$i];
>     }
> 
>     close $deffh;
>@@ -910,7 +912,11 @@ Print a brief help message and exits.
> 
> =item B<--child-processes, -p>
> 
>-Specify number of child processes.
>+Specify the number of child processes.
>+
>+=item B<--max-files, -m>
>+
>+Specify the max number of files per each child process.
> 
> =item B<--t262, -t>
>

Actions: View | Formatted Diff | Diff

Attachments on bug 186443: 342318 | 342324 | 342601 | 342602 | 342607 | 342646 | 342681 | 342699 | 342745 | 342782 | 342934 | 342951