Attachment 342324 Details for Bug 186443 – Patch

[patch] Patch

bug-186443-20180608171017.patch (text/plain), 6.10 KB, created by Leo Balter on 2018-06-08 14:10:18 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Leo Balter

Created: 2018-06-08 14:10:18 PDT

Size: 6.10 KB

patch

obsolete

>Subversion Revision: 232619
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index b785bb2d42e07ff8f59faf994fed5bcfd0a33a6f..057349906b87bcdd7b2f2c0dd7c1b867ed8c4ba2 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,19 @@
>+2018-06-08  Leo Balter  <leonardo.balter@gmail.com>
>+
>+        Test262-Runner: Improve files queue to optimize CPU usage/balancing
>+        https://bugs.webkit.org/show_bug.cgi?id=186443
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        This patch creates an option to set a max number of files per child process in each run. The default value is 96.
>+        This allows optimizations for each running target, setting custom amount of processes and how many files received
>+        by each. This allows some load balancing to avoid a single process to be stacked up for too long. While running a
>+        single test file is not worth enough of a single child process, this new option allow the user to define chunks of
>+        tests by their size.
>+        * Scripts/test262/Runner.pm:
>+        (processCLI):
>+        (main):
>+
> 2018-06-07  Yusuke Suzuki  <utatane.tea@gmail.com>
> 
>         [WTF] Add WorkerPool
>diff --git a/Tools/Scripts/test262/Runner.pm b/Tools/Scripts/test262/Runner.pm
>index dab4b1fdb9223f6311fc15094ecd5258166096b3..73bd7112155e5b6700b7a209bacc87e32fe1b1a9 100755
>--- a/Tools/Scripts/test262/Runner.pm
>+++ b/Tools/Scripts/test262/Runner.pm
>@@ -37,6 +37,7 @@ use File::Temp qw(tempfile tempdir);
> use File::Spec::Functions qw(abs2rel);
> use File::Basename qw(dirname);
> use File::Path qw(mkpath);
>+use List::Util qw(min max);
> use Cwd qw(abs_path);
> use FindBin;
> use Env qw(DYLD_FRAMEWORK_PATH);
>@@ -74,7 +75,7 @@ if (eval {require Pod::Usage; 1;}) {
> }
> 
> # Commandline settings
>-my $max_process;
>+my $maxProcesses;
> my @cliTestDirs;
> my $verbose;
> my $JSC;
>@@ -90,6 +91,7 @@ my $failingOnly;
> my $latestImport;
> my $runningAllTests;
> my $timeout;
>+my $maxFiles;
> 
> my $expectationsFile = abs_path("$Bin/../../../JSTests/test262/expectations.yaml");
> my $configFile = abs_path("$Bin/../../../JSTests/test262/config.yaml");
>@@ -128,7 +130,8 @@ sub processCLI {
>         'j|jsc=s' => \$JSC,
>         't|t262=s' => \$test262Dir,
>         'o|test-only=s@' => \@cliTestDirs,
>-        'p|child-processes=i' => \$max_process,
>+        'p|child-processes=i' => \$maxProcesses,
>+        'm|max-files=i' => \$maxFiles,
>         'h|help' => \$help,
>         'release' => \$release,
>         'v|verbose' => \$verbose,
>@@ -234,12 +237,14 @@ sub processCLI {
>         %filterFeatures = map { $_ => 1 } @features;
>     }
> 
>-    $max_process ||= getProcesses();
>+    $maxProcesses ||= getProcesses();
>+    $maxFiles ||= 96;
> 
>     print "\n-------------------------Settings------------------------\n"
>         . "Test262 Dir: $test262Dir\n"
>         . "JSC: $JSC\n"
>-        . "Child Processes: $max_process\n";
>+        . "Child Processes: $maxProcesses\n"
>+        . "Max files per queue: $maxFiles\n";
> 
>     print "Test timeout: $timeout\n" if $timeout;
>     print "DYLD_FRAMEWORK_PATH: $DYLD_FRAMEWORK_PATH\n" if $DYLD_FRAMEWORK_PATH;
>@@ -289,52 +294,46 @@ sub main {
>         }
>     }
> 
>-    # If we are processing many files, fork process
>-    if (scalar @files > $max_process * 5) {
>+    my $pm = Parallel::ForkManager->new($maxProcesses);
> 
>-        # Make temporary files to record results
>-        my @resultsfhs;
>-        for (my $i = 0; $i <= $max_process-1; $i++) {
>-            my ($fh, $filename) = getTempFile();
>-            $resultsfhs[$i] = $fh;
>-        }
>+    # How many processes are required
>+    my $reqProcesses = ceil(scalar @files / $maxFiles);
> 
>-        my $pm = Parallel::ForkManager->new($max_process);
>-        my $filesperprocess = int(scalar @files / $max_process);
>+    # Make temporary files to record results
>+    my @resultsfhs;
>+    for (my $i = 0; $i < $reqProcesses; $i++) {
>+        my ($fh, $filename) = getTempFile();
>+        $resultsfhs[$i] = $fh;
>+    }
> 
>-        FILES:
>-        for (my $i = 0; $i <= $max_process-1; $i++) {
>-            $pm->start and next FILES; # do the fork
>-            srand(time ^ $$); # Creates a new seed for each fork
>+    FILES:
>+    for (my $i = 0; $i < $reqProcesses; $i++) {
>+        $pm->start and next FILES; # do the fork
>+        srand(time ^ $$); # Creates a new seed for each fork
> 
>-            my $first = $filesperprocess * $i;
>-            my $last = $i == $max_process-1 ? scalar @files : $filesperprocess * ($i+1);
>+        my $first = $maxFiles * $i;
>+        my $last = min($maxFiles * ($i+1), scalar @files);
> 
>-            for (my $j = $first; $j < $last; $j++) {
>-                processFile($files[$j], $resultsfhs[$i]);
>-            };
>+        # End loop if it's not necessary to create more processes
>+        if ($first > $last) {
>+            $pm->finish;
>+            last;
>+        }
> 
>-            $pm->finish; # do the exit in the child process
>-        };
>+        for (my $j = $first; $j < $last; $j++) {
>+            processFile($files[$j], $resultsfhs[$i]);
>+        }
> 
>-        $pm->wait_all_children;
>+        $pm->finish; # do the exit in the child process
>+    };
> 
>-        # Read results from file into @results and close
>-        for (my $i = 0; $i <= $max_process-1; $i++) {
>-            seek($resultsfhs[$i], 0, 0);
>-            push @results, LoadFile($resultsfhs[$i]);
>-            close $resultsfhs[$i];
>-        }
>-    }
>-    # Otherwising, running sequentially is fine
>-    else {
>-        my ($resfh, $resfilename) = getTempFile();
>-        foreach my $file (@files) {
>-            processFile($file, $resfh);
>-        };
>-        seek($resfh, 0, 0);
>-        @results = LoadFile($resfh);
>-        close $resfh;
>+    $pm->wait_all_children;
>+
>+    # Read results from file into @results and close
>+    for (my $i = 0; $i < $reqProcesses; $i++) {
>+        seek($resultsfhs[$i], 0, 0);
>+        push @results, LoadFile($resultsfhs[$i]);
>+        close $resultsfhs[$i];
>     }
> 
>     close $deffh;
>@@ -910,7 +909,11 @@ Print a brief help message and exits.
> 
> =item B<--child-processes, -p>
> 
>-Specify number of child processes.
>+Specify the number of child processes.
>+
>+=item B<--max-files, -m>
>+
>+Specify the max number of files per each child process.
> 
> =item B<--t262, -t>
>

Actions: View | Formatted Diff | Diff

Attachments on bug 186443: 342318 | 342324 | 342601 | 342602 | 342607 | 342646 | 342681 | 342699 | 342745 | 342782 | 342934 | 342951