Attachment 343103 Details for Bug 186790 – Patch

[patch] Patch

bug-186790-20180619175019.patch (text/plain), 13.40 KB, created by Leo Balter on 2018-06-19 14:50:21 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Leo Balter

Created: 2018-06-19 14:50:21 PDT

Size: 13.40 KB

patch

obsolete

>Subversion Revision: 232973
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index c369afeaab63a452c14a34a19d41aa2669302ca8..6b46ddc7d4b64b92d6de0466b14aeb4580c03d1c 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,16 @@
>+2018-06-18  Leo Balter  <leonardo.balter@gmail.com>
>+
>+        [test262-runner] Print results report to HTML
>+        https://bugs.webkit.org/show_bug.cgi?id=186790
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        After runing the tests, 2 html files are created with a list to failing tests and summaries.
>+        * Scripts/test262/Runner.pm:
>+        (main):
>+        (summarizeResults):
>+        (printHTML):
>+
> 2018-06-19  Daniel Bates  <dabates@apple.com>
> 
>         QueueStatusServer: "'NoneType' object has no attribute 'message'" in ReleasePatch.get()
>diff --git a/Tools/Scripts/test262/Runner.pm b/Tools/Scripts/test262/Runner.pm
>index 4d9e822d5b2dc15f0b32031062e5462c32f7edcd..ae521ba28705f851eab96b81e4caf4f0d567ffaa 100755
>--- a/Tools/Scripts/test262/Runner.pm
>+++ b/Tools/Scripts/test262/Runner.pm
>@@ -186,7 +186,6 @@ sub processCLI {
>         }
>     }
> 
>-
>     if ($JSC) {
>         $JSC = abs_path($JSC);
>         # Make sure the path and file jsc exist
>@@ -292,8 +291,11 @@ sub main {
> 
>     print $deffh getHarness(\@defaultHarnessFiles);
> 
>-    # If not commandline test path supplied, use the root directory of all tests.
>-    push(@cliTestDirs, 'test') if not @cliTestDirs;
>+    if (!@cliTestDirs) {
>+        # If not commandline test path supplied, use the root directory of all tests.
>+        push(@cliTestDirs, 'test') if not @cliTestDirs;
>+        $runningAllTests = 1;
>+    }
> 
>     if ($latestImport) {
>         @files = loadImportFile();
>@@ -301,7 +303,6 @@ sub main {
>         # If we only want to re-run failure, only run tests in results file
>         findAllFailing();
>     } else {
>-        $runningAllTests = 1;
>         # Otherwise, get all files from directory
>         foreach my $testsDir (@cliTestDirs) {
>             find(
>@@ -501,36 +502,38 @@ sub main {
>         print "---------------------------------------------------------\n";
>     }
> 
>-    print("\n");
>+    my $totalRun = scalar @results - $skipfilecount;
>+    print "\n$totalRun tests run\n";
>+    print "$skipfilecount test files skipped\n";
>+
>+    if (!$expect) {
>+        print "$failcount tests failed\n";
>+        print "$newpasscount tests newly pass\n" if $skippedOnly;
>+    } else {
>+        print "$failcount tests failed in total\n";
>+        print "$newfailcount tests newly fail\n";
>+        print "$newpasscount tests newly pass\n";
>+    }
> 
>     if ($saveExpectations) {
>         DumpFile($expectationsFile, \%failed);
>         print "Saved expectation file in: $expectationsFile\n";
>     }
>-    if ($runningAllTests) {
>-        if (! -e $resultsDir) {
>-            mkpath($resultsDir);
>-        }
>-        $resultsFile = abs_path("$resultsDir/results.yaml");
>-
>-        DumpFile($resultsFile, \@results);
>-        print "Saved all the results in $resultsFile\n";
>-        summarizeResults();
>+    if (! -e $resultsDir) {
>+        mkpath($resultsDir);
>     }
> 
>-    my $total = scalar @results - $skipfilecount;
>-    print "\n" . $total . " tests ran\n";
>+    $resultsFile = abs_path("$resultsDir/results.yaml");
> 
>-    if ( !$expect ) {
>-        print $failcount . " tests failed\n";
>-        print $newpasscount . " tests newly pass\n" if $skippedOnly;
>-    } else {
>-        print $failcount . " tests failed in total\n";
>-        print $newfailcount . " tests newly fail\n";
>-        print $newpasscount . " tests newly pass\n";
>-    }
>+    DumpFile($resultsFile, \@results);
>+    print "Saved all the results in $resultsFile\n";
>+
>+    my $styleCss = abs_path("$Bin/report.css");
>+    qx/cp $styleCss $resultsDir/;
>+    summarizeResults();
>+    printHTMLResults(\%failed, $totalRun, $failcount, $newfailcount, $skipfilecount);
> 
>-    print $skipfilecount . " test files skipped\n";
>+    print "See the summaries and results in the $resultsDir.\n\n";
> 
>     printf("Done in %.2f seconds!\n", time() - $startTime);
> 
>@@ -916,6 +919,7 @@ sub summarizeResults {
>     }
>     $summaryTxtFile = abs_path("$resultsDir/summary.txt");
>     $summaryFile = abs_path("$resultsDir/summary.yaml");
>+    my $summaryHTMLFile = abs_path("$resultsDir/summary.html");
> 
>     my %byfeature;
>     my %bypath;
>@@ -972,66 +976,137 @@ sub summarizeResults {
>     }
> 
>     open(my $sfh, '>', $summaryTxtFile) or die $!;
>+    open(my $htmlfh, '>', $summaryHTMLFile) or die $!;
>+
>+    print $htmlfh qq{<html><head>
>+        <title>Test262 Summaries</title>
>+        <link rel="stylesheet" href="report.css">
>+        </head>
>+        <body>
>+        <h1>Test262 Summaries</h1>
>+        <div class="visit">Visit <a href="index.html">the index</a> for a report of failures.</div>
>+        <h2>By Features</h2>
>+        <table class="summary-table">
>+            <thead>
>+                <th>Feature</th>
>+                <th>%</th>
>+                <th>Total</th>
>+                <th>Run</th>
>+                <th>Passed</th>
>+                <th>Failed</th>
>+                <th>Skipped</th>
>+                <th>Exec. time</th>
>+                <th>Avg. time</th>
>+            </thead>
>+            <tbody>};
>+
>+    print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RUN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FEATURE');
> 
>-    print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RAN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FOLDER');
>-    foreach my $key (sort keys %bypath) {
>-        my $totalFilesRan = $bypath{$key}->[0] + $bypath{$key}->[1];
>-        my $totalFiles = $totalFilesRan + $bypath{$key}->[2];
>+    foreach my $key (sort keys %byfeature) {
>+        my $totalFilesRun = $byfeature{$key}->[0] + $byfeature{$key}->[1];
>+        my $totalFiles = $totalFilesRun + $byfeature{$key}->[2];
> 
>-        my $per = sprintf("%.0f", ($bypath{$key}->[0] / $totalFiles) * 100) . "%";
>+        my $iper = ($byfeature{$key}->[0] / $totalFiles) * 100;
>+        my $per = sprintf("%.0f", $iper) . "%";
> 
>-        my $time = sprintf("%.1f", $bypath{$key}->[3]) . "s";
>+        my $time = sprintf("%.1f", $byfeature{$key}->[3]) . "s";
>         my $avgTime;
> 
>-        if ($totalFilesRan) {
>-            $avgTime = sprintf("%.2f", $bypath{$key}->[3] / $totalFilesRan) . "s";
>+        if ($totalFilesRun) {
>+            $avgTime = sprintf("%.2f", $byfeature{$key}->[3] / $totalFilesRun) . "s";
>         } else {
>             $avgTime = "0s";
>         }
> 
>         print $sfh sprintf("%-6s %-6s %-6s %-6d %-6d %-6d %-7s %-6s %s\n",
>                            $totalFiles,
>-                           $totalFilesRan,
>+                           $totalFilesRun,
>                            $per,
>-                           $bypath{$key}->[0],
>-                           $bypath{$key}->[1],
>-                           $bypath{$key}->[2],
>+                           $byfeature{$key}->[0],
>+                           $byfeature{$key}->[1],
>+                           $byfeature{$key}->[2],
>                            $time,
>                            $avgTime,
>                            $key);
>-    }
> 
>-    print $sfh "\n\n";
>-    print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RAN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FEATURE');
>+        print $htmlfh qq{
>+            <tr class="per-$iper">
>+                <td>$key</td>
>+                <td>$per</td>
>+                <td>$totalFiles</td>
>+                <td>$totalFilesRun</td>
>+                <td>$byfeature{$key}->[0]</td>
>+                <td>$byfeature{$key}->[1]</td>
>+                <td>$byfeature{$key}->[2]</td>
>+                <td>$time</td>
>+                <td>$avgTime</td>
>+            </tr>};
>+    }
> 
>-    foreach my $key (sort keys %byfeature) {
>-        my $totalFilesRan = $byfeature{$key}->[0] + $byfeature{$key}->[1];
>-        my $totalFiles = $totalFilesRan + $byfeature{$key}->[2];
>+    print $htmlfh qq{</tbody></table>
>+        <h2>By Path</h2>
>+        <table class="summary-table">
>+            <thead>
>+                <th>Folder</th>
>+                <th>%</th>
>+                <th>Total</th>
>+                <th>Run</th>
>+                <th>Passed</th>
>+                <th>Failed</th>
>+                <th>Skipped</th>
>+                <th>Exec. time</th>
>+                <th>Avg. time</th>
>+            </thead>
>+            <tbody>};
>+
>+    print $sfh sprintf("\n\n%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RUN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FOLDER'); 
>+    foreach my $key (sort keys %bypath) {
>+        my $totalFilesRun = $bypath{$key}->[0] + $bypath{$key}->[1];
>+        my $totalFiles = $totalFilesRun + $bypath{$key}->[2];
> 
>-        my $per = sprintf("%.0f", ($byfeature{$key}->[0] / $totalFiles) * 100) . "%";
>+        my $iper = ($bypath{$key}->[0] / $totalFiles) * 100;
>+        my $per = sprintf("%.0f", $iper) . "%";
> 
>-        my $time = sprintf("%.1f", $byfeature{$key}->[3]) . "s";
>+        my $time = sprintf("%.1f", $bypath{$key}->[3]) . "s";
>         my $avgTime;
> 
>-        if ($totalFilesRan) {
>-            $avgTime = sprintf("%.2f", $byfeature{$key}->[3] / $totalFilesRan) . "s";
>+        if ($totalFilesRun) {
>+            $avgTime = sprintf("%.2f", $bypath{$key}->[3] / $totalFilesRun) . "s";
>         } else {
>             $avgTime = "0s";
>         }
> 
>         print $sfh sprintf("%-6s %-6s %-6s %-6d %-6d %-6d %-7s %-6s %s\n",
>                            $totalFiles,
>-                           $totalFilesRan,
>+                           $totalFilesRun,
>                            $per,
>-                           $byfeature{$key}->[0],
>-                           $byfeature{$key}->[1],
>-                           $byfeature{$key}->[2],
>+                           $bypath{$key}->[0],
>+                           $bypath{$key}->[1],
>+                           $bypath{$key}->[2],
>                            $time,
>                            $avgTime,
>                            $key);
>+
>+        print $htmlfh qq{
>+            <tr class="per-$iper">
>+                <td>$key</td>
>+                <td>$per</td>
>+                <td>$totalFiles</td>
>+                <td>$totalFilesRun</td>
>+                <td>$bypath{$key}->[0]</td>
>+                <td>$bypath{$key}->[1]</td>
>+                <td>$bypath{$key}->[2]</td>
>+                <td>$time</td>
>+                <td>$avgTime</td>
>+            </tr>};
>     }
> 
>+    print $htmlfh qq{</tbody></table>
>+        <div class="visit">Visit <a href="index.html">the index</a> for a report of failures.</div>
>+        </body></html>};
>+
>     close($sfh);
>+    close($htmlfh);
> 
>     my %resultsyaml = (
>         byFolder => \%bypath,
>@@ -1039,8 +1114,6 @@ sub summarizeResults {
>     );
> 
>     DumpFile($summaryFile, \%resultsyaml);
>-
>-    print "See summarized results in $summaryTxtFile\n";
> }
> 
> sub findAllFailing {
>@@ -1057,6 +1130,57 @@ sub findAllFailing {
>     @files = map { qq($test262Dir/$_) } keys %filedictionary;
> }
> 
>+sub printHTMLResults {
>+    my %failed = %{shift()};
>+    my ($total, $failcount, $newfailcount, $skipcount) = @_;
>+
>+    # Create test262-results folder if it does not exits
>+    if (! -e $resultsDir) {
>+        mkpath($resultsDir);
>+    }
>+
>+    my $indexHTML = abs_path("$resultsDir/index.html");
>+    open(my $htmlfh, '>', $indexHTML) or die $!;
>+
>+    print $htmlfh qq{<html><head>
>+        <title>Test262 Results</title>
>+        <link rel="stylesheet" href="report.css">
>+        </head>
>+        <body>
>+        <h1>Test262 Results</h1>
>+        <div class="visit">Visit <a href="summary.html">the summary</a> for statistics.</div>};
>+
>+    print $htmlfh qq{<h2>Stats</h2><ul>};
>+
>+    {
>+        my $failedFiles = scalar (keys %failed);
>+        my $totalPlus = $total + $skipcount;
>+        print $htmlfh qq{
>+            <li>$total test files run from $totalPlus files, $skipcount skipped test files</li>
>+            <li>$failcount failures from $failedFiles distinct files, $newfailcount new failures</li>
>+        };
>+    }
>+
>+    print $htmlfh qq{</ul><h2>Failures</h2><ul>};
>+
>+    while (my ($path, $scenarios) = each %failed) {
>+        print $htmlfh qq{<li class="list-item">
>+            <label for="$path" class="expander-control">$path</label>
>+            <input type="checkbox" id="$path" class="expander">
>+            <ul class="expand">};
>+        while (my ($scenario, $value) = each %{$scenarios}) {
>+            print $htmlfh qq{<li>$scenario: $value</li>};
>+        }
>+        print $htmlfh qq{</ul></li>}
>+    }
>+
>+    print $htmlfh qq{</ul>
>+    <div class="visit">Visit <a href="summary.html">the summary</a> for statistics.</div>
>+    </body></html>};
>+
>+    close $htmlfh;
>+}
>+
> __END__
> 
> =head1 DESCRIPTION
>diff --git a/Tools/Scripts/test262/report.css b/Tools/Scripts/test262/report.css
>new file mode 100644
>index 0000000000000000000000000000000000000000..911f473f785ebdc312ab211e0e835693dddabe3f
>--- /dev/null
>+++ b/Tools/Scripts/test262/report.css
>@@ -0,0 +1,43 @@
>+html {
>+    line-height: 1.3em;
>+}
>+
>+.visit {
>+    margin: 1em 0;
>+}
>+
>+.expander, .expand {
>+    display: none;
>+}
>+
>+.expander:checked + .expand {
>+    display: block;
>+}
>+
>+.expander-control {
>+    text-decoration: underline;
>+    cursor: pointer;
>+}
>+
>+.list-item {
>+    line-height: 1.5em;
>+}
>+
>+.summary-table {
>+    border-collapse: collapse;
>+}
>+
>+.summary-table th, .summary-table td{
>+    text-align: left;
>+    padding: 0 .4em;
>+    line-height: 2em;
>+    border: 1px solid #333;
>+}
>+
>+.per-100 {
>+    background-color: #e8ffde;
>+}
>+
>+.per-0 {
>+    background-color: #fac8c8;
>+}
>\ No newline at end of file

Actions: View | Formatted Diff | Diff

Attachments on bug 186790: 342975 | 342994 | 343103 | 343109