WebKit Bugzilla
Attachment 343109 Details for
Bug 186790
: [test262-runner] Print results report to HTML
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
Remember
[x]
|
Forgot Password
Login:
[x]
[patch]
Patch
bug-186790-20180619180908.patch (text/plain), 13.50 KB, created by
Leo Balter
on 2018-06-19 15:09:09 PDT
(
hide
)
Description:
Patch
Filename:
MIME Type:
Creator:
Leo Balter
Created:
2018-06-19 15:09:09 PDT
Size:
13.50 KB
patch
obsolete
>Subversion Revision: 232973 >diff --git a/Tools/ChangeLog b/Tools/ChangeLog >index c369afeaab63a452c14a34a19d41aa2669302ca8..6b46ddc7d4b64b92d6de0466b14aeb4580c03d1c 100644 >--- a/Tools/ChangeLog >+++ b/Tools/ChangeLog >@@ -1,3 +1,16 @@ >+2018-06-18 Leo Balter <leonardo.balter@gmail.com> >+ >+ [test262-runner] Print results report to HTML >+ https://bugs.webkit.org/show_bug.cgi?id=186790 >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ After runing the tests, 2 html files are created with a list to failing tests and summaries. >+ * Scripts/test262/Runner.pm: >+ (main): >+ (summarizeResults): >+ (printHTML): >+ > 2018-06-19 Daniel Bates <dabates@apple.com> > > QueueStatusServer: "'NoneType' object has no attribute 'message'" in ReleasePatch.get() >diff --git a/Tools/Scripts/test262/Runner.pm b/Tools/Scripts/test262/Runner.pm >index 4d9e822d5b2dc15f0b32031062e5462c32f7edcd..ae521ba28705f851eab96b81e4caf4f0d567ffaa 100755 >--- a/Tools/Scripts/test262/Runner.pm >+++ b/Tools/Scripts/test262/Runner.pm >@@ -186,7 +186,6 @@ sub processCLI { > } > } > >- > if ($JSC) { > $JSC = abs_path($JSC); > # Make sure the path and file jsc exist >@@ -292,8 +291,11 @@ sub main { > > print $deffh getHarness(\@defaultHarnessFiles); > >- # If not commandline test path supplied, use the root directory of all tests. >- push(@cliTestDirs, 'test') if not @cliTestDirs; >+ if (!@cliTestDirs) { >+ # If not commandline test path supplied, use the root directory of all tests. >+ push(@cliTestDirs, 'test') if not @cliTestDirs; >+ $runningAllTests = 1; >+ } > > if ($latestImport) { > @files = loadImportFile(); >@@ -301,7 +303,6 @@ sub main { > # If we only want to re-run failure, only run tests in results file > findAllFailing(); > } else { >- $runningAllTests = 1; > # Otherwise, get all files from directory > foreach my $testsDir (@cliTestDirs) { > find( >@@ -501,36 +502,38 @@ sub main { > print "---------------------------------------------------------\n"; > } > >- print("\n"); >+ my $totalRun = scalar @results - $skipfilecount; >+ print "\n$totalRun tests run\n"; >+ print "$skipfilecount test files skipped\n"; >+ >+ if (!$expect) { >+ print "$failcount tests failed\n"; >+ print "$newpasscount tests newly pass\n" if $skippedOnly; >+ } else { >+ print "$failcount tests failed in total\n"; >+ print "$newfailcount tests newly fail\n"; >+ print "$newpasscount tests newly pass\n"; >+ } > > if ($saveExpectations) { > DumpFile($expectationsFile, \%failed); > print "Saved expectation file in: $expectationsFile\n"; > } >- if ($runningAllTests) { >- if (! -e $resultsDir) { >- mkpath($resultsDir); >- } >- $resultsFile = abs_path("$resultsDir/results.yaml"); >- >- DumpFile($resultsFile, \@results); >- print "Saved all the results in $resultsFile\n"; >- summarizeResults(); >+ if (! -e $resultsDir) { >+ mkpath($resultsDir); > } > >- my $total = scalar @results - $skipfilecount; >- print "\n" . $total . " tests ran\n"; >+ $resultsFile = abs_path("$resultsDir/results.yaml"); > >- if ( !$expect ) { >- print $failcount . " tests failed\n"; >- print $newpasscount . " tests newly pass\n" if $skippedOnly; >- } else { >- print $failcount . " tests failed in total\n"; >- print $newfailcount . " tests newly fail\n"; >- print $newpasscount . " tests newly pass\n"; >- } >+ DumpFile($resultsFile, \@results); >+ print "Saved all the results in $resultsFile\n"; >+ >+ my $styleCss = abs_path("$Bin/report.css"); >+ qx/cp $styleCss $resultsDir/; >+ summarizeResults(); >+ printHTMLResults(\%failed, $totalRun, $failcount, $newfailcount, $skipfilecount); > >- print $skipfilecount . " test files skipped\n"; >+ print "See the summaries and results in the $resultsDir.\n\n"; > > printf("Done in %.2f seconds!\n", time() - $startTime); > >@@ -916,6 +919,7 @@ sub summarizeResults { > } > $summaryTxtFile = abs_path("$resultsDir/summary.txt"); > $summaryFile = abs_path("$resultsDir/summary.yaml"); >+ my $summaryHTMLFile = abs_path("$resultsDir/summary.html"); > > my %byfeature; > my %bypath; >@@ -972,66 +976,137 @@ sub summarizeResults { > } > > open(my $sfh, '>', $summaryTxtFile) or die $!; >+ open(my $htmlfh, '>', $summaryHTMLFile) or die $!; >+ >+ print $htmlfh qq{<html><head> >+ <title>Test262 Summaries</title> >+ <link rel="stylesheet" href="report.css"> >+ </head> >+ <body> >+ <h1>Test262 Summaries</h1> >+ <div class="visit">Visit <a href="index.html">the index</a> for a report of failures.</div> >+ <h2>By Features</h2> >+ <table class="summary-table"> >+ <thead> >+ <th>Feature</th> >+ <th>%</th> >+ <th>Total</th> >+ <th>Run</th> >+ <th>Passed</th> >+ <th>Failed</th> >+ <th>Skipped</th> >+ <th>Exec. time</th> >+ <th>Avg. time</th> >+ </thead> >+ <tbody>}; >+ >+ print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RUN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FEATURE'); > >- print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RAN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FOLDER'); >- foreach my $key (sort keys %bypath) { >- my $totalFilesRan = $bypath{$key}->[0] + $bypath{$key}->[1]; >- my $totalFiles = $totalFilesRan + $bypath{$key}->[2]; >+ foreach my $key (sort keys %byfeature) { >+ my $totalFilesRun = $byfeature{$key}->[0] + $byfeature{$key}->[1]; >+ my $totalFiles = $totalFilesRun + $byfeature{$key}->[2]; > >- my $per = sprintf("%.0f", ($bypath{$key}->[0] / $totalFiles) * 100) . "%"; >+ my $iper = ($byfeature{$key}->[0] / $totalFiles) * 100; >+ my $per = sprintf("%.0f", $iper) . "%"; > >- my $time = sprintf("%.1f", $bypath{$key}->[3]) . "s"; >+ my $time = sprintf("%.1f", $byfeature{$key}->[3]) . "s"; > my $avgTime; > >- if ($totalFilesRan) { >- $avgTime = sprintf("%.2f", $bypath{$key}->[3] / $totalFilesRan) . "s"; >+ if ($totalFilesRun) { >+ $avgTime = sprintf("%.2f", $byfeature{$key}->[3] / $totalFilesRun) . "s"; > } else { > $avgTime = "0s"; > } > > print $sfh sprintf("%-6s %-6s %-6s %-6d %-6d %-6d %-7s %-6s %s\n", > $totalFiles, >- $totalFilesRan, >+ $totalFilesRun, > $per, >- $bypath{$key}->[0], >- $bypath{$key}->[1], >- $bypath{$key}->[2], >+ $byfeature{$key}->[0], >+ $byfeature{$key}->[1], >+ $byfeature{$key}->[2], > $time, > $avgTime, > $key); >- } > >- print $sfh "\n\n"; >- print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RAN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FEATURE'); >+ print $htmlfh qq{ >+ <tr class="per-$iper"> >+ <td>$key</td> >+ <td>$per</td> >+ <td>$totalFiles</td> >+ <td>$totalFilesRun</td> >+ <td>$byfeature{$key}->[0]</td> >+ <td>$byfeature{$key}->[1]</td> >+ <td>$byfeature{$key}->[2]</td> >+ <td>$time</td> >+ <td>$avgTime</td> >+ </tr>}; >+ } > >- foreach my $key (sort keys %byfeature) { >- my $totalFilesRan = $byfeature{$key}->[0] + $byfeature{$key}->[1]; >- my $totalFiles = $totalFilesRan + $byfeature{$key}->[2]; >+ print $htmlfh qq{</tbody></table> >+ <h2>By Path</h2> >+ <table class="summary-table"> >+ <thead> >+ <th>Folder</th> >+ <th>%</th> >+ <th>Total</th> >+ <th>Run</th> >+ <th>Passed</th> >+ <th>Failed</th> >+ <th>Skipped</th> >+ <th>Exec. time</th> >+ <th>Avg. time</th> >+ </thead> >+ <tbody>}; >+ >+ print $sfh sprintf("\n\n%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RUN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FOLDER'); >+ foreach my $key (sort keys %bypath) { >+ my $totalFilesRun = $bypath{$key}->[0] + $bypath{$key}->[1]; >+ my $totalFiles = $totalFilesRun + $bypath{$key}->[2]; > >- my $per = sprintf("%.0f", ($byfeature{$key}->[0] / $totalFiles) * 100) . "%"; >+ my $iper = ($bypath{$key}->[0] / $totalFiles) * 100; >+ my $per = sprintf("%.0f", $iper) . "%"; > >- my $time = sprintf("%.1f", $byfeature{$key}->[3]) . "s"; >+ my $time = sprintf("%.1f", $bypath{$key}->[3]) . "s"; > my $avgTime; > >- if ($totalFilesRan) { >- $avgTime = sprintf("%.2f", $byfeature{$key}->[3] / $totalFilesRan) . "s"; >+ if ($totalFilesRun) { >+ $avgTime = sprintf("%.2f", $bypath{$key}->[3] / $totalFilesRun) . "s"; > } else { > $avgTime = "0s"; > } > > print $sfh sprintf("%-6s %-6s %-6s %-6d %-6d %-6d %-7s %-6s %s\n", > $totalFiles, >- $totalFilesRan, >+ $totalFilesRun, > $per, >- $byfeature{$key}->[0], >- $byfeature{$key}->[1], >- $byfeature{$key}->[2], >+ $bypath{$key}->[0], >+ $bypath{$key}->[1], >+ $bypath{$key}->[2], > $time, > $avgTime, > $key); >+ >+ print $htmlfh qq{ >+ <tr class="per-$iper"> >+ <td>$key</td> >+ <td>$per</td> >+ <td>$totalFiles</td> >+ <td>$totalFilesRun</td> >+ <td>$bypath{$key}->[0]</td> >+ <td>$bypath{$key}->[1]</td> >+ <td>$bypath{$key}->[2]</td> >+ <td>$time</td> >+ <td>$avgTime</td> >+ </tr>}; > } > >+ print $htmlfh qq{</tbody></table> >+ <div class="visit">Visit <a href="index.html">the index</a> for a report of failures.</div> >+ </body></html>}; >+ > close($sfh); >+ close($htmlfh); > > my %resultsyaml = ( > byFolder => \%bypath, >@@ -1039,8 +1114,6 @@ sub summarizeResults { > ); > > DumpFile($summaryFile, \%resultsyaml); >- >- print "See summarized results in $summaryTxtFile\n"; > } > > sub findAllFailing { >@@ -1057,6 +1130,57 @@ sub findAllFailing { > @files = map { qq($test262Dir/$_) } keys %filedictionary; > } > >+sub printHTMLResults { >+ my %failed = %{shift()}; >+ my ($total, $failcount, $newfailcount, $skipcount) = @_; >+ >+ # Create test262-results folder if it does not exits >+ if (! -e $resultsDir) { >+ mkpath($resultsDir); >+ } >+ >+ my $indexHTML = abs_path("$resultsDir/index.html"); >+ open(my $htmlfh, '>', $indexHTML) or die $!; >+ >+ print $htmlfh qq{<html><head> >+ <title>Test262 Results</title> >+ <link rel="stylesheet" href="report.css"> >+ </head> >+ <body> >+ <h1>Test262 Results</h1> >+ <div class="visit">Visit <a href="summary.html">the summary</a> for statistics.</div>}; >+ >+ print $htmlfh qq{<h2>Stats</h2><ul>}; >+ >+ { >+ my $failedFiles = scalar (keys %failed); >+ my $totalPlus = $total + $skipcount; >+ print $htmlfh qq{ >+ <li>$total test files run from $totalPlus files, $skipcount skipped test files</li> >+ <li>$failcount failures from $failedFiles distinct files, $newfailcount new failures</li> >+ }; >+ } >+ >+ print $htmlfh qq{</ul><h2>Failures</h2><ul>}; >+ >+ while (my ($path, $scenarios) = each %failed) { >+ print $htmlfh qq{<li class="list-item"> >+ <label for="$path" class="expander-control">$path</label> >+ <input type="checkbox" id="$path" class="expander"> >+ <ul class="expand">}; >+ while (my ($scenario, $value) = each %{$scenarios}) { >+ print $htmlfh qq{<li>$scenario: $value</li>}; >+ } >+ print $htmlfh qq{</ul></li>} >+ } >+ >+ print $htmlfh qq{</ul> >+ <div class="visit">Visit <a href="summary.html">the summary</a> for statistics.</div> >+ </body></html>}; >+ >+ close $htmlfh; >+} >+ > __END__ > > =head1 DESCRIPTION >diff --git a/Tools/Scripts/test262/report.css b/Tools/Scripts/test262/report.css >new file mode 100644 >index 0000000000000000000000000000000000000000..30edebe30e7bd2ec8b57f4f6a7757934c670cf52 >--- /dev/null >+++ b/Tools/Scripts/test262/report.css >@@ -0,0 +1,47 @@ >+html { >+ line-height: 1.3em; >+} >+ >+.visit { >+ margin: 1em 0; >+} >+ >+.expander, .expand { >+ display: none; >+} >+ >+.expander:checked + .expand { >+ display: block; >+} >+ >+.expander-control { >+ text-decoration: underline; >+ cursor: pointer; >+} >+ >+.list-item { >+ line-height: 1.5em; >+} >+ >+.summary-table { >+ border-collapse: collapse; >+} >+ >+.summary-table th, .summary-table td{ >+ text-align: left; >+ padding: 0 .4em; >+ line-height: 2em; >+ border: 1px solid #333; >+} >+ >+.summary-table tr:nth-child(even) td { >+ box-shadow: inset 100em 1em rgba(0,0,0,.04); >+} >+ >+.per-100 { >+ background-color: #e8ffde; >+} >+ >+.per-0 { >+ background-color: #fac8c8; >+} >\ No newline at end of file
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 186790
:
342975
|
342994
|
343103
| 343109