Attachment 342847 Details for Bug 186660 – Patch

[patch] Patch

bug-186660-20180615173438.patch (text/plain), 8.38 KB, created by valerie on 2018-06-15 14:34:38 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: valerie

Created: 2018-06-15 14:34:38 PDT

Size: 8.38 KB

patch

obsolete

>Subversion Revision: 232868
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index 5f401386257755b7d087a375fccb382504100ea9..737550f02f2209c1135e8f4000caa7f3c204898c 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,20 @@
>+2018-06-15  Valerie R Young  <valerie@bocoup.com>
>+
>+        test262/Runner.pm: option to run test skipped in config file
>+        https://bugs.webkit.org/show_bug.cgi?id=186660
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        Adds option --skipped-files, -S, to run all files skipped in the
>+        config.yaml file.
>+
>+        * Scripts/test262/Runner.pm:
>+        (processCLI):
>+        (main):
>+        (getBuildPath):
>+        (processFile):
>+        (processResult):
>+
> 2018-06-14  Saam Barati  <sbarati@apple.com>
> 
>         sun spider compile time benchmark fails in run-jsc-benchmarks with JSC_validateOptions=1
>diff --git a/Tools/Scripts/test262/Runner.pm b/Tools/Scripts/test262/Runner.pm
>index 3afa3268c977e4a7589b8b301e3ba42672c2115d..61674860dba924b76b1b2b27765fb8ebe385dca6 100755
>--- a/Tools/Scripts/test262/Runner.pm
>+++ b/Tools/Scripts/test262/Runner.pm
>@@ -89,6 +89,7 @@ my $failingOnly;
> my $latestImport;
> my $runningAllTests;
> my $timeout;
>+my $skippedOnly;
> 
> my $test262Dir;
> my $webkitTest262Dir = abs_path("$Bin/../../../JSTests/test262");
>@@ -140,6 +141,7 @@ sub processCLI {
>         'stats' => \$stats,
>         'r|results=s' => \$specifiedResultsFile,
>         'timeout=i' => \$timeout,
>+        'S|skipped-files' => \$skippedOnly,
>     );
> 
>     if ($help) {
>@@ -219,7 +221,7 @@ sub processCLI {
>     if (! $ignoreConfig) {
>         if ($configFile && ! -e $configFile) {
>             die "Error: Config file $configFile does not exist!\n" .
>-                "Run without config file with -i or supply with --config.\n"
>+                "Run without config file with -i or supply with --config.\n";
>         }
>         $config = LoadFile($configFile) or die $!;
>         if ($config->{skip} && $config->{skip}->{files}) {
>@@ -236,10 +238,19 @@ sub processCLI {
>     }
> 
>     # If the expectation file doesn't exist and is not specified, run all tests.
>-    if (! $ignoreExpectations && -e $expectationsFile) {
>+    # If we are running only skipped files, then ignore the expectation file.
>+    if (! $ignoreExpectations && -e $expectationsFile && !$skippedOnly) {
>         $expect = LoadFile($expectationsFile) or die $!;
>     }
> 
>+    # If running only the skipped files from the config list
>+    if ($skippedOnly) {
>+        if (! -e $configFile) {
>+            die "Error: Config file $configFile does not exist!\n" .
>+                "Cannot run skipped tests from config. Supply file with --config.\n";
>+        }
>+    }
>+
>     if (@features) {
>         %filterFeatures = map { $_ => 1 } @features;
>     }
>@@ -259,8 +270,9 @@ sub processCLI {
>     print "Expectations file: " . abs2rel($expectationsFile) . "\n" if $expect;
>     print "Results file: ". abs2rel($resultsFile) . "\n" if $stats || $failingOnly;
> 
>+    print "Running only the failing files in expectations.yaml\n" if $failingOnly;
>     print "Running only the latest imported files\n" if $latestImport;
>-
>+    print "Running only the skipped files in config.yaml\n" if $skippedOnly;
>     print "Verbose mode\n" if $verbose;
> 
>     print "---\n\n";
>@@ -397,10 +409,10 @@ sub main {
>         }
>         elsif ($test->{result} eq 'PASS') {
>             # If this is an newly passing test
>-            if ($expectedFailure) {
>+            if ($expectedFailure || $skippedOnly) {
>                 $newpasscount++;
> 
>-                if ($verbose) {
>+                if ($verbose || $skippedOnly) {
>                     my $path = $test->{path};
>                     my $mode = $test->{mode};
>                     $newpassreport .= "PASS $path ($mode)\n";
>@@ -423,16 +435,22 @@ sub main {
>             print "---------------NEW PASSING TESTS SUMMARY---------------\n\n";
>             print "$newpassreport\n";
>         }
>+        print "---------------------------------------------------------\n\n";
>+    }
>+
>+    # If we are running only skipped tests, report all the new passing tests
>+    if ($skippedOnly && $newpassreport) {
>+        print "---------------NEW PASSING TESTS SUMMARY---------------\n";
>+        print "\n$newpassreport\n";
>         print "---------------------------------------------------------\n";
>     }
> 
>+    print("\n");
>+
>     if ($saveExpectations) {
>         DumpFile($expectationsFile, \%failed);
>-        print "\nSaved results in: $expectationsFile\n";
>-    } else {
>-        print "\nRun with --save to save a new expectations file\n";
>+        print "Saved expectation file in: $expectationsFile\n";
>     }
>-
>     if ($runningAllTests) {
>         if (! -e $resultsDir) {
>             mkpath($resultsDir);
>@@ -449,6 +467,7 @@ sub main {
> 
>     if ( !$expect ) {
>         print $failcount . " tests failed\n";
>+        print $newpasscount . " tests newly pass\n" if $skippedOnly;
>     } else {
>         print $failcount . " tests failed in total\n";
>         print $newfailcount . " tests newly fail\n";
>@@ -521,8 +540,8 @@ sub getBuildPath {
>     my $jsc;
> 
>     if ($webkitdirIsAvailable) {
>-        my $config = $release ? 'Release' : 'Debug';
>-        setConfiguration($config);
>+        my $webkit_config = $release ? 'Release' : 'Debug';
>+        setConfiguration($webkit_config);
>         my $jscDir = executableProductDir();
> 
>         $jsc = $jscDir . '/jsc';
>@@ -554,27 +573,35 @@ sub processFile {
> 
>     # Check test against filters in config file
>     my $file = abs2rel( $filename, $test262Dir );
>-    if (shouldSkip($file, $data)) {
>-        $resultsdata = processResult($filename, $data, "skip");
>-        DumpFile($resultsfh, $resultsdata);
>+    my $skipTest = shouldSkip($file, $data);
>+
>+    # If we only want to run skipped tests, invert filter
>+    $skipTest = !$skipTest if $skippedOnly;
>+
>+    if ($skipTest) {
>+        if (! $skippedOnly) {
>+            $resultsdata = processResult($filename, $data, "skip");
>+            DumpFile($resultsfh, $resultsdata);
>+        }
>         return;
>     }
>+    else {
>+        my @scenarios = getScenarios(@{ $data->{flags} });
> 
>-    my @scenarios = getScenarios(@{ $data->{flags} });
>+        my $includes = $data->{includes};
>+        my ($includesfh, $includesfile);
> 
>-    my $includes = $data->{includes};
>-    my ($includesfh, $includesfile);
>+        ($includesfh, $includesfile) = compileTest($includes) if defined $includes;
> 
>-    ($includesfh, $includesfile) = compileTest($includes) if defined $includes;
>+        foreach my $scenario (@scenarios) {
>+            my ($result, $execTime) = runTest($includesfile, $filename, $scenario, $data);
> 
>-    foreach my $scenario (@scenarios) {
>-        my ($result, $execTime) = runTest($includesfile, $filename, $scenario, $data);
>+            $resultsdata = processResult($filename, $data, $scenario, $result, $execTime);
>+            DumpFile($resultsfh, $resultsdata);
>+        }
> 
>-        $resultsdata = processResult($filename, $data, $scenario, $result, $execTime);
>-        DumpFile($resultsfh, $resultsdata);
>+        close $includesfh if defined $includesfh;
>     }
>-
>-    close $includesfh if defined $includesfh;
> }
> 
> sub shouldSkip {
>@@ -723,12 +750,12 @@ sub processResult {
> 
>         # Print the failure if we haven't loaded an expectation file
>         # or the failure is new.
>-        my $printFailure = !$expect || $isnewfailure;
>+        my $printFailure = (!$expect || $isnewfailure) && !$skippedOnly;
> 
>         my $newFail = '';
>         $newFail = '! NEW ' if $isnewfailure;
>         my $failMsg = '';
>-        $failMsg = "FAIL $file ($scenario)\n" if ($printFailure or $verbose);
>+        $failMsg = "FAIL $file ($scenario)\n";
> 
>         my $suffixMsg = '';
> 
>@@ -738,7 +765,7 @@ sub processResult {
>             $suffixMsg = "$result$featuresList\n";
>         }
> 
>-        print "$newFail$failMsg$suffixMsg";
>+        print "$newFail$failMsg$suffixMsg" if ($printFailure || $verbose);
> 
>         $resultdata{result} = 'FAIL';
>         $resultdata{error} = $currentfailure;
>@@ -1059,6 +1086,10 @@ Runs all test files that failed in a given results file (specifc with --results)
> 
> Runs the test files listed in the last import (./JSTests/test262/latest-changes-summary.txt).
> 
>+=item B<--skipped-files, -S>
>+
>+Runs all test files that are skipped according to the config.yaml file.
>+
> =item B<--stats>
> 
> Calculate conformance statistics from results/results.yaml file or a supplied results file (--results). Saves results in results/summary.txt and results/summary.yaml.

Actions: View | Formatted Diff | Diff

Attachments on bug 186660: 342847