Attachment 339917 Details for Bug 185395 – Patch

[patch] Patch

bug-185395-20180508222541.patch (text/plain), 7.95 KB, created by valerie on 2018-05-08 19:25:43 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: valerie

Created: 2018-05-08 19:25:43 PDT

Size: 7.95 KB

patch

obsolete

>Subversion Revision: 231534
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index 2f5e3475708ec7396e41ef14b60e67dbb920f332..f4fd47f1ca82cc30ce5bfc06b330e6d218ea5658 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,22 @@
>+2018-05-08  Valerie R Young  <valerie@bocoup.com>
>+
>+        test262/Runner.pm: --failing-files uses results file not expectations
>+        https://bugs.webkit.org/show_bug.cgi?id=185395
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        --failing-files arg now uses results file to find failing tests
>+        to rerun. It will look in the test262-results directory from whereever
>+        the script is run, or a results.yaml file can be supplied via command
>+        line argument.
>+        --expectations arg added for suppling expectation file.
>+
>+        * Scripts/test262/Runner.pm:
>+        (processCLI):
>+        (main):
>+        (loadImportFile):
>+        (findAllFailing):
>+
> 2018-05-08  Wenson Hsieh  <wenson_hsieh@apple.com>
> 
>         Consolidate WebContentReaderIOS and WebContentReaderMac into WebContentReaderCocoa
>diff --git a/Tools/Scripts/test262/Runner.pm b/Tools/Scripts/test262/Runner.pm
>index d79a3e0c132677c7f346ff428548a341431ac610..373505c775623d79db12a0979f3320649fa90bb0 100755
>--- a/Tools/Scripts/test262/Runner.pm
>+++ b/Tools/Scripts/test262/Runner.pm
>@@ -86,9 +86,7 @@ my $runningAllTests;
> my $expectationsFile = abs_path("$Bin/../../../JSTests/test262/expectations.yaml");
> my $configFile = abs_path("$Bin/../../../JSTests/test262/config.yaml");
> 
>-my $resultsDir = `pwd`;
>-chomp $resultsDir;
>-$resultsDir = $resultsDir . "/test262-results";
>+my $resultsDir = $ENV{PWD} . "/test262-results";
> mkpath($resultsDir);
> 
> my $resultsFile = abs_path("$resultsDir/results.yaml");
>@@ -114,6 +112,7 @@ sub processCLI {
>     my @features;
>     my $stats;
>     my $specifiedResultsFile;
>+    my $specifiedExpectationsFile;
> 
>     # If adding a new commandline argument, you must update the POD
>     # documentation at the end of the file.
>@@ -129,8 +128,9 @@ sub processCLI {
>         'c|config=s' => \$configFile,
>         'i|ignore-config' => \$ignoreConfig,
>         's|save' => \$saveExpectations,
>+        'e|expectations=s' => \$specifiedExpectationsFile,
>         'x|ignore-expectations' => \$ignoreExpectations,
>-        'failing-files' => \$failingOnly,
>+        'f|failing-files' => \$failingOnly,
>         'l|latest-import' => \$latestImport,
>         'stats' => \$stats,
>         'r|results=s' => \$specifiedResultsFile,
>@@ -146,7 +146,7 @@ sub processCLI {
>     }
> 
>     if ($specifiedResultsFile) {
>-        if (!$stats) {
>+        if (!$stats && !$failingOnly) {
>             print "Waring: supplied results file not used for this command.\n";
>         }
>         elsif (-e $specifiedResultsFile) {
>@@ -159,7 +159,8 @@ sub processCLI {
> 
>     if ($stats) {
>         if (! -e $resultsFile) {
>-            die "Error: cannot find results file, please specify with --results.";
>+            die "Error: cannot find results file to summarize," .
>+                "please specify with --results.";
>         }
>         summarizeResults();
>         exit;
>@@ -193,8 +194,9 @@ sub processCLI {
>     $harnessDir = "$test262Dir/harness";
> 
>     if (! $ignoreConfig) {
>-        if ($configFile and not -e $configFile) {
>-            die "Config file $configFile does not exist!";
>+        if ($configFile && ! -e $configFile) {
>+            die "Error: Config file $configFile does not exist!\n" .
>+                "Run without config file with -i or supply with --config.\n"
>         }
>         $config = LoadFile($configFile) or die $!;
>         if ($config->{skip} && $config->{skip}->{files}) {
>@@ -202,17 +204,23 @@ sub processCLI {
>         }
>     }
> 
>-    if (! $ignoreExpectations) {
>-        # If expectations file doesn't exist yet, just run tests, UNLESS
>-        # --failures-only option supplied.
>-        if ( $failingOnly && ! -e $expectationsFile ) {
>-            print "Error: Cannot run failing tests if expectation.yaml file does not exist.\n";
>-            die;
>-        } elsif (-e $expectationsFile) {
>-            $expect = LoadFile($expectationsFile) or die $!;
>+    if ( $failingOnly && ! -e $resultsFile ) {
>+        die "Error: cannot find results file to run failing tests," .
>+            " please specify with --results.";
>+    }
>+
>+    if (! $ignoreExpectations && $specifiedExpectationsFile) {
>+        $expectationsFile = abs_path($specifiedExpectationsFile);
>+        if (! -e $expectationsFile) {
>+            die "Error: Supplied expectations file $expectationsFile does not exist!";
>         }
>     }
> 
>+    # If the expectation file doesn't exist and is not specified, run all tests.
>+    if (! $ignoreExpectations && -e $expectationsFile) {
>+        $expect = LoadFile($expectationsFile) or die $!;
>+    }
>+
>     if (@features) {
>         %filterFeatures = map { $_ => 1 } @features;
>     }
>@@ -229,6 +237,7 @@ sub processCLI {
>     print "Paths:  " . join(', ', @cliTestDirs) . "\n" if @cliTestDirs;
>     print "Config file: $configFile\n" if $config;
>     print "Expectations file: $expectationsFile\n" if $expect;
>+    print "Results file: $resultsFile\n" if $stats || $failingOnly;
> 
>     print "Running only the latest imported files\n" if $latestImport;
> 
>@@ -254,8 +263,8 @@ sub main {
>     if ($latestImport) {
>         @files = loadImportFile();
>     } elsif ($failingOnly) {
>-        # If we only want to re-run failure, only run tests in expectation file
>-        @files = map { qq($test262Dir/$_) } keys %{$expect};
>+        # If we only want to re-run failure, only run tests in results file
>+        findAllFailing();
>     } else {
>         $runningAllTests = 1;
>         # Otherwise, get all files from directory
>@@ -398,7 +407,7 @@ sub main {
> 
> sub loadImportFile {
>     my $importFile = abs_path("$Bin/../../../JSTests/test262/latest-changes-summary.txt");
>-    die "Import file not found at $importFile.\n" if ! -e $importFile;
>+    die "Error: Import file not found at $importFile.\n" if ! -e $importFile;
> 
>     open(my $fh, "<", $importFile) or die $!;
> 
>@@ -822,6 +831,20 @@ sub summarizeResults {
>     print "See summarized results in $summaryTxtFile\n";
> }
> 
>+sub findAllFailing {
>+    my @allresults = LoadFile($resultsFile) or die $!;
>+     @allresults = @{$allresults[0]};
>+
>+    my %filedictionary;
>+    foreach my $test (@allresults) {
>+        if ($test->{result} eq 'FAIL') {
>+            $filedictionary{$test->{path}} = 1;
>+        }
>+    }
>+
>+    @files = map { qq($test262Dir/$_) } keys %filedictionary;
>+}
>+
> __END__
> 
> =head1 DESCRIPTION
>@@ -876,7 +899,7 @@ Verbose output for test results. Includes error message for test.
> 
> =item B<--config, -c>
> 
>-Specify a config file. If not provided, script will load local test262-config.yaml
>+Specify a config file. If not provided, script will load local JSTests/test262/config.yaml
> 
> =item B<--ignore-config, -i>
> 
>@@ -894,13 +917,17 @@ Specify one or more specific test262 directory of test to run, relative to the r
> 
> Overwrites the test262-expectations.yaml file with the current list of test262 files and test results.
> 
>+=item B<--expectations, -e>
>+
>+Specify a expectations file.  If not provided, script will load local JSTests/test262/expectations.yaml
>+
> =item B<--ignore-expectations, -x>
> 
> Ignores the test262-expectations.yaml file and outputs all failures, instead of only unexpected failures.
> 
>-=item B<--failing-files>
>+=item B<--failing-files, -f>
> 
>-Runs all test files that expect to fail according to the expectation file. This option will run the rests in both strict and non-strict modes, even if the test only fails in one of the two modes.
>+Runs all test files that failed in a given results file (specifc with --results). This option will run the rests in both strict and non-strict modes, even if the test only fails in one of the two modes.
> 
> =item B<--latest-import, -l>
> 
>@@ -912,7 +939,7 @@ Calculate conformance statistics from results/results.yaml file or a supplied re
> 
> =item B<--results, -r>
> 
>-Specifies a results file the --stats option.
>+Specifies a results file for the --stats or --failing-files options.
> 
> =back
>

Actions: View | Formatted Diff | Diff

Attachments on bug 185395: 339917 | 339940