Attachment 340867 Details for Bug 185783 – Patch

[patch] Patch

bug-185783-20180521145733.patch (text/plain), 16.18 KB, created by valerie on 2018-05-21 11:57:34 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: valerie

Created: 2018-05-21 11:57:34 PDT

Size: 16.18 KB

patch

obsolete

>Subversion Revision: 232018
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index ef6de93cde92ec45fa270ddf4430ca1520e8cef6..7960e25951e7e4502dd87d8fcad5d644a06fa382 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,34 @@
>+2018-05-21  Valerie R Young  <valerie@bocoup.com>
>+
>+        test262/Runner.pm: add unit tests
>+        https://bugs.webkit.org/show_bug.cgi?id=185783
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        Add unit tests to be run by test-webkitperl
>+
>+        * Scripts/test262/Runner.pm:
>+        (main):
>+          Add environment variable (T262_RUNNER_MODE) which can be set to
>+          "TESTING", which defaults to "PRODUCTION". The "TESTING" option
>+          allows for a segment of test262 to be run and does not load
>+          harness files.
>+        (compileTest):
>+        (processResult):
>+        (getHarness):
>+        * Scripts/webkitperl/test262_unittest/README: Added.
>+        * Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml: Added.
>+        * Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml: Added.
>+        * Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js: Added.
>+        * Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js: Added.
>+        * Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js: Added.
>+        (f):
>+        * Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js: Added.
>+        * Scripts/webkitperl/test262_unittest/fixtures/test/fail.js: Added.
>+        * Scripts/webkitperl/test262_unittest/fixtures/test/pass.js: Added.
>+        (f):
>+        * Scripts/webkitperl/test262_unittest/test262-runner-tests.pl: Added.
>+
> 2018-05-21  Carlos Garcia Campos  <cgarcia@igalia.com>
> 
>         Unreviewed, rolling out r222967.
>diff --git a/Tools/Scripts/test262/Runner.pm b/Tools/Scripts/test262/Runner.pm
>index ec30d01fd4cd58365d2ba0cac2e3156d2959b619..c7d39a0ba7b579c8ee8ccd2a5daffc2a48a76ad2 100755
>--- a/Tools/Scripts/test262/Runner.pm
>+++ b/Tools/Scripts/test262/Runner.pm
>@@ -44,6 +44,7 @@ use Config;
> use Time::HiRes qw(time);
> 
> my $Bin;
>+my $Mode;
> BEGIN {
>     $ENV{DBIC_OVERWRITE_HELPER_METHODS_OK} = 1;
> 
>@@ -56,6 +57,8 @@ BEGIN {
>     unshift @INC, "$Bin/..";
> 
>     $ENV{LOAD_ROUTES} = 1;
>+
>+    $Mode = $ENV{T262_RUNNER_MODE} || "PRODUCTION";
> }
> 
> use YAML qw(Load LoadFile Dump DumpFile Bless);
>@@ -107,7 +110,7 @@ my @files;
> my $tempdir = tempdir();
> my ($deffh, $deffile) = getTempFile();
> 
>-my @default_harnesses;
>+my @default_harnesses = ();
> 
> my $startTime = time();
> 
>@@ -259,13 +262,15 @@ sub processCLI {
> sub main {
>     processCLI();
> 
>-    @default_harnesses = (
>-        "$harnessDir/sta.js",
>-        "$harnessDir/assert.js",
>-        "$harnessDir/doneprintHandle.js",
>-        "$Bin/agent.js"
>-    );
>-    print $deffh getHarness(<@default_harnesses>);
>+    if ($Mode ne 'TESTING') { # if testing, no need to load harness files
>+        @default_harnesses = (
>+            "$harnessDir/sta.js",
>+            "$harnessDir/assert.js",
>+            "$harnessDir/doneprintHandle.js",
>+            "$Bin/agent.js"
>+            );
>+    }
>+    print $deffh getHarness(\@default_harnesses);
> 
>     # If not commandline test path supplied, use the root directory of all tests.
>     push(@cliTestDirs, 'test') if not @cliTestDirs;
>@@ -592,7 +597,8 @@ sub compileTest {
>     my $includes = shift;
>     my ($tfh, $tfname) = getTempFile();
> 
>-    my $includesContent = getHarness(map { "$harnessDir/$_" } @{ $includes });
>+    my @includes = map { "$harnessDir/$_" } @{ $includes };
>+    my $includesContent = getHarness(\@includes);
>     print $tfh $includesContent;
> 
>     return ($tfh, $tfname);
>@@ -664,10 +670,11 @@ sub processResult {
> 
>     if ($scenario ne 'skip' && $currentfailure) {
> 
>-        # We have a new failure if we have loaded an expectation file
>-        # AND (there is no expected failure OR the failure has changed).
>-        my $isnewfailure = $expect
>-            && (!$expectedfailure || $expectedfailure ne $currentfailure);
>+        # We have a new failure if we haven't loaded an expectation file
>+        # (all fails are new) OR we have loaded an expectation fail and
>+        # (there is no expected failure OR the failure has changed).
>+        my $isnewfailure = ! $expect
>+            || !$expectedfailure || $expectedfailure ne $currentfailure;
> 
>         # Print the failure if we haven't loaded an expectation file
>         # or the failure is new.
>@@ -743,9 +750,10 @@ sub parseData {
> }
> 
> sub getHarness {
>-    my @files = @_;
>+    my ($filesref) = @_;
>+
>     my $content;
>-    for (@files) {
>+    for (@{$filesref}) {
>         my $file = $_;
> 
>         open(my $harness_file, '<', $file)
>@@ -756,7 +764,7 @@ sub getHarness {
>         close $harness_file;
>     };
> 
>-    return $content;
>+    return $content || '';
> }
> 
> sub summarizeResults {
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/README b/Tools/Scripts/webkitperl/test262_unittest/README
>new file mode 100644
>index 0000000000000000000000000000000000000000..f5c6ed4b6ebcf0aa4a34130d94d35d15d43ca9ed
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/README
>@@ -0,0 +1,6 @@
>+These unit tests test:
>+  webkit/Tools/Scripts/test262-runner
>+  webkit/Tools/Scripts/test262-import
>+
>+Helper perl modules for these scripts are located in:
>+  webkit/Tools/Scripts/test262/
>\ No newline at end of file
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml b/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml
>new file mode 100644
>index 0000000000000000000000000000000000000000..e1b85d49b19fdd52642a3d33888de179bcaf626f
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml
>@@ -0,0 +1,4 @@
>+---
>+test/fail.js:
>+  default: 'Test262: This test fails.'
>+  strict mode: 'Test262: This test fails.'
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml b/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml
>new file mode 100644
>index 0000000000000000000000000000000000000000..b662d4e32b255cbc1d976d86e22131447462053c
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml
>@@ -0,0 +1,10 @@
>+---
>+test/expected-to-fail-now-passing.js:
>+  default: 'Test262: This test fails.'
>+  strict mode: 'Test262: This test fails.'
>+test/expected-to-fail-now-failing.js:
>+  default: 'Test262: This test fails.'
>+  strict mode: 'Test262: This test fails.'
>+test/expected-to-fail-now-failing-with-new-error.js:
>+  default: 'Test262: This test fails.'
>+  strict mode: 'Test262: This test fails.'
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js
>new file mode 100644
>index 0000000000000000000000000000000000000000..9c6cbab4cb0b4602c42b3a21b27c99f486a1def9
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js
>@@ -0,0 +1,8 @@
>+// Copyright (C) 2018 Valerie Young. All rights reserved.
>+// This code is governed by the BSD license found in the LICENSE file.
>+/*---
>+esid: sec-function-definitions
>+description: Minimal test
>+---*/
>+
>+throw "Test262Unexpected: This test fails WITH AN UNEXPECTED ERROR.";
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js
>new file mode 100644
>index 0000000000000000000000000000000000000000..712d5a112a43207ad248fb03c08cb370c5380811
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js
>@@ -0,0 +1,8 @@
>+// Copyright (C) 2018 Valerie Young. All rights reserved.
>+// This code is governed by the BSD license found in the LICENSE file.
>+/*---
>+esid: sec-function-definitions
>+description: Minimal test
>+---*/
>+
>+throw "Test262: This test fails.";
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js
>new file mode 100644
>index 0000000000000000000000000000000000000000..fa637be7f706a78aad7139705bcfc2c0c992bb6c
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js
>@@ -0,0 +1,9 @@
>+// Copyright (C) 2018 Valerie Young. All rights reserved.
>+// This code is governed by the BSD license found in the LICENSE file.
>+/*---
>+esid: sec-function-definitions
>+description: Minimal test
>+---*/
>+
>+function f() {};
>+
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js
>new file mode 100644
>index 0000000000000000000000000000000000000000..712d5a112a43207ad248fb03c08cb370c5380811
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js
>@@ -0,0 +1,8 @@
>+// Copyright (C) 2018 Valerie Young. All rights reserved.
>+// This code is governed by the BSD license found in the LICENSE file.
>+/*---
>+esid: sec-function-definitions
>+description: Minimal test
>+---*/
>+
>+throw "Test262: This test fails.";
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js
>new file mode 100644
>index 0000000000000000000000000000000000000000..712d5a112a43207ad248fb03c08cb370c5380811
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js
>@@ -0,0 +1,8 @@
>+// Copyright (C) 2018 Valerie Young. All rights reserved.
>+// This code is governed by the BSD license found in the LICENSE file.
>+/*---
>+esid: sec-function-definitions
>+description: Minimal test
>+---*/
>+
>+throw "Test262: This test fails.";
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js
>new file mode 100644
>index 0000000000000000000000000000000000000000..fa637be7f706a78aad7139705bcfc2c0c992bb6c
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js
>@@ -0,0 +1,9 @@
>+// Copyright (C) 2018 Valerie Young. All rights reserved.
>+// This code is governed by the BSD license found in the LICENSE file.
>+/*---
>+esid: sec-function-definitions
>+description: Minimal test
>+---*/
>+
>+function f() {};
>+
>diff --git a/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl b/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl
>new file mode 100755
>index 0000000000000000000000000000000000000000..acb407d7be8ed61f73e515328610ffa48d184151
>--- /dev/null
>+++ b/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl
>@@ -0,0 +1,148 @@
>+#!/usr/bin/env perl
>+
>+# Copyright (C) 2018 Bocoup LLC. All rights reserved.
>+#
>+# Redistribution and use in source and binary forms, with or without
>+# modification, are permitted provided that the following conditions
>+# are met:
>+#
>+# 1. Redistributions of source code must retain the above
>+#    copyright notice, this list of conditions and the following
>+#    disclaimer.
>+# 2. Redistributions in binary form must reproduce the above
>+#    copyright notice, this list of conditions and the following
>+#    disclaimer in the documentation and/or other materials
>+#    provided with the distribution.
>+#
>+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
>+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
>+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
>+# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
>+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
>+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
>+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
>+# SUCH DAMAGE.
>+
>+use strict;
>+use warnings;
>+
>+use Test::More;
>+use File::Spec;
>+use FindBin;
>+use Cwd qw(abs_path);
>+use File::Path qw(rmtree);
>+use File::Temp qw(tempfile);
>+use File::Compare qw(compare);
>+
>+# This test should not be run on Windows
>+if ($^O eq 'MSWin32' || $^O eq 'cygwin' || $^O eq 'WinCairo') {
>+    plan(tests => 1);
>+    is(1, 1, 'do nothing for Windows builds.');
>+    exit 0;
>+}
>+
>+$ENV{T262_RUNNER_MODE} = 'TESTING';
>+
>+my $ToolsPath = $ENV{'WEBKIT_LIBRARIES'};
>+my $Runner = File::Spec->catfile($ToolsPath, 'Tools', 'Scripts', 'test262-runner');
>+my $Bin = $FindBin::Bin;
>+my $MockTest262 = abs_path("$Bin/fixtures");
>+
>+my @testCases = (
>+    {
>+        "TESTNAME" => "test262 test failed, ignore expectations",
>+        "EXITSTATUS" => 1,
>+        "TEST262TESTS_ARG" => "--test-only test/fail.js",
>+        "EXPECTATION_ARG" => "--ignore-expectations",
>+        "EXPECTED_NEW_FAIL_COUNT" => 2,
>+    },
>+    {
>+        "TESTNAME" => "test262 test passed, ignore expectations",
>+        "EXITSTATUS" => 0,
>+        "TEST262TESTS_ARG" => "--test-only test/pass.js",
>+        "EXPECTATION_ARG" => "--ignore-expectations",
>+        "EXPECTED_NEW_FAIL_COUNT" => 0,
>+    },
>+    {
>+        "TESTNAME" => "test262 tests newly failed",
>+        "EXITSTATUS" => 1,
>+        "TEST262TESTS_ARG" => "--test-only test/expected-to-pass-now-failing.js",
>+        "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
>+        "EXPECTED_NEW_FAIL_COUNT" => 2,
>+    },
>+    {
>+        "TESTNAME" => "test262 tests newly passed",
>+        "EXITSTATUS" => 0,
>+        "TEST262TESTS_ARG" => "--test-only test/expected-to-fail-now-passing.js",
>+        "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
>+        "EXPECTED_NEW_FAIL_COUNT" => 0,
>+    },
>+    {
>+        "TESTNAME" => "test262 tests fails, expected failure",
>+        "EXITSTATUS" => 0,
>+        "TEST262TESTS_ARG" => "--test-only  test/expected-to-fail-now-failing.js",
>+        "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
>+        "EXPECTED_NEW_FAIL_COUNT" => 0,
>+    },
>+    {
>+        "TESTNAME" => "test262 tests fails, with unexpected error string",
>+        "EXITSTATUS" => 1,
>+        "TEST262TESTS_ARG" => "--test-only test/expected-to-fail-now-failing-with-new-error.js",
>+        "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
>+        "EXPECTED_NEW_FAIL_COUNT" => 2,
>+    },
>+);
>+
>+my $testCasesCount = (scalar(@testCases) * 2) + 1;
>+plan(tests => $testCasesCount);
>+
>+## Test error codes and expected output messages of tests ##
>+
>+foreach my $testcase (@testCases) {
>+
>+    my $test = $testcase->{TEST262TESTS_ARG};
>+    my $expectation = $testcase->{EXPECTATION_ARG};
>+    my $test262loc = "--t262 $MockTest262";
>+
>+    my $cmd = qq($Runner $test262loc $test $expectation);
>+    my $output = qx($cmd);
>+
>+    # Test the resulting exit code
>+    my $exitcode = $? >> 8;
>+    my $expectedexitcode = $testcase->{EXITSTATUS};
>+    my $testname =  $testcase->{TESTNAME} . " (exit code: $expectedexitcode)";
>+    is($exitcode, $testcase->{EXITSTATUS}, $testname);
>+
>+    # Test the number of occurences of string "! NEW FAIL"
>+    my @newfailcount = $output =~ /! NEW FAIL/g;
>+    my $expectednewfailures = $testcase->{EXPECTED_NEW_FAIL_COUNT};
>+    $testname =  $testcase->{TESTNAME} . " (new failures: $expectednewfailures)";
>+    is(scalar(@newfailcount), $expectednewfailures, $testname);
>+}
>+
>+## Test format of saved expectations file ##
>+
>+my $test262loc = "--t262 $MockTest262";
>+my ($expectationsfh, $expectationsfile) = tempfile();
>+my $expect = "--expectations $expectationsfile";
>+my $test = "--test-only test/fail.js";
>+my $cmd = qq($Runner --save --ignore-expectations $test262loc $test $expect);
>+qx($cmd);
>+
>+my $expectedexpectationsfile = "$Bin/fixtures/expectations-compare.yaml";
>+my $filesmatch = compare($expectationsfile, $expectedexpectationsfile);
>+ok($filesmatch == 0, "expectations yaml file format");
>+
>+close $expectationsfh;
>+
>+END {
>+    # Clean up test262 results directory after running tests
>+    my $resultsDir = $ENV{PWD} . "/test262-results";
>+    if (-e $resultsDir) {
>+        rmtree($resultsDir);
>+    }
>+}

Actions: View | Formatted Diff | Diff

Attachments on bug 185783: 340736 | 340867 | 340957 | 341002 | 341027 | 341028 | 341105 | 342196 | 342205 | 342543 | 342544