Tools/ChangeLog

2015-03-20  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        W3C test importer should generate the modules installed dynamically to run wpt tests

        https://bugs.webkit.org/show_bug.cgi?id=142738

        Reviewed by NOBODY (OOPS!).

        The test importer can now generate the submodules description file based on information extracted from the corresponding git repository.

        The implementation is done within TestDownloader and it is activated for the WPT repository.

        LayoutTests/imported/w3c/resources/WPTModules is renamed as LayoutTests/imported/w3c/resources/web-platform-tests-modules.json for that reason.

        * Scripts/webkitpy/layout_tests/servers/web_platform_test_server.py:

        (WebPlatformTestServer._install_modules): Updated to cope with path as array.

        * Scripts/webkitpy/w3c/test_downloader.py:

        (TestDownloader._git_submodules_status): Added so that it can be overriden for unit tests.

        (TestDownloader):

        (TestDownloader._git_submodules_description): Computes submodule description.

        (TestDownloader.generate_git_submodules_description): Write submodule description in a file.

        * Scripts/webkitpy/w3c/test_importer.py:

        (TestImporter.do_import): Added the possibility to post process the tests.

        (TestImporter):

        (TestImporter.process_test_repositories_import_options): Enable generating module description file.

        (TestImporter.should_convert_test_harness_links): Updated according new options format.

        * Scripts/webkitpy/w3c/test_importer_unittest.py:

        (TestImporterTest.import_downloaded_tests): Making use of TestDownloaderMock. 

        (TestImporterTest.import_downloaded_tests.TestDownloaderMock): Added to override submodule status gathering.

        (TestImporterTest.import_downloaded_tests.TestDownloaderMock.__init__):

        (TestImporterTest.import_downloaded_tests.TestDownloaderMock._git_submodules_status):

        (TestImporterTest.test_submodules_generation): Added to check that modules description files works.

2015-03-18  Csaba Osztrogonác  <ossy@webkit.org>

        [EFL] Bump gstreamer version to 1.4.4

Tools/Scripts/webkitpy/layout_tests/servers/web_platform_test_server.py

@@class WebPlatformTestServer(http_server_base.HttpServerBase):

        self._doc_root_path = port_obj.path_from_webkit_base("LayoutTests", self._doc_root)

    def _install_modules(self):

82 modules_file_path = self._filesystem.join(self._layout_root, "imported", "w3c", "resources", "~~WPTM~~odules")

82 modules_file_path = self._filesystem.join(self._layout_root, "imported", "w3c", "resources", "web-platform-tests-modules.json")

        if not self._filesystem.isfile(modules_file_path):

            _log.warning("Cannot read " + modules_file_path)

            return

        modules = json.loads(self._filesystem.read_text_file(modules_file_path))

        for module in modules:

88 AutoInstaller(target_dir=self._filesystem.join(self._doc_root, module["path"])).install(url=module["url"], url_subpath=module["url_subpath"], target_name=module["name"])

88 AutoInstaller(target_dir=self._filesystem.join(self._doc_root, self._filesystem.sep.join(module["path"]))).install(url=module["url"], url_subpath=module["url_subpath"], target_name=module["name"])

    def _copy_webkit_test_files(self):

        _log.debug('Copying WebKit resources files')

Tools/Scripts/webkitpy/w3c/test_downloader.py

@@class TestDownloader(object):

            if self._filesystem.isfile(destination_path):

                self._filesystem.remove(destination_path)

    def _git_submodules_status(self, repository_directory):

        return self.git(repository_directory)._run_git(['submodule', 'status'])

    def _git_submodules_description(self, test_repository):

        submodules = []

        repository_directory = self._filesystem.join(self.repository_directory, test_repository['name'])

        if self._filesystem.isfile(self._filesystem.join(repository_directory, '.gitmodules')):

            submodule = {}

            for line in self._filesystem.read_text_file(self._filesystem.join(repository_directory, '.gitmodules')).splitlines():

                line = line.strip()

                if line.startswith('path = '):

                    submodule['path'] = line[7:]

                elif line.startswith('url = '):

                    submodule['url'] = line[6:]

                    if not submodule['url'].startswith('https://github.com/'):

                        _log.warning('Submodule %s is not hosted on github' % submodule['path'])

                        _log.warning('Please ensure that generated URL points to an archive of the module or manually edit its value after the import')

                    submodules.append(submodule)

                    submodule = {}

        submodules_status = [line[1:].split(' ') for line in self._git_submodules_status(repository_directory).splitlines()]

        for submodule in submodules:

            for status in submodules_status:

                if submodule['path'] == status[1]:

                    url = submodule['url'][:-4]

                    version = status[0]

                    repository_name = url.split('/').pop()

                    submodule['url'] = url + '/archive/' + version + '.tar.gz'

                    submodule['url_subpath'] = repository_name + '-' + version

            if '/' in submodule['path']:

                steps = submodule['path'].split('/')

                submodule['name'] = steps.pop()

                submodule['path'] = steps

            else:

                submodule['name'] = submodule['path']

                submodule['path'] = ['.']

        return submodules

    def generate_git_submodules_description(self, test_repository, filepath):

        self._filesystem.write_text_file(filepath, json.dumps(self._git_submodules_description(test_repository), sort_keys=True, indent=4))

    def download_tests(self, destination_directory, test_paths=[]):

        for test_repository in self.test_repositories:

            self.checkout_test_repository(test_repository['revision'], test_repository['url'], self._filesystem.join(self.repository_directory, test_repository['name']))

Tools/Scripts/webkitpy/w3c/test_importer.py

@@class TestImporter(object):

        self.import_tests()

        if self._importing_downloaded_tests:

            self.process_test_repositories_import_options()

    def process_test_repositories_import_options(self):

        for test_repository in self._test_downloader.test_repositories:

            if 'generate_git_submodules_description' in test_repository['import_options']:

                self.filesystem.maybe_make_directory(self.filesystem.join(self.destination_directory, 'resources'))

                self._test_downloader.generate_git_submodules_description(test_repository, self.filesystem.join(self.destination_directory, 'resources', test_repository['name'] + '-modules.json'))

            # FIXME: Generate WPT .gitignore and  main __init__.py

    def test_downloader(self):

        if not self._test_downloader:

            download_options = TestDownloader.default_options()

@@class TestImporter(object):

        if self._importing_downloaded_tests:

            for test_repository in self.test_downloader().test_repositories:

                if test.startswith(test_repository['name']):

284 return ~~test_repository[~~'convert_test_harness_links']

294 return 'convert_test_harness_links' in test_repository['import_options']

            return True

        return self.options.convert_test_harness_links

Tools/Scripts/webkitpy/w3c/test_importer_unittest.py

@@from webkitpy.common.host_mock import MockHost

from webkitpy.common.system.filesystem_mock import MockFileSystem

from webkitpy.common.system.executive_mock import MockExecutive2, ScriptError

from webkitpy.common.system.outputcapture import OutputCapture

37from webkitpy.w3c.test_downloader import TestDownloader

from webkitpy.w3c.test_importer import parse_args, TestImporter

@@class TestImporterTest(unittest.TestCase):

    def import_downloaded_tests(self, args, files):

        # files are passed as parameter as we cannot clone/fetch/checkout a repo in mock system.

        class TestDownloaderMock(TestDownloader):

            def __init__(self, repository_directory, host, options):

                TestDownloader.__init__(self, repository_directory, host, options)

            def _git_submodules_status(self, repository_directory):

                return 'adb4d391a69877d4a1eaaf51d1725c99a5b8ed84 tools/resources'

        host = MockHost()

        host.executive = MockExecutive2()

        host.filesystem = MockFileSystem(files=files)

        options, args = parse_args(args)

        importer = TestImporter(host, None, options)

110 importer._test_downloader = TestDownloaderMock(importer.tests_download_path, importer.host, importer.options)

        importer.do_import()

        return host.filesystem

@@class TestImporterTest(unittest.TestCase):

        self.assertTrue('src="/resources/testharness.js"' in fs.read_text_file('/mock-checkout/LayoutTests/w3c/web-platform-tests/t/test.html'))

        self.assertTrue('src="../' in fs.read_text_file('/mock-checkout/LayoutTests/w3c/csswg-tests/t/test.html'))

    def test_submodules_generation(self):

        FAKE_FILES = {

            '/mock-checkout/WebKitBuild/w3c-tests/csswg-tests/.gitmodules': '[submodule "tools/resources"]\n	path = tools/resources\n	url = https://github.com/w3c/testharness.js.git\n  ignore = dirty\n',

            '/mock-checkout/WebKitBuild/w3c-tests/web-platform-tests/.gitmodules': '[submodule "tools/resources"]\n	path = tools/resources\n	url = https://github.com/w3c/testharness.js.git\n  ignore = dirty\n',

        }

        fs = self.import_downloaded_tests(['--no-fetch', '--import-all', '-d', 'w3c'], FAKE_FILES)

        self.assertFalse(fs.exists('/mock-checkout/LayoutTests/w3c/resources/csswg-tests-modules.json'))

        self.assertTrue(fs.exists('/mock-checkout/LayoutTests/w3c/resources/web-platform-tests-modules.json'))

        self.assertTrue('https://github.com/w3c/testharness.js/archive/db4d391a69877d4a1eaaf51d1725c99a5b8ed84.tar.gz' in fs.read_text_file('/mock-checkout/LayoutTests/w3c/resources/web-platform-tests-modules.json'))

119142 # FIXME: Needs more tests.

LayoutTests/imported/w3c/ChangeLog

2015-03-20  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        W3C test importer should generate the modules installed dynamically to run wpt tests

        https://bugs.webkit.org/show_bug.cgi?id=142738

        Reviewed by NOBODY (OOPS!).

        Renamed WPTModules to web-platform-test-modules.json

        Updated TestRepositories to ask the importer to generate web-platform-test-modules.json at import time.  

        * resources/TestRepositories:

        * resources/WPTModules: Removed.

        * resources/web-platform-tests-modules.json: Added.

2015-03-13  Youenn Fablet  <youenn.fablet@crf.canon.fr>

        WebKit test infrastructure should automate the process of cloning W3C test suite and importing tests from it

LayoutTests/imported/w3c/resources/TestRepositories

        "paths_to_import": [

            "support"

        ],

13 "convert_test_harness_links"~~: true~~

13 "import_options": ["convert_test_harness_links"]

    },

    {

        "name": "web-platform-tests",

            "config.default.json",

            "serve.py"

        ],

33 "c~~onve~~rt_~~test_harness_link~~s": false

33 "import_options": ["generate_git_submodules_description"]

3434 }

3535]

LayoutTests/imported/w3c/resources/WPTModules

[

{

    "name": "html5lib",

    "path": "tools",

    "url": "https://github.com/html5lib/html5lib-python/archive/7cce65bbaa78411f98b8b37eeefc9db03c580097.tar.gz",

    "url_subpath": "html5lib-python-7cce65bbaa78411f98b8b37eeefc9db03c580097"

},

{

    "name": "pywebsocket",

    "path": "tools",

    "url": "https://github.com/w3c/pywebsocket/archive/8c285d9015121e8c1c40be852439fc03b4a18112.tar.gz",

    "url_subpath": "pywebsocket-8c285d9015121e8c1c40be852439fc03b4a18112"

},

{

    "name": "six",

    "path": "tools",

    "url": "https://github.com/jgraham/six/archive/3b6173c833d217ab0186c355804f5925cbcfca47.tar.gz",

    "url_subpath": "six-3b6173c833d217ab0186c355804f5925cbcfca47"

},

{

    "name": "wptserve",

    "path": "tools",

    "url": "https://github.com/w3c/wptserve/archive/91e764ffa0f587090341a26af457dfdc6762eb0d.tar.gz",

    "url_subpath": "wptserve-91e764ffa0f587090341a26af457dfdc6762eb0d"

},

{

    "name": "resources",

    "path": ".",

    "url": "https://github.com/w3c/testharness.js/archive/74ba13d2fbb19c6e7494960bd7724873e88c523c.tar.gz",

    "url_subpath": "testharness.js-74ba13d2fbb19c6e7494960bd7724873e88c523c"

}

]

LayoutTests/imported/w3c/resources/web-platform-tests-modules.json

[

    {

        "name": "resources", 

        "path": [

            "."

        ], 

        "url": "https://github.com/w3c/testharness.js/archive/2a1da264f6718db04c3925a9956b635426827aef.tar.gz", 

        "url_subpath": "testharness.js-2a1da264f6718db04c3925a9956b635426827aef"

    }, 

    {

        "name": "wptserve", 

        "path": [

            "tools"

        ], 

        "url": "https://github.com/w3c/wptserve/archive/91e764ffa0f587090341a26af457dfdc6762eb0d.tar.gz", 

        "url_subpath": "wptserve-91e764ffa0f587090341a26af457dfdc6762eb0d"

    }, 

    {

        "name": "pywebsocket", 

        "path": [

            "tools"

        ], 

        "url": "https://github.com/w3c/pywebsocket/archive/8c285d9015121e8c1c40be852439fc03b4a18112.tar.gz", 

        "url_subpath": "pywebsocket-8c285d9015121e8c1c40be852439fc03b4a18112"

    }, 

    {

        "name": "html5lib", 

        "path": [

            "tools"

        ], 

        "url": "https://github.com/html5lib/html5lib-python/archive/7cce65bbaa78411f98b8b37eeefc9db03c580097.tar.gz", 

        "url_subpath": "html5lib-python-7cce65bbaa78411f98b8b37eeefc9db03c580097"

    }, 

    {

        "name": "six", 

        "path": [

            "tools"

        ], 

        "url": "https://github.com/jgraham/six/archive/3b6173c833d217ab0186c355804f5925cbcfca47.tar.gz", 

        "url_subpath": "six-3b6173c833d217ab0186c355804f5925cbcfca47"

    }

]