Attachment 339013 Details for Bug 185045 – Patch

[patch] Patch

bug-185045-20180427152223.patch (text/plain), 13.51 KB, created by Amal Hussein on 2018-04-27 12:22:24 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Amal Hussein

Created: 2018-04-27 12:22:24 PDT

Size: 13.51 KB

patch

obsolete

>Subversion Revision: 230968
>diff --git a/Tools/ChangeLog b/Tools/ChangeLog
>index 3586e6b51d4734e65929734a823258a1b8b4ddf8..2e053f06b647dced5ee8b17d70867120f07d2af7 100644
>--- a/Tools/ChangeLog
>+++ b/Tools/ChangeLog
>@@ -1,3 +1,30 @@
>+2018-04-27  Amal Hussein  <amal@bocoup.com>
>+
>+        Add Support for a Download Build Product Script
>+        https://bugs.webkit.org/show_bug.cgi?id=185045
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        * Scripts/download-build-product.py: Added.
>+        (parse_args):
>+        (main):
>+        * Scripts/webkitpy/common/build_binaries_fetcher.py: Added.
>+        (BuildBinariesFetcher):
>+        (BuildBinariesFetcher.__init__):
>+        (BuildBinariesFetcher.downloaded_binaries_dir):
>+        (BuildBinariesFetcher.should_default_to_latest_revision):
>+        (BuildBinariesFetcher.s3_build_type):
>+        (BuildBinariesFetcher.s3_build_binaries_url):
>+        (BuildBinariesFetcher.local_build_binaries_dir):
>+        (BuildBinariesFetcher.local_zip_path):
>+        (BuildBinariesFetcher._get_os_version_name):
>+        (BuildBinariesFetcher.get_path):
>+        (BuildBinariesFetcher._get_build_binaries_json):
>+        (BuildBinariesFetcher._get_latest_build_revision):
>+        (BuildBinariesFetcher._fetch_build_binaries_json):
>+        (BuildBinariesFetcher._fetch_build_binaries_zip):
>+        (BuildBinariesFetcher.clean_up_on_error):
>+
> 2018-04-24  John Wilander  <wilander@apple.com>
> 
>         From-Origin: Support for 'same' and 'same-site' response header, nested frame origin check
>diff --git a/Tools/Scripts/download-build-product.py b/Tools/Scripts/download-build-product.py
>new file mode 100755
>index 0000000000000000000000000000000000000000..95569b858eeac2fce2d21385f60e6f3957efd4bf
>--- /dev/null
>+++ b/Tools/Scripts/download-build-product.py
>@@ -0,0 +1,77 @@
>+#!/usr/bin/env python
>+
>+# Copyright (C) 2018 Apple Inc.  All rights reserved.
>+# Redistribution and use in source and binary forms, with or without
>+# modification, are permitted provided that the following conditions
>+# are met:
>+#
>+# 1.  Redistributions of source code must retain the above copyright
>+#     notice, this list of conditions and the following disclaimer.
>+# 2.  Redistributions in binary form must reproduce the above copyright
>+#     notice, this list of conditions and the following disclaimer in the
>+#     documentation and/or other materials provided with the distribution.
>+# 3.  Neither the name of Apple Inc. ("Apple") nor the names of
>+#     its contributors may be used to endorse or promote products derived
>+#     from this software without specific prior written permission.
>+#
>+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
>+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
>+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
>+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
>+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
>+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
>+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
>+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
>+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>+
>+import sys
>+import optparse
>+import logging
>+
>+from webkitpy.port import platform_options, configuration_options
>+from webkitpy.common.build_binaries_fetcher import BuildBinariesFetcher
>+from webkitpy.common.host import Host
>+
>+EXCEPTIONAL_EXIT_STATUS = 254
>+
>+_log = logging.getLogger(__name__)
>+logging.basicConfig()
>+
>+
>+def parse_args(args):
>+
>+    option_group_definitions = [
>+        ("Platform options", platform_options()),
>+        ("Configuration options", configuration_options()),
>+        ("Build Binary options", [
>+            optparse.make_option('--revision', type='str', default=None,
>+                                 help=('Pass in a build revision number from WebKit archives to download binaries. Revision=None will download the latest build.'))]),
>+    ]
>+
>+    option_parser = optparse.OptionParser(usage="%prog [options] [<path>...]")
>+
>+    for group_name, group_options in option_group_definitions:
>+        option_group = optparse.OptionGroup(option_parser, group_name)
>+        option_group.add_options(group_options)
>+        option_parser.add_option_group(option_group)
>+
>+    return option_parser.parse_args(args)
>+
>+
>+def main(argv):
>+    options, args = parse_args(argv)
>+    host = Host()
>+
>+    try:
>+        port = host.port_factory.get(options.platform, options)
>+        build_binaries_fetcher = BuildBinariesFetcher(host, port.port_name, options.architecture, options.configuration,
>+                                                      options.revision)
>+        build_binaries_fetcher.get_path()
>+    except Exception as error:
>+        _log.error('%s : %s' % (type(error), error))
>+        return EXCEPTIONAL_EXIT_STATUS
>+
>+
>+if __name__ == '__main__':
>+    sys.exit(main(sys.argv[1:]))
>diff --git a/Tools/Scripts/webkitpy/common/build_binaries_fetcher.py b/Tools/Scripts/webkitpy/common/build_binaries_fetcher.py
>new file mode 100644
>index 0000000000000000000000000000000000000000..d3be0374079be6022cfe1242fff4977bff018a36
>--- /dev/null
>+++ b/Tools/Scripts/webkitpy/common/build_binaries_fetcher.py
>@@ -0,0 +1,186 @@
>+# Copyright (C) 2014-2018 Apple Inc. All rights reserved.
>+#
>+# Redistribution and use in source and binary forms, with or without
>+# modification, are permitted provided that the following conditions
>+# are met:
>+# 1.  Redistributions of source code must retain the above copyright
>+#     notice, this list of conditions and the following disclaimer.
>+# 2.  Redistributions in binary form must reproduce the above copyright
>+#     notice, this list of conditions and the following disclaimer in the
>+#     documentation and/or other materials provided with the distribution.
>+#
>+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
>+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
>+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
>+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
>+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
>+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
>+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
>+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
>+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>+
>+import os
>+import json
>+import sys
>+import subprocess
>+import shutil
>+import logging
>+from zipfile import ZipFile, BadZipfile
>+from urllib2 import urlopen, HTTPError, URLError
>+from webkitpy.common.webkit_finder import WebKitFinder
>+from webkitpy.port.ios import IOSPort
>+
>+
>+_log = logging.getLogger(__name__)
>+
>+logging.basicConfig()
>+
>+_log.setLevel(logging.INFO)
>+
>+
>+class BuildBinariesFetcher:
>+    """A class to which automates the fetching of the build binaries revisions."""
>+
>+    def __init__(self, host, port_name, architecture, configuration, build_binaries_revision=None):
>+        """ Initialize the build url options needed to construct paths"""
>+        self.host = host
>+        self.port_name = port_name
>+        self.os_version_name = self._get_os_version_name()
>+        self.architecture = architecture
>+        self.configuration = configuration
>+        self.build_binaries_revision = build_binaries_revision
>+        self.s3_zip_url = None
>+
>+        # FIXME find version of this endpoint which returns more than the latest 30 builds
>+        self.s3_build_binaries_api_base_path = 'https://q1tzqfy48e.execute-api.us-west-2.amazonaws.com/v2/latest'
>+
>+    @property
>+    def downloaded_binaries_dir(self):
>+        webkit_finder = WebKitFinder(self.host.filesystem)
>+        return webkit_finder.path_from_webkit_base('WebKitBuild', 'downloaded_binaries')
>+
>+    @property
>+    def should_default_to_latest_revision(self):
>+        return self.build_binaries_revision == None
>+
>+    @property
>+    def s3_build_type(self):
>+        return "{self.port_name}-{self.os_version_name}-{self.architecture}-{self.configuration}".format(self=self).lower()
>+
>+    @property
>+    def s3_build_binaries_url(self):
>+        return self.host.filesystem.join(self.s3_build_binaries_api_base_path, self.s3_build_type)
>+
>+    @property
>+    def local_build_binaries_dir(self):
>+        return self.host.filesystem.join(self.downloaded_binaries_dir, self.s3_build_type, self.build_binaries_revision)
>+
>+    @property
>+    def local_zip_path(self):
>+        return "{self.local_build_binaries_dir}.zip".format(self=self)
>+
>+    def _get_os_version_name(self):
>+        if self.port_name == "mac":
>+            return self.host.platform.os_version_name().lower().replace(' ', '')
>+        elif self.port_name == "ios-simulator":
>+            return IOSPort.CURRENT_VERSION.major
>+        else:
>+            raise NotImplementedError('Downloading binaries for the %s is not currently supported' % self.port_name)
>+
>+    def get_path(self):
>+        try:
>+            if not self.build_binaries_revision:
>+                self.build_binaries_revision = self._get_latest_build_revision()
>+
>+            # check to see if previously downloaded local version exists before downloading
>+            if self.host.filesystem.exists(self.local_build_binaries_dir):
>+                _log.info('Build Binary has been previously dowloaded and can be found at: %s' % self.local_build_binaries_dir)
>+                return self.local_build_binaries_dir
>+
>+            _log.info('Fetching %s builds' % self.s3_build_type)
>+            return self._fetch_build_binaries_json()
>+        except Exception as error:
>+            self.clean_up_on_error()
>+            raise error
>+
>+    def _get_build_binaries_json(self):
>+        response = urlopen(self.s3_build_binaries_url)
>+        build_binaries_json = json.load(response)
>+
>+        if build_binaries_json['Count'] == 0:
>+            raise Exception('No build revisions found at: %s' % self.s3_build_binaries_url)
>+
>+        return build_binaries_json
>+
>+    def _get_latest_build_revision(self):
>+        build_binaries_json = self._get_build_binaries_json()
>+        latest_build_revision = build_binaries_json['Items'][0]['revision']['N']
>+
>+        _log.info('Defaulting to fetching the latest build revision: %s' % latest_build_revision)
>+        return build_binaries_json['Items'][0]['revision']['N']
>+
>+    def _fetch_build_binaries_json(self):
>+        build_binaries_json = self._get_build_binaries_json()
>+
>+        if self.should_default_to_latest_revision:
>+            self.s3_zip_url = build_binaries_json['Items'][0]['revision']['N']
>+        else:
>+            for item in build_binaries_json['Items']:
>+                revision = item['revision']['N']
>+                if revision == self.build_binaries_revision:
>+                    self.s3_zip_url = item['s3_url']['S']
>+                    break
>+
>+        if self.s3_zip_url:
>+            return self._fetch_build_binaries_zip()
>+        else:
>+            raise Exception('Could not find revision %s in the constructed API path: %s'
>+                            % (self.build_binaries_revision, self.s3_build_binaries_url))
>+
>+    def _fetch_build_binaries_zip(self):
>+
>+        try:
>+            # attempt to download the zip file
>+            _log.info("Starting ZipFile Download: %s" % self.s3_zip_url)
>+            build_zip = urlopen(self.s3_zip_url)
>+
>+            self.host.filesystem.maybe_make_directory(self.local_build_binaries_dir)
>+
>+            with open(self.local_zip_path, "wb") as local_build_binaries:
>+                _log.info("Writing ZipFile To Local Drive: %s" % self.local_zip_path)
>+                local_build_binaries.write(build_zip.read())
>+
>+            _log.info("Extracting ZipFile")
>+
>+            extract_exception = Exception('Cannot extract zipfile %s' % self.local_zip_path)
>+            if sys.platform == 'darwin':
>+                if subprocess.call(["ditto", "-x", "-k", self.local_zip_path, self.local_build_binaries_dir]):
>+                    raise extract_exception
>+            elif sys.platform == 'cygwin' or sys.platform.startswith('linux'):
>+                if subprocess.call(["unzip", "-o", self.local_zip_path], cwd=self.local_build_binaries_dir):
>+                    raise extract_exception
>+            elif sys.platform == 'win32':
>+                archive = ZipFile(self.local_zip_path, "r")
>+                archive.extractall(self.local_build_binaries_dir)
>+                archive.close()
>+
>+            _log.info("Deleting ZipFile Extracted Binaries Can Be Found Here: %s" % self.local_build_binaries_dir)
>+            os.remove(self.local_zip_path)
>+
>+            return self.local_build_binaries_dir
>+
>+        except BadZipfile:
>+            raise Exception('BadZipfile Error: could not exact ZipFile')
>+        except HTTPError:
>+            raise Exception('HTTP Error: internet connectivity is required fetch binary file')
>+        except URLError:
>+            raise Exception('URLError Error: please make sure %s is a valid link' % self.s3_build_binaries_url)
>+
>+    def clean_up_on_error(self):
>+        """ Delete newly creating files & directories which may be corrupt or incomplete"""
>+        if self.host.filesystem.exists(self.local_build_binaries_dir):
>+            shutil.rmtree(self.local_build_binaries_dir)
>+
>+        if self.host.filesystem.exists(self.local_zip_path):
>+            os.remove(self.local_zip_path)

Actions: View | Formatted Diff | Diff

Attachments on bug 185045: 339013 | 340416 | 340445 | 340459