2019-03-15  Jonathan Bedard  <jbedard@apple.com>

        webkitpy: Upload test results

        https://bugs.webkit.org/show_bug.cgi?id=195755

        <rdar://problem/48896182>

        Reviewed by NOBODY (OOPS!).

        Establish a new format for uploading results that is not tied to layout tests, apply

        that format to webkitpy tests.

        * Scripts/webkitpy/common/checkout/scm/git.py:

        (Git.native_branch): Return what branch the current checkout is on.

        * Scripts/webkitpy/common/checkout/scm/scm_mock.py:

        * Scripts/webkitpy/common/checkout/scm/scm_unittest.py:

        * Scripts/webkitpy/common/checkout/scm/svn.py:

        (SVN.native_branch): Ditto.

        * Scripts/webkitpy/common/system/platforminfo.py:

        (PlatformInfo.build_version): Return a build version for Mac.

        * Scripts/webkitpy/common/system/platforminfo_mock.py:

        (MockPlatformInfo.__init__):

        (MockPlatformInfo.build_version):

        * Scripts/webkitpy/results: Added.

        * Scripts/webkitpy/results/__init__.py: Added.

        * Scripts/webkitpy/results/options.py: Added.

        (upload_options): OptParse list for upload options.

        * Scripts/webkitpy/results/upload.py: Added.

        (Upload): Class which enforces the upload format expected by the results server.

        (Upload.Expectations):

        (Upload.create_configuration):

        (Upload.create_commit):

        (Upload.create_details):

        (Upload.create_run_stats):

        (Upload.create_test_result):

        (Upload.__init__):

        (Upload.Encoder): Encode Upload object as json.

        (Upload.upload): Upload results to the results server, returning 'True' if the upload is successful.

        * Scripts/webkitpy/results/upload_unittest.py: Added.

        * Scripts/webkitpy/test/main.py:

        (Tester._parse_args): Add upload arguments.

        (Tester._run_tests): Allow results to be uploaded.

        * Scripts/webkitpy/test/runner.py:

        (Runner.__init__): Record which tests were run, rather than just counting them.

        (Runner.handle):

        * Scripts/webkitpy/test/runner_unittest.py:

        (RunnerTest.test_run):

        * Scripts/webkitpy/thirdparty/__init__.py:

        (AutoinstallImportHook.find_module): Add requests auto-install.

        (AutoinstallImportHook._install_requests):

        * Scripts/webkitpy/tool/commands/queues_unittest.py:

        (PatchProcessingQueueTest.test_upload_results_archive_for_patch): Update os name for testing.

# Required for Python to search this directory for module files

# Keep this file free of any code or import statements that could

# cause either an error to occur or a log message to be logged.

# This ensures that calling code can import initialization code from

# webkitpy before any errors or log messages due to code in this file.

# Initialization code can include things like version-checking code and

# logging configuration code.

#

# We do not execute any version-checking code or logging configuration

# code in this file so that callers can opt-in as they want.  This also

# allows different callers to choose different initialization code,

# as necessary.

# Copyright (C) 2019 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND

# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import optparse

def upload_options():

    return [

        optparse.make_option('--report', action='append', dest='report_urls', help='URL (or URLs) to report test results to'),

        optparse.make_option('--buildbot-master', help='The url of the buildbot master.'),

        optparse.make_option('--builder-name', help='The name of the buildbot builder tests were run on.'),

        optparse.make_option('--build-number', help='The buildbot build number tests are associated with.'),

        optparse.make_option('--buildbot-worker', help='The buildbot worker tests were run on.'),

    ]

# Copyright (C) 2019 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND

# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import webkitpy.thirdparty.autoinstalled.requests

import json

import requests

import sys

import platform as host_platform

class Upload(object):

    UPLOAD_ENDPOINT = '/api/upload'

    BUILDBOT_DETAILS = ['buildbot-master', 'builder-name', 'build-number', 'buildbot-worker']

    VERSION = 0

    class Expectations:

        # These are ordered by priority, meaning that a test which both crashes and has

        # a warning should be considered to have crashed.

        ORDER = [

            'CRASH',

            'TIMEOUT',

            'IMAGE',   # Image-diff

            'AUDIO',   # Audio-diff

            'TEXT',    # Text-diff

            'FAIL',

            'ERROR',

            'WARNING',

            'PASS',

        ]

        CRASH, TIMEOUT, IMAGE, AUDIO, TEXT, FAIL, ERROR, WARNING, PASS = ORDER

    class Encoder(json.JSONEncoder):

        def default(self, obj):

            if not isinstance(obj, Upload):

                return super(Upload.Encoder, self).default(obj)

            if not obj.suite:

                raise ValueError('No suite specified to results upload')

            if not obj.commits:

                raise ValueError('No commits specified to results upload')

            details = obj.details or obj.create_details()

            buildbot_args = [details.get(arg, None) is None for arg in obj.BUILDBOT_DETAILS]

            if any(buildbot_args) and not all(buildbot_args):

                raise ValueError('All buildbot details must be defined for upload, details missing: {}'.format(', '.join(

                    [obj.BUILDBOT_DETAILS[i] for i in xrange(len(obj.BUILDBOT_DETAILS)) if buildbot_args[i]],

                )))

            def unpack_test(current, path_to_test, data):

                if len(path_to_test) == 1:

                    current[path_to_test[0]] = data

                    return

                if not current.get(path_to_test[0]):

                    current[path_to_test[0]] = {}

                unpack_test(current[path_to_test[0]], path_to_test[1:], data)

            results = {}

            for test, data in obj.results.iteritems():

                unpack_test(results, test.split('/'), data)

            result = dict(

                version=obj.VERSION,

                suite=obj.suite,

                configuration=obj.configuration or obj.create_configuration(),

                commits=obj.commits,

                test_results=dict(

                    details=details,

                    run_stats=obj.run_stats or obj.create_run_stats(),

                    results=results,

                ),

            )

            if obj.timestamp:

                result['timestamp'] = obj.timestamp

            return result

    @staticmethod

    def create_configuration(

            platform=None,

            is_simulator=False,

            version=None,

            architecture=None,

            version_name=None,

            model=None,

            style=None,   # Debug/Production/Release

            flavor=None,  # Dumping ground suite-wide configuration changes (ie, GuardMalloc)

            sdk=None,

        ):

        # This deviates slightly from the rest of webkitpy, but it allows this file to be entirely portable.

        config = dict(

            platform=platform or (host_platform.system() if host_platform.system() != 'Darwin' else 'mac').lower(),

            is_simulator=is_simulator,

            version=version or (host_platform.release() if host_platform.system() != 'Darwin' else host_platform.mac_ver()[0]),

            architecture=architecture or host_platform.machine(),

        )

        optional_data = dict(version_name=version_name, model=model, style=style, flavor=flavor, sdk=sdk)

        config.update({key: value for key, value in optional_data.iteritems() if value is not None})

        return config

    def __init__(self, suite=None, configuration=None, commits=[], timestamp=None, details=None, run_stats=None, results={}):

        self.suite = suite

        self.configuration = configuration

        self.commits = commits

        self.timestamp = timestamp

        self.details = details

        self.run_stats = run_stats

        self.results = results

    @staticmethod

    def create_commit(repository_id, id, branch=None):

        commit = dict(repository_id=repository_id, id=id)

        if branch:

            commit['branch'] = branch

        return commit

    @staticmethod

    def create_details(link=None, options=None, **kwargs):

        result = dict(**kwargs)

        if link:

            result['link'] = link

        if not options:

            return result

        for element in Upload.BUILDBOT_DETAILS:

            value = getattr(options, element.replace('-', '_'), None)

            if value is not None:

                result[element] = value

        return result

    @staticmethod

    def create_run_stats(start_time=None, end_time=None, tests_skipped=None, **kwargs):

        stats = dict(**kwargs)

        optional_data = dict(start_time=start_time, end_time=end_time, tests_skipped=tests_skipped)

        stats.update({key: value for key, value in optional_data.iteritems() if value is not None})

        return stats

    @staticmethod

    def create_test_result(expected=None, actual=None, log=None, **kwargs):

        result = dict(**kwargs)

        # Tests which don't declare expectations or results are assumed to have passed.

        optional_data = dict(expected=expected, actual=actual, log=log)

        result.update({key: value for key, value in optional_data.iteritems() if value is not None})

        return result

    def upload(self, url, log_line_func=lambda val: sys.stdout.write(val + '\n')):

        try:

            response = requests.post(url + self.UPLOAD_ENDPOINT, data=json.dumps(self, cls=Upload.Encoder))

        except requests.exceptions.ConnectionError:

            log_line_func(' ' * 4 + 'Failed to upload to {}, results server not online'.format(url))

            return False

        except ValueError as e:

            log_line_func(' ' * 4 + 'Failed to encode upload data: {}'.format(e))

            return False

        if response.status_code != 200:

            log_line_func(' ' * 4 + 'Error uploading to {}:'.format(url))

            log_line_func(' ' * 8 + response.json()['description'])

            return False

        log_line_func(' ' * 4 + 'Uploaded results to {}'.format(url))

        return True

# Copyright (C) 2019 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND

# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import webkitpy.thirdparty.autoinstalled.requests

import collections

import json

import requests

import time

import unittest

from webkitpy.results.upload import Upload

from webkitpy.thirdparty import mock

class UploadTest(unittest.TestCase):

    class Options(object):

        def __init__(self, **kwargs):

            for key, value in kwargs.iteritems():

                setattr(self, key, value)

    class MockResponse(object):

        def __init__(self, status_code=200, text=''):

            self.status_code = status_code

            self.text = text

        def json(self):

            return json.loads(self.text)

    @staticmethod

    def normalize(data):

        if isinstance(data, basestring):

            return str(data)

        elif isinstance(data, collections.Mapping):

            return dict(map(UploadTest.normalize, data.iteritems()))

        elif isinstance(data, collections.Iterable):

            return type(data)(map(UploadTest.normalize, data))

        return data

    @staticmethod

    def raise_requests_ConnectionError():

        raise requests.exceptions.ConnectionError()

    def test_encoding(self):

        start_time, end_time = time.time() - 3, time.time()

        upload = Upload(

            suite='webkitpy-tests',

            configuration=Upload.create_configuration(

                platform='mac',

                version='10.13.0',

                version_name='High Sierra',

                architecture='x86_64',

                sdk='17A405',

            ),

            details=Upload.create_details(link='https://webkit.org'),

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

            run_stats=Upload.create_run_stats(

                start_time=start_time,

                end_time=end_time,

                tests_skipped=0,

            ),

            results={

                'webkitpy.test1': {},

                'webkitpy.test2': Upload.create_test_result(expected=Upload.Expectations.PASS, actual=Upload.Expectations.FAIL),

            },

        )

        generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))

        self.assertEqual(generated_dict['version'], 0)

        self.assertEqual(generated_dict['suite'], 'webkitpy-tests')

        self.assertEqual(generated_dict['configuration'], self.normalize(dict(

            platform='mac',

            is_simulator=False,

            version='10.13.0',

            version_name='High Sierra',

            architecture='x86_64',

            sdk='17A405',

        )))

        self.assertEqual(generated_dict['commits'], [dict(

            repository_id='webkit',

            id='5',

            branch='trunk',

        )])

        self.assertEqual(generated_dict['test_results']['details'], self.normalize(dict(link='https://webkit.org')))

        self.assertEqual(generated_dict['test_results']['run_stats'], self.normalize(dict(

            start_time=start_time,

            end_time=end_time,

            tests_skipped=0,

        )))

        self.assertEqual(generated_dict['test_results']['results'], self.normalize({

            'webkitpy.test1': {},

            'webkitpy.test2': Upload.create_test_result(expected=Upload.Expectations.PASS, actual=Upload.Expectations.FAIL),

        }))

    def test_upload(self):

        upload = Upload(

            suite='webkitpy-tests',

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

        )

        with mock.patch('requests.post', new=lambda url, data: self.MockResponse()):

            self.assertTrue(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))

        with mock.patch('requests.post', new=lambda url, data: self.raise_requests_ConnectionError()):

            self.assertFalse(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))

        mock_404 = mock.patch('requests.post', new=lambda url, data: self.MockResponse(

            status_code=404,

            text=json.dumps(dict(description='No such address')),

        ))

        with mock_404:

            self.assertFalse(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))

    def test_packed_test(self):

        upload = Upload(

            suite='webkitpy-tests',

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

            results={

                'dir1/sub-dir1/test1': Upload.create_test_result(actual=Upload.Expectations.FAIL),

                'dir1/sub-dir1/test2': Upload.create_test_result(actual=Upload.Expectations.TIMEOUT),

                'dir1/sub-dir2/test3': {},

                'dir1/sub-dir2/test4': {},

                'dir2/sub-dir3/test5': {},

                'dir2/test6': {},

            }

        )

        generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))

        self.assertEqual(generated_dict['test_results']['results'], self.normalize({

            'dir1': {

                'sub-dir1': {

                    'test1': {'actual': Upload.Expectations.FAIL},

                    'test2': {'actual': Upload.Expectations.TIMEOUT},

                }, 'sub-dir2': {

                    'test3': {},

                    'test4': {},

                }

            }, 'dir2': {

                'sub-dir3': {'test5': {}},

                'test6': {},

            },

        }))

    def test_no_suite(self):

        upload = Upload(

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

        )

        with self.assertRaises(ValueError):

            json.dumps(upload, cls=Upload.Encoder)

    def test_no_commits(self):

        upload = Upload(

            suite='webkitpy-tests',

        )

        with self.assertRaises(ValueError):

            json.dumps(upload, cls=Upload.Encoder)

    def test_buildbot(self):

        upload = Upload(

            suite='webkitpy-tests',

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

            details=Upload.create_details(options=self.Options(

                buildbot_master='webkit.org',

                builder_name='Queue-1',

                build_number=1,

        )))

        with self.assertRaises(ValueError):

            json.dumps(upload, cls=Upload.Encoder)

        upload.details['buildbot-worker'] = 'bot123'

        generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))

        self.assertEqual(generated_dict['test_results']['details'], self.normalize({

            'buildbot-master': 'webkit.org',

            'builder-name': 'Queue-1',

            'build-number': 1,

            'buildbot-worker': 'bot123',

        }))

    def native_branch(self, path):

        result = self._run_git(['-C', self.find_checkout_root(path), 'rev-parse', '--abbrev-ref', 'HEAD']).rstrip()

        # For git-svn

        if result.startswith('heads'):

            return result[6:]

        return result

    def native_branch(self, path):

        return 'trunk'

    def test_native_branch(self):

        self.assertEqual(self.scm.native_branch('.'), 'trunk')

    def test_native_branch(self):

        scm = self.tracking_scm

        self.assertEqual('master', scm.native_branch(scm.checkout_root))

    def test_native_branch(self):

        self.assertEqual('trunk', self.scm.native_branch(self.git_checkout_path))

    def native_branch(self, path):

        relative_url = self.value_from_svn_info(path, 'Relative URL')[2:]

        if relative_url.startswith('trunk'):

            return 'trunk'

        elif relative_url.startswith('branch'):

            return relative_url.split('/')[1]

        raise Exception('{} is not a branch'.format(relative_url.split('/')[0]))

    def build_version(self):

        if self.is_mac():

            return self._executive.run_command(['/usr/bin/sw_vers', '-buildVersion'], return_stderr=False, ignore_errors=True).rstrip()

        return None

    def build_version(self):

        if self.is_mac():

            return '17A405'

        return None

# Required for Python to search this directory for module files

# Keep this file free of any code or import statements that could

# cause either an error to occur or a log message to be logged.

# This ensures that calling code can import initialization code from

# webkitpy before any errors or log messages due to code in this file.

# Initialization code can include things like version-checking code and

# logging configuration code.

#

# We do not execute any version-checking code or logging configuration

# code in this file so that callers can opt-in as they want.  This also

# allows different callers to choose different initialization code,

# as necessary.

# Copyright (C) 2019 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND

# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import optparse

def upload_options():

    return [

        optparse.make_option('--report', action='append', dest='report_urls', help='URL (or URLs) to report test results to'),

        optparse.make_option('--buildbot-master', help='The url of the buildbot master.'),

        optparse.make_option('--builder-name', help='The name of the buildbot builder tests were run on.'),

        optparse.make_option('--build-number', help='The buildbot build number tests are associated with.'),

        optparse.make_option('--buildbot-worker', help='The buildbot worker tests were run on.'),

    ]

# Copyright (C) 2019 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND

# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import webkitpy.thirdparty.autoinstalled.requests

import json

import requests

import sys

import platform as host_platform

class Upload(object):

    UPLOAD_ENDPOINT = '/api/upload'

    BUILDBOT_DETAILS = ['buildbot-master', 'builder-name', 'build-number', 'buildbot-worker']

    VERSION = 0

    class Expectations:

        # These are ordered by priority, meaning that a test which both crashes and has

        # a warning should be considered to have crashed.

        ORDER = [

            'CRASH',

            'TIMEOUT',

            'IMAGE',   # Image-diff

            'AUDIO',   # Audio-diff

            'TEXT',    # Text-diff

            'FAIL',

            'ERROR',

            'WARNING',

            'PASS',

        ]

        CRASH, TIMEOUT, IMAGE, AUDIO, TEXT, FAIL, ERROR, WARNING, PASS = ORDER

    class Encoder(json.JSONEncoder):

        def default(self, obj):

            if not isinstance(obj, Upload):

                return super(Upload.Encoder, self).default(obj)

            if not obj.suite:

                raise ValueError('No suite specified to results upload')

            if not obj.commits:

                raise ValueError('No commits specified to results upload')

            details = obj.details or obj.create_details()

            buildbot_args = [details.get(arg, None) is None for arg in obj.BUILDBOT_DETAILS]

            if any(buildbot_args) and not all(buildbot_args):

                raise ValueError('All buildbot details must be defined for upload, details missing: {}'.format(', '.join(

                    [obj.BUILDBOT_DETAILS[i] for i in xrange(len(obj.BUILDBOT_DETAILS)) if buildbot_args[i]],

                )))

            def unpack_test(current, path_to_test, data):

                if len(path_to_test) == 1:

                    current[path_to_test[0]] = data

                    return

                if not current.get(path_to_test[0]):

                    current[path_to_test[0]] = {}

                unpack_test(current[path_to_test[0]], path_to_test[1:], data)

            results = {}

            for test, data in obj.results.iteritems():

                unpack_test(results, test.split('/'), data)

            result = dict(

                version=obj.VERSION,

                suite=obj.suite,

                configuration=obj.configuration or obj.create_configuration(),

                commits=obj.commits,

                test_results=dict(

                    details=details,

                    run_stats=obj.run_stats or obj.create_run_stats(),

                    results=results,

                ),

            )

            if obj.timestamp:

                result['timestamp'] = obj.timestamp

            return result

    @staticmethod

    def create_configuration(

            platform=None,

            is_simulator=False,

            version=None,

            architecture=None,

            version_name=None,

            model=None,

            style=None,   # Debug/Production/Release

            flavor=None,  # Dumping ground suite-wide configuration changes (ie, GuardMalloc)

            sdk=None,

        ):

        # This deviates slightly from the rest of webkitpy, but it allows this file to be entirely portable.

        config = dict(

            platform=platform or (host_platform.system() if host_platform.system() != 'Darwin' else 'mac').lower(),

            is_simulator=is_simulator,

            version=version or (host_platform.release() if host_platform.system() != 'Darwin' else host_platform.mac_ver()[0]),

            architecture=architecture or host_platform.machine(),

        )

        optional_data = dict(version_name=version_name, model=model, style=style, flavor=flavor, sdk=sdk)

        config.update({key: value for key, value in optional_data.iteritems() if value is not None})

        return config

    def __init__(self, suite=None, configuration=None, commits=[], timestamp=None, details=None, run_stats=None, results={}):

        self.suite = suite

        self.configuration = configuration

        self.commits = commits

        self.timestamp = timestamp

        self.details = details

        self.run_stats = run_stats

        self.results = results

    @staticmethod

    def create_commit(repository_id, id, branch=None):

        commit = dict(repository_id=repository_id, id=id)

        if branch:

            commit['branch'] = branch

        return commit

    @staticmethod

    def create_details(link=None, options=None, **kwargs):

        result = dict(**kwargs)

        if link:

            result['link'] = link

        if not options:

            return result

        for element in Upload.BUILDBOT_DETAILS:

            value = getattr(options, element.replace('-', '_'), None)

            if value is not None:

                result[element] = value

        return result

    @staticmethod

    def create_run_stats(start_time=None, end_time=None, tests_skipped=None, **kwargs):

        stats = dict(**kwargs)

        optional_data = dict(start_time=start_time, end_time=end_time, tests_skipped=tests_skipped)

        stats.update({key: value for key, value in optional_data.iteritems() if value is not None})

        return stats

    @staticmethod

    def create_test_result(expected=None, actual=None, log=None, **kwargs):

        result = dict(**kwargs)

        # Tests which don't declare expectations or results are assumed to have passed.

        optional_data = dict(expected=expected, actual=actual, log=log)

        result.update({key: value for key, value in optional_data.iteritems() if value is not None})

        return result

    def upload(self, url, log_line_func=lambda val: sys.stdout.write(val + '\n')):

        try:

            response = requests.post(url + self.UPLOAD_ENDPOINT, data=json.dumps(self, cls=Upload.Encoder))

        except requests.exceptions.ConnectionError:

            log_line_func(' ' * 4 + 'Failed to upload to {}, results server not online'.format(url))

            return False

        except ValueError as e:

            log_line_func(' ' * 4 + 'Failed to encode upload data: {}'.format(e))

            return False

        if response.status_code != 200:

            log_line_func(' ' * 4 + 'Error uploading to {}:'.format(url))

            log_line_func(' ' * 8 + response.json()['description'])

            return False

        log_line_func(' ' * 4 + 'Uploaded results to {}'.format(url))

        return True

# Copyright (C) 2019 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer.

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND

# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR

# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import webkitpy.thirdparty.autoinstalled.requests

import collections

import json

import requests

import time

import unittest

from webkitpy.results.upload import Upload

from webkitpy.thirdparty import mock

class UploadTest(unittest.TestCase):

    class Options(object):

        def __init__(self, **kwargs):

            for key, value in kwargs.iteritems():

                setattr(self, key, value)

    class MockResponse(object):

        def __init__(self, status_code=200, text=''):

            self.status_code = status_code

            self.text = text

        def json(self):

            return json.loads(self.text)

    @staticmethod

    def normalize(data):

        if isinstance(data, basestring):

            return str(data)

        elif isinstance(data, collections.Mapping):

            return dict(map(UploadTest.normalize, data.iteritems()))

        elif isinstance(data, collections.Iterable):

            return type(data)(map(UploadTest.normalize, data))

        return data

    @staticmethod

    def raise_requests_ConnectionError():

        raise requests.exceptions.ConnectionError()

    def test_encoding(self):

        start_time, end_time = time.time() - 3, time.time()

        upload = Upload(

            suite='webkitpy-tests',

            configuration=Upload.create_configuration(

                platform='mac',

                version='10.13.0',

                version_name='High Sierra',

                architecture='x86_64',

                sdk='17A405',

            ),

            details=Upload.create_details(link='https://webkit.org'),

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

            run_stats=Upload.create_run_stats(

                start_time=start_time,

                end_time=end_time,

                tests_skipped=0,

            ),

            results={

                'webkitpy.test1': {},

                'webkitpy.test2': Upload.create_test_result(expected=Upload.Expectations.PASS, actual=Upload.Expectations.FAIL),

            },

        )

        generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))

        self.assertEqual(generated_dict['version'], 0)

        self.assertEqual(generated_dict['suite'], 'webkitpy-tests')

        self.assertEqual(generated_dict['configuration'], self.normalize(dict(

            platform='mac',

            is_simulator=False,

            version='10.13.0',

            version_name='High Sierra',

            architecture='x86_64',

            sdk='17A405',

        )))

        self.assertEqual(generated_dict['commits'], [dict(

            repository_id='webkit',

            id='5',

            branch='trunk',

        )])

        self.assertEqual(generated_dict['test_results']['details'], self.normalize(dict(link='https://webkit.org')))

        self.assertEqual(generated_dict['test_results']['run_stats'], self.normalize(dict(

            start_time=start_time,

            end_time=end_time,

            tests_skipped=0,

        )))

        self.assertEqual(generated_dict['test_results']['results'], self.normalize({

            'webkitpy.test1': {},

            'webkitpy.test2': Upload.create_test_result(expected=Upload.Expectations.PASS, actual=Upload.Expectations.FAIL),

        }))

    def test_upload(self):

        upload = Upload(

            suite='webkitpy-tests',

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

        )

        with mock.patch('requests.post', new=lambda url, data: self.MockResponse()):

            self.assertTrue(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))

        with mock.patch('requests.post', new=lambda url, data: self.raise_requests_ConnectionError()):

            self.assertFalse(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))

        mock_404 = mock.patch('requests.post', new=lambda url, data: self.MockResponse(

            status_code=404,

            text=json.dumps(dict(description='No such address')),

        ))

        with mock_404:

            self.assertFalse(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))

    def test_packed_test(self):

        upload = Upload(

            suite='webkitpy-tests',

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

            results={

                'dir1/sub-dir1/test1': Upload.create_test_result(actual=Upload.Expectations.FAIL),

                'dir1/sub-dir1/test2': Upload.create_test_result(actual=Upload.Expectations.TIMEOUT),

                'dir1/sub-dir2/test3': {},

                'dir1/sub-dir2/test4': {},

                'dir2/sub-dir3/test5': {},

                'dir2/test6': {},

            }

        )

        generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))

        self.assertEqual(generated_dict['test_results']['results'], self.normalize({

            'dir1': {

                'sub-dir1': {

                    'test1': {'actual': Upload.Expectations.FAIL},

                    'test2': {'actual': Upload.Expectations.TIMEOUT},

                }, 'sub-dir2': {

                    'test3': {},

                    'test4': {},

                }

            }, 'dir2': {

                'sub-dir3': {'test5': {}},

                'test6': {},

            },

        }))

    def test_no_suite(self):

        upload = Upload(

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

        )

        with self.assertRaises(ValueError):

            json.dumps(upload, cls=Upload.Encoder)

    def test_no_commits(self):

        upload = Upload(

            suite='webkitpy-tests',

        )

        with self.assertRaises(ValueError):

            json.dumps(upload, cls=Upload.Encoder)

    def test_buildbot(self):

        upload = Upload(

            suite='webkitpy-tests',

            commits=[Upload.create_commit(

                repository_id='webkit',

                id='5',

                branch='trunk',

            )],

            details=Upload.create_details(options=self.Options(

                buildbot_master='webkit.org',

                builder_name='Queue-1',

                build_number=1,

        )))

        with self.assertRaises(ValueError):

            json.dumps(upload, cls=Upload.Encoder)

        upload.details['buildbot-worker'] = 'bot123'

        generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))

        self.assertEqual(generated_dict['test_results']['details'], self.normalize({

            'buildbot-master': 'webkit.org',

            'builder-name': 'Queue-1',

            'build-number': 1,

            'buildbot-worker': 'bot123',

        }))

from webkitpy.results.upload import Upload

from webkitpy.results.options import upload_options

        upload_group = optparse.OptionGroup(parser, 'Upload Options')

        upload_group.add_options(upload_options())

        parser.add_option_group(upload_group)

        start_time = time.time()

        config = Config(_host.executive, self.finder.filesystem)

        configuration_to_use = self._options.configuration or config.default_configuration()

        end_time = time.time()

        failed_uploads = 0

        if self._options.report_urls:

            self.printer.meter.writeln('\n')

            self.printer.write_update('Preparing upload data ...')

            # Empty test results indicate a PASS.

            results = {test: {} for test in test_runner.tests_run}

            for test, errors in test_runner.errors:

                results[test] = Upload.create_test_result(actual=Upload.Expectations.ERROR, log='/n'.join(errors))

            for test, failures in test_runner.failures:

                results[test] = Upload.create_test_result(actual=Upload.Expectations.FAIL, log='/n'.join(failures))

            _host.initialize_scm()

            upload = Upload(

                suite='webkitpy-tests',

                configuration=Upload.create_configuration(

                    platform=_host.platform.os_name,

                    version=str(_host.platform.os_version),

                    version_name=_host.platform.os_version_name(),

                    style=configuration_to_use,

                    sdk=_host.platform.build_version(),

                ),

                details=Upload.create_details(options=self._options),

                commits=[Upload.create_commit(

                    repository_id='webkit',

                    id=_host.scm().native_revision(_webkit_root),

                    branch=_host.scm().native_branch(_webkit_root),

                )],

                run_stats=Upload.create_run_stats(

                    start_time=start_time,

                    end_time=end_time,

                    tests_skipped=len(test_runner.tests_run) - len(parallel_tests) - len(serial_tests),

                ),

                results=results,

            )

            for url in self._options.report_urls:

                self.printer.write_update('Uploading to {} ...'.format(url))

                failed_uploads = failed_uploads if upload.upload(url, log_line_func=self.printer.meter.writeln) else (failed_uploads + 1)

            self.printer.meter.writeln('Uploads completed!')

        if self._options.coverage:

# Copyright (C) 2019 Apple Inc. All rights reserved.

        elif '.requests' in fullname:

            self._install_requests()

    def _install_requests(self):

        self._ensure_autoinstalled_dir_is_in_sys_path()

        self._install("https://files.pythonhosted.org/packages/06/b8/d1ea38513c22e8c906275d135818fee16ad8495985956a9b7e2bb21942a1/certifi-2019.3.9.tar.gz",

                      "certifi-2019.3.9/certifi")

        self._install("https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz",

                      "chardet-3.0.4/chardet")

        self._install("https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz",

                      "chardet-3.0.4/chardet")

        self._install("https://files.pythonhosted.org/packages/ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7/idna-2.8.tar.gz",

                      "idna-2.8/idna")

        self._install("https://files.pythonhosted.org/packages/b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1/urllib3-1.24.1.tar.gz",

                      "urllib3-1.24.1/src/urllib3")

        self._install("https://files.pythonhosted.org/packages/52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38/requests-2.21.0.tar.gz",

                      "requests-2.21.0/requests")