Attachment 340011 Details for Bug 185484 – Proposed patch

[patch] Proposed patch

ews-buildbot-config.patch (text/plain), 13.95 KB, created by Aakash Jain on 2018-05-09 12:58:37 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Aakash Jain

Created: 2018-05-09 12:58:37 PDT

Size: 13.95 KB

patch

obsolete

>Index: Tools/ChangeLog
>===================================================================
>--- Tools/ChangeLog	(revision 231585)
>+++ Tools/ChangeLog	(working copy)
>@@ -1,3 +1,23 @@
>+2018-05-09  Aakash Jain  <aakash_jain@apple.com>
>+
>+        Add Buildbot configuration for Opensource EWS
>+        https://bugs.webkit.org/show_bug.cgi?id=185484
>+
>+        Reviewed by NOBODY (OOPS!).
>+
>+        * BuildSlaveSupport/ews-build/config.json: Added, config file.
>+        * BuildSlaveSupport/ews-build/factories.py: Added, basic template for factories.
>+        * BuildSlaveSupport/ews-build/loadConfig.py: Added.
>+        (loadBuilderConfig): Loads the configuration.
>+        (checkValidWorker): Checks if the worker is valid.
>+        (checkValidBuilder): Checks if the builder is valid.
>+        (checkWorkersAndBuildersForConsistency): Check if workers and builders are consistent.
>+        (checkWorkersAndBuildersForConsistency._find_worker_with_name):
>+        (getBlackListedTags): Returns a list of keywords which should not be listed as tags.
>+        (getValidTags): Returns a list of valid tags.
>+        (getTagsForBuilder): Return a list of tags for given builder.
>+        * BuildSlaveSupport/ews-build/master.cfg: Configuration for buildbot.
>+
> 2018-05-09  Thibault Saunier  <tsaunier@igalia.com>
> 
>         [tests][GStreamer]: Pass USE_PLAYBIN3 to the tests subprocess
>Index: Tools/BuildSlaveSupport/ews-build/config.json
>===================================================================
>--- Tools/BuildSlaveSupport/ews-build/config.json	(nonexistent)
>+++ Tools/BuildSlaveSupport/ews-build/config.json	(working copy)
>@@ -0,0 +1,276 @@
>+{
>+  "workers": [
>+    {
>+      "name": "webkit-misc",
>+      "platform": "*"
>+    },
>+    {
>+      "name": "tanty-gtk-wk2-ews",
>+      "platform": "gtk"
>+    },
>+    {
>+      "name": "ltilve-gtk-wk2-ews",
>+      "platform": "gtk"
>+    },
>+    {
>+      "name": "igalia-wpe-ews",
>+      "platform": "wpe"
>+    },
>+    {
>+      "name": "aperez-wpe-gcc5-ews",
>+      "platform": "wpe"
>+    },
>+    {
>+      "name": "wincairo-ews-001",
>+      "platform": "wincairo"
>+    },
>+    {
>+      "name": "wincairo-ews-002",
>+      "platform": "wincairo"
>+    },
>+    {
>+      "name": "wincairo-ews-003",
>+      "platform": "wincairo"
>+    },
>+    {
>+      "name": "wincairo-ews-004",
>+      "platform": "wincairo"
>+    },
>+    {
>+      "name": "ews100",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews101",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews102",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews103",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews104",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews105",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews106",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews107",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews108",
>+      "platform": "ios-11"
>+    },
>+    {
>+      "name": "ews109",
>+      "platform": "ios-11"
>+    },
>+    {
>+      "name": "ews112",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews113",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews114",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews115",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews116",
>+      "platform": "mac-sierra"
>+    },
>+    {
>+      "name": "ews117",
>+      "platform": "tbd"
>+    },
>+    {
>+      "name": "ews118",
>+      "platform": "mac-high-sierra"
>+    },
>+    {
>+      "name": "ews119",
>+      "platform": "mac-high-sierra"
>+    },
>+    {
>+      "name": "ews120",
>+      "platform": "mac-high-sierra"
>+    },
>+    {
>+      "name": "ews123",
>+      "platform": "ios-11-simulator"
>+    },
>+    {
>+      "name": "ews124",
>+      "platform": "ios-11-simulator"
>+    },
>+    {
>+      "name": "ews125",
>+      "platform": "ios-11-simulator"
>+    },
>+    {
>+      "name": "ews126",
>+      "platform": "ios-11-simulator"
>+    },
>+    {
>+      "name": "ews127",
>+      "platform": "*"
>+    },
>+    {
>+      "name": "ews128",
>+      "platform": "*"
>+    },
>+    {
>+      "name": "ews200",
>+      "platform": "win"
>+    },
>+    {
>+      "name": "ews201",
>+      "platform": "win"
>+    },
>+    {
>+      "name": "ews202",
>+      "platform": "win"
>+    },
>+    {
>+      "name": "ews203",
>+      "platform": "win"
>+    },
>+    {
>+      "name": "ews204",
>+      "platform": "win"
>+    },
>+    {
>+      "name": "ews205",
>+      "platform": "win"
>+    },
>+    {
>+      "name": "ews206",
>+      "platform": "win"
>+    },
>+    {
>+      "name": "ews207",
>+      "platform": "win"
>+    },
>+    {
>+      "name": "ews208",
>+      "platform": "win"
>+    }
>+  ],
>+  "builders": [
>+    {
>+      "name": "Style-EWS",
>+      "factory": "StyleFactory",
>+      "platform": "*",
>+      "workernames": ["webkit-misc"]
>+    },
>+    {
>+      "name": "GTK-Webkit2-EWS",
>+      "factory": "GTKFactory",
>+      "platform": "gtk",
>+      "workernames": ["tanty-gtk-wk2-ews", "ltilve-gtk-wk2-ews"]
>+    },
>+    {
>+      "name": "iOS-11-EWS",
>+      "factory": "iOSFactory",
>+      "platform": "ios-11",
>+      "workernames": ["ews108", "ews109"]
>+    },
>+    {
>+      "name": "iOS-11-Simulator-EWS",
>+      "factory": "iOSSimulatorFactory",
>+      "platform": "ios-11-simulator",
>+      "workernames": ["ews123", "ews124", "ews125", "ews126"]
>+    },
>+    {
>+      "name": "macOS-Sierra-Release-WK1-EWS",
>+      "factory": "MacWK1Factory",
>+      "platform": "mac-sierra",
>+      "configuration": "Release",
>+      "workernames": ["ews100", "ews101", "ews102", "ews103"]
>+    },
>+    {
>+      "name": "macOS-Sierra-Release-WK2-EWS",
>+      "factory": "MacWK2Factory",
>+      "platform": "mac-sierra",
>+      "configuration": "Release",
>+      "workernames": ["ews104", "ews105", "ews106", "ews107"]
>+    },
>+    {
>+      "name": "macOS-Sierra-Debug-WK1-EWS",
>+      "factory": "MacWK1Factory",
>+      "platform": "mac-sierra",
>+      "configuration": "Debug",
>+      "workernames": ["ews112", "ews113", "ews114", "ews115", "ews116"]
>+    },
>+    {
>+      "name": "macOS-High-Sierra-Release-32bit-WK2-EWS",
>+      "factory": "MacWK2Factory",
>+      "platform": "mac-high-sierra",
>+      "configuration": "Release",
>+      "workernames": ["ews118", "ews119", "ews120"]
>+    },
>+    {
>+      "name": "Windows-EWS",
>+      "factory": "WindowsFactory",
>+      "platform": "win",
>+      "workernames": ["ews200", "ews201", "ews202", "ews203", "ews204", "ews205", "ews206", "ews207", "ews208"]
>+    },
>+    {
>+      "name": "WinCairo-EWS",
>+      "factory": "WinCairoFactory",
>+      "platform": "wincairo",
>+      "workernames": ["wincairo-ews-001", "wincairo-ews-002", "wincairo-ews-003", "wincairo-ews-004"]
>+    },
>+    {
>+      "name": "WPE-EWS",
>+      "factory": "WPEFactory",
>+      "platform": "wpe",
>+      "workernames": ["igalia-wpe-ews", "aperez-wpe-gcc5-ews"]
>+    },
>+    {
>+      "name": "JSC-tests-EWS",
>+      "factory": "JSCTestsFactory",
>+      "platform": "*",
>+      "workernames": ["ews127", "ews128"]
>+    },
>+    {
>+      "name": "Bindings-tests-EWS",
>+      "factory": "BindingsFactory",
>+      "platform": "*",
>+      "workernames": ["webkit-misc"]
>+    },
>+    {
>+      "name": "Webkitpy-tests-EWS",
>+      "factory": "WebkitpyFactory",
>+      "platform": "*",
>+      "workernames": ["webkit-misc"]
>+    }
>+  ],
>+  "schedulers": [
>+    {
>+      "type": "AnyBranchScheduler",
>+      "name": "misc-trigger",
>+      "treeStableTimer": 0,
>+      "builderNames": ["Style-EWS", "JSC-tests-EWS", "macOS-Sierra-Release-WK1-EWS", "GTK-Webkit2-EWS", "macOS-Sierra-Release-WK2-EWS",
>+                       "macOS-High-Sierra-Release-32bit-WK2-EWS", "WPE-EWS", "Windows-EWS", "iOS-11-EWS", "WinCairo-EWS", "iOS-11-Simulator-EWS",
>+                       "Webkitpy-tests-EWS", "macOS-Sierra-Debug-WK1-EWS", "Bindings-tests-EWS"]
>+    }
>+  ]
>+}
>Index: Tools/BuildSlaveSupport/ews-build/factories.py
>===================================================================
>--- Tools/BuildSlaveSupport/ews-build/factories.py	(nonexistent)
>+++ Tools/BuildSlaveSupport/ews-build/factories.py	(working copy)
>@@ -0,0 +1,49 @@
>+from buildbot.process import factory
>+
>+
>+class StyleFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class GTKFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class iOSFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class iOSSimulatorFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class MacWK1Factory(factory.BuildFactory):
>+    pass
>+
>+
>+class MacWK2Factory(factory.BuildFactory):
>+    pass
>+
>+
>+class WindowsFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class WinCairoFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class WPEFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class JSCTestsFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class BindingsFactory(factory.BuildFactory):
>+    pass
>+
>+
>+class WebkitpyFactory(factory.BuildFactory):
>+    pass
>Index: Tools/BuildSlaveSupport/ews-build/loadConfig.py
>===================================================================
>--- Tools/BuildSlaveSupport/ews-build/loadConfig.py	(nonexistent)
>+++ Tools/BuildSlaveSupport/ews-build/loadConfig.py	(working copy)
>@@ -0,0 +1,104 @@
>+import json
>+import re
>+
>+from buildbot.scheduler import AnyBranchScheduler, Periodic, Dependent, Triggerable, Nightly
>+from buildbot.worker import Worker
>+from buildbot.util import identifiers as buildbot_identifiers
>+
>+from factories import *
>+
>+BUILDER_NAME_LENGTH_LIMIT = 70
>+STEP_NAME_LENGTH_LIMIT = 50
>+
>+
>+def loadBuilderConfig(c):
>+    config = json.load(open('config.json'))
>+    passwords = json.load(open('passwords.json'))
>+    checkWorkersAndBuildersForConsistency(config['workers'], config['builders'])
>+
>+    c['workers'] = [Worker(worker['name'], passwords.get(worker['name'], "tbd")) for worker in config['workers']]
>+    c['builders'] = []
>+    for builder in config['builders']:
>+        factory = globals()[builder["factory"]]
>+        builder["factory"] = factory()
>+        builder.pop('platform')
>+        if builder.get('configuration'):
>+            builder.pop('configuration')
>+        builder['tags'] = getTagsForBuilder(builder)
>+        c['builders'].append(builder)
>+
>+    c['schedulers'] = []
>+    for scheduler in config['schedulers']:
>+        kls = globals()[scheduler.pop('type')]
>+        scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
>+        c['schedulers'].append(kls(**scheduler))
>+
>+
>+def checkValidWorker(worker):
>+    if not worker.get('name'):
>+        raise Exception('Worker "{}" does not have name defined.'.format(worker))
>+
>+    if not worker.get('platform'):
>+        raise Exception('Worker "{}" does not have platform defined.'.format(worker['name']))
>+
>+
>+def checkValidBuilder(builder):
>+    if not builder.get('name'):
>+        raise Exception('Builder "{}" does not have name defined.'.format(builder))
>+
>+    if not buildbot_identifiers.ident_re.match(builder['name']):
>+        raise Exception('Builder name "{}" is not a valid buildbot identifier.'.format(builder['name']))
>+
>+    if len(builder['name']) > BUILDER_NAME_LENGTH_LIMIT:
>+        raise Exception('Builder name "{}" is longer than maximum allowed by Buildbot ({} characters).'.format(builder['name'], BUILDER_NAME_LENGTH_LIMIT))
>+
>+    if 'configuration' in builder and builder['configuration'] not in ['Debug', 'Production', 'Release']:
>+        raise Exception("Invalid configuration: {} for builder: {}".format(builder.get('configuration'), builder.get('name')))
>+
>+    if not builder.get('factory'):
>+        raise Exception('Builder "{}" does not have factory defined.'.format(builder['name']))
>+
>+    if not builder.get('platform'):
>+        raise Exception('Builder "{}" does not have platform defined.'.format(builder['name']))
>+
>+
>+def checkWorkersAndBuildersForConsistency(workers, builders):
>+    def _find_worker_with_name(workers, worker_name):
>+        for worker in workers:
>+            if worker['name'] == worker_name:
>+                return worker
>+        return None
>+
>+    for worker in workers:
>+        checkValidWorker(worker)
>+
>+    for builder in builders:
>+        checkValidBuilder(builder)
>+        for worker_name in builder['workernames']:
>+            worker = _find_worker_with_name(workers, worker_name)
>+            if not worker:
>+                raise Exception('Builder "{}" has worker "{}", which is not defined in workers list!'.format(builder['name'], worker_name))
>+
>+            if worker['platform'] != builder['platform']:
>+                raise Exception('Builder "{0}" is for platform "{1}", but has worker "{2}" for platform "{3}"!'.format(
>+                    builder['name'], builder['platform'], worker['name'], worker['platform']))
>+
>+
>+def getBlackListedTags():
>+    """
>+    We maintain a blacklist of words which we do not want to display as tag in buildbot.
>+    We generate a list of tags by splitting the builder name. We do not want certain words as tag.
>+    For e.g. we don't want '11'as tag for builder iOS-11-Simulator-EWS
>+    """
>+    tags_blacklist = [str(i) for i in range(0, 20)]
>+    tags_blacklist.extend(['EWS', 'TryBot'])
>+    return tags_blacklist
>+
>+
>+def getValidTags(tags):
>+    return list(set(tags) - set(getBlackListedTags()))
>+
>+
>+def getTagsForBuilder(builder):
>+    keywords = filter(None, re.split("[, \-_:()]+", str(builder['name'])))
>+    return getValidTags(keywords)
>Index: Tools/BuildSlaveSupport/ews-build/master.cfg
>===================================================================
>--- Tools/BuildSlaveSupport/ews-build/master.cfg	(nonexistent)
>+++ Tools/BuildSlaveSupport/ews-build/master.cfg	(working copy)
>@@ -0,0 +1,23 @@
>+import loadConfig
>+
>+
>+c = BuildmasterConfig = {}
>+
>+c['www'] = dict(port=8010,
>+                plugins=dict(waterfall_view={}, console_view={}))
>+
>+c['www']['ui_default_config'] = { 
>+    'Builders.show_workers_name': True,
>+    'Builders.buildFetchLimit': 1000,
>+    'Workers.showWorkerBuilders': True,
>+}
>+
>+c['protocols'] = {'pb': {'port': 9989}}
>+
>+c['projectName'] = 'Webkit EWS'
>+c['projectURL'] = 'https://ews-build.webkit.org/'
>+c['buildbotURL'] = 'https://ews-build.webkit.org/'
>+
>+c['buildbotNetUsageData'] = None
>+
>+loadConfig.loadBuilderConfig(c)

Actions: View | Formatted Diff | Diff

Attachments on bug 185484: 340011 | 341011 | 341100 | 341141