Attachment #447317 for bug #234378

View Details Raw Unified Return to Bug 234378



2021-12-15  Carlos Alberto Lopez Perez  <clopez@igalia.com>

        [GTK][WPE] Apply optimizations to speed up the layout-test EWS and add a few more workers to the build ones.
        https://bugs.webkit.org/show_bug.cgi?id=234378

        Reviewed by NOBODY (OOPS!).

        This applies two optimizations to speed up the testing on the layout-tests EWS added in r286405

        1. Pass --fully-parallel to the retry steps. This makes the tool to utilize all the available workers for running
        the tests instead of running them in serial. That makes the time needed to complete the retry steps to be considerable less,
        In this example https://ews-build.webkit.org/#/builders/35/builds/1757 it reduces the timing from 43 mins to 12 mins.

        2. When the first run fails without giving a list of failures and we have not reached the maximum retry count
        we are going to end retring the whole testing anyway, so there is no point in running the tests without patch
        in that case, we can skip doing that to save time. See: https://ews-build.webkit.org/#/builders/35/builds/1763

        On top of that this patch adds an extra worker to the EWS build queues of GTK and WPE
        because I detected that this queues sometimes are not fast enough.

        * CISupport/ews-build/config.json:
        * CISupport/ews-build/steps.py:
        (BugzillaMixin.send_email_for_infrastructure_issue):
        (RunWebKitTestsRedTree.evaluateCommand):
        (RunWebKitTestsRepeatFailuresRedTree.setLayoutTestCommand):
        (RunWebKitTestsRepeatFailuresWithoutPatchRedTree.setLayoutTestCommand):
        * CISupport/ews-build/steps_unittest.py:

2021-12-15  Jonathan Bedard  <jbedard@apple.com>

        [reporelaypy] Forward branches to alternate remote



    { "name": "igalia10-gtk-wk2-ews", "platform": "gtk" },
    { "name": "igalia11-gtk-wk2-ews", "platform": "gtk" },
    { "name": "igalia12-gtk-wk2-ews", "platform": "gtk" },
    { "name": "igalia13-gtk-wk2-ews", "platform": "gtk" },
    { "name": "aperez-gtk-ews", "platform": "gtk" },
    { "name": "igalia-wpe-ews", "platform": "wpe" },
    { "name": "igalia2-wpe-ews", "platform": "wpe" },
    { "name": "aperez-wpe-ews", "platform": "wpe" },
    { "name": "wincairo-ews-001", "platform": "wincairo" },
    { "name": "wincairo-ews-002", "platform": "wincairo" },

      "factory": "GTKBuildFactory", "platform": "gtk",
      "configuration": "release", "architectures": ["x86_64"],
      "triggers": ["api-tests-gtk-ews", "gtk-wk2-tests-ews"],
      "workernames": ["aperez-gtk-ews", "igalia1-gtk-wk2-ews", "igalia2-gtk-wk2-ews", "igalia13-gtk-wk2-ews"]
    },
    {
      "name": "GTK-WK2-Tests-EWS", "shortname": "gtk-wk2", "icon": "testOnly",

      "name": "WPE-EWS", "shortname": "wpe", "icon": "buildOnly",
      "factory": "WPEFactory", "platform": "wpe",
      "configuration": "release", "architectures": ["x86_64"],
      "workernames": ["aperez-wpe-ews", "igalia-wpe-ews", "igalia2-wpe-ews"]
    },
    {
      "name": "JSC-Tests-EWS", "shortname": "jsc", "icon": "buildAndTest",



            build_url = '{}#/builders/{}/builds/{}'.format(self.master.config.buildbotURL, self.build._builderid, self.build.number)
            email_subject = 'Infrastructure issue at {}'.format(builder_name)
            email_text = 'The following infrastructure issue happened at:\n\n'
            email_text += '    - Build : <a href="{}">{}</a>\n'.format(build_url, build_url)
            email_text += '    - Builder : {}\n'.format(builder_name)
            email_text += '    - Worker : {}\n'.format(worker_name)
            email_text += '    - Issue: {}\n'.format(infrastructure_issue_text)

            self.build.results = SUCCESS
            self.setProperty('build_summary', message)
        else:
            # We have a failure return code but not a list of failed or flaky tests, so we can't run the repeat steps.
            # If we are on the last retry then run the whole layout tests without patch.
            # If not, then go to analyze-layout-tests-results where we will retry everything hoping this was a random failure.
            # then we would not report a list of failed test, just a generic "unknown" failure.
            self.setProperty('patchFailedTests', True)
            retry_count = int(self.getProperty('retry_count', 0))
            if retry_count < AnalyzeLayoutTestsResultsRedTree.MAX_RETRY:
                next_steps.append(AnalyzeLayoutTestsResultsRedTree())
            else:
                next_steps.extend([UnApplyPatchIfRequired(), CompileWebKitWithoutPatch(retry_build_on_failure=True), ValidatePatch(verifyBugClosed=False, addURLs=False), RunWebKitTestsWithoutPatchRedTree()])
        if next_steps:
            self.build.addStepsAfterCurrentStep(next_steps)
        return rc

    def setLayoutTestCommand(self):
        super().setLayoutTestCommand()
        first_results_failing_tests = set(self.getProperty('first_run_failures', []))
        self.setCommand(self.command + ['--fully-parallel', '--repeat-each=%s' % self.NUM_REPEATS_PER_TEST] + sorted(first_results_failing_tests))

    def evaluateCommand(self, cmd):
        with_patch_repeat_failures_results_nonflaky_failures = set(self.getProperty('with_patch_repeat_failures_results_nonflaky_failures', []))

        # is skipped anyways if is marked as such on the Expectation files or if is marked
        # as failure (since we are passing also '--skip-failing-tests'). That way we ensure
        # to report the case of a patch removing an expectation that still fails with it.
        self.setCommand(self.command + ['--fully-parallel', '--repeat-each=%s' % self.NUM_REPEATS_PER_TEST, '--skipped=always'] + sorted(failures_to_repeat))

    def evaluateCommand(self, cmd):
        rc = self.evaluateResult(cmd)



                                 'Tools/Scripts/run-webkit-tests',
                                 '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',
                                 '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',
                                 '--skip-failing-tests', '--fully-parallel', '--repeat-each=10'] + sorted(first_run_failures)
                        )
            + 0,
        )

                                 'Tools/Scripts/run-webkit-tests',
                                 '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',
                                 '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',
                                 '--skip-failing-tests', '--fully-parallel', '--repeat-each=10', '--skipped=always'] + sorted(with_patch_repeat_failures_results_nonflaky_failures)
                        )
            + 0,
        )

                                 'Tools/Scripts/run-webkit-tests',
                                 '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',
                                 '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',
                                 '--skip-failing-tests', '--fully-parallel', '--repeat-each=10', '--skipped=always'] + sorted(first_run_failures)
                        )
            + 0,
        )

Return to Bug 234378

Lines 755-761 class BugzillaMixin(object): a/Tools/CISupport/ews-build/steps.py_sec1
- a/Tools/CISupport/ews-build/steps.py -8 / +11 lines
755	build_url = '{}#/builders/{}/builds/{}'.format(self.master.config.buildbotURL, self.build._builderid, self.build.number)	755	build_url = '{}#/builders/{}/builds/{}'.format(self.master.config.buildbotURL, self.build._builderid, self.build.number)
756	email_subject = 'Infrastructure issue at {}'.format(builder_name)	756	email_subject = 'Infrastructure issue at {}'.format(builder_name)
757	email_text = 'The following infrastructure issue happened at:\n\n'	757	email_text = 'The following infrastructure issue happened at:\n\n'
758	email_text += ' - Build : {}\n'.format(build_url)	758	email_text += ' - Build : <a href="{}">{}</a>\n'.format(build_url, build_url)
759	email_text += ' - Builder : {}\n'.format(builder_name)	759	email_text += ' - Builder : {}\n'.format(builder_name)
760	email_text += ' - Worker : {}\n'.format(worker_name)	760	email_text += ' - Worker : {}\n'.format(worker_name)
761	email_text += ' - Issue: {}\n'.format(infrastructure_issue_text)	761	email_text += ' - Issue: {}\n'.format(infrastructure_issue_text)
Lines 2764-2775 class RunWebKitTestsRedTree(RunWebKitTests): a/Tools/CISupport/ews-build/steps.py_sec2
2764	self.build.results = SUCCESS	2764	self.build.results = SUCCESS
2765	self.setProperty('build_summary', message)	2765	self.setProperty('build_summary', message)
2766	else:	2766	else:
2767	# We have a failure return code, but not a list of failed or flaky tests.	2767	# We have a failure return code but not a list of failed or flaky tests, so we can't run the repeat steps.
2768	# So retry re-running the _whole_ layout tests without-patch to see if	2768	# If we are on the last retry then run the whole layout tests without patch.
2769	# this unexpected failure was pre-existent. If the failure was not pre-existent,	2769	# If not, then go to analyze-layout-tests-results where we will retry everything hoping this was a random failure.
2770	# then we would not report a list of failed test, just a generic "unknown" failure.
2771	self.setProperty('patchFailedTests', True)	2770	self.setProperty('patchFailedTests', True)
2772	next_steps.extend([UnApplyPatchIfRequired(), CompileWebKitWithoutPatch(retry_build_on_failure=True), ValidatePatch(verifyBugClosed=False, addURLs=False), RunWebKitTestsWithoutPatchRedTree()])	2771	retry_count = int(self.getProperty('retry_count', 0))
		2772	if retry_count < AnalyzeLayoutTestsResultsRedTree.MAX_RETRY:
		2773	next_steps.append(AnalyzeLayoutTestsResultsRedTree())
		2774	else:
		2775	next_steps.extend([UnApplyPatchIfRequired(), CompileWebKitWithoutPatch(retry_build_on_failure=True), ValidatePatch(verifyBugClosed=False, addURLs=False), RunWebKitTestsWithoutPatchRedTree()])
2773	if next_steps:	2776	if next_steps:
2774	self.build.addStepsAfterCurrentStep(next_steps)	2777	self.build.addStepsAfterCurrentStep(next_steps)
2775	return rc	2778	return rc
Lines 2787-2793 class RunWebKitTestsRepeatFailuresRedTree(RunWebKitTestsRedTree): a/Tools/CISupport/ews-build/steps.py_sec3
2787	def setLayoutTestCommand(self):	2790	def setLayoutTestCommand(self):
2788	super().setLayoutTestCommand()	2791	super().setLayoutTestCommand()
2789	first_results_failing_tests = set(self.getProperty('first_run_failures', []))	2792	first_results_failing_tests = set(self.getProperty('first_run_failures', []))
2790	self.setCommand(self.command + ['--repeat-each=%s' % self.NUM_REPEATS_PER_TEST] + sorted(first_results_failing_tests))	2793	self.setCommand(self.command + ['--fully-parallel', '--repeat-each=%s' % self.NUM_REPEATS_PER_TEST] + sorted(first_results_failing_tests))
2791		2794
2792	def evaluateCommand(self, cmd):	2795	def evaluateCommand(self, cmd):
2793	with_patch_repeat_failures_results_nonflaky_failures = set(self.getProperty('with_patch_repeat_failures_results_nonflaky_failures', []))	2796	with_patch_repeat_failures_results_nonflaky_failures = set(self.getProperty('with_patch_repeat_failures_results_nonflaky_failures', []))
Lines 2846-2852 class RunWebKitTestsRepeatFailuresWithoutPatchRedTree(RunWebKitTestsRedTree): a/Tools/CISupport/ews-build/steps.py_sec4
2846	# is skipped anyways if is marked as such on the Expectation files or if is marked	2849	# is skipped anyways if is marked as such on the Expectation files or if is marked
2847	# as failure (since we are passing also '--skip-failing-tests'). That way we ensure	2850	# as failure (since we are passing also '--skip-failing-tests'). That way we ensure
2848	# to report the case of a patch removing an expectation that still fails with it.	2851	# to report the case of a patch removing an expectation that still fails with it.
2849	self.setCommand(self.command + ['--repeat-each=%s' % self.NUM_REPEATS_PER_TEST, '--skipped=always'] + sorted(failures_to_repeat))	2852	self.setCommand(self.command + ['--fully-parallel', '--repeat-each=%s' % self.NUM_REPEATS_PER_TEST, '--skipped=always'] + sorted(failures_to_repeat))
2850		2853
2851	def evaluateCommand(self, cmd):	2854	def evaluateCommand(self, cmd):
2852	rc = self.evaluateResult(cmd)	2855	rc = self.evaluateResult(cmd)

Lines 1-3 a/Tools/ChangeLog_sec1
- a/Tools/ChangeLog +28 lines
		1	2021-12-15 Carlos Alberto Lopez Perez <clopez@igalia.com>
		2
		3	[GTK][WPE] Apply optimizations to speed up the layout-test EWS and add a few more workers to the build ones.
		4	https://bugs.webkit.org/show_bug.cgi?id=234378
		5
		6	Reviewed by NOBODY (OOPS!).
		7
		8	This applies two optimizations to speed up the testing on the layout-tests EWS added in r286405
		9
		10	1. Pass --fully-parallel to the retry steps. This makes the tool to utilize all the available workers for running
		11	the tests instead of running them in serial. That makes the time needed to complete the retry steps to be considerable less,
		12	In this example https://ews-build.webkit.org/#/builders/35/builds/1757 it reduces the timing from 43 mins to 12 mins.
		13
		14	2. When the first run fails without giving a list of failures and we have not reached the maximum retry count
		15	we are going to end retring the whole testing anyway, so there is no point in running the tests without patch
		16	in that case, we can skip doing that to save time. See: https://ews-build.webkit.org/#/builders/35/builds/1763
		17
		18	On top of that this patch adds an extra worker to the EWS build queues of GTK and WPE
		19	because I detected that this queues sometimes are not fast enough.
		20
		21	* CISupport/ews-build/config.json:
		22	* CISupport/ews-build/steps.py:
		23	(BugzillaMixin.send_email_for_infrastructure_issue):
		24	(RunWebKitTestsRedTree.evaluateCommand):
		25	(RunWebKitTestsRepeatFailuresRedTree.setLayoutTestCommand):
		26	(RunWebKitTestsRepeatFailuresWithoutPatchRedTree.setLayoutTestCommand):
		27	* CISupport/ews-build/steps_unittest.py:
		28
1	2021-12-15 Jonathan Bedard <jbedard@apple.com>	29	2021-12-15 Jonathan Bedard <jbedard@apple.com>
2		30
3	[reporelaypy] Forward branches to alternate remote	31	[reporelaypy] Forward branches to alternate remote

Lines 19-26 a/Tools/CISupport/ews-build/config.json_sec1
- a/Tools/CISupport/ews-build/config.json -2 / +4 lines
19	{ "name": "igalia10-gtk-wk2-ews", "platform": "gtk" },	19	{ "name": "igalia10-gtk-wk2-ews", "platform": "gtk" },
20	{ "name": "igalia11-gtk-wk2-ews", "platform": "gtk" },	20	{ "name": "igalia11-gtk-wk2-ews", "platform": "gtk" },
21	{ "name": "igalia12-gtk-wk2-ews", "platform": "gtk" },	21	{ "name": "igalia12-gtk-wk2-ews", "platform": "gtk" },
		22	{ "name": "igalia13-gtk-wk2-ews", "platform": "gtk" },
22	{ "name": "aperez-gtk-ews", "platform": "gtk" },	23	{ "name": "aperez-gtk-ews", "platform": "gtk" },
23	{ "name": "igalia-wpe-ews", "platform": "wpe" },	24	{ "name": "igalia-wpe-ews", "platform": "wpe" },
		25	{ "name": "igalia2-wpe-ews", "platform": "wpe" },
24	{ "name": "aperez-wpe-ews", "platform": "wpe" },	26	{ "name": "aperez-wpe-ews", "platform": "wpe" },
25	{ "name": "wincairo-ews-001", "platform": "wincairo" },	27	{ "name": "wincairo-ews-001", "platform": "wincairo" },
26	{ "name": "wincairo-ews-002", "platform": "wincairo" },	28	{ "name": "wincairo-ews-002", "platform": "wincairo" },
Lines 115-121 a/Tools/CISupport/ews-build/config.json_sec2
115	"factory": "GTKBuildFactory", "platform": "gtk",	117	"factory": "GTKBuildFactory", "platform": "gtk",
116	"configuration": "release", "architectures": ["x86_64"],	118	"configuration": "release", "architectures": ["x86_64"],
117	"triggers": ["api-tests-gtk-ews", "gtk-wk2-tests-ews"],	119	"triggers": ["api-tests-gtk-ews", "gtk-wk2-tests-ews"],
118	"workernames": ["igalia1-gtk-wk2-ews", "igalia2-gtk-wk2-ews", "aperez-gtk-ews"]	120	"workernames": ["aperez-gtk-ews", "igalia1-gtk-wk2-ews", "igalia2-gtk-wk2-ews", "igalia13-gtk-wk2-ews"]
119	},	121	},
120	{	122	{
121	"name": "GTK-WK2-Tests-EWS", "shortname": "gtk-wk2", "icon": "testOnly",	123	"name": "GTK-WK2-Tests-EWS", "shortname": "gtk-wk2", "icon": "testOnly",
Lines 240-246 a/Tools/CISupport/ews-build/config.json_sec3
240	"name": "WPE-EWS", "shortname": "wpe", "icon": "buildOnly",	242	"name": "WPE-EWS", "shortname": "wpe", "icon": "buildOnly",
241	"factory": "WPEFactory", "platform": "wpe",	243	"factory": "WPEFactory", "platform": "wpe",
242	"configuration": "release", "architectures": ["x86_64"],	244	"configuration": "release", "architectures": ["x86_64"],
243	"workernames": ["igalia-wpe-ews", "aperez-wpe-ews"]	245	"workernames": ["aperez-wpe-ews", "igalia-wpe-ews", "igalia2-wpe-ews"]
244	},	246	},
245	{	247	{
246	"name": "JSC-Tests-EWS", "shortname": "jsc", "icon": "buildAndTest",	248	"name": "JSC-Tests-EWS", "shortname": "jsc", "icon": "buildAndTest",

Lines 2581-2587 class TestRunWebKitTestsRepeatFailuresRedTree(BuildStepMixinAdditions, unittest. a/Tools/CISupport/ews-build/steps_unittest.py_sec1
- a/Tools/CISupport/ews-build/steps_unittest.py -3 / +3 lines
2581	'Tools/Scripts/run-webkit-tests',	2581	'Tools/Scripts/run-webkit-tests',
2582	'--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',	2582	'--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',
2583	'--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',	2583	'--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',
2584	'--skip-failing-tests', '--repeat-each=10'] + sorted(first_run_failures)	2584	'--skip-failing-tests', '--fully-parallel', '--repeat-each=10'] + sorted(first_run_failures)
2585	)	2585	)
2586	+ 0,	2586	+ 0,
2587	)	2587	)
Lines 2622-2628 class TestRunWebKitTestsRepeatFailuresWithoutPatchRedTree(BuildStepMixinAddition a/Tools/CISupport/ews-build/steps_unittest.py_sec2
2622	'Tools/Scripts/run-webkit-tests',	2622	'Tools/Scripts/run-webkit-tests',
2623	'--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',	2623	'--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',
2624	'--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',	2624	'--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',
2625	'--skip-failing-tests', '--repeat-each=10', '--skipped=always'] + sorted(with_patch_repeat_failures_results_nonflaky_failures)	2625	'--skip-failing-tests', '--fully-parallel', '--repeat-each=10', '--skipped=always'] + sorted(with_patch_repeat_failures_results_nonflaky_failures)
2626	)	2626	)
2627	+ 0,	2627	+ 0,
2628	)	2628	)
Lines 2651-2657 class TestRunWebKitTestsRepeatFailuresWithoutPatchRedTree(BuildStepMixinAddition a/Tools/CISupport/ews-build/steps_unittest.py_sec3
2651	'Tools/Scripts/run-webkit-tests',	2651	'Tools/Scripts/run-webkit-tests',
2652	'--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',	2652	'--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results',
2653	'--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',	2653	'--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging',
2654	'--skip-failing-tests', '--repeat-each=10', '--skipped=always'] + sorted(first_run_failures)	2654	'--skip-failing-tests', '--fully-parallel', '--repeat-each=10', '--skipped=always'] + sorted(first_run_failures)
2655	)	2655	)
2656	+ 0,	2656	+ 0,
2657	)	2657	)