708 def evaluateCommand(self, cmd):
709 rc = super(RunAPITests, self).evaluateCommand(cmd)
710 if rc == SUCCESS:
711 self.finished(SUCCESS)
712 message = 'Passed API tests'
713 self.descriptionDone = message
714 self.build.results = SUCCESS
715 self.build.buildFinished([message], SUCCESS)
716 else:
717 self.build.addStepsAfterCurrentStep([ReRunAPITests()])
718
719 return rc
720
721
722class ReRunAPITests(RunAPITests):
723 name = 're-run-api-tests'
724
725 def evaluateCommand(self, cmd):
726 rc = TestWithFailureCount.evaluateCommand(self, cmd)
727 if rc == SUCCESS:
728 self.finished(SUCCESS)
729 message = 'Passed API tests'
730 self.descriptionDone = message
731 self.build.results = SUCCESS
732 self.build.buildFinished([message], SUCCESS)
733 else:
734 self.setProperty('patchFailedAPITests', True)
735 self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileWebKitToT(), RunAPITestsWithoutPatch(), AnalyzeAPITestsResults()])
736 return rc
737
738
739class RunAPITestsWithoutPatch(RunAPITests):
740 name = 'run-api-tests-without-patch'
741
742 def doStepIf(self, step):
743 return self.getProperty('patchFailedAPITests')
744
745 def hideStepIf(self, results, step):
746 return not self.doStepIf(step)
747
748 def evaluateCommand(self, cmd):
749 return TestWithFailureCount.evaluateCommand(self, cmd)
750
751
752class AnalyzeAPITestsResults(buildstep.BuildStep):
753 name = 'analyze-api-tests-results'
754 description = ['analyze-api-test-results']
755 descriptionDone = ['analyze-api-tests-results']
756
757 def start(self):
758 self.results = {}
759 d = self.getTestsResults(RunAPITests.name)
760 d.addCallback(lambda res: self.getTestsResults(ReRunAPITests.name))
761 d.addCallback(lambda res: self.getTestsResults(RunAPITestsWithoutPatch.name))
762 d.addCallback(lambda res: self.analyzeResults())
763 return defer.succeed(None)
764
765 def analyzeResults(self):
766 if not self.results or len(self.results) == 0:
767 self._addToLog('stderr', "Unable to parse API test results: {}".format(self.results))
768 self.finished(RETRY)
769 self.build.buildFinished(["Unable to parse API test results"], RETRY)
770 return -1
771
772 first_run_results = self.results.get(RunAPITests.name)
773 second_run_results = self.results.get(ReRunAPITests.name)
774 clean_tree_results = self.results.get(RunAPITestsWithoutPatch.name)
775
776 if not (first_run_results and second_run_results and clean_tree_results):
777 self.finished(RETRY)
778 self.build.buildFinished(["Unable to parse API test results"], RETRY)
779 return -1
780
781 def getAPITestFailures(result):
782 # TODO: Analyze Time-out, Crash and Failure independently
783 return set([failure.get('name') for failure in result.get("Timedout", [])] +
784 [failure.get('name') for failure in result.get("Crashed", [])] +
785 [failure.get('name') for failure in result.get("Failed", [])])
786
787 first_run_failures = getAPITestFailures(first_run_results)
788 second_run_failures = getAPITestFailures(second_run_results)
789 clean_tree_failures = getAPITestFailures(clean_tree_results)
790
791 self._addToLog('stderr', "\nFailures in API Test first run: {}".format(first_run_failures))
792 self._addToLog('stderr', "\nFailures in API Test second run: {}".format(first_run_failures))
793 self._addToLog('stderr', "\nFailures in API Test on clean tree: {}".format(clean_tree_failures))
794 failures_with_patch = first_run_failures.intersection(second_run_failures)
795 new_failures = failures_with_patch - clean_tree_failures
796 new_failures_string = ', '.join([failure_name.replace('TestWebKitAPI.', '') for failure_name in new_failures])
797
798 if new_failures:
799 self._addToLog('stderr', "\nNew failures: {}\n".format(new_failures))
800 self.finished(FAILURE)
801 self.build.results = FAILURE
802 message = 'Found {} new API Tests failures: {}'.format(len(new_failures), new_failures_string)
803 self.descriptionDone = message
804 self.build.buildFinished([message], FAILURE)
805 else:
806 self._addToLog('stderr', "\nNo new failures\n")
807 self.finished(SUCCESS)
808 self.build.results = SUCCESS
809 self.descriptionDone = "Passed API tests"
810 message = "Found {} pre-existing API tests failures".format(len(clean_tree_failures))
811 self.build.buildFinished([message], SUCCESS)
812
813 @defer.inlineCallbacks
814 def _addToLog(self, logName, message):
815 try:
816 log = self.getLog(logName)
817 except KeyError:
818 log = yield self.addLog(logName)
819 log.addStdout(message)
820
821 def getBuildStepByName(self, name):
822 for step in self.build.executedSteps:
823 if step.name == name:
824 return step
825 return None
826
827 @defer.inlineCallbacks
828 def getTestsResults(self, name):
829 step = self.getBuildStepByName(name)
830 if not step:
831 self._addToLog('stderr', "ERROR: step not found: {}".format(step))
832 defer.returnValue(None)
833
834 logs = yield self.master.db.logs.getLogs(step.stepid)
835 log = next((log for log in logs if log['name'] == u'json'), None)
836 if not log:
837 self._addToLog('stderr', "ERROR: log for step not found: {}".format(step))
838 defer.returnValue(None)
839
840 lastline = int(max(0, log['num_lines'] - 1))
841 logLines = yield self.master.db.logs.getLogLines(log['id'], 0, lastline)
842 if log['type'] == 's':
843 logLines = "".join([line[1:] for line in logLines.splitlines()])
844
845 try:
846 self.results[name] = json.loads(logLines)
847 except Exception as ex:
848 self._addToLog('stderr', "ERROR: unable to parse data, exception: {}".format(ex))
849