Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1 | # Copyright (C) 2016 Intel Corporation |
| 2 | # Released under the MIT license (see COPYING.MIT) |
| 3 | |
| 4 | import os |
| 5 | import time |
| 6 | import unittest |
| 7 | import logging |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 8 | import re |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 9 | import json |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 10 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 11 | from unittest import TextTestResult as _TestResult |
| 12 | from unittest import TextTestRunner as _TestRunner |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 13 | |
| 14 | class OEStreamLogger(object): |
| 15 | def __init__(self, logger): |
| 16 | self.logger = logger |
| 17 | self.buffer = "" |
| 18 | |
| 19 | def write(self, msg): |
| 20 | if len(msg) > 1 and msg[0] != '\n': |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 21 | if '...' in msg: |
| 22 | self.buffer += msg |
| 23 | elif self.buffer: |
| 24 | self.buffer += msg |
| 25 | self.logger.log(logging.INFO, self.buffer) |
| 26 | self.buffer = "" |
| 27 | else: |
| 28 | self.logger.log(logging.INFO, msg) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 29 | |
| 30 | def flush(self): |
| 31 | for handler in self.logger.handlers: |
| 32 | handler.flush() |
| 33 | |
| 34 | class OETestResult(_TestResult): |
| 35 | def __init__(self, tc, *args, **kwargs): |
| 36 | super(OETestResult, self).__init__(*args, **kwargs) |
| 37 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 38 | self.successes = [] |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 39 | self.starttime = {} |
| 40 | self.endtime = {} |
| 41 | self.progressinfo = {} |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 42 | |
| 43 | # Inject into tc so that TestDepends decorator can see results |
| 44 | tc.results = self |
| 45 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 46 | self.tc = tc |
| 47 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 48 | def startTest(self, test): |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 49 | # May have been set by concurrencytest |
| 50 | if test.id() not in self.starttime: |
| 51 | self.starttime[test.id()] = time.time() |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 52 | super(OETestResult, self).startTest(test) |
| 53 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 54 | def stopTest(self, test): |
| 55 | self.endtime[test.id()] = time.time() |
| 56 | super(OETestResult, self).stopTest(test) |
| 57 | if test.id() in self.progressinfo: |
| 58 | self.tc.logger.info(self.progressinfo[test.id()]) |
| 59 | |
| 60 | # Print the errors/failures early to aid/speed debugging, its a pain |
| 61 | # to wait until selftest finishes to see them. |
| 62 | for t in ['failures', 'errors', 'skipped', 'expectedFailures']: |
| 63 | for (scase, msg) in getattr(self, t): |
| 64 | if test.id() == scase.id(): |
| 65 | self.tc.logger.info(str(msg)) |
| 66 | break |
| 67 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 68 | def logSummary(self, component, context_msg=''): |
| 69 | elapsed_time = self.tc._run_end_time - self.tc._run_start_time |
| 70 | self.tc.logger.info("SUMMARY:") |
| 71 | self.tc.logger.info("%s (%s) - Ran %d test%s in %.3fs" % (component, |
| 72 | context_msg, self.testsRun, self.testsRun != 1 and "s" or "", |
| 73 | elapsed_time)) |
| 74 | |
| 75 | if self.wasSuccessful(): |
| 76 | msg = "%s - OK - All required tests passed" % component |
| 77 | else: |
| 78 | msg = "%s - FAIL - Required tests failed" % component |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 79 | msg += " (successes=%d, skipped=%d, failures=%d, errors=%d)" % (len(self.successes), len(self.skipped), len(self.failures), len(self.errors)) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 80 | self.tc.logger.info(msg) |
| 81 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 82 | def _getTestResultDetails(self, case): |
| 83 | result_types = {'failures': 'FAILED', 'errors': 'ERROR', 'skipped': 'SKIPPED', |
| 84 | 'expectedFailures': 'EXPECTEDFAIL', 'successes': 'PASSED'} |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 85 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 86 | for rtype in result_types: |
| 87 | found = False |
| 88 | for (scase, msg) in getattr(self, rtype): |
| 89 | if case.id() == scase.id(): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 90 | found = True |
| 91 | break |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 92 | scase_str = str(scase.id()) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 93 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 94 | # When fails at module or class level the class name is passed as string |
| 95 | # so figure out to see if match |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 96 | m = re.search(r"^setUpModule \((?P<module_name>.*)\)$", scase_str) |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 97 | if m: |
| 98 | if case.__class__.__module__ == m.group('module_name'): |
| 99 | found = True |
| 100 | break |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 101 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 102 | m = re.search(r"^setUpClass \((?P<class_name>.*)\)$", scase_str) |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 103 | if m: |
| 104 | class_name = "%s.%s" % (case.__class__.__module__, |
| 105 | case.__class__.__name__) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 106 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 107 | if class_name == m.group('class_name'): |
| 108 | found = True |
| 109 | break |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 110 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 111 | if found: |
| 112 | return result_types[rtype], msg |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 113 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 114 | return 'UNKNOWN', None |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 115 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 116 | def addSuccess(self, test): |
| 117 | #Added so we can keep track of successes too |
| 118 | self.successes.append((test, None)) |
| 119 | super(OETestResult, self).addSuccess(test) |
| 120 | |
| 121 | def logDetails(self, json_file_dir=None, configuration=None, result_id=None): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 122 | self.tc.logger.info("RESULTS:") |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 123 | |
| 124 | result = {} |
| 125 | logs = {} |
| 126 | if hasattr(self.tc, "extraresults"): |
| 127 | result = self.tc.extraresults |
| 128 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 129 | for case_name in self.tc._registry['cases']: |
| 130 | case = self.tc._registry['cases'][case_name] |
| 131 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 132 | (status, log) = self._getTestResultDetails(case) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 133 | |
| 134 | oeid = -1 |
| 135 | if hasattr(case, 'decorators'): |
| 136 | for d in case.decorators: |
| 137 | if hasattr(d, 'oeid'): |
| 138 | oeid = d.oeid |
| 139 | |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 140 | t = "" |
| 141 | if case.id() in self.starttime and case.id() in self.endtime: |
| 142 | t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)" |
| 143 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 144 | if status not in logs: |
| 145 | logs[status] = [] |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 146 | logs[status].append("RESULTS - %s - Testcase %s: %s%s" % (case.id(), oeid, status, t)) |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 147 | if log: |
| 148 | result[case.id()] = {'status': status, 'log': log} |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 149 | else: |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 150 | result[case.id()] = {'status': status} |
| 151 | |
| 152 | for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']: |
| 153 | if i not in logs: |
| 154 | continue |
| 155 | for l in logs[i]: |
| 156 | self.tc.logger.info(l) |
| 157 | |
| 158 | if json_file_dir: |
| 159 | tresultjsonhelper = OETestResultJSONHelper() |
| 160 | tresultjsonhelper.dump_testresult_file(json_file_dir, configuration, result_id, result) |
| 161 | |
| 162 | def wasSuccessful(self): |
| 163 | # Override as we unexpected successes aren't failures for us |
| 164 | return (len(self.failures) == len(self.errors) == 0) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 165 | |
| 166 | class OEListTestsResult(object): |
| 167 | def wasSuccessful(self): |
| 168 | return True |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 169 | |
| 170 | class OETestRunner(_TestRunner): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 171 | streamLoggerClass = OEStreamLogger |
| 172 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 173 | def __init__(self, tc, *args, **kwargs): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 174 | kwargs['stream'] = self.streamLoggerClass(tc.logger) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 175 | super(OETestRunner, self).__init__(*args, **kwargs) |
| 176 | self.tc = tc |
| 177 | self.resultclass = OETestResult |
| 178 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 179 | def _makeResult(self): |
| 180 | return self.resultclass(self.tc, self.stream, self.descriptions, |
| 181 | self.verbosity) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 182 | |
| 183 | def _walk_suite(self, suite, func): |
| 184 | for obj in suite: |
| 185 | if isinstance(obj, unittest.suite.TestSuite): |
| 186 | if len(obj._tests): |
| 187 | self._walk_suite(obj, func) |
| 188 | elif isinstance(obj, unittest.case.TestCase): |
| 189 | func(self.tc.logger, obj) |
| 190 | self._walked_cases = self._walked_cases + 1 |
| 191 | |
| 192 | def _list_tests_name(self, suite): |
| 193 | from oeqa.core.decorator.oeid import OETestID |
| 194 | from oeqa.core.decorator.oetag import OETestTag |
| 195 | |
| 196 | self._walked_cases = 0 |
| 197 | |
| 198 | def _list_cases_without_id(logger, case): |
| 199 | |
| 200 | found_id = False |
| 201 | if hasattr(case, 'decorators'): |
| 202 | for d in case.decorators: |
| 203 | if isinstance(d, OETestID): |
| 204 | found_id = True |
| 205 | |
| 206 | if not found_id: |
| 207 | logger.info('oeid missing for %s' % case.id()) |
| 208 | |
| 209 | def _list_cases(logger, case): |
| 210 | oeid = None |
| 211 | oetag = None |
| 212 | |
| 213 | if hasattr(case, 'decorators'): |
| 214 | for d in case.decorators: |
| 215 | if isinstance(d, OETestID): |
| 216 | oeid = d.oeid |
| 217 | elif isinstance(d, OETestTag): |
| 218 | oetag = d.oetag |
| 219 | |
| 220 | logger.info("%s\t%s\t\t%s" % (oeid, oetag, case.id())) |
| 221 | |
| 222 | self.tc.logger.info("Listing test cases that don't have oeid ...") |
| 223 | self._walk_suite(suite, _list_cases_without_id) |
| 224 | self.tc.logger.info("-" * 80) |
| 225 | |
| 226 | self.tc.logger.info("Listing all available tests:") |
| 227 | self._walked_cases = 0 |
| 228 | self.tc.logger.info("id\ttag\t\ttest") |
| 229 | self.tc.logger.info("-" * 80) |
| 230 | self._walk_suite(suite, _list_cases) |
| 231 | self.tc.logger.info("-" * 80) |
| 232 | self.tc.logger.info("Total found:\t%s" % self._walked_cases) |
| 233 | |
| 234 | def _list_tests_class(self, suite): |
| 235 | self._walked_cases = 0 |
| 236 | |
| 237 | curr = {} |
| 238 | def _list_classes(logger, case): |
| 239 | if not 'module' in curr or curr['module'] != case.__module__: |
| 240 | curr['module'] = case.__module__ |
| 241 | logger.info(curr['module']) |
| 242 | |
| 243 | if not 'class' in curr or curr['class'] != \ |
| 244 | case.__class__.__name__: |
| 245 | curr['class'] = case.__class__.__name__ |
| 246 | logger.info(" -- %s" % curr['class']) |
| 247 | |
| 248 | logger.info(" -- -- %s" % case._testMethodName) |
| 249 | |
| 250 | self.tc.logger.info("Listing all available test classes:") |
| 251 | self._walk_suite(suite, _list_classes) |
| 252 | |
| 253 | def _list_tests_module(self, suite): |
| 254 | self._walked_cases = 0 |
| 255 | |
| 256 | listed = [] |
| 257 | def _list_modules(logger, case): |
| 258 | if not case.__module__ in listed: |
| 259 | if case.__module__.startswith('_'): |
| 260 | logger.info("%s (hidden)" % case.__module__) |
| 261 | else: |
| 262 | logger.info(case.__module__) |
| 263 | listed.append(case.__module__) |
| 264 | |
| 265 | self.tc.logger.info("Listing all available test modules:") |
| 266 | self._walk_suite(suite, _list_modules) |
| 267 | |
| 268 | def list_tests(self, suite, display_type): |
| 269 | if display_type == 'name': |
| 270 | self._list_tests_name(suite) |
| 271 | elif display_type == 'class': |
| 272 | self._list_tests_class(suite) |
| 273 | elif display_type == 'module': |
| 274 | self._list_tests_module(suite) |
| 275 | |
| 276 | return OEListTestsResult() |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 277 | |
| 278 | class OETestResultJSONHelper(object): |
| 279 | |
| 280 | testresult_filename = 'testresults.json' |
| 281 | |
| 282 | def _get_existing_testresults_if_available(self, write_dir): |
| 283 | testresults = {} |
| 284 | file = os.path.join(write_dir, self.testresult_filename) |
| 285 | if os.path.exists(file): |
| 286 | with open(file, "r") as f: |
| 287 | testresults = json.load(f) |
| 288 | return testresults |
| 289 | |
| 290 | def _write_file(self, write_dir, file_name, file_content): |
| 291 | file_path = os.path.join(write_dir, file_name) |
| 292 | with open(file_path, 'w') as the_file: |
| 293 | the_file.write(file_content) |
| 294 | |
| 295 | def dump_testresult_file(self, write_dir, configuration, result_id, test_result): |
| 296 | bb.utils.mkdirhier(write_dir) |
| 297 | lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock')) |
| 298 | test_results = self._get_existing_testresults_if_available(write_dir) |
| 299 | test_results[result_id] = {'configuration': configuration, 'result': test_result} |
| 300 | json_testresults = json.dumps(test_results, sort_keys=True, indent=4) |
| 301 | self._write_file(write_dir, self.testresult_filename, json_testresults) |
| 302 | bb.utils.unlockfile(lf) |