Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1 | # Copyright (C) 2016 Intel Corporation |
| 2 | # Released under the MIT license (see COPYING.MIT) |
| 3 | |
| 4 | import os |
| 5 | import sys |
| 6 | import json |
| 7 | import time |
| 8 | import logging |
| 9 | import collections |
| 10 | import re |
| 11 | |
| 12 | from oeqa.core.loader import OETestLoader |
| 13 | from oeqa.core.runner import OETestRunner, OEStreamLogger, xmlEnabled |
| 14 | |
| 15 | class OETestContext(object): |
| 16 | loaderClass = OETestLoader |
| 17 | runnerClass = OETestRunner |
| 18 | streamLoggerClass = OEStreamLogger |
| 19 | |
| 20 | files_dir = os.path.abspath(os.path.join(os.path.dirname( |
| 21 | os.path.abspath(__file__)), "../files")) |
| 22 | |
| 23 | def __init__(self, td=None, logger=None): |
| 24 | if not type(td) is dict: |
| 25 | raise TypeError("td isn't dictionary type") |
| 26 | |
| 27 | self.td = td |
| 28 | self.logger = logger |
| 29 | self._registry = {} |
| 30 | self._registry['cases'] = collections.OrderedDict() |
| 31 | self._results = {} |
| 32 | |
| 33 | def _read_modules_from_manifest(self, manifest): |
| 34 | if not os.path.exists(manifest): |
| 35 | raise |
| 36 | |
| 37 | modules = [] |
| 38 | for line in open(manifest).readlines(): |
| 39 | line = line.strip() |
| 40 | if line and not line.startswith("#"): |
| 41 | modules.append(line) |
| 42 | |
| 43 | return modules |
| 44 | |
| 45 | def loadTests(self, module_paths, modules=[], tests=[], |
| 46 | modules_manifest="", modules_required=[], filters={}): |
| 47 | if modules_manifest: |
| 48 | modules = self._read_modules_from_manifest(modules_manifest) |
| 49 | |
| 50 | self.loader = self.loaderClass(self, module_paths, modules, tests, |
| 51 | modules_required, filters) |
| 52 | self.suites = self.loader.discover() |
| 53 | |
| 54 | def runTests(self): |
| 55 | streamLogger = self.streamLoggerClass(self.logger) |
| 56 | self.runner = self.runnerClass(self, stream=streamLogger, verbosity=2) |
| 57 | |
| 58 | self._run_start_time = time.time() |
| 59 | result = self.runner.run(self.suites) |
| 60 | self._run_end_time = time.time() |
| 61 | |
| 62 | return result |
| 63 | |
| 64 | def logSummary(self, result, component, context_msg=''): |
| 65 | self.logger.info("SUMMARY:") |
| 66 | self.logger.info("%s (%s) - Ran %d test%s in %.3fs" % (component, |
| 67 | context_msg, result.testsRun, result.testsRun != 1 and "s" or "", |
| 68 | (self._run_end_time - self._run_start_time))) |
| 69 | |
| 70 | if result.wasSuccessful(): |
| 71 | msg = "%s - OK - All required tests passed" % component |
| 72 | else: |
| 73 | msg = "%s - FAIL - Required tests failed" % component |
| 74 | skipped = len(self._results['skipped']) |
| 75 | if skipped: |
| 76 | msg += " (skipped=%d)" % skipped |
| 77 | self.logger.info(msg) |
| 78 | |
| 79 | def _getDetailsNotPassed(self, case, type, desc): |
| 80 | found = False |
| 81 | |
| 82 | for (scase, msg) in self._results[type]: |
| 83 | # XXX: When XML reporting is enabled scase is |
| 84 | # xmlrunner.result._TestInfo instance instead of |
| 85 | # string. |
| 86 | if xmlEnabled: |
| 87 | if case.id() == scase.test_id: |
| 88 | found = True |
| 89 | break |
| 90 | scase_str = scase.test_id |
| 91 | else: |
| 92 | if case == scase: |
| 93 | found = True |
| 94 | break |
| 95 | scase_str = str(scase) |
| 96 | |
| 97 | # When fails at module or class level the class name is passed as string |
| 98 | # so figure out to see if match |
| 99 | m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str) |
| 100 | if m: |
| 101 | if case.__class__.__module__ == m.group('module_name'): |
| 102 | found = True |
| 103 | break |
| 104 | |
| 105 | m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str) |
| 106 | if m: |
| 107 | class_name = "%s.%s" % (case.__class__.__module__, |
| 108 | case.__class__.__name__) |
| 109 | |
| 110 | if class_name == m.group('class_name'): |
| 111 | found = True |
| 112 | break |
| 113 | |
| 114 | if found: |
| 115 | return (found, msg) |
| 116 | |
| 117 | return (found, None) |
| 118 | |
| 119 | def logDetails(self): |
| 120 | self.logger.info("RESULTS:") |
| 121 | for case_name in self._registry['cases']: |
| 122 | case = self._registry['cases'][case_name] |
| 123 | |
| 124 | result_types = ['failures', 'errors', 'skipped', 'expectedFailures'] |
| 125 | result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL'] |
| 126 | |
| 127 | fail = False |
| 128 | desc = None |
| 129 | for idx, name in enumerate(result_types): |
| 130 | (fail, msg) = self._getDetailsNotPassed(case, result_types[idx], |
| 131 | result_desc[idx]) |
| 132 | if fail: |
| 133 | desc = result_desc[idx] |
| 134 | break |
| 135 | |
| 136 | oeid = -1 |
| 137 | for d in case.decorators: |
| 138 | if hasattr(d, 'oeid'): |
| 139 | oeid = d.oeid |
| 140 | |
| 141 | if fail: |
| 142 | self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), |
| 143 | oeid, desc)) |
| 144 | if msg: |
| 145 | self.logger.info(msg) |
| 146 | else: |
| 147 | self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), |
| 148 | oeid, 'PASSED')) |
| 149 | |
| 150 | class OETestContextExecutor(object): |
| 151 | _context_class = OETestContext |
| 152 | |
| 153 | name = 'core' |
| 154 | help = 'core test component example' |
| 155 | description = 'executes core test suite example' |
| 156 | |
| 157 | default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)), |
| 158 | 'cases/example')] |
| 159 | default_test_data = os.path.join(default_cases[0], 'data.json') |
| 160 | default_tests = None |
| 161 | |
| 162 | def register_commands(self, logger, subparsers): |
| 163 | self.parser = subparsers.add_parser(self.name, help=self.help, |
| 164 | description=self.description, group='components') |
| 165 | |
| 166 | self.default_output_log = '%s-results-%s.log' % (self.name, |
| 167 | time.strftime("%Y%m%d%H%M%S")) |
| 168 | self.parser.add_argument('--output-log', action='store', |
| 169 | default=self.default_output_log, |
| 170 | help="results output log, default: %s" % self.default_output_log) |
| 171 | self.parser.add_argument('--run-tests', action='store', |
| 172 | default=self.default_tests, |
| 173 | help="tests to run in <module>[.<class>[.<name>]] format. Just works for modules now") |
| 174 | |
| 175 | if self.default_test_data: |
| 176 | self.parser.add_argument('--test-data-file', action='store', |
| 177 | default=self.default_test_data, |
| 178 | help="data file to load, default: %s" % self.default_test_data) |
| 179 | else: |
| 180 | self.parser.add_argument('--test-data-file', action='store', |
| 181 | help="data file to load") |
| 182 | |
| 183 | if self.default_cases: |
| 184 | self.parser.add_argument('CASES_PATHS', action='store', |
| 185 | default=self.default_cases, nargs='*', |
| 186 | help="paths to directories with test cases, default: %s"\ |
| 187 | % self.default_cases) |
| 188 | else: |
| 189 | self.parser.add_argument('CASES_PATHS', action='store', |
| 190 | nargs='+', help="paths to directories with test cases") |
| 191 | |
| 192 | self.parser.set_defaults(func=self.run) |
| 193 | |
| 194 | def _setup_logger(self, logger, args): |
| 195 | formatter = logging.Formatter('%(asctime)s - ' + self.name + \ |
| 196 | ' - %(levelname)s - %(message)s') |
| 197 | sh = logger.handlers[0] |
| 198 | sh.setFormatter(formatter) |
| 199 | fh = logging.FileHandler(args.output_log) |
| 200 | fh.setFormatter(formatter) |
| 201 | logger.addHandler(fh) |
| 202 | |
| 203 | return logger |
| 204 | |
| 205 | def _process_args(self, logger, args): |
| 206 | self.tc_kwargs = {} |
| 207 | self.tc_kwargs['init'] = {} |
| 208 | self.tc_kwargs['load'] = {} |
| 209 | self.tc_kwargs['run'] = {} |
| 210 | |
| 211 | self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args) |
| 212 | if args.test_data_file: |
| 213 | self.tc_kwargs['init']['td'] = json.load( |
| 214 | open(args.test_data_file, "r")) |
| 215 | else: |
| 216 | self.tc_kwargs['init']['td'] = {} |
| 217 | |
| 218 | |
| 219 | if args.run_tests: |
| 220 | self.tc_kwargs['load']['modules'] = args.run_tests.split() |
| 221 | else: |
| 222 | self.tc_kwargs['load']['modules'] = None |
| 223 | |
| 224 | self.module_paths = args.CASES_PATHS |
| 225 | |
| 226 | def run(self, logger, args): |
| 227 | self._process_args(logger, args) |
| 228 | |
| 229 | self.tc = self._context_class(**self.tc_kwargs['init']) |
| 230 | self.tc.loadTests(self.module_paths, **self.tc_kwargs['load']) |
| 231 | rc = self.tc.runTests(**self.tc_kwargs['run']) |
| 232 | self.tc.logSummary(rc, self.name) |
| 233 | self.tc.logDetails() |
| 234 | |
| 235 | output_link = os.path.join(os.path.dirname(args.output_log), |
| 236 | "%s-results.log" % self.name) |
| 237 | if os.path.exists(output_link): |
| 238 | os.remove(output_link) |
| 239 | os.symlink(args.output_log, output_link) |
| 240 | |
| 241 | return rc |
| 242 | |
| 243 | _executor_class = OETestContextExecutor |