Brad Bishop | c342db3 | 2019-05-15 21:57:59 -0400 | [diff] [blame] | 1 | # |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 2 | # Copyright (C) 2016 Intel Corporation |
Brad Bishop | c342db3 | 2019-05-15 21:57:59 -0400 | [diff] [blame] | 3 | # |
| 4 | # SPDX-License-Identifier: MIT |
| 5 | # |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 6 | |
| 7 | import os |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 8 | import re |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 9 | import sys |
| 10 | import unittest |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 11 | import inspect |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 12 | |
| 13 | from oeqa.core.utils.path import findFile |
| 14 | from oeqa.core.utils.test import getSuiteModules, getCaseID |
| 15 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 16 | from oeqa.core.exception import OEQATestNotFound |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 17 | from oeqa.core.case import OETestCase |
| 18 | from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 19 | OETestDiscover |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 20 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 21 | # When loading tests, the unittest framework stores any exceptions and |
| 22 | # displays them only when the run method is called. |
| 23 | # |
| 24 | # For our purposes, it is better to raise the exceptions in the loading |
| 25 | # step rather than waiting to run the test suite. |
| 26 | # |
| 27 | # Generate the function definition because this differ across python versions |
| 28 | # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3 |
| 29 | # ueses four parameters so isn't incremental. |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 30 | _failed_test_args = inspect.getfullargspec(unittest.loader._make_failed_test).args |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 31 | exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args)) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 32 | unittest.loader._make_failed_test = _make_failed_test |
| 33 | |
| 34 | def _find_duplicated_modules(suite, directory): |
| 35 | for module in getSuiteModules(suite): |
| 36 | path = findFile('%s.py' % module, directory) |
| 37 | if path: |
| 38 | raise ImportError("Duplicated %s module found in %s" % (module, path)) |
| 39 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 40 | def _built_modules_dict(modules): |
| 41 | modules_dict = {} |
| 42 | |
| 43 | if modules == None: |
| 44 | return modules_dict |
| 45 | |
| 46 | for module in modules: |
| 47 | # Assumption: package and module names do not contain upper case |
| 48 | # characters, whereas class names do |
Andrew Geissler | 475cb72 | 2020-07-10 16:00:51 -0500 | [diff] [blame] | 49 | m = re.match(r'^([0-9a-z_.]+)(?:\.(\w[^.]*)(?:\.([^.]+))?)?$', module, flags=re.ASCII) |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame] | 50 | if not m: |
| 51 | continue |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 52 | |
| 53 | module_name, class_name, test_name = m.groups() |
| 54 | |
| 55 | if module_name and module_name not in modules_dict: |
| 56 | modules_dict[module_name] = {} |
| 57 | if class_name and class_name not in modules_dict[module_name]: |
| 58 | modules_dict[module_name][class_name] = [] |
| 59 | if test_name and test_name not in modules_dict[module_name][class_name]: |
| 60 | modules_dict[module_name][class_name].append(test_name) |
| 61 | |
| 62 | return modules_dict |
| 63 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 64 | class OETestLoader(unittest.TestLoader): |
| 65 | caseClass = OETestCase |
| 66 | |
| 67 | kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass', |
| 68 | '_top_level_dir'] |
| 69 | |
| 70 | def __init__(self, tc, module_paths, modules, tests, modules_required, |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 71 | *args, **kwargs): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 72 | self.tc = tc |
| 73 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 74 | self.modules = _built_modules_dict(modules) |
| 75 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 76 | self.tests = tests |
| 77 | self.modules_required = modules_required |
| 78 | |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 79 | self.tags_filter = kwargs.get("tags_filter", None) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 80 | |
| 81 | if isinstance(module_paths, str): |
| 82 | module_paths = [module_paths] |
| 83 | elif not isinstance(module_paths, list): |
| 84 | raise TypeError('module_paths must be a str or a list of str') |
| 85 | self.module_paths = module_paths |
| 86 | |
| 87 | for kwname in self.kwargs_names: |
| 88 | if kwname in kwargs: |
| 89 | setattr(self, kwname, kwargs[kwname]) |
| 90 | |
| 91 | self._patchCaseClass(self.caseClass) |
| 92 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 93 | super(OETestLoader, self).__init__() |
| 94 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 95 | def _patchCaseClass(self, testCaseClass): |
| 96 | # Adds custom attributes to the OETestCase class |
| 97 | setattr(testCaseClass, 'tc', self.tc) |
| 98 | setattr(testCaseClass, 'td', self.tc.td) |
| 99 | setattr(testCaseClass, 'logger', self.tc.logger) |
| 100 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 101 | def _registerTestCase(self, case): |
| 102 | case_id = case.id() |
| 103 | self.tc._registry['cases'][case_id] = case |
| 104 | |
| 105 | def _handleTestCaseDecorators(self, case): |
| 106 | def _handle(obj): |
| 107 | if isinstance(obj, OETestDecorator): |
| 108 | if not obj.__class__ in decoratorClasses: |
| 109 | raise Exception("Decorator %s isn't registered" \ |
| 110 | " in decoratorClasses." % obj.__name__) |
| 111 | obj.bind(self.tc._registry, case) |
| 112 | |
| 113 | def _walk_closure(obj): |
| 114 | if hasattr(obj, '__closure__') and obj.__closure__: |
| 115 | for f in obj.__closure__: |
| 116 | obj = f.cell_contents |
| 117 | _handle(obj) |
| 118 | _walk_closure(obj) |
| 119 | method = getattr(case, case._testMethodName, None) |
| 120 | _walk_closure(method) |
| 121 | |
| 122 | def _filterTest(self, case): |
| 123 | """ |
| 124 | Returns True if test case must be filtered, False otherwise. |
| 125 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 126 | # XXX; If the module has more than one namespace only use |
| 127 | # the first to support run the whole module specifying the |
| 128 | # <module_name>.[test_class].[test_name] |
| 129 | module_name_small = case.__module__.split('.')[0] |
| 130 | module_name = case.__module__ |
| 131 | |
| 132 | class_name = case.__class__.__name__ |
| 133 | test_name = case._testMethodName |
| 134 | |
Brad Bishop | d5ae7d9 | 2018-06-14 09:52:03 -0700 | [diff] [blame] | 135 | # 'auto' is a reserved key word to run test cases automatically |
| 136 | # warn users if their test case belong to a module named 'auto' |
| 137 | if module_name_small == "auto": |
| 138 | bb.warn("'auto' is a reserved key word for TEST_SUITES. " |
| 139 | "But test case '%s' is detected to belong to auto module. " |
| 140 | "Please condier using a new name for your module." % str(case)) |
| 141 | |
| 142 | # check if case belongs to any specified module |
| 143 | # if 'auto' is specified, such check is skipped |
| 144 | if self.modules and not 'auto' in self.modules: |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 145 | module = None |
| 146 | try: |
| 147 | module = self.modules[module_name_small] |
| 148 | except KeyError: |
| 149 | try: |
| 150 | module = self.modules[module_name] |
| 151 | except KeyError: |
| 152 | return True |
| 153 | |
| 154 | if module: |
| 155 | if not class_name in module: |
| 156 | return True |
| 157 | |
| 158 | if module[class_name]: |
| 159 | if test_name not in module[class_name]: |
| 160 | return True |
| 161 | |
| 162 | # Decorator filters |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 163 | if self.tags_filter is not None and callable(self.tags_filter): |
| 164 | alltags = set() |
| 165 | # pull tags from the case class |
| 166 | if hasattr(case, "__oeqa_testtags"): |
| 167 | for t in getattr(case, "__oeqa_testtags"): |
| 168 | alltags.add(t) |
| 169 | # pull tags from the method itself |
| 170 | if hasattr(case, test_name): |
| 171 | method = getattr(case, test_name) |
| 172 | if hasattr(method, "__oeqa_testtags"): |
| 173 | for t in getattr(method, "__oeqa_testtags"): |
| 174 | alltags.add(t) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 175 | |
Brad Bishop | 79641f2 | 2019-09-10 07:20:22 -0400 | [diff] [blame] | 176 | if self.tags_filter(alltags): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 177 | return True |
| 178 | |
| 179 | return False |
| 180 | |
| 181 | def _getTestCase(self, testCaseClass, tcName): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 182 | if not hasattr(testCaseClass, '__oeqa_loader') and \ |
| 183 | issubclass(testCaseClass, OETestCase): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 184 | # In order to support data_vars validation |
| 185 | # monkey patch the default setUp/tearDown{Class} to use |
| 186 | # the ones provided by OETestCase |
| 187 | setattr(testCaseClass, 'setUpClassMethod', |
| 188 | getattr(testCaseClass, 'setUpClass')) |
| 189 | setattr(testCaseClass, 'tearDownClassMethod', |
| 190 | getattr(testCaseClass, 'tearDownClass')) |
| 191 | setattr(testCaseClass, 'setUpClass', |
| 192 | testCaseClass._oeSetUpClass) |
| 193 | setattr(testCaseClass, 'tearDownClass', |
| 194 | testCaseClass._oeTearDownClass) |
| 195 | |
| 196 | # In order to support decorators initialization |
| 197 | # monkey patch the default setUp/tearDown to use |
| 198 | # a setUpDecorators/tearDownDecorators that methods |
| 199 | # will call setUp/tearDown original methods. |
| 200 | setattr(testCaseClass, 'setUpMethod', |
| 201 | getattr(testCaseClass, 'setUp')) |
| 202 | setattr(testCaseClass, 'tearDownMethod', |
| 203 | getattr(testCaseClass, 'tearDown')) |
| 204 | setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp) |
| 205 | setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown) |
| 206 | |
| 207 | setattr(testCaseClass, '__oeqa_loader', True) |
| 208 | |
| 209 | case = testCaseClass(tcName) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 210 | if isinstance(case, OETestCase): |
| 211 | setattr(case, 'decorators', []) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 212 | |
| 213 | return case |
| 214 | |
| 215 | def loadTestsFromTestCase(self, testCaseClass): |
| 216 | """ |
| 217 | Returns a suite of all tests cases contained in testCaseClass. |
| 218 | """ |
| 219 | if issubclass(testCaseClass, unittest.suite.TestSuite): |
| 220 | raise TypeError("Test cases should not be derived from TestSuite." \ |
| 221 | " Maybe you meant to derive %s from TestCase?" \ |
| 222 | % testCaseClass.__name__) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 223 | if not issubclass(testCaseClass, unittest.case.TestCase): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 224 | raise TypeError("Test %s is not derived from %s" % \ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 225 | (testCaseClass.__name__, unittest.case.TestCase.__name__)) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 226 | |
| 227 | testCaseNames = self.getTestCaseNames(testCaseClass) |
| 228 | if not testCaseNames and hasattr(testCaseClass, 'runTest'): |
| 229 | testCaseNames = ['runTest'] |
| 230 | |
| 231 | suite = [] |
| 232 | for tcName in testCaseNames: |
| 233 | case = self._getTestCase(testCaseClass, tcName) |
| 234 | # Filer by case id |
Brad Bishop | d5ae7d9 | 2018-06-14 09:52:03 -0700 | [diff] [blame] | 235 | if not (self.tests and not 'auto' in self.tests |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 236 | and not getCaseID(case) in self.tests): |
| 237 | self._handleTestCaseDecorators(case) |
| 238 | |
| 239 | # Filter by decorators |
| 240 | if not self._filterTest(case): |
| 241 | self._registerTestCase(case) |
| 242 | suite.append(case) |
| 243 | |
| 244 | return self.suiteClass(suite) |
| 245 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 246 | def _required_modules_validation(self): |
| 247 | """ |
| 248 | Search in Test context registry if a required |
| 249 | test is found, raise an exception when not found. |
| 250 | """ |
| 251 | |
| 252 | for module in self.modules_required: |
| 253 | found = False |
| 254 | |
| 255 | # The module name is splitted to only compare the |
| 256 | # first part of a test case id. |
| 257 | comp_len = len(module.split('.')) |
| 258 | for case in self.tc._registry['cases']: |
| 259 | case_comp = '.'.join(case.split('.')[0:comp_len]) |
| 260 | if module == case_comp: |
| 261 | found = True |
| 262 | break |
| 263 | |
| 264 | if not found: |
| 265 | raise OEQATestNotFound("Not found %s in loaded test cases" % \ |
| 266 | module) |
| 267 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 268 | def discover(self): |
| 269 | big_suite = self.suiteClass() |
| 270 | for path in self.module_paths: |
| 271 | _find_duplicated_modules(big_suite, path) |
| 272 | suite = super(OETestLoader, self).discover(path, |
| 273 | pattern='*.py', top_level_dir=path) |
| 274 | big_suite.addTests(suite) |
| 275 | |
| 276 | cases = None |
| 277 | discover_classes = [clss for clss in decoratorClasses |
| 278 | if issubclass(clss, OETestDiscover)] |
| 279 | for clss in discover_classes: |
| 280 | cases = clss.discover(self.tc._registry) |
| 281 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 282 | if self.modules_required: |
| 283 | self._required_modules_validation() |
| 284 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 285 | return self.suiteClass(cases) if cases else big_suite |
| 286 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 287 | def _filterModule(self, module): |
| 288 | if module.__name__ in sys.builtin_module_names: |
| 289 | msg = 'Tried to import %s test module but is a built-in' |
| 290 | raise ImportError(msg % module.__name__) |
| 291 | |
| 292 | # XXX; If the module has more than one namespace only use |
| 293 | # the first to support run the whole module specifying the |
| 294 | # <module_name>.[test_class].[test_name] |
| 295 | module_name_small = module.__name__.split('.')[0] |
| 296 | module_name = module.__name__ |
| 297 | |
| 298 | # Normal test modules are loaded if no modules were specified, |
Brad Bishop | d5ae7d9 | 2018-06-14 09:52:03 -0700 | [diff] [blame] | 299 | # if module is in the specified module list or if 'auto' is in |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 300 | # module list. |
| 301 | # Underscore modules are loaded only if specified in module list. |
| 302 | load_module = True if not module_name.startswith('_') \ |
| 303 | and (not self.modules \ |
| 304 | or module_name in self.modules \ |
| 305 | or module_name_small in self.modules \ |
Brad Bishop | d5ae7d9 | 2018-06-14 09:52:03 -0700 | [diff] [blame] | 306 | or 'auto' in self.modules) \ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 307 | else False |
| 308 | |
| 309 | load_underscore = True if module_name.startswith('_') \ |
| 310 | and (module_name in self.modules or \ |
| 311 | module_name_small in self.modules) \ |
| 312 | else False |
| 313 | |
| 314 | return (load_module, load_underscore) |
| 315 | |
| 316 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 317 | # XXX After Python 3.5, remove backward compatibility hacks for |
| 318 | # use_load_tests deprecation via *args and **kws. See issue 16662. |
| 319 | if sys.version_info >= (3,5): |
| 320 | def loadTestsFromModule(self, module, *args, pattern=None, **kws): |
| 321 | """ |
| 322 | Returns a suite of all tests cases contained in module. |
| 323 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 324 | load_module, load_underscore = self._filterModule(module) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 325 | |
| 326 | if load_module or load_underscore: |
| 327 | return super(OETestLoader, self).loadTestsFromModule( |
| 328 | module, *args, pattern=pattern, **kws) |
| 329 | else: |
| 330 | return self.suiteClass() |
| 331 | else: |
| 332 | def loadTestsFromModule(self, module, use_load_tests=True): |
| 333 | """ |
| 334 | Returns a suite of all tests cases contained in module. |
| 335 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 336 | load_module, load_underscore = self._filterModule(module) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 337 | |
| 338 | if load_module or load_underscore: |
| 339 | return super(OETestLoader, self).loadTestsFromModule( |
| 340 | module, use_load_tests) |
| 341 | else: |
| 342 | return self.suiteClass() |