Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1 | # Copyright (C) 2016 Intel Corporation |
| 2 | # Released under the MIT license (see COPYING.MIT) |
| 3 | |
| 4 | import os |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 5 | import re |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 6 | import sys |
| 7 | import unittest |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 8 | import inspect |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 9 | |
| 10 | from oeqa.core.utils.path import findFile |
| 11 | from oeqa.core.utils.test import getSuiteModules, getCaseID |
| 12 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 13 | from oeqa.core.exception import OEQATestNotFound |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 14 | from oeqa.core.case import OETestCase |
| 15 | from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ |
| 16 | OETestFilter, OETestDiscover |
| 17 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 18 | # When loading tests, the unittest framework stores any exceptions and |
| 19 | # displays them only when the run method is called. |
| 20 | # |
| 21 | # For our purposes, it is better to raise the exceptions in the loading |
| 22 | # step rather than waiting to run the test suite. |
| 23 | # |
| 24 | # Generate the function definition because this differ across python versions |
| 25 | # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3 |
| 26 | # ueses four parameters so isn't incremental. |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 27 | _failed_test_args = inspect.getfullargspec(unittest.loader._make_failed_test).args |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 28 | exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args)) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 29 | unittest.loader._make_failed_test = _make_failed_test |
| 30 | |
| 31 | def _find_duplicated_modules(suite, directory): |
| 32 | for module in getSuiteModules(suite): |
| 33 | path = findFile('%s.py' % module, directory) |
| 34 | if path: |
| 35 | raise ImportError("Duplicated %s module found in %s" % (module, path)) |
| 36 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 37 | def _built_modules_dict(modules): |
| 38 | modules_dict = {} |
| 39 | |
| 40 | if modules == None: |
| 41 | return modules_dict |
| 42 | |
| 43 | for module in modules: |
| 44 | # Assumption: package and module names do not contain upper case |
| 45 | # characters, whereas class names do |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame] | 46 | m = re.match(r'^(\w+)(?:\.(\w[^.]*)(?:\.([^.]+))?)?$', module, flags=re.ASCII) |
Brad Bishop | 1a4b7ee | 2018-12-16 17:11:34 -0800 | [diff] [blame^] | 47 | if not m: |
| 48 | continue |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 49 | |
| 50 | module_name, class_name, test_name = m.groups() |
| 51 | |
| 52 | if module_name and module_name not in modules_dict: |
| 53 | modules_dict[module_name] = {} |
| 54 | if class_name and class_name not in modules_dict[module_name]: |
| 55 | modules_dict[module_name][class_name] = [] |
| 56 | if test_name and test_name not in modules_dict[module_name][class_name]: |
| 57 | modules_dict[module_name][class_name].append(test_name) |
| 58 | |
| 59 | return modules_dict |
| 60 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 61 | class OETestLoader(unittest.TestLoader): |
| 62 | caseClass = OETestCase |
| 63 | |
| 64 | kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass', |
| 65 | '_top_level_dir'] |
| 66 | |
| 67 | def __init__(self, tc, module_paths, modules, tests, modules_required, |
| 68 | filters, *args, **kwargs): |
| 69 | self.tc = tc |
| 70 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 71 | self.modules = _built_modules_dict(modules) |
| 72 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 73 | self.tests = tests |
| 74 | self.modules_required = modules_required |
| 75 | |
| 76 | self.filters = filters |
| 77 | self.decorator_filters = [d for d in decoratorClasses if \ |
| 78 | issubclass(d, OETestFilter)] |
| 79 | self._validateFilters(self.filters, self.decorator_filters) |
| 80 | self.used_filters = [d for d in self.decorator_filters |
| 81 | for f in self.filters |
| 82 | if f in d.attrs] |
| 83 | |
| 84 | if isinstance(module_paths, str): |
| 85 | module_paths = [module_paths] |
| 86 | elif not isinstance(module_paths, list): |
| 87 | raise TypeError('module_paths must be a str or a list of str') |
| 88 | self.module_paths = module_paths |
| 89 | |
| 90 | for kwname in self.kwargs_names: |
| 91 | if kwname in kwargs: |
| 92 | setattr(self, kwname, kwargs[kwname]) |
| 93 | |
| 94 | self._patchCaseClass(self.caseClass) |
| 95 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 96 | super(OETestLoader, self).__init__() |
| 97 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 98 | def _patchCaseClass(self, testCaseClass): |
| 99 | # Adds custom attributes to the OETestCase class |
| 100 | setattr(testCaseClass, 'tc', self.tc) |
| 101 | setattr(testCaseClass, 'td', self.tc.td) |
| 102 | setattr(testCaseClass, 'logger', self.tc.logger) |
| 103 | |
| 104 | def _validateFilters(self, filters, decorator_filters): |
| 105 | # Validate if filter isn't empty |
| 106 | for key,value in filters.items(): |
| 107 | if not value: |
| 108 | raise TypeError("Filter %s specified is empty" % key) |
| 109 | |
| 110 | # Validate unique attributes |
| 111 | attr_filters = [attr for clss in decorator_filters \ |
| 112 | for attr in clss.attrs] |
| 113 | dup_attr = [attr for attr in attr_filters |
| 114 | if attr_filters.count(attr) > 1] |
| 115 | if dup_attr: |
| 116 | raise TypeError('Detected duplicated attribute(s) %s in filter' |
| 117 | ' decorators' % ' ,'.join(dup_attr)) |
| 118 | |
| 119 | # Validate if filter is supported |
| 120 | for f in filters: |
| 121 | if f not in attr_filters: |
| 122 | classes = ', '.join([d.__name__ for d in decorator_filters]) |
| 123 | raise TypeError('Found "%s" filter but not declared in any of ' |
| 124 | '%s decorators' % (f, classes)) |
| 125 | |
| 126 | def _registerTestCase(self, case): |
| 127 | case_id = case.id() |
| 128 | self.tc._registry['cases'][case_id] = case |
| 129 | |
| 130 | def _handleTestCaseDecorators(self, case): |
| 131 | def _handle(obj): |
| 132 | if isinstance(obj, OETestDecorator): |
| 133 | if not obj.__class__ in decoratorClasses: |
| 134 | raise Exception("Decorator %s isn't registered" \ |
| 135 | " in decoratorClasses." % obj.__name__) |
| 136 | obj.bind(self.tc._registry, case) |
| 137 | |
| 138 | def _walk_closure(obj): |
| 139 | if hasattr(obj, '__closure__') and obj.__closure__: |
| 140 | for f in obj.__closure__: |
| 141 | obj = f.cell_contents |
| 142 | _handle(obj) |
| 143 | _walk_closure(obj) |
| 144 | method = getattr(case, case._testMethodName, None) |
| 145 | _walk_closure(method) |
| 146 | |
| 147 | def _filterTest(self, case): |
| 148 | """ |
| 149 | Returns True if test case must be filtered, False otherwise. |
| 150 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 151 | # XXX; If the module has more than one namespace only use |
| 152 | # the first to support run the whole module specifying the |
| 153 | # <module_name>.[test_class].[test_name] |
| 154 | module_name_small = case.__module__.split('.')[0] |
| 155 | module_name = case.__module__ |
| 156 | |
| 157 | class_name = case.__class__.__name__ |
| 158 | test_name = case._testMethodName |
| 159 | |
Brad Bishop | d5ae7d9 | 2018-06-14 09:52:03 -0700 | [diff] [blame] | 160 | # 'auto' is a reserved key word to run test cases automatically |
| 161 | # warn users if their test case belong to a module named 'auto' |
| 162 | if module_name_small == "auto": |
| 163 | bb.warn("'auto' is a reserved key word for TEST_SUITES. " |
| 164 | "But test case '%s' is detected to belong to auto module. " |
| 165 | "Please condier using a new name for your module." % str(case)) |
| 166 | |
| 167 | # check if case belongs to any specified module |
| 168 | # if 'auto' is specified, such check is skipped |
| 169 | if self.modules and not 'auto' in self.modules: |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 170 | module = None |
| 171 | try: |
| 172 | module = self.modules[module_name_small] |
| 173 | except KeyError: |
| 174 | try: |
| 175 | module = self.modules[module_name] |
| 176 | except KeyError: |
| 177 | return True |
| 178 | |
| 179 | if module: |
| 180 | if not class_name in module: |
| 181 | return True |
| 182 | |
| 183 | if module[class_name]: |
| 184 | if test_name not in module[class_name]: |
| 185 | return True |
| 186 | |
| 187 | # Decorator filters |
| 188 | if self.filters and isinstance(case, OETestCase): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 189 | filters = self.filters.copy() |
| 190 | case_decorators = [cd for cd in case.decorators |
| 191 | if cd.__class__ in self.used_filters] |
| 192 | |
| 193 | # Iterate over case decorators to check if needs to be filtered. |
| 194 | for cd in case_decorators: |
| 195 | if cd.filtrate(filters): |
| 196 | return True |
| 197 | |
| 198 | # Case is missing one or more decorators for all the filters |
| 199 | # being used, so filter test case. |
| 200 | if filters: |
| 201 | return True |
| 202 | |
| 203 | return False |
| 204 | |
| 205 | def _getTestCase(self, testCaseClass, tcName): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 206 | if not hasattr(testCaseClass, '__oeqa_loader') and \ |
| 207 | issubclass(testCaseClass, OETestCase): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 208 | # In order to support data_vars validation |
| 209 | # monkey patch the default setUp/tearDown{Class} to use |
| 210 | # the ones provided by OETestCase |
| 211 | setattr(testCaseClass, 'setUpClassMethod', |
| 212 | getattr(testCaseClass, 'setUpClass')) |
| 213 | setattr(testCaseClass, 'tearDownClassMethod', |
| 214 | getattr(testCaseClass, 'tearDownClass')) |
| 215 | setattr(testCaseClass, 'setUpClass', |
| 216 | testCaseClass._oeSetUpClass) |
| 217 | setattr(testCaseClass, 'tearDownClass', |
| 218 | testCaseClass._oeTearDownClass) |
| 219 | |
| 220 | # In order to support decorators initialization |
| 221 | # monkey patch the default setUp/tearDown to use |
| 222 | # a setUpDecorators/tearDownDecorators that methods |
| 223 | # will call setUp/tearDown original methods. |
| 224 | setattr(testCaseClass, 'setUpMethod', |
| 225 | getattr(testCaseClass, 'setUp')) |
| 226 | setattr(testCaseClass, 'tearDownMethod', |
| 227 | getattr(testCaseClass, 'tearDown')) |
| 228 | setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp) |
| 229 | setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown) |
| 230 | |
| 231 | setattr(testCaseClass, '__oeqa_loader', True) |
| 232 | |
| 233 | case = testCaseClass(tcName) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 234 | if isinstance(case, OETestCase): |
| 235 | setattr(case, 'decorators', []) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 236 | |
| 237 | return case |
| 238 | |
| 239 | def loadTestsFromTestCase(self, testCaseClass): |
| 240 | """ |
| 241 | Returns a suite of all tests cases contained in testCaseClass. |
| 242 | """ |
| 243 | if issubclass(testCaseClass, unittest.suite.TestSuite): |
| 244 | raise TypeError("Test cases should not be derived from TestSuite." \ |
| 245 | " Maybe you meant to derive %s from TestCase?" \ |
| 246 | % testCaseClass.__name__) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 247 | if not issubclass(testCaseClass, unittest.case.TestCase): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 248 | raise TypeError("Test %s is not derived from %s" % \ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 249 | (testCaseClass.__name__, unittest.case.TestCase.__name__)) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 250 | |
| 251 | testCaseNames = self.getTestCaseNames(testCaseClass) |
| 252 | if not testCaseNames and hasattr(testCaseClass, 'runTest'): |
| 253 | testCaseNames = ['runTest'] |
| 254 | |
| 255 | suite = [] |
| 256 | for tcName in testCaseNames: |
| 257 | case = self._getTestCase(testCaseClass, tcName) |
| 258 | # Filer by case id |
Brad Bishop | d5ae7d9 | 2018-06-14 09:52:03 -0700 | [diff] [blame] | 259 | if not (self.tests and not 'auto' in self.tests |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 260 | and not getCaseID(case) in self.tests): |
| 261 | self._handleTestCaseDecorators(case) |
| 262 | |
| 263 | # Filter by decorators |
| 264 | if not self._filterTest(case): |
| 265 | self._registerTestCase(case) |
| 266 | suite.append(case) |
| 267 | |
| 268 | return self.suiteClass(suite) |
| 269 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 270 | def _required_modules_validation(self): |
| 271 | """ |
| 272 | Search in Test context registry if a required |
| 273 | test is found, raise an exception when not found. |
| 274 | """ |
| 275 | |
| 276 | for module in self.modules_required: |
| 277 | found = False |
| 278 | |
| 279 | # The module name is splitted to only compare the |
| 280 | # first part of a test case id. |
| 281 | comp_len = len(module.split('.')) |
| 282 | for case in self.tc._registry['cases']: |
| 283 | case_comp = '.'.join(case.split('.')[0:comp_len]) |
| 284 | if module == case_comp: |
| 285 | found = True |
| 286 | break |
| 287 | |
| 288 | if not found: |
| 289 | raise OEQATestNotFound("Not found %s in loaded test cases" % \ |
| 290 | module) |
| 291 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 292 | def discover(self): |
| 293 | big_suite = self.suiteClass() |
| 294 | for path in self.module_paths: |
| 295 | _find_duplicated_modules(big_suite, path) |
| 296 | suite = super(OETestLoader, self).discover(path, |
| 297 | pattern='*.py', top_level_dir=path) |
| 298 | big_suite.addTests(suite) |
| 299 | |
| 300 | cases = None |
| 301 | discover_classes = [clss for clss in decoratorClasses |
| 302 | if issubclass(clss, OETestDiscover)] |
| 303 | for clss in discover_classes: |
| 304 | cases = clss.discover(self.tc._registry) |
| 305 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 306 | if self.modules_required: |
| 307 | self._required_modules_validation() |
| 308 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 309 | return self.suiteClass(cases) if cases else big_suite |
| 310 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 311 | def _filterModule(self, module): |
| 312 | if module.__name__ in sys.builtin_module_names: |
| 313 | msg = 'Tried to import %s test module but is a built-in' |
| 314 | raise ImportError(msg % module.__name__) |
| 315 | |
| 316 | # XXX; If the module has more than one namespace only use |
| 317 | # the first to support run the whole module specifying the |
| 318 | # <module_name>.[test_class].[test_name] |
| 319 | module_name_small = module.__name__.split('.')[0] |
| 320 | module_name = module.__name__ |
| 321 | |
| 322 | # Normal test modules are loaded if no modules were specified, |
Brad Bishop | d5ae7d9 | 2018-06-14 09:52:03 -0700 | [diff] [blame] | 323 | # if module is in the specified module list or if 'auto' is in |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 324 | # module list. |
| 325 | # Underscore modules are loaded only if specified in module list. |
| 326 | load_module = True if not module_name.startswith('_') \ |
| 327 | and (not self.modules \ |
| 328 | or module_name in self.modules \ |
| 329 | or module_name_small in self.modules \ |
Brad Bishop | d5ae7d9 | 2018-06-14 09:52:03 -0700 | [diff] [blame] | 330 | or 'auto' in self.modules) \ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 331 | else False |
| 332 | |
| 333 | load_underscore = True if module_name.startswith('_') \ |
| 334 | and (module_name in self.modules or \ |
| 335 | module_name_small in self.modules) \ |
| 336 | else False |
| 337 | |
| 338 | return (load_module, load_underscore) |
| 339 | |
| 340 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 341 | # XXX After Python 3.5, remove backward compatibility hacks for |
| 342 | # use_load_tests deprecation via *args and **kws. See issue 16662. |
| 343 | if sys.version_info >= (3,5): |
| 344 | def loadTestsFromModule(self, module, *args, pattern=None, **kws): |
| 345 | """ |
| 346 | Returns a suite of all tests cases contained in module. |
| 347 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 348 | load_module, load_underscore = self._filterModule(module) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 349 | |
| 350 | if load_module or load_underscore: |
| 351 | return super(OETestLoader, self).loadTestsFromModule( |
| 352 | module, *args, pattern=pattern, **kws) |
| 353 | else: |
| 354 | return self.suiteClass() |
| 355 | else: |
| 356 | def loadTestsFromModule(self, module, use_load_tests=True): |
| 357 | """ |
| 358 | Returns a suite of all tests cases contained in module. |
| 359 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 360 | load_module, load_underscore = self._filterModule(module) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 361 | |
| 362 | if load_module or load_underscore: |
| 363 | return super(OETestLoader, self).loadTestsFromModule( |
| 364 | module, use_load_tests) |
| 365 | else: |
| 366 | return self.suiteClass() |