Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 1 | # Copyright (C) 2016 Intel Corporation |
| 2 | # Released under the MIT license (see COPYING.MIT) |
| 3 | |
| 4 | import os |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 5 | import re |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 6 | import sys |
| 7 | import unittest |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 8 | import inspect |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 9 | |
| 10 | from oeqa.core.utils.path import findFile |
| 11 | from oeqa.core.utils.test import getSuiteModules, getCaseID |
| 12 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 13 | from oeqa.core.exception import OEQATestNotFound |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 14 | from oeqa.core.case import OETestCase |
| 15 | from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ |
| 16 | OETestFilter, OETestDiscover |
| 17 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 18 | # When loading tests, the unittest framework stores any exceptions and |
| 19 | # displays them only when the run method is called. |
| 20 | # |
| 21 | # For our purposes, it is better to raise the exceptions in the loading |
| 22 | # step rather than waiting to run the test suite. |
| 23 | # |
| 24 | # Generate the function definition because this differ across python versions |
| 25 | # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3 |
| 26 | # ueses four parameters so isn't incremental. |
| 27 | _failed_test_args = inspect.getargspec(unittest.loader._make_failed_test).args |
| 28 | exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args)) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 29 | unittest.loader._make_failed_test = _make_failed_test |
| 30 | |
| 31 | def _find_duplicated_modules(suite, directory): |
| 32 | for module in getSuiteModules(suite): |
| 33 | path = findFile('%s.py' % module, directory) |
| 34 | if path: |
| 35 | raise ImportError("Duplicated %s module found in %s" % (module, path)) |
| 36 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 37 | def _built_modules_dict(modules): |
| 38 | modules_dict = {} |
| 39 | |
| 40 | if modules == None: |
| 41 | return modules_dict |
| 42 | |
| 43 | for module in modules: |
| 44 | # Assumption: package and module names do not contain upper case |
| 45 | # characters, whereas class names do |
Brad Bishop | 316dfdd | 2018-06-25 12:45:53 -0400 | [diff] [blame^] | 46 | m = re.match(r'^(\w+)(?:\.(\w[^.]*)(?:\.([^.]+))?)?$', module, flags=re.ASCII) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 47 | |
| 48 | module_name, class_name, test_name = m.groups() |
| 49 | |
| 50 | if module_name and module_name not in modules_dict: |
| 51 | modules_dict[module_name] = {} |
| 52 | if class_name and class_name not in modules_dict[module_name]: |
| 53 | modules_dict[module_name][class_name] = [] |
| 54 | if test_name and test_name not in modules_dict[module_name][class_name]: |
| 55 | modules_dict[module_name][class_name].append(test_name) |
| 56 | |
| 57 | return modules_dict |
| 58 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 59 | class OETestLoader(unittest.TestLoader): |
| 60 | caseClass = OETestCase |
| 61 | |
| 62 | kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass', |
| 63 | '_top_level_dir'] |
| 64 | |
| 65 | def __init__(self, tc, module_paths, modules, tests, modules_required, |
| 66 | filters, *args, **kwargs): |
| 67 | self.tc = tc |
| 68 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 69 | self.modules = _built_modules_dict(modules) |
| 70 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 71 | self.tests = tests |
| 72 | self.modules_required = modules_required |
| 73 | |
| 74 | self.filters = filters |
| 75 | self.decorator_filters = [d for d in decoratorClasses if \ |
| 76 | issubclass(d, OETestFilter)] |
| 77 | self._validateFilters(self.filters, self.decorator_filters) |
| 78 | self.used_filters = [d for d in self.decorator_filters |
| 79 | for f in self.filters |
| 80 | if f in d.attrs] |
| 81 | |
| 82 | if isinstance(module_paths, str): |
| 83 | module_paths = [module_paths] |
| 84 | elif not isinstance(module_paths, list): |
| 85 | raise TypeError('module_paths must be a str or a list of str') |
| 86 | self.module_paths = module_paths |
| 87 | |
| 88 | for kwname in self.kwargs_names: |
| 89 | if kwname in kwargs: |
| 90 | setattr(self, kwname, kwargs[kwname]) |
| 91 | |
| 92 | self._patchCaseClass(self.caseClass) |
| 93 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 94 | super(OETestLoader, self).__init__() |
| 95 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 96 | def _patchCaseClass(self, testCaseClass): |
| 97 | # Adds custom attributes to the OETestCase class |
| 98 | setattr(testCaseClass, 'tc', self.tc) |
| 99 | setattr(testCaseClass, 'td', self.tc.td) |
| 100 | setattr(testCaseClass, 'logger', self.tc.logger) |
| 101 | |
| 102 | def _validateFilters(self, filters, decorator_filters): |
| 103 | # Validate if filter isn't empty |
| 104 | for key,value in filters.items(): |
| 105 | if not value: |
| 106 | raise TypeError("Filter %s specified is empty" % key) |
| 107 | |
| 108 | # Validate unique attributes |
| 109 | attr_filters = [attr for clss in decorator_filters \ |
| 110 | for attr in clss.attrs] |
| 111 | dup_attr = [attr for attr in attr_filters |
| 112 | if attr_filters.count(attr) > 1] |
| 113 | if dup_attr: |
| 114 | raise TypeError('Detected duplicated attribute(s) %s in filter' |
| 115 | ' decorators' % ' ,'.join(dup_attr)) |
| 116 | |
| 117 | # Validate if filter is supported |
| 118 | for f in filters: |
| 119 | if f not in attr_filters: |
| 120 | classes = ', '.join([d.__name__ for d in decorator_filters]) |
| 121 | raise TypeError('Found "%s" filter but not declared in any of ' |
| 122 | '%s decorators' % (f, classes)) |
| 123 | |
| 124 | def _registerTestCase(self, case): |
| 125 | case_id = case.id() |
| 126 | self.tc._registry['cases'][case_id] = case |
| 127 | |
| 128 | def _handleTestCaseDecorators(self, case): |
| 129 | def _handle(obj): |
| 130 | if isinstance(obj, OETestDecorator): |
| 131 | if not obj.__class__ in decoratorClasses: |
| 132 | raise Exception("Decorator %s isn't registered" \ |
| 133 | " in decoratorClasses." % obj.__name__) |
| 134 | obj.bind(self.tc._registry, case) |
| 135 | |
| 136 | def _walk_closure(obj): |
| 137 | if hasattr(obj, '__closure__') and obj.__closure__: |
| 138 | for f in obj.__closure__: |
| 139 | obj = f.cell_contents |
| 140 | _handle(obj) |
| 141 | _walk_closure(obj) |
| 142 | method = getattr(case, case._testMethodName, None) |
| 143 | _walk_closure(method) |
| 144 | |
| 145 | def _filterTest(self, case): |
| 146 | """ |
| 147 | Returns True if test case must be filtered, False otherwise. |
| 148 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 149 | # XXX; If the module has more than one namespace only use |
| 150 | # the first to support run the whole module specifying the |
| 151 | # <module_name>.[test_class].[test_name] |
| 152 | module_name_small = case.__module__.split('.')[0] |
| 153 | module_name = case.__module__ |
| 154 | |
| 155 | class_name = case.__class__.__name__ |
| 156 | test_name = case._testMethodName |
| 157 | |
| 158 | if self.modules: |
| 159 | module = None |
| 160 | try: |
| 161 | module = self.modules[module_name_small] |
| 162 | except KeyError: |
| 163 | try: |
| 164 | module = self.modules[module_name] |
| 165 | except KeyError: |
| 166 | return True |
| 167 | |
| 168 | if module: |
| 169 | if not class_name in module: |
| 170 | return True |
| 171 | |
| 172 | if module[class_name]: |
| 173 | if test_name not in module[class_name]: |
| 174 | return True |
| 175 | |
| 176 | # Decorator filters |
| 177 | if self.filters and isinstance(case, OETestCase): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 178 | filters = self.filters.copy() |
| 179 | case_decorators = [cd for cd in case.decorators |
| 180 | if cd.__class__ in self.used_filters] |
| 181 | |
| 182 | # Iterate over case decorators to check if needs to be filtered. |
| 183 | for cd in case_decorators: |
| 184 | if cd.filtrate(filters): |
| 185 | return True |
| 186 | |
| 187 | # Case is missing one or more decorators for all the filters |
| 188 | # being used, so filter test case. |
| 189 | if filters: |
| 190 | return True |
| 191 | |
| 192 | return False |
| 193 | |
| 194 | def _getTestCase(self, testCaseClass, tcName): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 195 | if not hasattr(testCaseClass, '__oeqa_loader') and \ |
| 196 | issubclass(testCaseClass, OETestCase): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 197 | # In order to support data_vars validation |
| 198 | # monkey patch the default setUp/tearDown{Class} to use |
| 199 | # the ones provided by OETestCase |
| 200 | setattr(testCaseClass, 'setUpClassMethod', |
| 201 | getattr(testCaseClass, 'setUpClass')) |
| 202 | setattr(testCaseClass, 'tearDownClassMethod', |
| 203 | getattr(testCaseClass, 'tearDownClass')) |
| 204 | setattr(testCaseClass, 'setUpClass', |
| 205 | testCaseClass._oeSetUpClass) |
| 206 | setattr(testCaseClass, 'tearDownClass', |
| 207 | testCaseClass._oeTearDownClass) |
| 208 | |
| 209 | # In order to support decorators initialization |
| 210 | # monkey patch the default setUp/tearDown to use |
| 211 | # a setUpDecorators/tearDownDecorators that methods |
| 212 | # will call setUp/tearDown original methods. |
| 213 | setattr(testCaseClass, 'setUpMethod', |
| 214 | getattr(testCaseClass, 'setUp')) |
| 215 | setattr(testCaseClass, 'tearDownMethod', |
| 216 | getattr(testCaseClass, 'tearDown')) |
| 217 | setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp) |
| 218 | setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown) |
| 219 | |
| 220 | setattr(testCaseClass, '__oeqa_loader', True) |
| 221 | |
| 222 | case = testCaseClass(tcName) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 223 | if isinstance(case, OETestCase): |
| 224 | setattr(case, 'decorators', []) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 225 | |
| 226 | return case |
| 227 | |
| 228 | def loadTestsFromTestCase(self, testCaseClass): |
| 229 | """ |
| 230 | Returns a suite of all tests cases contained in testCaseClass. |
| 231 | """ |
| 232 | if issubclass(testCaseClass, unittest.suite.TestSuite): |
| 233 | raise TypeError("Test cases should not be derived from TestSuite." \ |
| 234 | " Maybe you meant to derive %s from TestCase?" \ |
| 235 | % testCaseClass.__name__) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 236 | if not issubclass(testCaseClass, unittest.case.TestCase): |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 237 | raise TypeError("Test %s is not derived from %s" % \ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 238 | (testCaseClass.__name__, unittest.case.TestCase.__name__)) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 239 | |
| 240 | testCaseNames = self.getTestCaseNames(testCaseClass) |
| 241 | if not testCaseNames and hasattr(testCaseClass, 'runTest'): |
| 242 | testCaseNames = ['runTest'] |
| 243 | |
| 244 | suite = [] |
| 245 | for tcName in testCaseNames: |
| 246 | case = self._getTestCase(testCaseClass, tcName) |
| 247 | # Filer by case id |
| 248 | if not (self.tests and not 'all' in self.tests |
| 249 | and not getCaseID(case) in self.tests): |
| 250 | self._handleTestCaseDecorators(case) |
| 251 | |
| 252 | # Filter by decorators |
| 253 | if not self._filterTest(case): |
| 254 | self._registerTestCase(case) |
| 255 | suite.append(case) |
| 256 | |
| 257 | return self.suiteClass(suite) |
| 258 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 259 | def _required_modules_validation(self): |
| 260 | """ |
| 261 | Search in Test context registry if a required |
| 262 | test is found, raise an exception when not found. |
| 263 | """ |
| 264 | |
| 265 | for module in self.modules_required: |
| 266 | found = False |
| 267 | |
| 268 | # The module name is splitted to only compare the |
| 269 | # first part of a test case id. |
| 270 | comp_len = len(module.split('.')) |
| 271 | for case in self.tc._registry['cases']: |
| 272 | case_comp = '.'.join(case.split('.')[0:comp_len]) |
| 273 | if module == case_comp: |
| 274 | found = True |
| 275 | break |
| 276 | |
| 277 | if not found: |
| 278 | raise OEQATestNotFound("Not found %s in loaded test cases" % \ |
| 279 | module) |
| 280 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 281 | def discover(self): |
| 282 | big_suite = self.suiteClass() |
| 283 | for path in self.module_paths: |
| 284 | _find_duplicated_modules(big_suite, path) |
| 285 | suite = super(OETestLoader, self).discover(path, |
| 286 | pattern='*.py', top_level_dir=path) |
| 287 | big_suite.addTests(suite) |
| 288 | |
| 289 | cases = None |
| 290 | discover_classes = [clss for clss in decoratorClasses |
| 291 | if issubclass(clss, OETestDiscover)] |
| 292 | for clss in discover_classes: |
| 293 | cases = clss.discover(self.tc._registry) |
| 294 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 295 | if self.modules_required: |
| 296 | self._required_modules_validation() |
| 297 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 298 | return self.suiteClass(cases) if cases else big_suite |
| 299 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 300 | def _filterModule(self, module): |
| 301 | if module.__name__ in sys.builtin_module_names: |
| 302 | msg = 'Tried to import %s test module but is a built-in' |
| 303 | raise ImportError(msg % module.__name__) |
| 304 | |
| 305 | # XXX; If the module has more than one namespace only use |
| 306 | # the first to support run the whole module specifying the |
| 307 | # <module_name>.[test_class].[test_name] |
| 308 | module_name_small = module.__name__.split('.')[0] |
| 309 | module_name = module.__name__ |
| 310 | |
| 311 | # Normal test modules are loaded if no modules were specified, |
| 312 | # if module is in the specified module list or if 'all' is in |
| 313 | # module list. |
| 314 | # Underscore modules are loaded only if specified in module list. |
| 315 | load_module = True if not module_name.startswith('_') \ |
| 316 | and (not self.modules \ |
| 317 | or module_name in self.modules \ |
| 318 | or module_name_small in self.modules \ |
| 319 | or 'all' in self.modules) \ |
| 320 | else False |
| 321 | |
| 322 | load_underscore = True if module_name.startswith('_') \ |
| 323 | and (module_name in self.modules or \ |
| 324 | module_name_small in self.modules) \ |
| 325 | else False |
| 326 | |
| 327 | return (load_module, load_underscore) |
| 328 | |
| 329 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 330 | # XXX After Python 3.5, remove backward compatibility hacks for |
| 331 | # use_load_tests deprecation via *args and **kws. See issue 16662. |
| 332 | if sys.version_info >= (3,5): |
| 333 | def loadTestsFromModule(self, module, *args, pattern=None, **kws): |
| 334 | """ |
| 335 | Returns a suite of all tests cases contained in module. |
| 336 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 337 | load_module, load_underscore = self._filterModule(module) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 338 | |
| 339 | if load_module or load_underscore: |
| 340 | return super(OETestLoader, self).loadTestsFromModule( |
| 341 | module, *args, pattern=pattern, **kws) |
| 342 | else: |
| 343 | return self.suiteClass() |
| 344 | else: |
| 345 | def loadTestsFromModule(self, module, use_load_tests=True): |
| 346 | """ |
| 347 | Returns a suite of all tests cases contained in module. |
| 348 | """ |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 349 | load_module, load_underscore = self._filterModule(module) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 350 | |
| 351 | if load_module or load_underscore: |
| 352 | return super(OETestLoader, self).loadTestsFromModule( |
| 353 | module, use_load_tests) |
| 354 | else: |
| 355 | return self.suiteClass() |