| # |
| # Copyright (C) 2016 Intel Corporation |
| # |
| # SPDX-License-Identifier: MIT |
| # |
| |
| import os |
| import re |
| import sys |
| import unittest |
| import inspect |
| |
| from oeqa.core.utils.path import findFile |
| from oeqa.core.utils.test import getSuiteModules, getCaseID |
| |
| from oeqa.core.exception import OEQATestNotFound |
| from oeqa.core.case import OETestCase |
| from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ |
| OETestDiscover |
| |
| # When loading tests, the unittest framework stores any exceptions and |
| # displays them only when the run method is called. |
| # |
| # For our purposes, it is better to raise the exceptions in the loading |
| # step rather than waiting to run the test suite. |
| # |
| # Generate the function definition because this differ across python versions |
| # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3 |
| # ueses four parameters so isn't incremental. |
| _failed_test_args = inspect.getfullargspec(unittest.loader._make_failed_test).args |
| exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args)) |
| unittest.loader._make_failed_test = _make_failed_test |
| |
| def _find_duplicated_modules(suite, directory): |
| for module in getSuiteModules(suite): |
| path = findFile('%s.py' % module, directory) |
| if path: |
| raise ImportError("Duplicated %s module found in %s" % (module, path)) |
| |
| def _built_modules_dict(modules, logger): |
| modules_dict = {} |
| |
| if modules == None: |
| return modules_dict |
| |
| for module in modules: |
| # Assumption: package and module names do not contain upper case |
| # characters, whereas class names do |
| m = re.match(r'^([0-9a-z_.]+)(?:\.(\w[^.]*)(?:\.([^.]+))?)?$', module, flags=re.ASCII) |
| if not m: |
| logger.warn("module '%s' was skipped from selected modules, "\ |
| "because it doesn't match with module name assumptions: "\ |
| "package and module names do not contain upper case characters, whereas class names do" % module) |
| continue |
| |
| module_name, class_name, test_name = m.groups() |
| |
| if module_name and module_name not in modules_dict: |
| modules_dict[module_name] = {} |
| if class_name and class_name not in modules_dict[module_name]: |
| modules_dict[module_name][class_name] = [] |
| if test_name and test_name not in modules_dict[module_name][class_name]: |
| modules_dict[module_name][class_name].append(test_name) |
| if modules and not modules_dict: |
| raise OEQATestNotFound("All selected modules were skipped, this would trigger selftest with all tests and -r ignored.") |
| |
| return modules_dict |
| |
| class OETestLoader(unittest.TestLoader): |
| caseClass = OETestCase |
| |
| kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass', |
| '_top_level_dir'] |
| |
| def __init__(self, tc, module_paths, modules, tests, modules_required, |
| *args, **kwargs): |
| self.tc = tc |
| |
| self.modules = _built_modules_dict(modules, tc.logger) |
| |
| self.tests = tests |
| self.modules_required = modules_required |
| |
| self.tags_filter = kwargs.get("tags_filter", None) |
| |
| if isinstance(module_paths, str): |
| module_paths = [module_paths] |
| elif not isinstance(module_paths, list): |
| raise TypeError('module_paths must be a str or a list of str') |
| self.module_paths = module_paths |
| |
| for kwname in self.kwargs_names: |
| if kwname in kwargs: |
| setattr(self, kwname, kwargs[kwname]) |
| |
| self._patchCaseClass(self.caseClass) |
| |
| super(OETestLoader, self).__init__() |
| |
| def _patchCaseClass(self, testCaseClass): |
| # Adds custom attributes to the OETestCase class |
| setattr(testCaseClass, 'tc', self.tc) |
| setattr(testCaseClass, 'td', self.tc.td) |
| setattr(testCaseClass, 'logger', self.tc.logger) |
| |
| def _registerTestCase(self, case): |
| case_id = case.id() |
| self.tc._registry['cases'][case_id] = case |
| |
| def _handleTestCaseDecorators(self, case): |
| def _handle(obj): |
| if isinstance(obj, OETestDecorator): |
| if not obj.__class__ in decoratorClasses: |
| raise Exception("Decorator %s isn't registered" \ |
| " in decoratorClasses." % obj.__name__) |
| obj.bind(self.tc._registry, case) |
| |
| def _walk_closure(obj): |
| if hasattr(obj, '__closure__') and obj.__closure__: |
| for f in obj.__closure__: |
| obj = f.cell_contents |
| _handle(obj) |
| _walk_closure(obj) |
| method = getattr(case, case._testMethodName, None) |
| _walk_closure(method) |
| |
| def _filterTest(self, case): |
| """ |
| Returns True if test case must be filtered, False otherwise. |
| """ |
| # XXX; If the module has more than one namespace only use |
| # the first to support run the whole module specifying the |
| # <module_name>.[test_class].[test_name] |
| module_name_small = case.__module__.split('.')[0] |
| module_name = case.__module__ |
| |
| class_name = case.__class__.__name__ |
| test_name = case._testMethodName |
| |
| # 'auto' is a reserved key word to run test cases automatically |
| # warn users if their test case belong to a module named 'auto' |
| if module_name_small == "auto": |
| bb.warn("'auto' is a reserved key word for TEST_SUITES. " |
| "But test case '%s' is detected to belong to auto module. " |
| "Please condier using a new name for your module." % str(case)) |
| |
| # check if case belongs to any specified module |
| # if 'auto' is specified, such check is skipped |
| if self.modules and not 'auto' in self.modules: |
| module = None |
| try: |
| module = self.modules[module_name_small] |
| except KeyError: |
| try: |
| module = self.modules[module_name] |
| except KeyError: |
| return True |
| |
| if module: |
| if not class_name in module: |
| return True |
| |
| if module[class_name]: |
| if test_name not in module[class_name]: |
| return True |
| |
| # Decorator filters |
| if self.tags_filter is not None and callable(self.tags_filter): |
| alltags = set() |
| # pull tags from the case class |
| if hasattr(case, "__oeqa_testtags"): |
| for t in getattr(case, "__oeqa_testtags"): |
| alltags.add(t) |
| # pull tags from the method itself |
| if hasattr(case, test_name): |
| method = getattr(case, test_name) |
| if hasattr(method, "__oeqa_testtags"): |
| for t in getattr(method, "__oeqa_testtags"): |
| alltags.add(t) |
| |
| if self.tags_filter(alltags): |
| return True |
| |
| return False |
| |
| def _getTestCase(self, testCaseClass, tcName): |
| if not hasattr(testCaseClass, '__oeqa_loader') and \ |
| issubclass(testCaseClass, OETestCase): |
| # In order to support data_vars validation |
| # monkey patch the default setUp/tearDown{Class} to use |
| # the ones provided by OETestCase |
| setattr(testCaseClass, 'setUpClassMethod', |
| getattr(testCaseClass, 'setUpClass')) |
| setattr(testCaseClass, 'tearDownClassMethod', |
| getattr(testCaseClass, 'tearDownClass')) |
| setattr(testCaseClass, 'setUpClass', |
| testCaseClass._oeSetUpClass) |
| setattr(testCaseClass, 'tearDownClass', |
| testCaseClass._oeTearDownClass) |
| |
| # In order to support decorators initialization |
| # monkey patch the default setUp/tearDown to use |
| # a setUpDecorators/tearDownDecorators that methods |
| # will call setUp/tearDown original methods. |
| setattr(testCaseClass, 'setUpMethod', |
| getattr(testCaseClass, 'setUp')) |
| setattr(testCaseClass, 'tearDownMethod', |
| getattr(testCaseClass, 'tearDown')) |
| setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp) |
| setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown) |
| |
| setattr(testCaseClass, '__oeqa_loader', True) |
| |
| case = testCaseClass(tcName) |
| if isinstance(case, OETestCase): |
| setattr(case, 'decorators', []) |
| |
| return case |
| |
| def loadTestsFromTestCase(self, testCaseClass): |
| """ |
| Returns a suite of all tests cases contained in testCaseClass. |
| """ |
| if issubclass(testCaseClass, unittest.suite.TestSuite): |
| raise TypeError("Test cases should not be derived from TestSuite." \ |
| " Maybe you meant to derive %s from TestCase?" \ |
| % testCaseClass.__name__) |
| if not issubclass(testCaseClass, unittest.case.TestCase): |
| raise TypeError("Test %s is not derived from %s" % \ |
| (testCaseClass.__name__, unittest.case.TestCase.__name__)) |
| |
| testCaseNames = self.getTestCaseNames(testCaseClass) |
| if not testCaseNames and hasattr(testCaseClass, 'runTest'): |
| testCaseNames = ['runTest'] |
| |
| suite = [] |
| for tcName in testCaseNames: |
| case = self._getTestCase(testCaseClass, tcName) |
| # Filer by case id |
| if not (self.tests and not 'auto' in self.tests |
| and not getCaseID(case) in self.tests): |
| self._handleTestCaseDecorators(case) |
| |
| # Filter by decorators |
| if not self._filterTest(case): |
| self._registerTestCase(case) |
| suite.append(case) |
| |
| return self.suiteClass(suite) |
| |
| def _required_modules_validation(self): |
| """ |
| Search in Test context registry if a required |
| test is found, raise an exception when not found. |
| """ |
| |
| for module in self.modules_required: |
| found = False |
| |
| # The module name is splitted to only compare the |
| # first part of a test case id. |
| comp_len = len(module.split('.')) |
| for case in self.tc._registry['cases']: |
| case_comp = '.'.join(case.split('.')[0:comp_len]) |
| if module == case_comp: |
| found = True |
| break |
| |
| if not found: |
| raise OEQATestNotFound("Not found %s in loaded test cases" % \ |
| module) |
| |
| def discover(self): |
| big_suite = self.suiteClass() |
| for path in self.module_paths: |
| _find_duplicated_modules(big_suite, path) |
| suite = super(OETestLoader, self).discover(path, |
| pattern='*.py', top_level_dir=path) |
| big_suite.addTests(suite) |
| |
| cases = None |
| discover_classes = [clss for clss in decoratorClasses |
| if issubclass(clss, OETestDiscover)] |
| for clss in discover_classes: |
| cases = clss.discover(self.tc._registry) |
| |
| if self.modules_required: |
| self._required_modules_validation() |
| |
| return self.suiteClass(cases) if cases else big_suite |
| |
| def _filterModule(self, module): |
| if module.__name__ in sys.builtin_module_names: |
| msg = 'Tried to import %s test module but is a built-in' |
| raise ImportError(msg % module.__name__) |
| |
| # XXX; If the module has more than one namespace only use |
| # the first to support run the whole module specifying the |
| # <module_name>.[test_class].[test_name] |
| module_name_small = module.__name__.split('.')[0] |
| module_name = module.__name__ |
| |
| # Normal test modules are loaded if no modules were specified, |
| # if module is in the specified module list or if 'auto' is in |
| # module list. |
| # Underscore modules are loaded only if specified in module list. |
| load_module = True if not module_name.startswith('_') \ |
| and (not self.modules \ |
| or module_name in self.modules \ |
| or module_name_small in self.modules \ |
| or 'auto' in self.modules) \ |
| else False |
| |
| load_underscore = True if module_name.startswith('_') \ |
| and (module_name in self.modules or \ |
| module_name_small in self.modules) \ |
| else False |
| |
| if any(c.isupper() for c in module.__name__): |
| raise SystemExit("Module '%s' contains uppercase characters and this isn't supported. Please fix the module name." % module.__name__) |
| |
| return (load_module, load_underscore) |
| |
| |
| # XXX After Python 3.5, remove backward compatibility hacks for |
| # use_load_tests deprecation via *args and **kws. See issue 16662. |
| if sys.version_info >= (3,5): |
| def loadTestsFromModule(self, module, *args, pattern=None, **kws): |
| """ |
| Returns a suite of all tests cases contained in module. |
| """ |
| load_module, load_underscore = self._filterModule(module) |
| |
| if load_module or load_underscore: |
| return super(OETestLoader, self).loadTestsFromModule( |
| module, *args, pattern=pattern, **kws) |
| else: |
| return self.suiteClass() |
| else: |
| def loadTestsFromModule(self, module, use_load_tests=True): |
| """ |
| Returns a suite of all tests cases contained in module. |
| """ |
| load_module, load_underscore = self._filterModule(module) |
| |
| if load_module or load_underscore: |
| return super(OETestLoader, self).loadTestsFromModule( |
| module, use_load_tests) |
| else: |
| return self.suiteClass() |