Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 1 | # Copyright (C) 2017 Intel Corporation |
| 2 | # Released under the MIT license (see COPYING.MIT) |
| 3 | |
| 4 | import os |
| 5 | import time |
| 6 | import glob |
| 7 | import sys |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 8 | import importlib |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 9 | import signal |
| 10 | from shutil import copyfile |
| 11 | from random import choice |
| 12 | |
| 13 | import oeqa |
| 14 | |
| 15 | from oeqa.core.context import OETestContext, OETestContextExecutor |
| 16 | from oeqa.core.exception import OEQAPreRun, OEQATestNotFound |
| 17 | |
| 18 | from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer |
| 19 | |
| 20 | class OESelftestTestContext(OETestContext): |
| 21 | def __init__(self, td=None, logger=None, machines=None, config_paths=None): |
| 22 | super(OESelftestTestContext, self).__init__(td, logger) |
| 23 | |
| 24 | self.machines = machines |
| 25 | self.custommachine = None |
| 26 | self.config_paths = config_paths |
| 27 | |
| 28 | def runTests(self, machine=None, skips=[]): |
| 29 | if machine: |
| 30 | self.custommachine = machine |
| 31 | if machine == 'random': |
| 32 | self.custommachine = choice(self.machines) |
| 33 | self.logger.info('Run tests with custom MACHINE set to: %s' % \ |
| 34 | self.custommachine) |
| 35 | return super(OESelftestTestContext, self).runTests(skips) |
| 36 | |
| 37 | def listTests(self, display_type, machine=None): |
| 38 | return super(OESelftestTestContext, self).listTests(display_type) |
| 39 | |
| 40 | class OESelftestTestContextExecutor(OETestContextExecutor): |
| 41 | _context_class = OESelftestTestContext |
| 42 | _script_executor = 'oe-selftest' |
| 43 | |
| 44 | name = 'oe-selftest' |
| 45 | help = 'oe-selftest test component' |
| 46 | description = 'Executes selftest tests' |
| 47 | |
| 48 | def register_commands(self, logger, parser): |
| 49 | group = parser.add_mutually_exclusive_group(required=True) |
| 50 | |
| 51 | group.add_argument('-a', '--run-all-tests', default=False, |
| 52 | action="store_true", dest="run_all_tests", |
| 53 | help='Run all (unhidden) tests') |
| 54 | group.add_argument('-R', '--skip-tests', required=False, action='store', |
| 55 | nargs='+', dest="skips", default=None, |
| 56 | help='Run all (unhidden) tests except the ones specified. Format should be <module>[.<class>[.<test_method>]]') |
| 57 | group.add_argument('-r', '--run-tests', required=False, action='store', |
| 58 | nargs='+', dest="run_tests", default=None, |
| 59 | help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>') |
| 60 | |
| 61 | group.add_argument('-m', '--list-modules', required=False, |
| 62 | action="store_true", default=False, |
| 63 | help='List all available test modules.') |
| 64 | group.add_argument('--list-classes', required=False, |
| 65 | action="store_true", default=False, |
| 66 | help='List all available test classes.') |
| 67 | group.add_argument('-l', '--list-tests', required=False, |
| 68 | action="store_true", default=False, |
| 69 | help='List all available tests.') |
| 70 | |
| 71 | parser.add_argument('--machine', required=False, choices=['random', 'all'], |
| 72 | help='Run tests on different machines (random/all).') |
| 73 | |
| 74 | parser.set_defaults(func=self.run) |
| 75 | |
| 76 | def _get_available_machines(self): |
| 77 | machines = [] |
| 78 | |
| 79 | bbpath = self.tc_kwargs['init']['td']['BBPATH'].split(':') |
| 80 | |
| 81 | for path in bbpath: |
| 82 | found_machines = glob.glob(os.path.join(path, 'conf', 'machine', '*.conf')) |
| 83 | if found_machines: |
| 84 | for i in found_machines: |
| 85 | # eg: '/home/<user>/poky/meta-intel/conf/machine/intel-core2-32.conf' |
| 86 | machines.append(os.path.splitext(os.path.basename(i))[0]) |
| 87 | |
| 88 | return machines |
| 89 | |
| 90 | def _get_cases_paths(self, bbpath): |
| 91 | cases_paths = [] |
| 92 | for layer in bbpath: |
| 93 | cases_dir = os.path.join(layer, 'lib', 'oeqa', 'selftest', 'cases') |
| 94 | if os.path.isdir(cases_dir): |
| 95 | cases_paths.append(cases_dir) |
| 96 | return cases_paths |
| 97 | |
| 98 | def _process_args(self, logger, args): |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 99 | |
| 100 | args.test_start_time = time.strftime("%Y%m%d%H%M%S") |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 101 | args.test_data_file = None |
| 102 | args.CASES_PATHS = None |
| 103 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 104 | bbvars = get_bb_vars() |
| 105 | logdir = os.environ.get("BUILDDIR") |
| 106 | if 'LOG_DIR' in bbvars: |
| 107 | logdir = bbvars['LOG_DIR'] |
| 108 | args.output_log = logdir + '/%s-results-%s.log' % (self.name, args.test_start_time) |
| 109 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 110 | super(OESelftestTestContextExecutor, self)._process_args(logger, args) |
| 111 | |
| 112 | if args.list_modules: |
| 113 | args.list_tests = 'module' |
| 114 | elif args.list_classes: |
| 115 | args.list_tests = 'class' |
| 116 | elif args.list_tests: |
| 117 | args.list_tests = 'name' |
| 118 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 119 | self.tc_kwargs['init']['td'] = bbvars |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 120 | self.tc_kwargs['init']['machines'] = self._get_available_machines() |
| 121 | |
| 122 | builddir = os.environ.get("BUILDDIR") |
| 123 | self.tc_kwargs['init']['config_paths'] = {} |
| 124 | self.tc_kwargs['init']['config_paths']['testlayer_path'] = \ |
| 125 | get_test_layer() |
| 126 | self.tc_kwargs['init']['config_paths']['builddir'] = builddir |
| 127 | self.tc_kwargs['init']['config_paths']['localconf'] = \ |
| 128 | os.path.join(builddir, "conf/local.conf") |
| 129 | self.tc_kwargs['init']['config_paths']['localconf_backup'] = \ |
| 130 | os.path.join(builddir, "conf/local.conf.orig") |
| 131 | self.tc_kwargs['init']['config_paths']['localconf_class_backup'] = \ |
| 132 | os.path.join(builddir, "conf/local.conf.bk") |
| 133 | self.tc_kwargs['init']['config_paths']['bblayers'] = \ |
| 134 | os.path.join(builddir, "conf/bblayers.conf") |
| 135 | self.tc_kwargs['init']['config_paths']['bblayers_backup'] = \ |
| 136 | os.path.join(builddir, "conf/bblayers.conf.orig") |
| 137 | self.tc_kwargs['init']['config_paths']['bblayers_class_backup'] = \ |
| 138 | os.path.join(builddir, "conf/bblayers.conf.bk") |
| 139 | |
| 140 | copyfile(self.tc_kwargs['init']['config_paths']['localconf'], |
| 141 | self.tc_kwargs['init']['config_paths']['localconf_backup']) |
| 142 | copyfile(self.tc_kwargs['init']['config_paths']['bblayers'], |
| 143 | self.tc_kwargs['init']['config_paths']['bblayers_backup']) |
| 144 | |
| 145 | self.tc_kwargs['run']['skips'] = args.skips |
| 146 | |
| 147 | def _pre_run(self): |
| 148 | def _check_required_env_variables(vars): |
| 149 | for var in vars: |
| 150 | if not os.environ.get(var): |
| 151 | self.tc.logger.error("%s is not set. Did you forget to source your build environment setup script?" % var) |
| 152 | raise OEQAPreRun |
| 153 | |
| 154 | def _check_presence_meta_selftest(): |
| 155 | builddir = os.environ.get("BUILDDIR") |
| 156 | if os.getcwd() != builddir: |
| 157 | self.tc.logger.info("Changing cwd to %s" % builddir) |
| 158 | os.chdir(builddir) |
| 159 | |
| 160 | if not "meta-selftest" in self.tc.td["BBLAYERS"]: |
| 161 | self.tc.logger.warn("meta-selftest layer not found in BBLAYERS, adding it") |
| 162 | meta_selftestdir = os.path.join( |
| 163 | self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest') |
| 164 | if os.path.isdir(meta_selftestdir): |
| 165 | runCmd("bitbake-layers add-layer %s" %meta_selftestdir) |
| 166 | # reload data is needed because a meta-selftest layer was add |
| 167 | self.tc.td = get_bb_vars() |
| 168 | self.tc.config_paths['testlayer_path'] = get_test_layer() |
| 169 | else: |
| 170 | self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir) |
| 171 | raise OEQAPreRun |
| 172 | |
| 173 | def _add_layer_libs(): |
| 174 | bbpath = self.tc.td['BBPATH'].split(':') |
| 175 | layer_libdirs = [p for p in (os.path.join(l, 'lib') \ |
| 176 | for l in bbpath) if os.path.exists(p)] |
| 177 | if layer_libdirs: |
| 178 | self.tc.logger.info("Adding layer libraries:") |
| 179 | for l in layer_libdirs: |
| 180 | self.tc.logger.info("\t%s" % l) |
| 181 | |
| 182 | sys.path.extend(layer_libdirs) |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 183 | importlib.reload(oeqa.selftest) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 184 | |
| 185 | _check_required_env_variables(["BUILDDIR"]) |
| 186 | _check_presence_meta_selftest() |
| 187 | |
| 188 | if "buildhistory.bbclass" in self.tc.td["BBINCLUDED"]: |
| 189 | self.tc.logger.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.") |
| 190 | raise OEQAPreRun |
| 191 | |
| 192 | if "PRSERV_HOST" in self.tc.td: |
| 193 | self.tc.logger.error("Please unset PRSERV_HOST in order to run oe-selftest") |
| 194 | raise OEQAPreRun |
| 195 | |
| 196 | if "SANITY_TESTED_DISTROS" in self.tc.td: |
| 197 | self.tc.logger.error("Please unset SANITY_TESTED_DISTROS in order to run oe-selftest") |
| 198 | raise OEQAPreRun |
| 199 | |
| 200 | _add_layer_libs() |
| 201 | |
| 202 | self.tc.logger.info("Running bitbake -p") |
| 203 | runCmd("bitbake -p") |
| 204 | |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 205 | def get_json_result_dir(self, args): |
| 206 | json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa') |
| 207 | if "OEQA_JSON_RESULT_DIR" in self.tc.td: |
| 208 | json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"] |
| 209 | |
| 210 | return json_result_dir |
| 211 | |
| 212 | def get_configuration(self, args): |
| 213 | import platform |
| 214 | from oeqa.utils.metadata import metadata_from_bb |
| 215 | metadata = metadata_from_bb() |
| 216 | configuration = {'TEST_TYPE': 'oeselftest', |
| 217 | 'STARTTIME': args.test_start_time, |
| 218 | 'MACHINE': self.tc.td["MACHINE"], |
| 219 | 'HOST_DISTRO': ('-'.join(platform.linux_distribution())).replace(' ', '-'), |
| 220 | 'HOST_NAME': metadata['hostname'], |
| 221 | 'LAYERS': metadata['layers']} |
| 222 | return configuration |
| 223 | |
| 224 | def get_result_id(self, configuration): |
| 225 | return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'], configuration['STARTTIME']) |
| 226 | |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 227 | def _internal_run(self, logger, args): |
| 228 | self.module_paths = self._get_cases_paths( |
| 229 | self.tc_kwargs['init']['td']['BBPATH'].split(':')) |
| 230 | |
| 231 | self.tc = self._context_class(**self.tc_kwargs['init']) |
| 232 | try: |
| 233 | self.tc.loadTests(self.module_paths, **self.tc_kwargs['load']) |
| 234 | except OEQATestNotFound as ex: |
| 235 | logger.error(ex) |
| 236 | sys.exit(1) |
| 237 | |
| 238 | if args.list_tests: |
| 239 | rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list']) |
| 240 | else: |
| 241 | self._pre_run() |
| 242 | rc = self.tc.runTests(**self.tc_kwargs['run']) |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 243 | configuration = self.get_configuration(args) |
| 244 | rc.logDetails(self.get_json_result_dir(args), |
| 245 | configuration, |
| 246 | self.get_result_id(configuration)) |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 247 | rc.logSummary(self.name) |
| 248 | |
| 249 | return rc |
| 250 | |
| 251 | def _signal_clean_handler(self, signum, frame): |
| 252 | sys.exit(1) |
| 253 | |
| 254 | def run(self, logger, args): |
| 255 | self._process_args(logger, args) |
| 256 | |
| 257 | signal.signal(signal.SIGTERM, self._signal_clean_handler) |
| 258 | |
| 259 | rc = None |
| 260 | try: |
| 261 | if args.machine: |
| 262 | logger.info('Custom machine mode enabled. MACHINE set to %s' % |
| 263 | args.machine) |
| 264 | |
| 265 | if args.machine == 'all': |
| 266 | results = [] |
| 267 | for m in self.tc_kwargs['init']['machines']: |
| 268 | self.tc_kwargs['run']['machine'] = m |
| 269 | results.append(self._internal_run(logger, args)) |
| 270 | |
| 271 | # XXX: the oe-selftest script only needs to know if one |
| 272 | # machine run fails |
| 273 | for r in results: |
| 274 | rc = r |
| 275 | if not r.wasSuccessful(): |
| 276 | break |
| 277 | |
| 278 | else: |
| 279 | self.tc_kwargs['run']['machine'] = args.machine |
| 280 | return self._internal_run(logger, args) |
| 281 | |
| 282 | else: |
| 283 | self.tc_kwargs['run']['machine'] = args.machine |
| 284 | rc = self._internal_run(logger, args) |
| 285 | finally: |
| 286 | config_paths = self.tc_kwargs['init']['config_paths'] |
| 287 | if os.path.exists(config_paths['localconf_backup']): |
| 288 | copyfile(config_paths['localconf_backup'], |
| 289 | config_paths['localconf']) |
| 290 | os.remove(config_paths['localconf_backup']) |
| 291 | |
| 292 | if os.path.exists(config_paths['bblayers_backup']): |
| 293 | copyfile(config_paths['bblayers_backup'], |
| 294 | config_paths['bblayers']) |
| 295 | os.remove(config_paths['bblayers_backup']) |
| 296 | |
| 297 | if os.path.exists(config_paths['localconf_class_backup']): |
| 298 | os.remove(config_paths['localconf_class_backup']) |
| 299 | if os.path.exists(config_paths['bblayers_class_backup']): |
| 300 | os.remove(config_paths['bblayers_class_backup']) |
| 301 | |
| 302 | output_link = os.path.join(os.path.dirname(args.output_log), |
| 303 | "%s-results.log" % self.name) |
Brad Bishop | f86d055 | 2018-12-04 14:18:15 -0800 | [diff] [blame] | 304 | if os.path.lexists(output_link): |
Brad Bishop | d7bf8c1 | 2018-02-25 22:55:05 -0500 | [diff] [blame] | 305 | os.remove(output_link) |
| 306 | os.symlink(args.output_log, output_link) |
| 307 | |
| 308 | return rc |
| 309 | |
| 310 | _executor_class = OESelftestTestContextExecutor |