Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1 | # Copyright (c) 2013-2014 Intel Corporation |
| 2 | # |
| 3 | # Released under the MIT license (see COPYING.MIT) |
| 4 | |
| 5 | # DESCRIPTION |
| 6 | # This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest |
| 7 | # It provides a class and methods for running commands on the host in a convienent way for tests. |
| 8 | |
| 9 | |
| 10 | |
| 11 | import os |
| 12 | import sys |
| 13 | import signal |
| 14 | import subprocess |
| 15 | import threading |
| 16 | import logging |
| 17 | from oeqa.utils import CommandError |
| 18 | from oeqa.utils import ftools |
| 19 | import re |
| 20 | import contextlib |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 21 | # Export test doesn't require bb |
| 22 | try: |
| 23 | import bb |
| 24 | except ImportError: |
| 25 | pass |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 26 | |
| 27 | class Command(object): |
| 28 | def __init__(self, command, bg=False, timeout=None, data=None, **options): |
| 29 | |
| 30 | self.defaultopts = { |
| 31 | "stdout": subprocess.PIPE, |
| 32 | "stderr": subprocess.STDOUT, |
| 33 | "stdin": None, |
| 34 | "shell": False, |
| 35 | "bufsize": -1, |
| 36 | } |
| 37 | |
| 38 | self.cmd = command |
| 39 | self.bg = bg |
| 40 | self.timeout = timeout |
| 41 | self.data = data |
| 42 | |
| 43 | self.options = dict(self.defaultopts) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 44 | if isinstance(self.cmd, str): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 45 | self.options["shell"] = True |
| 46 | if self.data: |
| 47 | self.options['stdin'] = subprocess.PIPE |
| 48 | self.options.update(options) |
| 49 | |
| 50 | self.status = None |
| 51 | self.output = None |
| 52 | self.error = None |
| 53 | self.thread = None |
| 54 | |
| 55 | self.log = logging.getLogger("utils.commands") |
| 56 | |
| 57 | def run(self): |
| 58 | self.process = subprocess.Popen(self.cmd, **self.options) |
| 59 | |
| 60 | def commThread(): |
| 61 | self.output, self.error = self.process.communicate(self.data) |
| 62 | |
| 63 | self.thread = threading.Thread(target=commThread) |
| 64 | self.thread.start() |
| 65 | |
| 66 | self.log.debug("Running command '%s'" % self.cmd) |
| 67 | |
| 68 | if not self.bg: |
| 69 | self.thread.join(self.timeout) |
| 70 | self.stop() |
| 71 | |
| 72 | def stop(self): |
| 73 | if self.thread.isAlive(): |
| 74 | self.process.terminate() |
| 75 | # let's give it more time to terminate gracefully before killing it |
| 76 | self.thread.join(5) |
| 77 | if self.thread.isAlive(): |
| 78 | self.process.kill() |
| 79 | self.thread.join() |
| 80 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 81 | if not self.output: |
| 82 | self.output = "" |
| 83 | else: |
| 84 | self.output = self.output.decode("utf-8", errors='replace').rstrip() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 85 | self.status = self.process.poll() |
| 86 | |
| 87 | self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) |
| 88 | # logging the complete output is insane |
| 89 | # bitbake -e output is really big |
| 90 | # and makes the log file useless |
| 91 | if self.status: |
| 92 | lout = "\n".join(self.output.splitlines()[-20:]) |
| 93 | self.log.debug("Last 20 lines:\n%s" % lout) |
| 94 | |
| 95 | |
| 96 | class Result(object): |
| 97 | pass |
| 98 | |
| 99 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 100 | def runCmd(command, ignore_status=False, timeout=None, assert_error=True, |
| 101 | native_sysroot=None, limit_exc_output=0, **options): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 102 | result = Result() |
| 103 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 104 | if native_sysroot: |
| 105 | extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ |
| 106 | (native_sysroot, native_sysroot, native_sysroot) |
| 107 | nenv = dict(options.get('env', os.environ)) |
| 108 | nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') |
| 109 | options['env'] = nenv |
| 110 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 111 | cmd = Command(command, timeout=timeout, **options) |
| 112 | cmd.run() |
| 113 | |
| 114 | result.command = command |
| 115 | result.status = cmd.status |
| 116 | result.output = cmd.output |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 117 | result.error = cmd.error |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 118 | result.pid = cmd.process.pid |
| 119 | |
| 120 | if result.status and not ignore_status: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 121 | exc_output = result.output |
| 122 | if limit_exc_output > 0: |
| 123 | split = result.output.splitlines() |
| 124 | if len(split) > limit_exc_output: |
| 125 | exc_output = "\n... (last %d lines of output)\n" % limit_exc_output + \ |
| 126 | '\n'.join(split[-limit_exc_output:]) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 127 | if assert_error: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 128 | raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 129 | else: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 130 | raise CommandError(result.status, command, exc_output) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 131 | |
| 132 | return result |
| 133 | |
| 134 | |
| 135 | def bitbake(command, ignore_status=False, timeout=None, postconfig=None, **options): |
| 136 | |
| 137 | if postconfig: |
| 138 | postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf') |
| 139 | ftools.write_file(postconfig_file, postconfig) |
| 140 | extra_args = "-R %s" % postconfig_file |
| 141 | else: |
| 142 | extra_args = "" |
| 143 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 144 | if isinstance(command, str): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 145 | cmd = "bitbake " + extra_args + " " + command |
| 146 | else: |
| 147 | cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]] |
| 148 | |
| 149 | try: |
| 150 | return runCmd(cmd, ignore_status, timeout, **options) |
| 151 | finally: |
| 152 | if postconfig: |
| 153 | os.remove(postconfig_file) |
| 154 | |
| 155 | |
| 156 | def get_bb_env(target=None, postconfig=None): |
| 157 | if target: |
| 158 | return bitbake("-e %s" % target, postconfig=postconfig).output |
| 159 | else: |
| 160 | return bitbake("-e", postconfig=postconfig).output |
| 161 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 162 | def get_bb_vars(variables=None, target=None, postconfig=None): |
| 163 | """Get values of multiple bitbake variables""" |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 164 | bbenv = get_bb_env(target, postconfig=postconfig) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 165 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 166 | if variables is not None: |
| 167 | variables = variables.copy() |
| 168 | var_re = re.compile(r'^(export )?(?P<var>\w+(_.*)?)="(?P<value>.*)"$') |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 169 | unset_re = re.compile(r'^unset (?P<var>\w+)$') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 170 | lastline = None |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 171 | values = {} |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 172 | for line in bbenv.splitlines(): |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 173 | match = var_re.match(line) |
| 174 | val = None |
| 175 | if match: |
| 176 | val = match.group('value') |
| 177 | else: |
| 178 | match = unset_re.match(line) |
| 179 | if match: |
| 180 | # Handle [unexport] variables |
| 181 | if lastline.startswith('# "'): |
| 182 | val = lastline.split('"')[1] |
| 183 | if val: |
| 184 | var = match.group('var') |
| 185 | if variables is None: |
| 186 | values[var] = val |
| 187 | else: |
| 188 | if var in variables: |
| 189 | values[var] = val |
| 190 | variables.remove(var) |
| 191 | # Stop after all required variables have been found |
| 192 | if not variables: |
| 193 | break |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 194 | lastline = line |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 195 | if variables: |
| 196 | # Fill in missing values |
| 197 | for var in variables: |
| 198 | values[var] = None |
| 199 | return values |
| 200 | |
| 201 | def get_bb_var(var, target=None, postconfig=None): |
| 202 | return get_bb_vars([var], target, postconfig)[var] |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 203 | |
| 204 | def get_test_layer(): |
| 205 | layers = get_bb_var("BBLAYERS").split() |
| 206 | testlayer = None |
| 207 | for l in layers: |
| 208 | if '~' in l: |
| 209 | l = os.path.expanduser(l) |
| 210 | if "/meta-selftest" in l and os.path.isdir(l): |
| 211 | testlayer = l |
| 212 | break |
| 213 | return testlayer |
| 214 | |
| 215 | def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'): |
| 216 | os.makedirs(os.path.join(templayerdir, 'conf')) |
| 217 | with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f: |
| 218 | f.write('BBPATH .= ":${LAYERDIR}"\n') |
| 219 | f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec) |
| 220 | f.write(' ${LAYERDIR}/%s/*.bbappend"\n' % recipepathspec) |
| 221 | f.write('BBFILE_COLLECTIONS += "%s"\n' % templayername) |
| 222 | f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername) |
| 223 | f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority)) |
| 224 | f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername) |
| 225 | |
| 226 | |
| 227 | @contextlib.contextmanager |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 228 | def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True): |
| 229 | """ |
| 230 | launch_cmd means directly run the command, don't need set rootfs or env vars. |
| 231 | """ |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 232 | |
| 233 | import bb.tinfoil |
| 234 | import bb.build |
| 235 | |
| 236 | tinfoil = bb.tinfoil.Tinfoil() |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 237 | tinfoil.prepare(config_only=False, quiet=True) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 238 | try: |
| 239 | tinfoil.logger.setLevel(logging.WARNING) |
| 240 | import oeqa.targetcontrol |
| 241 | tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") |
| 242 | tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 243 | # Tell QemuTarget() whether need find rootfs/kernel or not |
| 244 | if launch_cmd: |
| 245 | tinfoil.config_data.setVar("FIND_ROOTFS", '0') |
| 246 | else: |
| 247 | tinfoil.config_data.setVar("FIND_ROOTFS", '1') |
| 248 | |
| 249 | recipedata = tinfoil.parse_recipe(pn) |
| 250 | for key, value in overrides.items(): |
| 251 | recipedata.setVar(key, value) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 252 | |
| 253 | # The QemuRunner log is saved out, but we need to ensure it is at the right |
| 254 | # log level (and then ensure that since it's a child of the BitBake logger, |
| 255 | # we disable propagation so we don't then see the log events on the console) |
| 256 | logger = logging.getLogger('BitBake.QemuRunner') |
| 257 | logger.setLevel(logging.DEBUG) |
| 258 | logger.propagate = False |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 259 | logdir = recipedata.getVar("TEST_LOG_DIR") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 260 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 261 | qemu = oeqa.targetcontrol.QemuTarget(recipedata, image_fstype) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 262 | finally: |
| 263 | # We need to shut down tinfoil early here in case we actually want |
| 264 | # to run tinfoil-using utilities with the running QEMU instance. |
| 265 | # Luckily QemuTarget doesn't need it after the constructor. |
| 266 | tinfoil.shutdown() |
| 267 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 268 | # Setup bitbake logger as console handler is removed by tinfoil.shutdown |
| 269 | bblogger = logging.getLogger('BitBake') |
| 270 | bblogger.setLevel(logging.INFO) |
| 271 | console = logging.StreamHandler(sys.stdout) |
| 272 | bbformat = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") |
| 273 | if sys.stdout.isatty(): |
| 274 | bbformat.enable_color() |
| 275 | console.setFormatter(bbformat) |
| 276 | bblogger.addHandler(console) |
| 277 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 278 | try: |
| 279 | qemu.deploy() |
| 280 | try: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame] | 281 | qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 282 | except bb.build.FuncFailed: |
| 283 | raise Exception('Failed to start QEMU - see the logs in %s' % logdir) |
| 284 | |
| 285 | yield qemu |
| 286 | |
| 287 | finally: |
| 288 | try: |
| 289 | qemu.stop() |
| 290 | except: |
| 291 | pass |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 292 | |
| 293 | def updateEnv(env_file): |
| 294 | """ |
| 295 | Source a file and update environment. |
| 296 | """ |
| 297 | |
| 298 | cmd = ". %s; env -0" % env_file |
| 299 | result = runCmd(cmd) |
| 300 | |
| 301 | for line in result.output.split("\0"): |
| 302 | (key, _, value) = line.partition("=") |
| 303 | os.environ[key] = value |