blob: 0d9cf23fe42131df0c62c2f17ddac1831a738037 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Copyright (c) 2013-2014 Intel Corporation
2#
3# Released under the MIT license (see COPYING.MIT)
4
5# DESCRIPTION
6# This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest
7# It provides a class and methods for running commands on the host in a convienent way for tests.
8
9
10
11import os
12import sys
13import signal
14import subprocess
15import threading
Brad Bishopd7bf8c12018-02-25 22:55:05 -050016import time
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017import logging
18from oeqa.utils import CommandError
19from oeqa.utils import ftools
20import re
21import contextlib
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050022# Export test doesn't require bb
23try:
24 import bb
25except ImportError:
26 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027
28class Command(object):
Brad Bishopd7bf8c12018-02-25 22:55:05 -050029 def __init__(self, command, bg=False, timeout=None, data=None, output_log=None, **options):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030
31 self.defaultopts = {
32 "stdout": subprocess.PIPE,
33 "stderr": subprocess.STDOUT,
34 "stdin": None,
35 "shell": False,
36 "bufsize": -1,
37 }
38
39 self.cmd = command
40 self.bg = bg
41 self.timeout = timeout
42 self.data = data
43
44 self.options = dict(self.defaultopts)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060045 if isinstance(self.cmd, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046 self.options["shell"] = True
47 if self.data:
48 self.options['stdin'] = subprocess.PIPE
49 self.options.update(options)
50
51 self.status = None
Brad Bishopd7bf8c12018-02-25 22:55:05 -050052 # We collect chunks of output before joining them at the end.
53 self._output_chunks = []
54 self._error_chunks = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055 self.output = None
56 self.error = None
Brad Bishopd7bf8c12018-02-25 22:55:05 -050057 self.threads = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -050058
Brad Bishopd7bf8c12018-02-25 22:55:05 -050059 self.output_log = output_log
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060 self.log = logging.getLogger("utils.commands")
61
62 def run(self):
63 self.process = subprocess.Popen(self.cmd, **self.options)
64
Brad Bishopd7bf8c12018-02-25 22:55:05 -050065 def readThread(output, stream, logfunc):
66 if logfunc:
67 for line in stream:
68 output.append(line)
69 logfunc(line.decode("utf-8", errors='replace').rstrip())
70 else:
71 output.append(stream.read())
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
Brad Bishopd7bf8c12018-02-25 22:55:05 -050073 def readStderrThread():
74 readThread(self._error_chunks, self.process.stderr, self.output_log.error if self.output_log else None)
75
76 def readStdoutThread():
77 readThread(self._output_chunks, self.process.stdout, self.output_log.info if self.output_log else None)
78
79 def writeThread():
80 try:
81 self.process.stdin.write(self.data)
82 self.process.stdin.close()
83 except OSError as ex:
84 # It's not an error when the command does not consume all
85 # of our data. subprocess.communicate() also ignores that.
86 if ex.errno != EPIPE:
87 raise
88
89 # We write in a separate thread because then we can read
90 # without worrying about deadlocks. The additional thread is
91 # expected to terminate by itself and we mark it as a daemon,
92 # so even it should happen to not terminate for whatever
93 # reason, the main process will still exit, which will then
94 # kill the write thread.
95 if self.data:
96 threading.Thread(target=writeThread, daemon=True).start()
97 if self.process.stderr:
98 thread = threading.Thread(target=readStderrThread)
99 thread.start()
100 self.threads.append(thread)
101 if self.output_log:
102 self.output_log.info('Running: %s' % self.cmd)
103 thread = threading.Thread(target=readStdoutThread)
104 thread.start()
105 self.threads.append(thread)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500106
107 self.log.debug("Running command '%s'" % self.cmd)
108
109 if not self.bg:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500110 if self.timeout is None:
111 for thread in self.threads:
112 thread.join()
113 else:
114 deadline = time.time() + self.timeout
115 for thread in self.threads:
116 timeout = deadline - time.time()
117 if timeout < 0:
118 timeout = 0
119 thread.join(timeout)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120 self.stop()
121
122 def stop(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500123 for thread in self.threads:
124 if thread.isAlive():
125 self.process.terminate()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 # let's give it more time to terminate gracefully before killing it
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500127 thread.join(5)
128 if thread.isAlive():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500129 self.process.kill()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500130 thread.join()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500131
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500132 def finalize_output(data):
133 if not data:
134 data = ""
135 else:
136 data = b"".join(data)
137 data = data.decode("utf-8", errors='replace').rstrip()
138 return data
139
140 self.output = finalize_output(self._output_chunks)
141 self._output_chunks = None
142 # self.error used to be a byte string earlier, probably unintentionally.
143 # Now it is a normal string, just like self.output.
144 self.error = finalize_output(self._error_chunks)
145 self._error_chunks = None
146 # At this point we know that the process has closed stdout/stderr, so
147 # it is safe and necessary to wait for the actual process completion.
148 self.status = self.process.wait()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500149
150 self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status))
151 # logging the complete output is insane
152 # bitbake -e output is really big
153 # and makes the log file useless
154 if self.status:
155 lout = "\n".join(self.output.splitlines()[-20:])
156 self.log.debug("Last 20 lines:\n%s" % lout)
157
158
159class Result(object):
160 pass
161
162
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500163def runCmd(command, ignore_status=False, timeout=None, assert_error=True,
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500164 native_sysroot=None, limit_exc_output=0, output_log=None, **options):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500165 result = Result()
166
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500167 if native_sysroot:
168 extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \
169 (native_sysroot, native_sysroot, native_sysroot)
170 nenv = dict(options.get('env', os.environ))
171 nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '')
172 options['env'] = nenv
173
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500174 cmd = Command(command, timeout=timeout, output_log=output_log, **options)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500175 cmd.run()
176
177 result.command = command
178 result.status = cmd.status
179 result.output = cmd.output
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600180 result.error = cmd.error
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500181 result.pid = cmd.process.pid
182
183 if result.status and not ignore_status:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500184 exc_output = result.output
185 if limit_exc_output > 0:
186 split = result.output.splitlines()
187 if len(split) > limit_exc_output:
188 exc_output = "\n... (last %d lines of output)\n" % limit_exc_output + \
189 '\n'.join(split[-limit_exc_output:])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190 if assert_error:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500191 raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500193 raise CommandError(result.status, command, exc_output)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500194
195 return result
196
197
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500198def bitbake(command, ignore_status=False, timeout=None, postconfig=None, output_log=None, **options):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500199
200 if postconfig:
201 postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf')
202 ftools.write_file(postconfig_file, postconfig)
203 extra_args = "-R %s" % postconfig_file
204 else:
205 extra_args = ""
206
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600207 if isinstance(command, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500208 cmd = "bitbake " + extra_args + " " + command
209 else:
210 cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]]
211
212 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500213 return runCmd(cmd, ignore_status, timeout, output_log=output_log, **options)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500214 finally:
215 if postconfig:
216 os.remove(postconfig_file)
217
218
219def get_bb_env(target=None, postconfig=None):
220 if target:
221 return bitbake("-e %s" % target, postconfig=postconfig).output
222 else:
223 return bitbake("-e", postconfig=postconfig).output
224
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600225def get_bb_vars(variables=None, target=None, postconfig=None):
226 """Get values of multiple bitbake variables"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500227 bbenv = get_bb_env(target, postconfig=postconfig)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600228
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500229 if variables is not None:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400230 variables = list(variables)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500231 var_re = re.compile(r'^(export )?(?P<var>\w+(_.*)?)="(?P<value>.*)"$')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600232 unset_re = re.compile(r'^unset (?P<var>\w+)$')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500233 lastline = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600234 values = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500235 for line in bbenv.splitlines():
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600236 match = var_re.match(line)
237 val = None
238 if match:
239 val = match.group('value')
240 else:
241 match = unset_re.match(line)
242 if match:
243 # Handle [unexport] variables
244 if lastline.startswith('# "'):
245 val = lastline.split('"')[1]
246 if val:
247 var = match.group('var')
248 if variables is None:
249 values[var] = val
250 else:
251 if var in variables:
252 values[var] = val
253 variables.remove(var)
254 # Stop after all required variables have been found
255 if not variables:
256 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 lastline = line
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600258 if variables:
259 # Fill in missing values
260 for var in variables:
261 values[var] = None
262 return values
263
264def get_bb_var(var, target=None, postconfig=None):
265 return get_bb_vars([var], target, postconfig)[var]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500266
267def get_test_layer():
268 layers = get_bb_var("BBLAYERS").split()
269 testlayer = None
270 for l in layers:
271 if '~' in l:
272 l = os.path.expanduser(l)
273 if "/meta-selftest" in l and os.path.isdir(l):
274 testlayer = l
275 break
276 return testlayer
277
278def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'):
279 os.makedirs(os.path.join(templayerdir, 'conf'))
280 with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:
281 f.write('BBPATH .= ":${LAYERDIR}"\n')
282 f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec)
283 f.write(' ${LAYERDIR}/%s/*.bbappend"\n' % recipepathspec)
284 f.write('BBFILE_COLLECTIONS += "%s"\n' % templayername)
285 f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername)
286 f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority))
287 f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400288 f.write('LAYERSERIES_COMPAT_%s = "${LAYERSERIES_COMPAT_core}"\n' % templayername)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500289
290@contextlib.contextmanager
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500291def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True):
292 """
293 launch_cmd means directly run the command, don't need set rootfs or env vars.
294 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295
296 import bb.tinfoil
297 import bb.build
298
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500299 # Need a non-'BitBake' logger to capture the runner output
300 targetlogger = logging.getLogger('TargetRunner')
301 targetlogger.setLevel(logging.DEBUG)
302 handler = logging.StreamHandler(sys.stdout)
303 targetlogger.addHandler(handler)
304
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 tinfoil = bb.tinfoil.Tinfoil()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500306 tinfoil.prepare(config_only=False, quiet=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500307 try:
308 tinfoil.logger.setLevel(logging.WARNING)
309 import oeqa.targetcontrol
310 tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
311 tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500312 # Tell QemuTarget() whether need find rootfs/kernel or not
313 if launch_cmd:
314 tinfoil.config_data.setVar("FIND_ROOTFS", '0')
315 else:
316 tinfoil.config_data.setVar("FIND_ROOTFS", '1')
317
318 recipedata = tinfoil.parse_recipe(pn)
319 for key, value in overrides.items():
320 recipedata.setVar(key, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500322 logdir = recipedata.getVar("TEST_LOG_DIR")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500324 qemu = oeqa.targetcontrol.QemuTarget(recipedata, targetlogger, image_fstype)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500325 finally:
326 # We need to shut down tinfoil early here in case we actually want
327 # to run tinfoil-using utilities with the running QEMU instance.
328 # Luckily QemuTarget doesn't need it after the constructor.
329 tinfoil.shutdown()
330
331 try:
332 qemu.deploy()
333 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500334 qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335 except bb.build.FuncFailed:
336 raise Exception('Failed to start QEMU - see the logs in %s' % logdir)
337
338 yield qemu
339
340 finally:
341 try:
342 qemu.stop()
343 except:
344 pass
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500345 targetlogger.removeHandler(handler)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600346
347def updateEnv(env_file):
348 """
349 Source a file and update environment.
350 """
351
352 cmd = ". %s; env -0" % env_file
353 result = runCmd(cmd)
354
355 for line in result.output.split("\0"):
356 (key, _, value) = line.partition("=")
357 os.environ[key] = value