blob: f4f897e41a8571157756b7341656c06c682035c1 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# BitBake 'Build' implementation
3#
4# Core code for function execution and task handling in the
5# BitBake build tools.
6#
7# Copyright (C) 2003, 2004 Chris Larson
8#
9# Based on Gentoo's portage.py.
10#
Brad Bishopc342db32019-05-15 21:57:59 -040011# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012#
13# Based on functions from the base bb module, Copyright 2003 Holger Schurig
14
15import os
16import sys
17import logging
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018import glob
Andrew Geissler635e0e42020-08-21 15:58:33 -050019import itertools
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020import time
Andrew Geissler635e0e42020-08-21 15:58:33 -050021import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022import stat
23import bb
24import bb.msg
25import bb.process
Patrick Williamsc0f7c042017-02-23 20:41:17 -060026import bb.progress
27from bb import data, event, utils
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028
29bblogger = logging.getLogger('BitBake')
30logger = logging.getLogger('BitBake.Build')
31
Andrew Geisslerc9f78652020-09-18 14:11:35 -050032verboseShellLogging = False
33verboseStdoutLogging = False
34
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035__mtime_cache = {}
36
37def cached_mtime_noerror(f):
38 if f not in __mtime_cache:
39 try:
40 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
41 except OSError:
42 return 0
43 return __mtime_cache[f]
44
45def reset_cache():
46 global __mtime_cache
47 __mtime_cache = {}
48
49# When we execute a Python function, we'd like certain things
50# in all namespaces, hence we add them to __builtins__.
51# If we do not do this and use the exec globals, they will
52# not be available to subfunctions.
Patrick Williamsc0f7c042017-02-23 20:41:17 -060053if hasattr(__builtins__, '__setitem__'):
54 builtins = __builtins__
55else:
56 builtins = __builtins__.__dict__
57
58builtins['bb'] = bb
59builtins['os'] = os
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061class TaskBase(event.Event):
62 """Base class for task events"""
63
Andrew Geissler82c905d2020-04-13 13:39:40 -050064 def __init__(self, t, fn, logfile, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050065 self._task = t
Andrew Geissler82c905d2020-04-13 13:39:40 -050066 self._fn = fn
Brad Bishop6e60e8b2018-02-01 10:27:11 -050067 self._package = d.getVar("PF")
68 self._mc = d.getVar("BB_CURRENT_MC")
69 self.taskfile = d.getVar("FILE")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070 self.taskname = self._task
71 self.logfile = logfile
72 self.time = time.time()
Andrew Geissler82c905d2020-04-13 13:39:40 -050073 self.pn = d.getVar("PN")
74 self.pv = d.getVar("PV")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075 event.Event.__init__(self)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050076 self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName())
Patrick Williamsc124f4f2015-09-15 14:41:29 -050077
78 def getTask(self):
79 return self._task
80
81 def setTask(self, task):
82 self._task = task
83
84 def getDisplayName(self):
85 return bb.event.getName(self)[4:]
86
87 task = property(getTask, setTask, None, "task property")
88
89class TaskStarted(TaskBase):
90 """Task execution started"""
Andrew Geissler82c905d2020-04-13 13:39:40 -050091 def __init__(self, t, fn, logfile, taskflags, d):
92 super(TaskStarted, self).__init__(t, fn, logfile, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050093 self.taskflags = taskflags
94
95class TaskSucceeded(TaskBase):
96 """Task execution completed"""
97
98class TaskFailed(TaskBase):
99 """Task execution failed"""
100
Andrew Geissler82c905d2020-04-13 13:39:40 -0500101 def __init__(self, task, fn, logfile, metadata, errprinted = False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 self.errprinted = errprinted
Andrew Geissler82c905d2020-04-13 13:39:40 -0500103 super(TaskFailed, self).__init__(task, fn, logfile, metadata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500104
105class TaskFailedSilent(TaskBase):
106 """Task execution failed (silently)"""
107 def getDisplayName(self):
108 # Don't need to tell the user it was silent
109 return "Failed"
110
111class TaskInvalid(TaskBase):
112
Andrew Geissler82c905d2020-04-13 13:39:40 -0500113 def __init__(self, task, fn, metadata):
114 super(TaskInvalid, self).__init__(task, fn, None, metadata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 self._message = "No such task '%s'" % task
116
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600117class TaskProgress(event.Event):
118 """
119 Task made some progress that could be reported to the user, usually in
120 the form of a progress bar or similar.
121 NOTE: this class does not inherit from TaskBase since it doesn't need
122 to - it's fired within the task context itself, so we don't have any of
123 the context information that you do in the case of the other events.
124 The event PID can be used to determine which task it came from.
125 The progress value is normally 0-100, but can also be negative
126 indicating that progress has been made but we aren't able to determine
127 how much.
128 The rate is optional, this is simply an extra string to display to the
129 user if specified.
130 """
131 def __init__(self, progress, rate=None):
132 self.progress = progress
133 self.rate = rate
134 event.Event.__init__(self)
135
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500136
137class LogTee(object):
138 def __init__(self, logger, outfile):
139 self.outfile = outfile
140 self.logger = logger
141 self.name = self.outfile.name
142
143 def write(self, string):
144 self.logger.plain(string)
145 self.outfile.write(string)
146
147 def __enter__(self):
148 self.outfile.__enter__()
149 return self
150
151 def __exit__(self, *excinfo):
152 self.outfile.__exit__(*excinfo)
153
154 def __repr__(self):
155 return '<LogTee {0}>'.format(self.name)
Brad Bishop15ae2502019-06-18 21:44:24 -0400156
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 def flush(self):
158 self.outfile.flush()
159
Brad Bishop15ae2502019-06-18 21:44:24 -0400160
161class StdoutNoopContextManager:
162 """
163 This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods.
164 """
165 def __enter__(self):
166 return sys.stdout
167
168 def __exit__(self, *exc_info):
169 pass
170
171 def write(self, string):
172 return sys.stdout.write(string)
173
174 def flush(self):
175 sys.stdout.flush()
176
177 @property
178 def name(self):
179 return sys.stdout.name
180
181
Brad Bishop08902b02019-08-20 09:16:51 -0400182def exec_func(func, d, dirs = None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183 """Execute a BB 'function'"""
184
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600185 try:
186 oldcwd = os.getcwd()
187 except:
188 oldcwd = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500189
190 flags = d.getVarFlags(func)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500191 cleandirs = flags.get('cleandirs') if flags else None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192 if cleandirs:
193 for cdir in d.expand(cleandirs).split():
194 bb.utils.remove(cdir, True)
195 bb.utils.mkdirhier(cdir)
196
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500197 if flags and dirs is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500198 dirs = flags.get('dirs')
199 if dirs:
200 dirs = d.expand(dirs).split()
201
202 if dirs:
203 for adir in dirs:
204 bb.utils.mkdirhier(adir)
205 adir = dirs[-1]
206 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600207 adir = None
208
209 body = d.getVar(func, False)
210 if not body:
211 if body is None:
212 logger.warning("Function %s doesn't exist", func)
213 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500214
215 ispython = flags.get('python')
216
217 lockflag = flags.get('lockfiles')
218 if lockflag:
219 lockfiles = [f for f in d.expand(lockflag).split()]
220 else:
221 lockfiles = None
222
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500223 tempdir = d.getVar('T')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500224
225 # or func allows items to be executed outside of the normal
226 # task set, such as buildhistory
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500227 task = d.getVar('BB_RUNTASK') or func
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228 if task == func:
229 taskfunc = task
230 else:
231 taskfunc = "%s.%s" % (task, func)
232
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 runfmt = d.getVar('BB_RUNFMT') or "run.{func}.{pid}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234 runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid())
235 runfile = os.path.join(tempdir, runfn)
236 bb.utils.mkdirhier(os.path.dirname(runfile))
237
238 # Setup the courtesy link to the runfn, only for tasks
239 # we create the link 'just' before the run script is created
240 # if we create it after, and if the run script fails, then the
241 # link won't be created as an exception would be fired.
242 if task == func:
243 runlink = os.path.join(tempdir, 'run.{0}'.format(task))
244 if runlink:
245 bb.utils.remove(runlink)
246
247 try:
248 os.symlink(runfn, runlink)
249 except OSError:
250 pass
251
252 with bb.utils.fileslocked(lockfiles):
253 if ispython:
Brad Bishop08902b02019-08-20 09:16:51 -0400254 exec_func_python(func, d, runfile, cwd=adir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 else:
256 exec_func_shell(func, d, runfile, cwd=adir)
257
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600258 try:
259 curcwd = os.getcwd()
260 except:
261 curcwd = None
262
263 if oldcwd and curcwd != oldcwd:
264 try:
265 bb.warn("Task %s changed cwd to %s" % (func, curcwd))
266 os.chdir(oldcwd)
267 except:
268 pass
269
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500270_functionfmt = """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271{function}(d)
272"""
273logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
Brad Bishop08902b02019-08-20 09:16:51 -0400274def exec_func_python(func, d, runfile, cwd=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500275 """Execute a python BB 'function'"""
276
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500277 code = _functionfmt.format(function=func)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278 bb.utils.mkdirhier(os.path.dirname(runfile))
279 with open(runfile, 'w') as script:
280 bb.data.emit_func_python(func, script, d)
281
282 if cwd:
283 try:
284 olddir = os.getcwd()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600285 except OSError as e:
286 bb.warn("%s: Cannot get cwd: %s" % (func, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500287 olddir = None
288 os.chdir(cwd)
289
290 bb.debug(2, "Executing python function %s" % func)
291
292 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500293 text = "def %s(d):\n%s" % (func, d.getVar(func, False))
294 fn = d.getVarFlag(func, "filename", False)
295 lineno = int(d.getVarFlag(func, "lineno", False))
296 bb.methodpool.insert_method(func, text, fn, lineno - 1)
297
298 comp = utils.better_compile(code, func, "exec_python_func() autogenerated")
Brad Bishop08902b02019-08-20 09:16:51 -0400299 utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500300 finally:
301 bb.debug(2, "Python function %s finished" % func)
302
303 if cwd and olddir:
304 try:
305 os.chdir(olddir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600306 except OSError as e:
307 bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308
309def shell_trap_code():
310 return '''#!/bin/sh\n
Andrew Geissler635e0e42020-08-21 15:58:33 -0500311__BITBAKE_LAST_LINE=0
312
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500313# Emit a useful diagnostic if something fails:
Andrew Geissler635e0e42020-08-21 15:58:33 -0500314bb_sh_exit_handler() {
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315 ret=$?
Andrew Geissler635e0e42020-08-21 15:58:33 -0500316 if [ "$ret" != 0 ]; then
317 echo "WARNING: exit code $ret from a shell command."
318 fi
319 exit $ret
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320}
Andrew Geissler635e0e42020-08-21 15:58:33 -0500321
322bb_bash_exit_handler() {
323 ret=$?
324 { set +x; } > /dev/null
325 trap "" DEBUG
326 if [ "$ret" != 0 ]; then
327 echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'"
328
329 echo "WARNING: Backtrace (BB generated script): "
330 for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do
331 if [ "$i" -eq 1 ]; then
332 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}"
333 else
334 echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}"
335 fi
336 done
337 fi
338 exit $ret
339}
340
341bb_bash_debug_handler() {
342 local line=${BASH_LINENO[0]}
343 # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it
344 if [ "$line" -eq 1 ]; then
345 return
346 fi
347
348 # Track the line number of commands as they execute. This is so we can have access to the failing line number
349 # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html
350 if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then
351 __BITBAKE_LAST_LINE=$line
352 fi
353}
354
355case $BASH_VERSION in
356"") trap 'bb_sh_exit_handler' 0
357 set -e
358 ;;
359*) trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0
360 trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG
361 set -e
362 shopt -s extdebug
363 ;;
364esac
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500365'''
366
Brad Bishop15ae2502019-06-18 21:44:24 -0400367def create_progress_handler(func, progress, logfile, d):
368 if progress == 'percent':
369 # Use default regex
370 return bb.progress.BasicProgressHandler(d, outfile=logfile)
371 elif progress.startswith('percent:'):
372 # Use specified regex
373 return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
374 elif progress.startswith('outof:'):
375 # Use specified regex
376 return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
377 elif progress.startswith("custom:"):
378 # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__
379 import functools
380 from types import ModuleType
381
382 parts = progress.split(":", 2)
383 _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None
384 if cls:
385 def resolve(x, y):
386 if not x:
387 return None
388 if isinstance(x, ModuleType):
389 return getattr(x, y, None)
390 return x.get(y)
391 cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context)
392 if not cls_obj:
393 # Fall-back on __builtins__
Andrew Geissler635e0e42020-08-21 15:58:33 -0500394 cls_obj = functools.reduce(resolve, cls.split("."), __builtins__)
Brad Bishop15ae2502019-06-18 21:44:24 -0400395 if cls_obj:
396 return cls_obj(d, outfile=logfile, otherargs=otherargs)
397 bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls))
398 else:
399 bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
400
401 return logfile
402
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500403def exec_func_shell(func, d, runfile, cwd=None):
404 """Execute a shell function from the metadata
405
406 Note on directory behavior. The 'dirs' varflag should contain a list
407 of the directories you need created prior to execution. The last
408 item in the list is where we will chdir/cd to.
409 """
410
411 # Don't let the emitted shell script override PWD
412 d.delVarFlag('PWD', 'export')
413
414 with open(runfile, 'w') as script:
415 script.write(shell_trap_code())
416
417 bb.data.emit_func(func, script, d)
418
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500419 if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500420 script.write("set -x\n")
421 if cwd:
422 script.write("cd '%s'\n" % cwd)
423 script.write("%s\n" % func)
424 script.write('''
425# cleanup
426ret=$?
427trap '' 0
428exit $ret
429''')
430
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600431 os.chmod(runfile, 0o775)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500432
433 cmd = runfile
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500434 if d.getVarFlag(func, 'fakeroot', False):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500435 fakerootcmd = d.getVar('FAKEROOT')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436 if fakerootcmd:
437 cmd = [fakerootcmd, runfile]
438
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500439 if verboseStdoutLogging:
Brad Bishop15ae2502019-06-18 21:44:24 -0400440 logfile = LogTee(logger, StdoutNoopContextManager())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500441 else:
Brad Bishop15ae2502019-06-18 21:44:24 -0400442 logfile = StdoutNoopContextManager()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500443
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500444 progress = d.getVarFlag(func, 'progress')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600445 if progress:
Andrew Geissler635e0e42020-08-21 15:58:33 -0500446 try:
447 logfile = create_progress_handler(func, progress, logfile, d)
448 except:
449 from traceback import format_exc
450 logger.error("Failed to create progress handler")
451 logger.error(format_exc())
452 raise
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600453
454 fifobuffer = bytearray()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500455 def readfifo(data):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600456 nonlocal fifobuffer
457 fifobuffer.extend(data)
458 while fifobuffer:
459 message, token, nextmsg = fifobuffer.partition(b"\00")
460 if token:
461 splitval = message.split(b' ', 1)
462 cmd = splitval[0].decode("utf-8")
463 if len(splitval) > 1:
464 value = splitval[1].decode("utf-8")
465 else:
466 value = ''
467 if cmd == 'bbplain':
468 bb.plain(value)
469 elif cmd == 'bbnote':
470 bb.note(value)
Brad Bishopc342db32019-05-15 21:57:59 -0400471 elif cmd == 'bbverbnote':
472 bb.verbnote(value)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600473 elif cmd == 'bbwarn':
474 bb.warn(value)
475 elif cmd == 'bberror':
476 bb.error(value)
477 elif cmd == 'bbfatal':
478 # The caller will call exit themselves, so bb.error() is
479 # what we want here rather than bb.fatal()
480 bb.error(value)
481 elif cmd == 'bbfatal_log':
482 bb.error(value, forcelog=True)
483 elif cmd == 'bbdebug':
484 splitval = value.split(' ', 1)
485 level = int(splitval[0])
486 value = splitval[1]
487 bb.debug(level, value)
488 else:
489 bb.warn("Unrecognised command '%s' on FIFO" % cmd)
490 fifobuffer = nextmsg
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600492 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500493
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500494 tempdir = d.getVar('T')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
496 if os.path.exists(fifopath):
497 os.unlink(fifopath)
498 os.mkfifo(fifopath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600499 with open(fifopath, 'r+b', buffering=0) as fifo:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500500 try:
501 bb.debug(2, "Executing shell function %s" % func)
Brad Bishop08902b02019-08-20 09:16:51 -0400502 with open(os.devnull, 'r+') as stdin, logfile:
503 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
Andrew Geissler635e0e42020-08-21 15:58:33 -0500504 except bb.process.ExecutionError as exe:
505 # Find the backtrace that the shell trap generated
506 backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)")
507 stdout_lines = (exe.stdout or "").split("\n")
508 backtrace_start_line = None
509 for i, line in enumerate(reversed(stdout_lines)):
510 if backtrace_marker_regex.search(line):
511 backtrace_start_line = len(stdout_lines) - i
512 break
513
514 # Read the backtrace frames, starting at the location we just found
515 backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line ("
516 r"?P<lineno>\d+)")
517 backtrace_frames = []
518 if backtrace_start_line:
519 for line in itertools.islice(stdout_lines, backtrace_start_line, None):
520 match = backtrace_entry_regex.search(line)
521 if match:
522 backtrace_frames.append(match.groupdict())
523
524 with open(runfile, "r") as script:
525 script_lines = [line.rstrip() for line in script.readlines()]
526
527 # For each backtrace frame, search backwards in the script (from the line number called out by the frame),
528 # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata
529 # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined.
530 script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)")
531 better_frames = []
532 # Skip the very last frame since it's just the call to the shell task in the body of the script
533 for frame in backtrace_frames[:-1]:
534 # Check whether the frame corresponds to a function defined in the script vs external script.
535 if os.path.samefile(frame["file"], runfile):
536 # Search backwards from the frame lineno to locate the comment that BB injected
537 i = int(frame["lineno"]) - 1
538 while i >= 0:
539 match = script_metadata_comment_regex.match(script_lines[i])
540 if match:
541 # Calculate the relative line in the function itself
542 relative_line_in_function = int(frame["lineno"]) - i - 2
543 # Calculate line in the function as declared in the metadata
544 metadata_function_line = relative_line_in_function + int(match["lineno"])
545 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(
546 frameno=frame["frameno"],
547 funcname=frame["funcname"],
548 file=match["file"],
549 lineno=metadata_function_line
550 ))
551 break
552 i -= 1
553 else:
554 better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame))
555
556 if better_frames:
557 better_frames = ("\t{0}".format(frame) for frame in better_frames)
558 exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames))
559 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500560 finally:
561 os.unlink(fifopath)
562
563 bb.debug(2, "Shell function %s finished" % func)
564
565def _task_data(fn, task, d):
566 localdata = bb.data.createCopy(d)
567 localdata.setVar('BB_FILENAME', fn)
568 localdata.setVar('BB_CURRENTTASK', task[3:])
569 localdata.setVar('OVERRIDES', 'task-%s:%s' %
570 (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False)))
571 localdata.finalize()
572 bb.data.expandKeys(localdata)
573 return localdata
574
575def _exec_task(fn, task, d, quieterr):
576 """Execute a BB 'task'
577
578 Execution of a task involves a bit more setup than executing a function,
579 running it with its own local metadata, and with some useful variables set.
580 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500581 if not d.getVarFlag(task, 'task', False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500582 event.fire(TaskInvalid(task, d), d)
583 logger.error("No such task: %s" % task)
584 return 1
585
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600586 logger.debug("Executing task %s", task)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500587
588 localdata = _task_data(fn, task, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500589 tempdir = localdata.getVar('T')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500590 if not tempdir:
591 bb.fatal("T variable not set, unable to build")
592
593 # Change nice level if we're asked to
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500594 nice = localdata.getVar("BB_TASK_NICE_LEVEL")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500595 if nice:
596 curnice = os.nice(0)
597 nice = int(nice) - curnice
598 newnice = os.nice(nice)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600599 logger.debug("Renice to %s " % newnice)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500600 ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500601 if ionice:
602 try:
603 cls, prio = ionice.split(".", 1)
604 bb.utils.ioprio_set(os.getpid(), int(cls), int(prio))
605 except:
606 bb.warn("Invalid ionice level %s" % ionice)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607
608 bb.utils.mkdirhier(tempdir)
609
610 # Determine the logfile to generate
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500611 logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500612 logbase = logfmt.format(task=task, pid=os.getpid())
613
614 # Document the order of the tasks...
615 logorder = os.path.join(tempdir, 'log.task_order')
616 try:
617 with open(logorder, 'a') as logorderfile:
618 logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
619 except OSError:
620 logger.exception("Opening log file '%s'", logorder)
621 pass
622
623 # Setup the courtesy link to the logfn
624 loglink = os.path.join(tempdir, 'log.{0}'.format(task))
625 logfn = os.path.join(tempdir, logbase)
626 if loglink:
627 bb.utils.remove(loglink)
628
629 try:
630 os.symlink(logbase, loglink)
631 except OSError:
632 pass
633
634 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
635 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
636
637 class ErrorCheckHandler(logging.Handler):
638 def __init__(self):
639 self.triggered = False
640 logging.Handler.__init__(self, logging.ERROR)
641 def emit(self, record):
642 if getattr(record, 'forcelog', False):
643 self.triggered = False
644 else:
645 self.triggered = True
646
647 # Handle logfiles
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500648 try:
649 bb.utils.mkdirhier(os.path.dirname(logfn))
650 logfile = open(logfn, 'w')
651 except OSError:
652 logger.exception("Opening log file '%s'", logfn)
653 pass
654
655 # Dup the existing fds so we dont lose them
656 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
657 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
658 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
659
660 # Replace those fds with our own
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800661 with open('/dev/null', 'r') as si:
662 os.dup2(si.fileno(), osi[1])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500663 os.dup2(logfile.fileno(), oso[1])
664 os.dup2(logfile.fileno(), ose[1])
665
666 # Ensure Python logging goes to the logfile
667 handler = logging.StreamHandler(logfile)
668 handler.setFormatter(logformatter)
669 # Always enable full debug output into task logfiles
670 handler.setLevel(logging.DEBUG - 2)
671 bblogger.addHandler(handler)
672
673 errchk = ErrorCheckHandler()
674 bblogger.addHandler(errchk)
675
676 localdata.setVar('BB_LOGFILE', logfn)
677 localdata.setVar('BB_RUNTASK', task)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500678 localdata.setVar('BB_TASK_LOGGER', bblogger)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500679
680 flags = localdata.getVarFlags(task)
681
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500682 try:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600683 try:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500684 event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600685 except (bb.BBHandledException, SystemExit):
686 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600687
688 try:
689 for func in (prefuncs or '').split():
690 exec_func(func, localdata)
691 exec_func(task, localdata)
692 for func in (postfuncs or '').split():
693 exec_func(func, localdata)
Brad Bishop08902b02019-08-20 09:16:51 -0400694 except bb.BBHandledException:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500695 event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata)
Brad Bishop08902b02019-08-20 09:16:51 -0400696 return 1
697 except Exception as exc:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600698 if quieterr:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500699 event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600700 else:
701 errprinted = errchk.triggered
702 logger.error(str(exc))
Andrew Geissler82c905d2020-04-13 13:39:40 -0500703 event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600704 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500705 finally:
706 sys.stdout.flush()
707 sys.stderr.flush()
708
709 bblogger.removeHandler(handler)
710
711 # Restore the backup fds
712 os.dup2(osi[0], osi[1])
713 os.dup2(oso[0], oso[1])
714 os.dup2(ose[0], ose[1])
715
716 # Close the backup fds
717 os.close(osi[0])
718 os.close(oso[0])
719 os.close(ose[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500720
721 logfile.close()
722 if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600723 logger.debug2("Zero size logfn %s, removing", logfn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500724 bb.utils.remove(logfn)
725 bb.utils.remove(loglink)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500726 event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500727
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500728 if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500729 make_stamp(task, localdata)
730
731 return 0
732
733def exec_task(fn, task, d, profile = False):
734 try:
735 quieterr = False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500736 if d.getVarFlag(task, "quieterrors", False) is not None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500737 quieterr = True
738
739 if profile:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500740 profname = "profile-%s.log" % (d.getVar("PN") + "-" + task)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741 try:
742 import cProfile as profile
743 except:
744 import profile
745 prof = profile.Profile()
746 ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr)
747 prof.dump_stats(profname)
748 bb.utils.process_profilelog(profname)
749
750 return ret
751 else:
752 return _exec_task(fn, task, d, quieterr)
753
754 except Exception:
755 from traceback import format_exc
756 if not quieterr:
757 logger.error("Build of %s failed" % (task))
758 logger.error(format_exc())
759 failedevent = TaskFailed(task, None, d, True)
760 event.fire(failedevent, d)
761 return 1
762
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600763def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500764 """
765 Internal stamp helper function
766 Makes sure the stamp directory exists
767 Returns the stamp path+filename
768
769 In the bitbake core, d can be a CacheData and file_name will be set.
770 When called in task context, d will be a data store, file_name will not be set
771 """
772 taskflagname = taskname
773 if taskname.endswith("_setscene") and taskname != "do_setscene":
774 taskflagname = taskname.replace("_setscene", "")
775
776 if file_name:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500777 stamp = d.stamp[file_name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500778 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
779 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500780 stamp = d.getVar('STAMP')
781 file_name = d.getVar('BB_FILENAME')
782 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500783
784 if baseonly:
785 return stamp
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600786 if noextra:
787 extrainfo = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500788
789 if not stamp:
790 return
791
792 stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo)
793
794 stampdir = os.path.dirname(stamp)
795 if cached_mtime_noerror(stampdir) == 0:
796 bb.utils.mkdirhier(stampdir)
797
798 return stamp
799
800def stamp_cleanmask_internal(taskname, d, file_name):
801 """
802 Internal stamp helper function to generate stamp cleaning mask
803 Returns the stamp path+filename
804
805 In the bitbake core, d can be a CacheData and file_name will be set.
806 When called in task context, d will be a data store, file_name will not be set
807 """
808 taskflagname = taskname
809 if taskname.endswith("_setscene") and taskname != "do_setscene":
810 taskflagname = taskname.replace("_setscene", "")
811
812 if file_name:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500813 stamp = d.stampclean[file_name]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500814 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
815 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500816 stamp = d.getVar('STAMPCLEAN')
817 file_name = d.getVar('BB_FILENAME')
818 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500819
820 if not stamp:
821 return []
822
823 cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo)
824
825 return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
826
827def make_stamp(task, d, file_name = None):
828 """
829 Creates/updates a stamp for a given task
830 (d can be a data dict or dataCache)
831 """
832 cleanmask = stamp_cleanmask_internal(task, d, file_name)
833 for mask in cleanmask:
834 for name in glob.glob(mask):
835 # Preserve sigdata files in the stamps directory
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500836 if "sigdata" in name or "sigbasedata" in name:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500837 continue
838 # Preserve taint files in the stamps directory
839 if name.endswith('.taint'):
840 continue
841 os.unlink(name)
842
843 stamp = stamp_internal(task, d, file_name)
844 # Remove the file and recreate to force timestamp
845 # change on broken NFS filesystems
846 if stamp:
847 bb.utils.remove(stamp)
848 open(stamp, "w").close()
849
850 # If we're in task context, write out a signature file for each task
851 # as it completes
852 if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
853 stampbase = stamp_internal(task, d, None, True)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500854 file_name = d.getVar('BB_FILENAME')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500855 bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
856
857def del_stamp(task, d, file_name = None):
858 """
859 Removes a stamp for a given task
860 (d can be a data dict or dataCache)
861 """
862 stamp = stamp_internal(task, d, file_name)
863 bb.utils.remove(stamp)
864
865def write_taint(task, d, file_name = None):
866 """
867 Creates a "taint" file which will force the specified task and its
868 dependents to be re-run the next time by influencing the value of its
869 taskhash.
870 (d can be a data dict or dataCache)
871 """
872 import uuid
873 if file_name:
874 taintfn = d.stamp[file_name] + '.' + task + '.taint'
875 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500876 taintfn = d.getVar('STAMP') + '.' + task + '.taint'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500877 bb.utils.mkdirhier(os.path.dirname(taintfn))
878 # The specific content of the taint file is not really important,
879 # we just need it to be random, so a random UUID is used
880 with open(taintfn, 'w') as taintf:
881 taintf.write(str(uuid.uuid4()))
882
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600883def stampfile(taskname, d, file_name = None, noextra=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500884 """
885 Return the stamp for a given task
886 (d can be a data dict or dataCache)
887 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600888 return stamp_internal(taskname, d, file_name, noextra=noextra)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500889
890def add_tasks(tasklist, d):
891 task_deps = d.getVar('_task_deps', False)
892 if not task_deps:
893 task_deps = {}
894 if not 'tasks' in task_deps:
895 task_deps['tasks'] = []
896 if not 'parents' in task_deps:
897 task_deps['parents'] = {}
898
899 for task in tasklist:
900 task = d.expand(task)
901
902 d.setVarFlag(task, 'task', 1)
903
904 if not task in task_deps['tasks']:
905 task_deps['tasks'].append(task)
906
907 flags = d.getVarFlags(task)
908 def getTask(name):
909 if not name in task_deps:
910 task_deps[name] = {}
911 if name in flags:
912 deptask = d.expand(flags[name])
913 task_deps[name][task] = deptask
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800914 getTask('mcdepends')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500915 getTask('depends')
916 getTask('rdepends')
917 getTask('deptask')
918 getTask('rdeptask')
919 getTask('recrdeptask')
920 getTask('recideptask')
921 getTask('nostamp')
922 getTask('fakeroot')
923 getTask('noexec')
924 getTask('umask')
925 task_deps['parents'][task] = []
926 if 'deps' in flags:
927 for dep in flags['deps']:
Brad Bishopc342db32019-05-15 21:57:59 -0400928 # Check and warn for "addtask task after foo" while foo does not exist
929 #if not dep in tasklist:
930 # bb.warn('%s: dependent task %s for %s does not exist' % (d.getVar('PN'), dep, task))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500931 dep = d.expand(dep)
932 task_deps['parents'][task].append(dep)
933
934 # don't assume holding a reference
935 d.setVar('_task_deps', task_deps)
936
937def addtask(task, before, after, d):
938 if task[:3] != "do_":
939 task = "do_" + task
940
941 d.setVarFlag(task, "task", 1)
942 bbtasks = d.getVar('__BBTASKS', False) or []
943 if task not in bbtasks:
944 bbtasks.append(task)
945 d.setVar('__BBTASKS', bbtasks)
946
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500947 existing = d.getVarFlag(task, "deps", False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500948 if after is not None:
949 # set up deps for function
950 for entry in after.split():
951 if entry not in existing:
952 existing.append(entry)
953 d.setVarFlag(task, "deps", existing)
954 if before is not None:
955 # set up things that depend on this func
956 for entry in before.split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500957 existing = d.getVarFlag(entry, "deps", False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500958 if task not in existing:
959 d.setVarFlag(entry, "deps", [task] + existing)
960
961def deltask(task, d):
962 if task[:3] != "do_":
963 task = "do_" + task
964
965 bbtasks = d.getVar('__BBTASKS', False) or []
966 if task in bbtasks:
967 bbtasks.remove(task)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600968 d.delVarFlag(task, 'task')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500969 d.setVar('__BBTASKS', bbtasks)
970
971 d.delVarFlag(task, 'deps')
972 for bbtask in d.getVar('__BBTASKS', False) or []:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500973 deps = d.getVarFlag(bbtask, 'deps', False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500974 if task in deps:
975 deps.remove(task)
976 d.setVarFlag(bbtask, 'deps', deps)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500977
978def preceedtask(task, with_recrdeptasks, d):
979 """
980 Returns a set of tasks in the current recipe which were specified as
981 precondition by the task itself ("after") or which listed themselves
982 as precondition ("before"). Preceeding tasks specified via the
983 "recrdeptask" are included in the result only if requested. Beware
984 that this may lead to the task itself being listed.
985 """
986 preceed = set()
Brad Bishop316dfdd2018-06-25 12:45:53 -0400987
988 # Ignore tasks which don't exist
989 tasks = d.getVar('__BBTASKS', False)
990 if task not in tasks:
991 return preceed
992
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500993 preceed.update(d.getVarFlag(task, 'deps') or [])
994 if with_recrdeptasks:
995 recrdeptask = d.getVarFlag(task, 'recrdeptask')
996 if recrdeptask:
997 preceed.update(recrdeptask.split())
998 return preceed
999
1000def tasksbetween(task_start, task_end, d):
1001 """
1002 Return the list of tasks between two tasks in the current recipe,
1003 where task_start is to start at and task_end is the task to end at
1004 (and task_end has a dependency chain back to task_start).
1005 """
1006 outtasks = []
1007 tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys()))
1008 def follow_chain(task, endtask, chain=None):
1009 if not chain:
1010 chain = []
1011 chain.append(task)
1012 for othertask in tasks:
1013 if othertask == task:
1014 continue
1015 if task == endtask:
1016 for ctask in chain:
1017 if ctask not in outtasks:
1018 outtasks.append(ctask)
1019 else:
1020 deps = d.getVarFlag(othertask, 'deps', False)
1021 if task in deps:
1022 follow_chain(othertask, endtask, chain)
1023 chain.pop()
1024 follow_chain(task_start, task_end)
1025 return outtasks