blob: 0618e46fe67f0c4636b1322769c1748c1ddbeff5 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake Utility Functions
3"""
4
5# Copyright (C) 2004 Michael Lauer
6#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import re, fcntl, os, string, stat, shutil, time
11import sys
12import errno
13import logging
14import bb
15import bb.msg
16import multiprocessing
17import fcntl
Brad Bishop19323692019-04-05 15:28:33 -040018import importlib
19from importlib import machinery
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020import itertools
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021import subprocess
22import glob
23import fnmatch
24import traceback
25import errno
26import signal
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050027import ast
Patrick Williamsc0f7c042017-02-23 20:41:17 -060028import collections
29import copy
30from subprocess import getstatusoutput
Patrick Williamsc124f4f2015-09-15 14:41:29 -050031from contextlib import contextmanager
32from ctypes import cdll
33
Patrick Williamsc124f4f2015-09-15 14:41:29 -050034logger = logging.getLogger("BitBake.Util")
Brad Bishop19323692019-04-05 15:28:33 -040035python_extensions = importlib.machinery.all_suffixes()
Patrick Williamsc0f7c042017-02-23 20:41:17 -060036
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037
38def clean_context():
39 return {
40 "os": os,
41 "bb": bb,
42 "time": time,
43 }
44
45def get_context():
46 return _context
47
48
49def set_context(ctx):
50 _context = ctx
51
52# Context used in better_exec, eval
53_context = clean_context()
54
55class VersionStringException(Exception):
56 """Exception raised when an invalid version specification is found"""
57
58def explode_version(s):
59 r = []
Brad Bishop19323692019-04-05 15:28:33 -040060 alpha_regexp = re.compile(r'^([a-zA-Z]+)(.*)$')
61 numeric_regexp = re.compile(r'^(\d+)(.*)$')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050062 while (s != ''):
63 if s[0] in string.digits:
64 m = numeric_regexp.match(s)
65 r.append((0, int(m.group(1))))
66 s = m.group(2)
67 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -060068 if s[0] in string.ascii_letters:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069 m = alpha_regexp.match(s)
70 r.append((1, m.group(1)))
71 s = m.group(2)
72 continue
73 if s[0] == '~':
74 r.append((-1, s[0]))
75 else:
76 r.append((2, s[0]))
77 s = s[1:]
78 return r
79
80def split_version(s):
81 """Split a version string into its constituent parts (PE, PV, PR)"""
82 s = s.strip(" <>=")
83 e = 0
84 if s.count(':'):
85 e = int(s.split(":")[0])
86 s = s.split(":")[1]
87 r = ""
88 if s.count('-'):
89 r = s.rsplit("-", 1)[1]
90 s = s.rsplit("-", 1)[0]
91 v = s
92 return (e, v, r)
93
94def vercmp_part(a, b):
95 va = explode_version(a)
96 vb = explode_version(b)
97 while True:
98 if va == []:
99 (oa, ca) = (0, None)
100 else:
101 (oa, ca) = va.pop(0)
102 if vb == []:
103 (ob, cb) = (0, None)
104 else:
105 (ob, cb) = vb.pop(0)
106 if (oa, ca) == (0, None) and (ob, cb) == (0, None):
107 return 0
108 if oa < ob:
109 return -1
110 elif oa > ob:
111 return 1
Brad Bishop19323692019-04-05 15:28:33 -0400112 elif ca is None:
113 return -1
114 elif cb is None:
115 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500116 elif ca < cb:
117 return -1
118 elif ca > cb:
119 return 1
120
121def vercmp(ta, tb):
122 (ea, va, ra) = ta
123 (eb, vb, rb) = tb
124
125 r = int(ea or 0) - int(eb or 0)
126 if (r == 0):
127 r = vercmp_part(va, vb)
128 if (r == 0):
129 r = vercmp_part(ra, rb)
130 return r
131
132def vercmp_string(a, b):
133 ta = split_version(a)
134 tb = split_version(b)
135 return vercmp(ta, tb)
136
137def vercmp_string_op(a, b, op):
138 """
139 Compare two versions and check if the specified comparison operator matches the result of the comparison.
140 This function is fairly liberal about what operators it will accept since there are a variety of styles
141 depending on the context.
142 """
143 res = vercmp_string(a, b)
144 if op in ('=', '=='):
145 return res == 0
146 elif op == '<=':
147 return res <= 0
148 elif op == '>=':
149 return res >= 0
150 elif op in ('>', '>>'):
151 return res > 0
152 elif op in ('<', '<<'):
153 return res < 0
154 elif op == '!=':
155 return res != 0
156 else:
157 raise VersionStringException('Unsupported comparison operator "%s"' % op)
158
159def explode_deps(s):
160 """
161 Take an RDEPENDS style string of format:
162 "DEPEND1 (optional version) DEPEND2 (optional version) ..."
163 and return a list of dependencies.
164 Version information is ignored.
165 """
166 r = []
167 l = s.split()
168 flag = False
169 for i in l:
170 if i[0] == '(':
171 flag = True
172 #j = []
173 if not flag:
174 r.append(i)
175 #else:
176 # j.append(i)
177 if flag and i.endswith(')'):
178 flag = False
179 # Ignore version
180 #r[-1] += ' ' + ' '.join(j)
181 return r
182
Brad Bishop316dfdd2018-06-25 12:45:53 -0400183def explode_dep_versions2(s, *, sort=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500184 """
185 Take an RDEPENDS style string of format:
186 "DEPEND1 (optional version) DEPEND2 (optional version) ..."
187 and return a dictionary of dependencies and versions.
188 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600189 r = collections.OrderedDict()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190 l = s.replace(",", "").split()
191 lastdep = None
192 lastcmp = ""
193 lastver = ""
194 incmp = False
195 inversion = False
196 for i in l:
197 if i[0] == '(':
198 incmp = True
199 i = i[1:].strip()
200 if not i:
201 continue
202
203 if incmp:
204 incmp = False
205 inversion = True
206 # This list is based on behavior and supported comparisons from deb, opkg and rpm.
207 #
208 # Even though =<, <<, ==, !=, =>, and >> may not be supported,
209 # we list each possibly valid item.
210 # The build system is responsible for validation of what it supports.
211 if i.startswith(('<=', '=<', '<<', '==', '!=', '>=', '=>', '>>')):
212 lastcmp = i[0:2]
213 i = i[2:]
214 elif i.startswith(('<', '>', '=')):
215 lastcmp = i[0:1]
216 i = i[1:]
217 else:
218 # This is an unsupported case!
219 raise VersionStringException('Invalid version specification in "(%s" - invalid or missing operator' % i)
220 lastcmp = (i or "")
221 i = ""
222 i.strip()
223 if not i:
224 continue
225
226 if inversion:
227 if i.endswith(')'):
228 i = i[:-1] or ""
229 inversion = False
230 if lastver and i:
231 lastver += " "
232 if i:
233 lastver += i
234 if lastdep not in r:
235 r[lastdep] = []
236 r[lastdep].append(lastcmp + " " + lastver)
237 continue
238
239 #if not inversion:
240 lastdep = i
241 lastver = ""
242 lastcmp = ""
243 if not (i in r and r[i]):
244 r[lastdep] = []
245
Brad Bishop316dfdd2018-06-25 12:45:53 -0400246 if sort:
247 r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248 return r
249
250def explode_dep_versions(s):
251 r = explode_dep_versions2(s)
252 for d in r:
253 if not r[d]:
254 r[d] = None
255 continue
256 if len(r[d]) > 1:
257 bb.warn("explode_dep_versions(): Item %s appeared in dependency string '%s' multiple times with different values. explode_dep_versions cannot cope with this." % (d, s))
258 r[d] = r[d][0]
259 return r
260
261def join_deps(deps, commasep=True):
262 """
263 Take the result from explode_dep_versions and generate a dependency string
264 """
265 result = []
266 for dep in deps:
267 if deps[dep]:
268 if isinstance(deps[dep], list):
269 for v in deps[dep]:
270 result.append(dep + " (" + v + ")")
271 else:
272 result.append(dep + " (" + deps[dep] + ")")
273 else:
274 result.append(dep)
275 if commasep:
276 return ", ".join(result)
277 else:
278 return " ".join(result)
279
280def _print_trace(body, line):
281 """
282 Print the Environment of a Text Body
283 """
284 error = []
285 # print the environment of the method
286 min_line = max(1, line-4)
287 max_line = min(line + 4, len(body))
288 for i in range(min_line, max_line + 1):
289 if line == i:
290 error.append(' *** %.4d:%s' % (i, body[i-1].rstrip()))
291 else:
292 error.append(' %.4d:%s' % (i, body[i-1].rstrip()))
293 return error
294
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500295def better_compile(text, file, realfile, mode = "exec", lineno = 0):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296 """
297 A better compile method. This method
298 will print the offending lines.
299 """
300 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500301 cache = bb.methodpool.compile_cache(text)
302 if cache:
303 return cache
304 # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
305 text2 = "\n" * int(lineno) + text
306 code = compile(text2, realfile, mode)
307 bb.methodpool.compile_cache_add(text, code)
308 return code
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309 except Exception as e:
310 error = []
311 # split the text into lines again
312 body = text.split('\n')
Brad Bishop19323692019-04-05 15:28:33 -0400313 error.append("Error in compiling python function in %s, line %s:\n" % (realfile, e.lineno))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500314 if hasattr(e, "lineno"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315 error.append("The code lines resulting in this error were:")
Brad Bishop19323692019-04-05 15:28:33 -0400316 # e.lineno: line's position in reaflile
317 # lineno: function name's "position -1" in realfile
318 # e.lineno - lineno: line's relative position in function
319 error.extend(_print_trace(body, e.lineno - lineno))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320 else:
321 error.append("The function causing this error was:")
322 for line in body:
323 error.append(line)
324 error.append("%s: %s" % (e.__class__.__name__, str(e)))
325
326 logger.error("\n".join(error))
327
328 e = bb.BBHandledException(e)
329 raise e
330
331def _print_exception(t, value, tb, realfile, text, context):
332 error = []
333 try:
334 exception = traceback.format_exception_only(t, value)
335 error.append('Error executing a python function in %s:\n' % realfile)
336
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500337 # Strip 'us' from the stack (better_exec call) unless that was where the
338 # error came from
339 if tb.tb_next is not None:
340 tb = tb.tb_next
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500341
342 textarray = text.split('\n')
343
344 linefailed = tb.tb_lineno
345
346 tbextract = traceback.extract_tb(tb)
347 tbformat = traceback.format_list(tbextract)
348 error.append("The stack trace of python calls that resulted in this exception/failure was:")
349 error.append("File: '%s', lineno: %s, function: %s" % (tbextract[0][0], tbextract[0][1], tbextract[0][2]))
350 error.extend(_print_trace(textarray, linefailed))
351
352 # See if this is a function we constructed and has calls back into other functions in
353 # "text". If so, try and improve the context of the error by diving down the trace
354 level = 0
355 nexttb = tb.tb_next
356 while nexttb is not None and (level+1) < len(tbextract):
357 error.append("File: '%s', lineno: %s, function: %s" % (tbextract[level+1][0], tbextract[level+1][1], tbextract[level+1][2]))
358 if tbextract[level][0] == tbextract[level+1][0] and tbextract[level+1][2] == tbextract[level][0]:
359 # The code was possibly in the string we compiled ourselves
360 error.extend(_print_trace(textarray, tbextract[level+1][1]))
361 elif tbextract[level+1][0].startswith("/"):
362 # The code looks like it might be in a file, try and load it
363 try:
364 with open(tbextract[level+1][0], "r") as f:
365 text = f.readlines()
366 error.extend(_print_trace(text, tbextract[level+1][1]))
367 except:
368 error.append(tbformat[level+1])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369 else:
370 error.append(tbformat[level+1])
371 nexttb = tb.tb_next
372 level = level + 1
373
374 error.append("Exception: %s" % ''.join(exception))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600375
376 # If the exception is from spwaning a task, let's be helpful and display
377 # the output (which hopefully includes stderr).
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500378 if isinstance(value, subprocess.CalledProcessError) and value.output:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600379 error.append("Subprocess output:")
380 error.append(value.output.decode("utf-8", errors="ignore"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381 finally:
382 logger.error("\n".join(error))
383
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500384def better_exec(code, context, text = None, realfile = "<code>", pythonexception=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385 """
386 Similiar to better_compile, better_exec will
387 print the lines that are responsible for the
388 error.
389 """
390 import bb.parse
391 if not text:
392 text = code
393 if not hasattr(code, "co_filename"):
394 code = better_compile(code, realfile, realfile)
395 try:
396 exec(code, get_context(), context)
Brad Bishop08902b02019-08-20 09:16:51 -0400397 except (bb.BBHandledException, bb.parse.SkipRecipe, bb.data_smart.ExpansionError):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500398 # Error already shown so passthrough, no need for traceback
399 raise
400 except Exception as e:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500401 if pythonexception:
402 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500403 (t, value, tb) = sys.exc_info()
404 try:
405 _print_exception(t, value, tb, realfile, text, context)
406 except Exception as e:
407 logger.error("Exception handler error: %s" % str(e))
408
409 e = bb.BBHandledException(e)
410 raise e
411
412def simple_exec(code, context):
413 exec(code, get_context(), context)
414
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600415def better_eval(source, locals, extraglobals = None):
416 ctx = get_context()
417 if extraglobals:
418 ctx = copy.copy(ctx)
419 for g in extraglobals:
420 ctx[g] = extraglobals[g]
421 return eval(source, ctx, locals)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500422
423@contextmanager
424def fileslocked(files):
425 """Context manager for locking and unlocking file locks."""
426 locks = []
427 if files:
428 for lockfile in files:
429 locks.append(bb.utils.lockfile(lockfile))
430
431 yield
432
433 for lock in locks:
434 bb.utils.unlockfile(lock)
435
436@contextmanager
437def timeout(seconds):
438 def timeout_handler(signum, frame):
439 pass
440
441 original_handler = signal.signal(signal.SIGALRM, timeout_handler)
442
443 try:
444 signal.alarm(seconds)
445 yield
446 finally:
447 signal.alarm(0)
448 signal.signal(signal.SIGALRM, original_handler)
449
450def lockfile(name, shared=False, retry=True, block=False):
451 """
452 Use the specified file as a lock file, return when the lock has
453 been acquired. Returns a variable to pass to unlockfile().
454 Parameters:
455 retry: True to re-try locking if it fails, False otherwise
456 block: True to block until the lock succeeds, False otherwise
457 The retry and block parameters are kind of equivalent unless you
458 consider the possibility of sending a signal to the process to break
459 out - at which point you want block=True rather than retry=True.
460 """
461 dirname = os.path.dirname(name)
462 mkdirhier(dirname)
463
464 if not os.access(dirname, os.W_OK):
465 logger.error("Unable to acquire lock '%s', directory is not writable",
466 name)
467 sys.exit(1)
468
469 op = fcntl.LOCK_EX
470 if shared:
471 op = fcntl.LOCK_SH
472 if not retry and not block:
473 op = op | fcntl.LOCK_NB
474
475 while True:
476 # If we leave the lockfiles lying around there is no problem
477 # but we should clean up after ourselves. This gives potential
478 # for races though. To work around this, when we acquire the lock
479 # we check the file we locked was still the lock file on disk.
480 # by comparing inode numbers. If they don't match or the lockfile
481 # no longer exists, we start again.
482
483 # This implementation is unfair since the last person to request the
484 # lock is the most likely to win it.
485
486 try:
487 lf = open(name, 'a+')
488 fileno = lf.fileno()
489 fcntl.flock(fileno, op)
490 statinfo = os.fstat(fileno)
491 if os.path.exists(lf.name):
492 statinfo2 = os.stat(lf.name)
493 if statinfo.st_ino == statinfo2.st_ino:
494 return lf
495 lf.close()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800496 except OSError as e:
497 if e.errno == errno.EACCES:
498 logger.error("Unable to acquire lock '%s', %s",
499 e.strerror, name)
500 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500501 try:
502 lf.close()
503 except Exception:
504 pass
505 pass
506 if not retry:
507 return None
508
509def unlockfile(lf):
510 """
511 Unlock a file locked using lockfile()
512 """
513 try:
514 # If we had a shared lock, we need to promote to exclusive before
515 # removing the lockfile. Attempt this, ignore failures.
516 fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
517 os.unlink(lf.name)
518 except (IOError, OSError):
519 pass
520 fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
521 lf.close()
522
523def md5_file(filename):
524 """
525 Return the hex string representation of the MD5 checksum of filename.
526 """
Brad Bishop220d5532018-08-14 00:59:39 +0100527 import hashlib, mmap
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500528
529 with open(filename, "rb") as f:
Brad Bishop220d5532018-08-14 00:59:39 +0100530 m = hashlib.md5()
531 try:
532 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
533 for chunk in iter(lambda: mm.read(8192), b''):
534 m.update(chunk)
535 except ValueError:
536 # You can't mmap() an empty file so silence this exception
537 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500538 return m.hexdigest()
539
540def sha256_file(filename):
541 """
542 Return the hex string representation of the 256-bit SHA checksum of
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500543 filename.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500544 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500545 import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500546
547 s = hashlib.sha256()
548 with open(filename, "rb") as f:
549 for line in f:
550 s.update(line)
551 return s.hexdigest()
552
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500553def sha1_file(filename):
554 """
555 Return the hex string representation of the SHA1 checksum of the filename
556 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500557 import hashlib
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500558
559 s = hashlib.sha1()
560 with open(filename, "rb") as f:
561 for line in f:
562 s.update(line)
563 return s.hexdigest()
564
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500565def preserved_envvars_exported():
566 """Variables which are taken from the environment and placed in and exported
567 from the metadata"""
568 return [
569 'BB_TASKHASH',
570 'HOME',
571 'LOGNAME',
572 'PATH',
573 'PWD',
574 'SHELL',
575 'TERM',
576 'USER',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600577 'LC_ALL',
578 'BBSERVER',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579 ]
580
581def preserved_envvars():
582 """Variables which are taken from the environment and placed in the metadata"""
583 v = [
584 'BBPATH',
585 'BB_PRESERVE_ENV',
586 'BB_ENV_WHITELIST',
587 'BB_ENV_EXTRAWHITE',
588 ]
589 return v + preserved_envvars_exported()
590
591def filter_environment(good_vars):
592 """
593 Create a pristine environment for bitbake. This will remove variables that
594 are not known and may influence the build in a negative way.
595 """
596
597 removed_vars = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600598 for key in list(os.environ):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500599 if key in good_vars:
600 continue
601
602 removed_vars[key] = os.environ[key]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500603 del os.environ[key]
604
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600605 # If we spawn a python process, we need to have a UTF-8 locale, else python's file
606 # access methods will use ascii. You can't change that mode once the interpreter is
607 # started so we have to ensure a locale is set. Ideally we'd use C.UTF-8 but not all
608 # distros support that and we need to set something.
609 os.environ["LC_ALL"] = "en_US.UTF-8"
610
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611 if removed_vars:
612 logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
613
614 return removed_vars
615
616def approved_variables():
617 """
618 Determine and return the list of whitelisted variables which are approved
619 to remain in the environment.
620 """
621 if 'BB_PRESERVE_ENV' in os.environ:
622 return os.environ.keys()
623 approved = []
624 if 'BB_ENV_WHITELIST' in os.environ:
625 approved = os.environ['BB_ENV_WHITELIST'].split()
626 approved.extend(['BB_ENV_WHITELIST'])
627 else:
628 approved = preserved_envvars()
629 if 'BB_ENV_EXTRAWHITE' in os.environ:
630 approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split())
631 if 'BB_ENV_EXTRAWHITE' not in approved:
632 approved.extend(['BB_ENV_EXTRAWHITE'])
633 return approved
634
635def clean_environment():
636 """
637 Clean up any spurious environment variables. This will remove any
638 variables the user hasn't chosen to preserve.
639 """
640 if 'BB_PRESERVE_ENV' not in os.environ:
641 good_vars = approved_variables()
642 return filter_environment(good_vars)
643
644 return {}
645
646def empty_environment():
647 """
648 Remove all variables from the environment.
649 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600650 for s in list(os.environ.keys()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500651 os.unsetenv(s)
652 del os.environ[s]
653
654def build_environment(d):
655 """
656 Build an environment from all exported variables.
657 """
658 import bb.data
659 for var in bb.data.keys(d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500660 export = d.getVarFlag(var, "export", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500661 if export:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500662 os.environ[var] = d.getVar(var) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500663
664def _check_unsafe_delete_path(path):
665 """
666 Basic safeguard against recursively deleting something we shouldn't. If it returns True,
667 the caller should raise an exception with an appropriate message.
668 NOTE: This is NOT meant to be a security mechanism - just a guard against silly mistakes
669 with potentially disastrous results.
670 """
671 extra = ''
672 # HOME might not be /home/something, so in case we can get it, check against it
673 homedir = os.environ.get('HOME', '')
674 if homedir:
675 extra = '|%s' % homedir
676 if re.match('(/|//|/home|/home/[^/]*%s)$' % extra, os.path.abspath(path)):
677 return True
678 return False
679
680def remove(path, recurse=False):
681 """Equivalent to rm -f or rm -rf"""
682 if not path:
683 return
684 if recurse:
685 for name in glob.glob(path):
686 if _check_unsafe_delete_path(path):
687 raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
688 # shutil.rmtree(name) would be ideal but its too slow
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500689 subprocess.check_call(['rm', '-rf'] + glob.glob(path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500690 return
691 for name in glob.glob(path):
692 try:
693 os.unlink(name)
694 except OSError as exc:
695 if exc.errno != errno.ENOENT:
696 raise
697
698def prunedir(topdir):
699 # Delete everything reachable from the directory named in 'topdir'.
700 # CAUTION: This is dangerous!
701 if _check_unsafe_delete_path(topdir):
702 raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
Brad Bishop19323692019-04-05 15:28:33 -0400703 remove(topdir, recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500704
705#
706# Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var)
707# but thats possibly insane and suffixes is probably going to be small
708#
709def prune_suffix(var, suffixes, d):
710 # See if var ends with any of the suffixes listed and
711 # remove it if found
712 for suffix in suffixes:
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400713 if suffix and var.endswith(suffix):
714 return var[:-len(suffix)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500715 return var
716
717def mkdirhier(directory):
718 """Create a directory like 'mkdir -p', but does not complain if
719 directory already exists like os.makedirs
720 """
721
722 try:
723 os.makedirs(directory)
724 except OSError as e:
Brad Bishopc342db32019-05-15 21:57:59 -0400725 if e.errno != errno.EEXIST or not os.path.isdir(directory):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500726 raise e
727
728def movefile(src, dest, newmtime = None, sstat = None):
729 """Moves a file from src to dest, preserving all permissions and
730 attributes; mtime will be preserved even when moving across
731 filesystems. Returns true on success and false on failure. Move is
732 atomic.
733 """
734
735 #print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
736 try:
737 if not sstat:
738 sstat = os.lstat(src)
739 except Exception as e:
740 print("movefile: Stating source file failed...", e)
741 return None
742
743 destexists = 1
744 try:
745 dstat = os.lstat(dest)
746 except:
747 dstat = os.lstat(os.path.dirname(dest))
748 destexists = 0
749
750 if destexists:
751 if stat.S_ISLNK(dstat[stat.ST_MODE]):
752 try:
753 os.unlink(dest)
754 destexists = 0
755 except Exception as e:
756 pass
757
758 if stat.S_ISLNK(sstat[stat.ST_MODE]):
759 try:
760 target = os.readlink(src)
761 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
762 os.unlink(dest)
763 os.symlink(target, dest)
764 #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
765 os.unlink(src)
766 return os.lstat(dest)
767 except Exception as e:
768 print("movefile: failed to properly create symlink:", dest, "->", target, e)
769 return None
770
771 renamefailed = 1
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500772 # os.rename needs to know the dest path ending with file name
773 # so append the file name to a path only if it's a dir specified
774 srcfname = os.path.basename(src)
775 destpath = os.path.join(dest, srcfname) if os.path.isdir(dest) \
776 else dest
777
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500778 if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
779 try:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500780 os.rename(src, destpath)
781 renamefailed = 0
782 except Exception as e:
783 if e[0] != errno.EXDEV:
784 # Some random error.
785 print("movefile: Failed to move", src, "to", dest, e)
786 return None
787 # Invalid cross-device-link 'bind' mounted or actually Cross-Device
788
789 if renamefailed:
790 didcopy = 0
791 if stat.S_ISREG(sstat[stat.ST_MODE]):
792 try: # For safety copy then move it over.
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500793 shutil.copyfile(src, destpath + "#new")
794 os.rename(destpath + "#new", destpath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500795 didcopy = 1
796 except Exception as e:
797 print('movefile: copy', src, '->', dest, 'failed.', e)
798 return None
799 else:
800 #we don't yet handle special, so we need to fall back to /bin/mv
801 a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'")
802 if a[0] != 0:
803 print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
804 return None # failure
805 try:
806 if didcopy:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400807 os.lchown(destpath, sstat[stat.ST_UID], sstat[stat.ST_GID])
808 os.chmod(destpath, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500809 os.unlink(src)
810 except Exception as e:
811 print("movefile: Failed to chown/chmod/unlink", dest, e)
812 return None
813
814 if newmtime:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500815 os.utime(destpath, (newmtime, newmtime))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500816 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500817 os.utime(destpath, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500818 newmtime = sstat[stat.ST_MTIME]
819 return newmtime
820
821def copyfile(src, dest, newmtime = None, sstat = None):
822 """
823 Copies a file from src to dest, preserving all permissions and
824 attributes; mtime will be preserved even when moving across
825 filesystems. Returns true on success and false on failure.
826 """
827 #print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
828 try:
829 if not sstat:
830 sstat = os.lstat(src)
831 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600832 logger.warning("copyfile: stat of %s failed (%s)" % (src, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500833 return False
834
835 destexists = 1
836 try:
837 dstat = os.lstat(dest)
838 except:
839 dstat = os.lstat(os.path.dirname(dest))
840 destexists = 0
841
842 if destexists:
843 if stat.S_ISLNK(dstat[stat.ST_MODE]):
844 try:
845 os.unlink(dest)
846 destexists = 0
847 except Exception as e:
848 pass
849
850 if stat.S_ISLNK(sstat[stat.ST_MODE]):
851 try:
852 target = os.readlink(src)
853 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
854 os.unlink(dest)
855 os.symlink(target, dest)
856 #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
857 return os.lstat(dest)
858 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600859 logger.warning("copyfile: failed to create symlink %s to %s (%s)" % (dest, target, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500860 return False
861
862 if stat.S_ISREG(sstat[stat.ST_MODE]):
863 try:
864 srcchown = False
865 if not os.access(src, os.R_OK):
866 # Make sure we can read it
867 srcchown = True
868 os.chmod(src, sstat[stat.ST_MODE] | stat.S_IRUSR)
869
870 # For safety copy then move it over.
871 shutil.copyfile(src, dest + "#new")
872 os.rename(dest + "#new", dest)
873 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600874 logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500875 return False
876 finally:
877 if srcchown:
878 os.chmod(src, sstat[stat.ST_MODE])
879 os.utime(src, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
880
881 else:
882 #we don't yet handle special, so we need to fall back to /bin/mv
883 a = getstatusoutput("/bin/cp -f " + "'" + src + "' '" + dest + "'")
884 if a[0] != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600885 logger.warning("copyfile: failed to copy special file %s to %s (%s)" % (src, dest, a))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500886 return False # failure
887 try:
888 os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
889 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
890 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600891 logger.warning("copyfile: failed to chown/chmod %s (%s)" % (dest, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500892 return False
893
894 if newmtime:
895 os.utime(dest, (newmtime, newmtime))
896 else:
897 os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
898 newmtime = sstat[stat.ST_MTIME]
899 return newmtime
900
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800901def break_hardlinks(src, sstat = None):
902 """
903 Ensures src is the only hardlink to this file. Other hardlinks,
904 if any, are not affected (other than in their st_nlink value, of
905 course). Returns true on success and false on failure.
906
907 """
908 try:
909 if not sstat:
910 sstat = os.lstat(src)
911 except Exception as e:
912 logger.warning("break_hardlinks: stat of %s failed (%s)" % (src, e))
913 return False
914 if sstat[stat.ST_NLINK] == 1:
915 return True
916 return copyfile(src, src, sstat=sstat)
917
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500918def which(path, item, direction = 0, history = False, executable=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500919 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500920 Locate `item` in the list of paths `path` (colon separated string like $PATH).
921 If `direction` is non-zero then the list is reversed.
922 If `history` is True then the list of candidates also returned as result,history.
923 If `executable` is True then the candidate has to be an executable file,
924 otherwise the candidate simply has to exist.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500925 """
926
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500927 if executable:
928 is_candidate = lambda p: os.path.isfile(p) and os.access(p, os.X_OK)
929 else:
930 is_candidate = lambda p: os.path.exists(p)
931
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500932 hist = []
933 paths = (path or "").split(':')
934 if direction != 0:
935 paths.reverse()
936
937 for p in paths:
938 next = os.path.join(p, item)
939 hist.append(next)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500940 if is_candidate(next):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500941 if not os.path.isabs(next):
942 next = os.path.abspath(next)
943 if history:
944 return next, hist
945 return next
946
947 if history:
948 return "", hist
949 return ""
950
951def to_boolean(string, default=None):
952 if not string:
953 return default
954
955 normalized = string.lower()
956 if normalized in ("y", "yes", "1", "true"):
957 return True
958 elif normalized in ("n", "no", "0", "false"):
959 return False
960 else:
961 raise ValueError("Invalid value for to_boolean: %s" % string)
962
963def contains(variable, checkvalues, truevalue, falsevalue, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500964 """Check if a variable contains all the values specified.
965
966 Arguments:
967
968 variable -- the variable name. This will be fetched and expanded (using
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500969 d.getVar(variable)) and then split into a set().
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500970
971 checkvalues -- if this is a string it is split on whitespace into a set(),
972 otherwise coerced directly into a set().
973
974 truevalue -- the value to return if checkvalues is a subset of variable.
975
976 falsevalue -- the value to return if variable is empty or if checkvalues is
977 not a subset of variable.
978
979 d -- the data store.
980 """
981
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500982 val = d.getVar(variable)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500983 if not val:
984 return falsevalue
985 val = set(val.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600986 if isinstance(checkvalues, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500987 checkvalues = set(checkvalues.split())
988 else:
989 checkvalues = set(checkvalues)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500990 if checkvalues.issubset(val):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500991 return truevalue
992 return falsevalue
993
994def contains_any(variable, checkvalues, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500995 val = d.getVar(variable)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500996 if not val:
997 return falsevalue
998 val = set(val.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600999 if isinstance(checkvalues, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001000 checkvalues = set(checkvalues.split())
1001 else:
1002 checkvalues = set(checkvalues)
1003 if checkvalues & val:
1004 return truevalue
1005 return falsevalue
1006
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001007def filter(variable, checkvalues, d):
1008 """Return all words in the variable that are present in the checkvalues.
1009
1010 Arguments:
1011
1012 variable -- the variable name. This will be fetched and expanded (using
1013 d.getVar(variable)) and then split into a set().
1014
1015 checkvalues -- if this is a string it is split on whitespace into a set(),
1016 otherwise coerced directly into a set().
1017
1018 d -- the data store.
1019 """
1020
1021 val = d.getVar(variable)
1022 if not val:
1023 return ''
1024 val = set(val.split())
1025 if isinstance(checkvalues, str):
1026 checkvalues = set(checkvalues.split())
1027 else:
1028 checkvalues = set(checkvalues)
1029 return ' '.join(sorted(checkvalues & val))
1030
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001031def cpu_count():
1032 return multiprocessing.cpu_count()
1033
1034def nonblockingfd(fd):
1035 fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
1036
1037def process_profilelog(fn, pout = None):
1038 # Either call with a list of filenames and set pout or a filename and optionally pout.
1039 if not pout:
1040 pout = fn + '.processed'
1041 pout = open(pout, 'w')
1042
1043 import pstats
1044 if isinstance(fn, list):
1045 p = pstats.Stats(*fn, stream=pout)
1046 else:
1047 p = pstats.Stats(fn, stream=pout)
1048 p.sort_stats('time')
1049 p.print_stats()
1050 p.print_callers()
1051 p.sort_stats('cumulative')
1052 p.print_stats()
1053
1054 pout.flush()
1055 pout.close()
1056
1057#
1058# Was present to work around multiprocessing pool bugs in python < 2.7.3
1059#
1060def multiprocessingpool(*args, **kwargs):
1061
1062 import multiprocessing.pool
1063 #import multiprocessing.util
1064 #multiprocessing.util.log_to_stderr(10)
1065 # Deal with a multiprocessing bug where signals to the processes would be delayed until the work
1066 # completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed.
1067 def wrapper(func):
1068 def wrap(self, timeout=None):
1069 return func(self, timeout=timeout if timeout is not None else 1e100)
1070 return wrap
1071 multiprocessing.pool.IMapIterator.next = wrapper(multiprocessing.pool.IMapIterator.next)
1072
1073 return multiprocessing.Pool(*args, **kwargs)
1074
1075def exec_flat_python_func(func, *args, **kwargs):
1076 """Execute a flat python function (defined with def funcname(args):...)"""
1077 # Prepare a small piece of python code which calls the requested function
1078 # To do this we need to prepare two things - a set of variables we can use to pass
1079 # the values of arguments into the calling function, and the list of arguments for
1080 # the function being called
1081 context = {}
1082 funcargs = []
1083 # Handle unnamed arguments
1084 aidx = 1
1085 for arg in args:
1086 argname = 'arg_%s' % aidx
1087 context[argname] = arg
1088 funcargs.append(argname)
1089 aidx += 1
1090 # Handle keyword arguments
1091 context.update(kwargs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001092 funcargs.extend(['%s=%s' % (arg, arg) for arg in kwargs.keys()])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001093 code = 'retval = %s(%s)' % (func, ', '.join(funcargs))
1094 comp = bb.utils.better_compile(code, '<string>', '<string>')
1095 bb.utils.better_exec(comp, context, code, '<string>')
1096 return context['retval']
1097
1098def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
1099 """Edit lines from a recipe or config file and modify one or more
1100 specified variable values set in the file using a specified callback
1101 function. Lines are expected to have trailing newlines.
1102 Parameters:
1103 meta_lines: lines from the file; can be a list or an iterable
1104 (e.g. file pointer)
1105 variables: a list of variable names to look for. Functions
1106 may also be specified, but must be specified with '()' at
1107 the end of the name. Note that the function doesn't have
1108 any intrinsic understanding of _append, _prepend, _remove,
1109 or overrides, so these are considered as part of the name.
1110 These values go into a regular expression, so regular
1111 expression syntax is allowed.
1112 varfunc: callback function called for every variable matching
1113 one of the entries in the variables parameter. The function
1114 should take four arguments:
1115 varname: name of variable matched
1116 origvalue: current value in file
1117 op: the operator (e.g. '+=')
1118 newlines: list of lines up to this point. You can use
1119 this to prepend lines before this variable setting
1120 if you wish.
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001121 and should return a four-element tuple:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001122 newvalue: new value to substitute in, or None to drop
1123 the variable setting entirely. (If the removal
1124 results in two consecutive blank lines, one of the
1125 blank lines will also be dropped).
1126 newop: the operator to use - if you specify None here,
1127 the original operation will be used.
1128 indent: number of spaces to indent multi-line entries,
1129 or -1 to indent up to the level of the assignment
1130 and opening quote, or a string to use as the indent.
1131 minbreak: True to allow the first element of a
1132 multi-line value to continue on the same line as
1133 the assignment, False to indent before the first
1134 element.
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001135 To clarify, if you wish not to change the value, then you
1136 would return like this: return origvalue, None, 0, True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001137 match_overrides: True to match items with _overrides on the end,
1138 False otherwise
1139 Returns a tuple:
1140 updated:
1141 True if changes were made, False otherwise.
1142 newlines:
1143 Lines after processing
1144 """
1145
1146 var_res = {}
1147 if match_overrides:
Brad Bishop19323692019-04-05 15:28:33 -04001148 override_re = r'(_[a-zA-Z0-9-_$(){}]+)?'
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001149 else:
1150 override_re = ''
1151 for var in variables:
1152 if var.endswith('()'):
Brad Bishop19323692019-04-05 15:28:33 -04001153 var_res[var] = re.compile(r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001154 else:
Brad Bishop19323692019-04-05 15:28:33 -04001155 var_res[var] = re.compile(r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001156
1157 updated = False
1158 varset_start = ''
1159 varlines = []
1160 newlines = []
1161 in_var = None
1162 full_value = ''
1163 var_end = ''
1164
1165 def handle_var_end():
1166 prerun_newlines = newlines[:]
1167 op = varset_start[len(in_var):].strip()
1168 (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines)
1169 changed = (prerun_newlines != newlines)
1170
1171 if newvalue is None:
1172 # Drop the value
1173 return True
1174 elif newvalue != full_value or (newop not in [None, op]):
1175 if newop not in [None, op]:
1176 # Callback changed the operator
1177 varset_new = "%s %s" % (in_var, newop)
1178 else:
1179 varset_new = varset_start
1180
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001181 if isinstance(indent, int):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001182 if indent == -1:
1183 indentspc = ' ' * (len(varset_new) + 2)
1184 else:
1185 indentspc = ' ' * indent
1186 else:
1187 indentspc = indent
1188 if in_var.endswith('()'):
1189 # A function definition
1190 if isinstance(newvalue, list):
1191 newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue)))
1192 else:
1193 if not newvalue.startswith('\n'):
1194 newvalue = '\n' + newvalue
1195 if not newvalue.endswith('\n'):
1196 newvalue = newvalue + '\n'
1197 newlines.append('%s {%s}\n' % (varset_new, newvalue))
1198 else:
1199 # Normal variable
1200 if isinstance(newvalue, list):
1201 if not newvalue:
1202 # Empty list -> empty string
1203 newlines.append('%s ""\n' % varset_new)
1204 elif minbreak:
1205 # First item on first line
1206 if len(newvalue) == 1:
1207 newlines.append('%s "%s"\n' % (varset_new, newvalue[0]))
1208 else:
1209 newlines.append('%s "%s \\\n' % (varset_new, newvalue[0]))
1210 for item in newvalue[1:]:
1211 newlines.append('%s%s \\\n' % (indentspc, item))
1212 newlines.append('%s"\n' % indentspc)
1213 else:
1214 # No item on first line
1215 newlines.append('%s " \\\n' % varset_new)
1216 for item in newvalue:
1217 newlines.append('%s%s \\\n' % (indentspc, item))
1218 newlines.append('%s"\n' % indentspc)
1219 else:
1220 newlines.append('%s "%s"\n' % (varset_new, newvalue))
1221 return True
1222 else:
1223 # Put the old lines back where they were
1224 newlines.extend(varlines)
1225 # If newlines was touched by the function, we'll need to return True
1226 return changed
1227
1228 checkspc = False
1229
1230 for line in meta_lines:
1231 if in_var:
1232 value = line.rstrip()
1233 varlines.append(line)
1234 if in_var.endswith('()'):
1235 full_value += '\n' + value
1236 else:
1237 full_value += value[:-1]
1238 if value.endswith(var_end):
1239 if in_var.endswith('()'):
1240 if full_value.count('{') - full_value.count('}') >= 0:
1241 continue
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001242 full_value = full_value[:-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001243 if handle_var_end():
1244 updated = True
1245 checkspc = True
1246 in_var = None
1247 else:
1248 skip = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001249 for (varname, var_re) in var_res.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001250 res = var_re.match(line)
1251 if res:
1252 isfunc = varname.endswith('()')
1253 if isfunc:
1254 splitvalue = line.split('{', 1)
1255 var_end = '}'
1256 else:
1257 var_end = res.groups()[-1]
1258 splitvalue = line.split(var_end, 1)
1259 varset_start = splitvalue[0].rstrip()
1260 value = splitvalue[1].rstrip()
1261 if not isfunc and value.endswith('\\'):
1262 value = value[:-1]
1263 full_value = value
1264 varlines = [line]
1265 in_var = res.group(1)
1266 if isfunc:
1267 in_var += '()'
1268 if value.endswith(var_end):
1269 full_value = full_value[:-1]
1270 if handle_var_end():
1271 updated = True
1272 checkspc = True
1273 in_var = None
1274 skip = True
1275 break
1276 if not skip:
1277 if checkspc:
1278 checkspc = False
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001279 if newlines and newlines[-1] == '\n' and line == '\n':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001280 # Squash blank line if there are two consecutive blanks after a removal
1281 continue
1282 newlines.append(line)
1283 return (updated, newlines)
1284
1285
1286def edit_metadata_file(meta_file, variables, varfunc):
1287 """Edit a recipe or config file and modify one or more specified
1288 variable values set in the file using a specified callback function.
1289 The file is only written to if the value(s) actually change.
1290 This is basically the file version of edit_metadata(), see that
1291 function's description for parameter/usage information.
1292 Returns True if the file was written to, False otherwise.
1293 """
1294 with open(meta_file, 'r') as f:
1295 (updated, newlines) = edit_metadata(f, variables, varfunc)
1296 if updated:
1297 with open(meta_file, 'w') as f:
1298 f.writelines(newlines)
1299 return updated
1300
1301
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001302def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001303 """Edit bblayers.conf, adding and/or removing layers
1304 Parameters:
1305 bblayers_conf: path to bblayers.conf file to edit
1306 add: layer path (or list of layer paths) to add; None or empty
1307 list to add nothing
1308 remove: layer path (or list of layer paths) to remove; None or
1309 empty list to remove nothing
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001310 edit_cb: optional callback function that will be called after
1311 processing adds/removes once per existing entry.
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001312 Returns a tuple:
1313 notadded: list of layers specified to be added but weren't
1314 (because they were already in the list)
1315 notremoved: list of layers that were specified to be removed
1316 but weren't (because they weren't in the list)
1317 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001318
1319 import fnmatch
1320
1321 def remove_trailing_sep(pth):
1322 if pth and pth[-1] == os.sep:
1323 pth = pth[:-1]
1324 return pth
1325
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001326 approved = bb.utils.approved_variables()
1327 def canonicalise_path(pth):
1328 pth = remove_trailing_sep(pth)
1329 if 'HOME' in approved and '~' in pth:
1330 pth = os.path.expanduser(pth)
1331 return pth
1332
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001333 def layerlist_param(value):
1334 if not value:
1335 return []
1336 elif isinstance(value, list):
1337 return [remove_trailing_sep(x) for x in value]
1338 else:
1339 return [remove_trailing_sep(value)]
1340
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001341 addlayers = layerlist_param(add)
1342 removelayers = layerlist_param(remove)
1343
1344 # Need to use a list here because we can't set non-local variables from a callback in python 2.x
1345 bblayercalls = []
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001346 removed = []
1347 plusequals = False
1348 orig_bblayers = []
1349
1350 def handle_bblayers_firstpass(varname, origvalue, op, newlines):
1351 bblayercalls.append(op)
1352 if op == '=':
1353 del orig_bblayers[:]
1354 orig_bblayers.extend([canonicalise_path(x) for x in origvalue.split()])
1355 return (origvalue, None, 2, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001356
1357 def handle_bblayers(varname, origvalue, op, newlines):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001358 updated = False
1359 bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
1360 if removelayers:
1361 for removelayer in removelayers:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001362 for layer in bblayers:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001363 if fnmatch.fnmatch(canonicalise_path(layer), canonicalise_path(removelayer)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001364 updated = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001365 bblayers.remove(layer)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001366 removed.append(removelayer)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001367 break
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001368 if addlayers and not plusequals:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001369 for addlayer in addlayers:
1370 if addlayer not in bblayers:
1371 updated = True
1372 bblayers.append(addlayer)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001373 del addlayers[:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001374
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001375 if edit_cb:
1376 newlist = []
1377 for layer in bblayers:
1378 res = edit_cb(layer, canonicalise_path(layer))
1379 if res != layer:
1380 newlist.append(res)
1381 updated = True
1382 else:
1383 newlist.append(layer)
1384 bblayers = newlist
1385
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001386 if updated:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001387 if op == '+=' and not bblayers:
1388 bblayers = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001389 return (bblayers, None, 2, False)
1390 else:
1391 return (origvalue, None, 2, False)
1392
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001393 with open(bblayers_conf, 'r') as f:
1394 (_, newlines) = edit_metadata(f, ['BBLAYERS'], handle_bblayers_firstpass)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001395
1396 if not bblayercalls:
1397 raise Exception('Unable to find BBLAYERS in %s' % bblayers_conf)
1398
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001399 # Try to do the "smart" thing depending on how the user has laid out
1400 # their bblayers.conf file
1401 if bblayercalls.count('+=') > 1:
1402 plusequals = True
1403
1404 removelayers_canon = [canonicalise_path(layer) for layer in removelayers]
1405 notadded = []
1406 for layer in addlayers:
1407 layer_canon = canonicalise_path(layer)
1408 if layer_canon in orig_bblayers and not layer_canon in removelayers_canon:
1409 notadded.append(layer)
1410 notadded_canon = [canonicalise_path(layer) for layer in notadded]
1411 addlayers[:] = [layer for layer in addlayers if canonicalise_path(layer) not in notadded_canon]
1412
1413 (updated, newlines) = edit_metadata(newlines, ['BBLAYERS'], handle_bblayers)
1414 if addlayers:
1415 # Still need to add these
1416 for addlayer in addlayers:
1417 newlines.append('BBLAYERS += "%s"\n' % addlayer)
1418 updated = True
1419
1420 if updated:
1421 with open(bblayers_conf, 'w') as f:
1422 f.writelines(newlines)
1423
1424 notremoved = list(set(removelayers) - set(removed))
1425
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001426 return (notadded, notremoved)
1427
1428
1429def get_file_layer(filename, d):
1430 """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001431 collections = (d.getVar('BBFILE_COLLECTIONS') or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001432 collection_res = {}
1433 for collection in collections:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001434 collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection) or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001435
1436 def path_to_layer(path):
1437 # Use longest path so we handle nested layers
1438 matchlen = 0
1439 match = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001440 for collection, regex in collection_res.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001441 if len(regex) > matchlen and re.match(regex, path):
1442 matchlen = len(regex)
1443 match = collection
1444 return match
1445
1446 result = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001447 bbfiles = (d.getVar('BBFILES') or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001448 bbfilesmatch = False
1449 for bbfilesentry in bbfiles:
1450 if fnmatch.fnmatch(filename, bbfilesentry):
1451 bbfilesmatch = True
1452 result = path_to_layer(bbfilesentry)
1453
1454 if not bbfilesmatch:
1455 # Probably a bbclass
1456 result = path_to_layer(filename)
1457
1458 return result
1459
1460
1461# Constant taken from http://linux.die.net/include/linux/prctl.h
1462PR_SET_PDEATHSIG = 1
1463
1464class PrCtlError(Exception):
1465 pass
1466
1467def signal_on_parent_exit(signame):
1468 """
1469 Trigger signame to be sent when the parent process dies
1470 """
1471 signum = getattr(signal, signame)
1472 # http://linux.die.net/man/2/prctl
1473 result = cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG, signum)
1474 if result != 0:
1475 raise PrCtlError('prctl failed with error code %s' % result)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001476
1477#
1478# Manually call the ioprio syscall. We could depend on other libs like psutil
1479# however this gets us enough of what we need to bitbake for now without the
1480# dependency
1481#
1482_unamearch = os.uname()[4]
1483IOPRIO_WHO_PROCESS = 1
1484IOPRIO_CLASS_SHIFT = 13
1485
1486def ioprio_set(who, cls, value):
1487 NR_ioprio_set = None
1488 if _unamearch == "x86_64":
1489 NR_ioprio_set = 251
1490 elif _unamearch[0] == "i" and _unamearch[2:3] == "86":
1491 NR_ioprio_set = 289
Brad Bishop19323692019-04-05 15:28:33 -04001492 elif _unamearch == "aarch64":
1493 NR_ioprio_set = 30
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001494
1495 if NR_ioprio_set:
1496 ioprio = value | (cls << IOPRIO_CLASS_SHIFT)
1497 rc = cdll['libc.so.6'].syscall(NR_ioprio_set, IOPRIO_WHO_PROCESS, who, ioprio)
1498 if rc != 0:
1499 raise ValueError("Unable to set ioprio, syscall returned %s" % rc)
1500 else:
1501 bb.warn("Unable to set IO Prio for arch %s" % _unamearch)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001502
1503def set_process_name(name):
1504 from ctypes import cdll, byref, create_string_buffer
1505 # This is nice to have for debugging, not essential
1506 try:
1507 libc = cdll.LoadLibrary('libc.so.6')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001508 buf = create_string_buffer(bytes(name, 'utf-8'))
1509 libc.prctl(15, byref(buf), 0, 0, 0)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001510 except:
1511 pass
1512
1513# export common proxies variables from datastore to environment
1514def export_proxies(d):
1515 import os
1516
1517 variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001518 'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
1519 'GIT_PROXY_COMMAND']
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001520 exported = False
1521
1522 for v in variables:
1523 if v in os.environ.keys():
1524 exported = True
1525 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001526 v_proxy = d.getVar(v)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001527 if v_proxy is not None:
1528 os.environ[v] = v_proxy
1529 exported = True
1530
1531 return exported
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001532
1533
1534def load_plugins(logger, plugins, pluginpath):
1535 def load_plugin(name):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001536 logger.debug(1, 'Loading plugin %s' % name)
Brad Bishop19323692019-04-05 15:28:33 -04001537 spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
1538 if spec:
1539 return spec.loader.load_module()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001540
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001541 logger.debug(1, 'Loading plugins from %s...' % pluginpath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001542
1543 expanded = (glob.glob(os.path.join(pluginpath, '*' + ext))
1544 for ext in python_extensions)
1545 files = itertools.chain.from_iterable(expanded)
1546 names = set(os.path.splitext(os.path.basename(fn))[0] for fn in files)
1547 for name in names:
1548 if name != '__init__':
1549 plugin = load_plugin(name)
1550 if hasattr(plugin, 'plugin_init'):
1551 obj = plugin.plugin_init(plugins)
1552 plugins.append(obj or plugin)
1553 else:
1554 plugins.append(plugin)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001555
1556
1557class LogCatcher(logging.Handler):
1558 """Logging handler for collecting logged messages so you can check them later"""
1559 def __init__(self):
1560 self.messages = []
1561 logging.Handler.__init__(self, logging.WARNING)
1562 def emit(self, record):
1563 self.messages.append(bb.build.logformatter.format(record))
1564 def contains(self, message):
1565 return (message in self.messages)