blob: f73d31fb73c53e34f55b08848ec3095897e35e45 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake Utility Functions
3"""
4
5# Copyright (C) 2004 Michael Lauer
6#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
10import re, fcntl, os, string, stat, shutil, time
11import sys
12import errno
13import logging
14import bb
15import bb.msg
16import multiprocessing
17import fcntl
Brad Bishop19323692019-04-05 15:28:33 -040018import importlib
19from importlib import machinery
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020import itertools
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021import subprocess
22import glob
23import fnmatch
24import traceback
25import errno
26import signal
Patrick Williamsc0f7c042017-02-23 20:41:17 -060027import collections
28import copy
29from subprocess import getstatusoutput
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030from contextlib import contextmanager
31from ctypes import cdll
32
Patrick Williamsc124f4f2015-09-15 14:41:29 -050033logger = logging.getLogger("BitBake.Util")
Brad Bishop19323692019-04-05 15:28:33 -040034python_extensions = importlib.machinery.all_suffixes()
Patrick Williamsc0f7c042017-02-23 20:41:17 -060035
Patrick Williamsc124f4f2015-09-15 14:41:29 -050036
37def clean_context():
38 return {
39 "os": os,
40 "bb": bb,
41 "time": time,
42 }
43
44def get_context():
45 return _context
46
47
48def set_context(ctx):
49 _context = ctx
50
51# Context used in better_exec, eval
52_context = clean_context()
53
54class VersionStringException(Exception):
55 """Exception raised when an invalid version specification is found"""
56
57def explode_version(s):
58 r = []
Brad Bishop19323692019-04-05 15:28:33 -040059 alpha_regexp = re.compile(r'^([a-zA-Z]+)(.*)$')
60 numeric_regexp = re.compile(r'^(\d+)(.*)$')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061 while (s != ''):
62 if s[0] in string.digits:
63 m = numeric_regexp.match(s)
64 r.append((0, int(m.group(1))))
65 s = m.group(2)
66 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -060067 if s[0] in string.ascii_letters:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050068 m = alpha_regexp.match(s)
69 r.append((1, m.group(1)))
70 s = m.group(2)
71 continue
72 if s[0] == '~':
73 r.append((-1, s[0]))
74 else:
75 r.append((2, s[0]))
76 s = s[1:]
77 return r
78
79def split_version(s):
80 """Split a version string into its constituent parts (PE, PV, PR)"""
81 s = s.strip(" <>=")
82 e = 0
83 if s.count(':'):
84 e = int(s.split(":")[0])
85 s = s.split(":")[1]
86 r = ""
87 if s.count('-'):
88 r = s.rsplit("-", 1)[1]
89 s = s.rsplit("-", 1)[0]
90 v = s
91 return (e, v, r)
92
93def vercmp_part(a, b):
94 va = explode_version(a)
95 vb = explode_version(b)
96 while True:
97 if va == []:
98 (oa, ca) = (0, None)
99 else:
100 (oa, ca) = va.pop(0)
101 if vb == []:
102 (ob, cb) = (0, None)
103 else:
104 (ob, cb) = vb.pop(0)
105 if (oa, ca) == (0, None) and (ob, cb) == (0, None):
106 return 0
107 if oa < ob:
108 return -1
109 elif oa > ob:
110 return 1
Brad Bishop19323692019-04-05 15:28:33 -0400111 elif ca is None:
112 return -1
113 elif cb is None:
114 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 elif ca < cb:
116 return -1
117 elif ca > cb:
118 return 1
119
120def vercmp(ta, tb):
121 (ea, va, ra) = ta
122 (eb, vb, rb) = tb
123
124 r = int(ea or 0) - int(eb or 0)
125 if (r == 0):
126 r = vercmp_part(va, vb)
127 if (r == 0):
128 r = vercmp_part(ra, rb)
129 return r
130
131def vercmp_string(a, b):
132 ta = split_version(a)
133 tb = split_version(b)
134 return vercmp(ta, tb)
135
136def vercmp_string_op(a, b, op):
137 """
138 Compare two versions and check if the specified comparison operator matches the result of the comparison.
139 This function is fairly liberal about what operators it will accept since there are a variety of styles
140 depending on the context.
141 """
142 res = vercmp_string(a, b)
143 if op in ('=', '=='):
144 return res == 0
145 elif op == '<=':
146 return res <= 0
147 elif op == '>=':
148 return res >= 0
149 elif op in ('>', '>>'):
150 return res > 0
151 elif op in ('<', '<<'):
152 return res < 0
153 elif op == '!=':
154 return res != 0
155 else:
156 raise VersionStringException('Unsupported comparison operator "%s"' % op)
157
158def explode_deps(s):
159 """
160 Take an RDEPENDS style string of format:
161 "DEPEND1 (optional version) DEPEND2 (optional version) ..."
162 and return a list of dependencies.
163 Version information is ignored.
164 """
165 r = []
166 l = s.split()
167 flag = False
168 for i in l:
169 if i[0] == '(':
170 flag = True
171 #j = []
172 if not flag:
173 r.append(i)
174 #else:
175 # j.append(i)
176 if flag and i.endswith(')'):
177 flag = False
178 # Ignore version
179 #r[-1] += ' ' + ' '.join(j)
180 return r
181
Brad Bishop316dfdd2018-06-25 12:45:53 -0400182def explode_dep_versions2(s, *, sort=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183 """
184 Take an RDEPENDS style string of format:
185 "DEPEND1 (optional version) DEPEND2 (optional version) ..."
186 and return a dictionary of dependencies and versions.
187 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600188 r = collections.OrderedDict()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500189 l = s.replace(",", "").split()
190 lastdep = None
191 lastcmp = ""
192 lastver = ""
193 incmp = False
194 inversion = False
195 for i in l:
196 if i[0] == '(':
197 incmp = True
198 i = i[1:].strip()
199 if not i:
200 continue
201
202 if incmp:
203 incmp = False
204 inversion = True
205 # This list is based on behavior and supported comparisons from deb, opkg and rpm.
206 #
207 # Even though =<, <<, ==, !=, =>, and >> may not be supported,
208 # we list each possibly valid item.
209 # The build system is responsible for validation of what it supports.
210 if i.startswith(('<=', '=<', '<<', '==', '!=', '>=', '=>', '>>')):
211 lastcmp = i[0:2]
212 i = i[2:]
213 elif i.startswith(('<', '>', '=')):
214 lastcmp = i[0:1]
215 i = i[1:]
216 else:
217 # This is an unsupported case!
218 raise VersionStringException('Invalid version specification in "(%s" - invalid or missing operator' % i)
219 lastcmp = (i or "")
220 i = ""
221 i.strip()
222 if not i:
223 continue
224
225 if inversion:
226 if i.endswith(')'):
227 i = i[:-1] or ""
228 inversion = False
229 if lastver and i:
230 lastver += " "
231 if i:
232 lastver += i
233 if lastdep not in r:
234 r[lastdep] = []
235 r[lastdep].append(lastcmp + " " + lastver)
236 continue
237
238 #if not inversion:
239 lastdep = i
240 lastver = ""
241 lastcmp = ""
242 if not (i in r and r[i]):
243 r[lastdep] = []
244
Brad Bishop316dfdd2018-06-25 12:45:53 -0400245 if sort:
246 r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247 return r
248
249def explode_dep_versions(s):
250 r = explode_dep_versions2(s)
251 for d in r:
252 if not r[d]:
253 r[d] = None
254 continue
255 if len(r[d]) > 1:
256 bb.warn("explode_dep_versions(): Item %s appeared in dependency string '%s' multiple times with different values. explode_dep_versions cannot cope with this." % (d, s))
257 r[d] = r[d][0]
258 return r
259
260def join_deps(deps, commasep=True):
261 """
262 Take the result from explode_dep_versions and generate a dependency string
263 """
264 result = []
265 for dep in deps:
266 if deps[dep]:
267 if isinstance(deps[dep], list):
268 for v in deps[dep]:
269 result.append(dep + " (" + v + ")")
270 else:
271 result.append(dep + " (" + deps[dep] + ")")
272 else:
273 result.append(dep)
274 if commasep:
275 return ", ".join(result)
276 else:
277 return " ".join(result)
278
279def _print_trace(body, line):
280 """
281 Print the Environment of a Text Body
282 """
283 error = []
284 # print the environment of the method
285 min_line = max(1, line-4)
286 max_line = min(line + 4, len(body))
287 for i in range(min_line, max_line + 1):
288 if line == i:
289 error.append(' *** %.4d:%s' % (i, body[i-1].rstrip()))
290 else:
291 error.append(' %.4d:%s' % (i, body[i-1].rstrip()))
292 return error
293
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500294def better_compile(text, file, realfile, mode = "exec", lineno = 0):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 """
296 A better compile method. This method
297 will print the offending lines.
298 """
299 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500300 cache = bb.methodpool.compile_cache(text)
301 if cache:
302 return cache
303 # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
304 text2 = "\n" * int(lineno) + text
305 code = compile(text2, realfile, mode)
306 bb.methodpool.compile_cache_add(text, code)
307 return code
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308 except Exception as e:
309 error = []
310 # split the text into lines again
311 body = text.split('\n')
Brad Bishop19323692019-04-05 15:28:33 -0400312 error.append("Error in compiling python function in %s, line %s:\n" % (realfile, e.lineno))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500313 if hasattr(e, "lineno"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 error.append("The code lines resulting in this error were:")
Brad Bishop19323692019-04-05 15:28:33 -0400315 # e.lineno: line's position in reaflile
316 # lineno: function name's "position -1" in realfile
317 # e.lineno - lineno: line's relative position in function
318 error.extend(_print_trace(body, e.lineno - lineno))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319 else:
320 error.append("The function causing this error was:")
321 for line in body:
322 error.append(line)
323 error.append("%s: %s" % (e.__class__.__name__, str(e)))
324
325 logger.error("\n".join(error))
326
327 e = bb.BBHandledException(e)
328 raise e
329
330def _print_exception(t, value, tb, realfile, text, context):
331 error = []
332 try:
333 exception = traceback.format_exception_only(t, value)
334 error.append('Error executing a python function in %s:\n' % realfile)
335
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500336 # Strip 'us' from the stack (better_exec call) unless that was where the
337 # error came from
338 if tb.tb_next is not None:
339 tb = tb.tb_next
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500340
341 textarray = text.split('\n')
342
343 linefailed = tb.tb_lineno
344
345 tbextract = traceback.extract_tb(tb)
346 tbformat = traceback.format_list(tbextract)
347 error.append("The stack trace of python calls that resulted in this exception/failure was:")
348 error.append("File: '%s', lineno: %s, function: %s" % (tbextract[0][0], tbextract[0][1], tbextract[0][2]))
349 error.extend(_print_trace(textarray, linefailed))
350
351 # See if this is a function we constructed and has calls back into other functions in
352 # "text". If so, try and improve the context of the error by diving down the trace
353 level = 0
354 nexttb = tb.tb_next
355 while nexttb is not None and (level+1) < len(tbextract):
356 error.append("File: '%s', lineno: %s, function: %s" % (tbextract[level+1][0], tbextract[level+1][1], tbextract[level+1][2]))
357 if tbextract[level][0] == tbextract[level+1][0] and tbextract[level+1][2] == tbextract[level][0]:
358 # The code was possibly in the string we compiled ourselves
359 error.extend(_print_trace(textarray, tbextract[level+1][1]))
360 elif tbextract[level+1][0].startswith("/"):
361 # The code looks like it might be in a file, try and load it
362 try:
363 with open(tbextract[level+1][0], "r") as f:
364 text = f.readlines()
365 error.extend(_print_trace(text, tbextract[level+1][1]))
366 except:
367 error.append(tbformat[level+1])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500368 else:
369 error.append(tbformat[level+1])
370 nexttb = tb.tb_next
371 level = level + 1
372
373 error.append("Exception: %s" % ''.join(exception))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600374
375 # If the exception is from spwaning a task, let's be helpful and display
376 # the output (which hopefully includes stderr).
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500377 if isinstance(value, subprocess.CalledProcessError) and value.output:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600378 error.append("Subprocess output:")
379 error.append(value.output.decode("utf-8", errors="ignore"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500380 finally:
381 logger.error("\n".join(error))
382
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500383def better_exec(code, context, text = None, realfile = "<code>", pythonexception=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384 """
385 Similiar to better_compile, better_exec will
386 print the lines that are responsible for the
387 error.
388 """
389 import bb.parse
390 if not text:
391 text = code
392 if not hasattr(code, "co_filename"):
393 code = better_compile(code, realfile, realfile)
394 try:
395 exec(code, get_context(), context)
Brad Bishop08902b02019-08-20 09:16:51 -0400396 except (bb.BBHandledException, bb.parse.SkipRecipe, bb.data_smart.ExpansionError):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500397 # Error already shown so passthrough, no need for traceback
398 raise
399 except Exception as e:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500400 if pythonexception:
401 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 (t, value, tb) = sys.exc_info()
403 try:
404 _print_exception(t, value, tb, realfile, text, context)
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500405 except Exception as e2:
406 logger.error("Exception handler error: %s" % str(e2))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500407
408 e = bb.BBHandledException(e)
409 raise e
410
411def simple_exec(code, context):
412 exec(code, get_context(), context)
413
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600414def better_eval(source, locals, extraglobals = None):
415 ctx = get_context()
416 if extraglobals:
417 ctx = copy.copy(ctx)
418 for g in extraglobals:
419 ctx[g] = extraglobals[g]
420 return eval(source, ctx, locals)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500421
422@contextmanager
423def fileslocked(files):
424 """Context manager for locking and unlocking file locks."""
425 locks = []
426 if files:
427 for lockfile in files:
428 locks.append(bb.utils.lockfile(lockfile))
429
Andrew Geissler82c905d2020-04-13 13:39:40 -0500430 try:
431 yield
432 finally:
433 for lock in locks:
434 bb.utils.unlockfile(lock)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436def lockfile(name, shared=False, retry=True, block=False):
437 """
438 Use the specified file as a lock file, return when the lock has
439 been acquired. Returns a variable to pass to unlockfile().
440 Parameters:
441 retry: True to re-try locking if it fails, False otherwise
442 block: True to block until the lock succeeds, False otherwise
443 The retry and block parameters are kind of equivalent unless you
444 consider the possibility of sending a signal to the process to break
445 out - at which point you want block=True rather than retry=True.
446 """
447 dirname = os.path.dirname(name)
448 mkdirhier(dirname)
449
450 if not os.access(dirname, os.W_OK):
451 logger.error("Unable to acquire lock '%s', directory is not writable",
452 name)
453 sys.exit(1)
454
455 op = fcntl.LOCK_EX
456 if shared:
457 op = fcntl.LOCK_SH
458 if not retry and not block:
459 op = op | fcntl.LOCK_NB
460
461 while True:
462 # If we leave the lockfiles lying around there is no problem
463 # but we should clean up after ourselves. This gives potential
464 # for races though. To work around this, when we acquire the lock
465 # we check the file we locked was still the lock file on disk.
466 # by comparing inode numbers. If they don't match or the lockfile
467 # no longer exists, we start again.
468
469 # This implementation is unfair since the last person to request the
470 # lock is the most likely to win it.
471
472 try:
473 lf = open(name, 'a+')
474 fileno = lf.fileno()
475 fcntl.flock(fileno, op)
476 statinfo = os.fstat(fileno)
477 if os.path.exists(lf.name):
478 statinfo2 = os.stat(lf.name)
479 if statinfo.st_ino == statinfo2.st_ino:
480 return lf
481 lf.close()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800482 except OSError as e:
483 if e.errno == errno.EACCES:
484 logger.error("Unable to acquire lock '%s', %s",
485 e.strerror, name)
486 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500487 try:
488 lf.close()
489 except Exception:
490 pass
491 pass
492 if not retry:
493 return None
494
495def unlockfile(lf):
496 """
497 Unlock a file locked using lockfile()
498 """
499 try:
500 # If we had a shared lock, we need to promote to exclusive before
501 # removing the lockfile. Attempt this, ignore failures.
502 fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
503 os.unlink(lf.name)
504 except (IOError, OSError):
505 pass
506 fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
507 lf.close()
508
Brad Bishop6dbb3162019-11-25 09:41:34 -0500509def _hasher(method, filename):
510 import mmap
511
512 with open(filename, "rb") as f:
513 try:
514 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
515 for chunk in iter(lambda: mm.read(8192), b''):
516 method.update(chunk)
517 except ValueError:
518 # You can't mmap() an empty file so silence this exception
519 pass
520 return method.hexdigest()
521
522
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500523def md5_file(filename):
524 """
525 Return the hex string representation of the MD5 checksum of filename.
526 """
Brad Bishop6dbb3162019-11-25 09:41:34 -0500527 import hashlib
528 return _hasher(hashlib.md5(), filename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500529
530def sha256_file(filename):
531 """
532 Return the hex string representation of the 256-bit SHA checksum of
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500533 filename.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500534 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500535 import hashlib
Brad Bishop6dbb3162019-11-25 09:41:34 -0500536 return _hasher(hashlib.sha256(), filename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500537
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500538def sha1_file(filename):
539 """
540 Return the hex string representation of the SHA1 checksum of the filename
541 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500542 import hashlib
Brad Bishop6dbb3162019-11-25 09:41:34 -0500543 return _hasher(hashlib.sha1(), filename)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500544
Andrew Geissler82c905d2020-04-13 13:39:40 -0500545def sha384_file(filename):
546 """
547 Return the hex string representation of the SHA384 checksum of the filename
548 """
549 import hashlib
550 return _hasher(hashlib.sha384(), filename)
551
552def sha512_file(filename):
553 """
554 Return the hex string representation of the SHA512 checksum of the filename
555 """
556 import hashlib
557 return _hasher(hashlib.sha512(), filename)
558
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500559def preserved_envvars_exported():
560 """Variables which are taken from the environment and placed in and exported
561 from the metadata"""
562 return [
563 'BB_TASKHASH',
564 'HOME',
565 'LOGNAME',
566 'PATH',
567 'PWD',
568 'SHELL',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500569 'USER',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600570 'LC_ALL',
571 'BBSERVER',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500572 ]
573
574def preserved_envvars():
575 """Variables which are taken from the environment and placed in the metadata"""
576 v = [
577 'BBPATH',
578 'BB_PRESERVE_ENV',
579 'BB_ENV_WHITELIST',
580 'BB_ENV_EXTRAWHITE',
581 ]
582 return v + preserved_envvars_exported()
583
584def filter_environment(good_vars):
585 """
586 Create a pristine environment for bitbake. This will remove variables that
587 are not known and may influence the build in a negative way.
588 """
589
590 removed_vars = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600591 for key in list(os.environ):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500592 if key in good_vars:
593 continue
594
595 removed_vars[key] = os.environ[key]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500596 del os.environ[key]
597
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600598 # If we spawn a python process, we need to have a UTF-8 locale, else python's file
599 # access methods will use ascii. You can't change that mode once the interpreter is
600 # started so we have to ensure a locale is set. Ideally we'd use C.UTF-8 but not all
601 # distros support that and we need to set something.
602 os.environ["LC_ALL"] = "en_US.UTF-8"
603
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500604 if removed_vars:
605 logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
606
607 return removed_vars
608
609def approved_variables():
610 """
611 Determine and return the list of whitelisted variables which are approved
612 to remain in the environment.
613 """
614 if 'BB_PRESERVE_ENV' in os.environ:
615 return os.environ.keys()
616 approved = []
617 if 'BB_ENV_WHITELIST' in os.environ:
618 approved = os.environ['BB_ENV_WHITELIST'].split()
619 approved.extend(['BB_ENV_WHITELIST'])
620 else:
621 approved = preserved_envvars()
622 if 'BB_ENV_EXTRAWHITE' in os.environ:
623 approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split())
624 if 'BB_ENV_EXTRAWHITE' not in approved:
625 approved.extend(['BB_ENV_EXTRAWHITE'])
626 return approved
627
628def clean_environment():
629 """
630 Clean up any spurious environment variables. This will remove any
631 variables the user hasn't chosen to preserve.
632 """
633 if 'BB_PRESERVE_ENV' not in os.environ:
634 good_vars = approved_variables()
635 return filter_environment(good_vars)
636
637 return {}
638
639def empty_environment():
640 """
641 Remove all variables from the environment.
642 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600643 for s in list(os.environ.keys()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500644 os.unsetenv(s)
645 del os.environ[s]
646
647def build_environment(d):
648 """
649 Build an environment from all exported variables.
650 """
651 import bb.data
652 for var in bb.data.keys(d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500653 export = d.getVarFlag(var, "export", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500654 if export:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500655 os.environ[var] = d.getVar(var) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500656
657def _check_unsafe_delete_path(path):
658 """
659 Basic safeguard against recursively deleting something we shouldn't. If it returns True,
660 the caller should raise an exception with an appropriate message.
661 NOTE: This is NOT meant to be a security mechanism - just a guard against silly mistakes
662 with potentially disastrous results.
663 """
664 extra = ''
665 # HOME might not be /home/something, so in case we can get it, check against it
666 homedir = os.environ.get('HOME', '')
667 if homedir:
668 extra = '|%s' % homedir
669 if re.match('(/|//|/home|/home/[^/]*%s)$' % extra, os.path.abspath(path)):
670 return True
671 return False
672
Brad Bishopa34c0302019-09-23 22:34:48 -0400673def remove(path, recurse=False, ionice=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500674 """Equivalent to rm -f or rm -rf"""
675 if not path:
676 return
677 if recurse:
678 for name in glob.glob(path):
679 if _check_unsafe_delete_path(path):
680 raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
681 # shutil.rmtree(name) would be ideal but its too slow
Brad Bishopa34c0302019-09-23 22:34:48 -0400682 cmd = []
683 if ionice:
684 cmd = ['ionice', '-c', '3']
685 subprocess.check_call(cmd + ['rm', '-rf'] + glob.glob(path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500686 return
687 for name in glob.glob(path):
688 try:
689 os.unlink(name)
690 except OSError as exc:
691 if exc.errno != errno.ENOENT:
692 raise
693
Brad Bishopa34c0302019-09-23 22:34:48 -0400694def prunedir(topdir, ionice=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500695 # Delete everything reachable from the directory named in 'topdir'.
696 # CAUTION: This is dangerous!
697 if _check_unsafe_delete_path(topdir):
698 raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
Brad Bishopa34c0302019-09-23 22:34:48 -0400699 remove(topdir, recurse=True, ionice=ionice)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500700
701#
702# Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var)
703# but thats possibly insane and suffixes is probably going to be small
704#
705def prune_suffix(var, suffixes, d):
706 # See if var ends with any of the suffixes listed and
707 # remove it if found
708 for suffix in suffixes:
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400709 if suffix and var.endswith(suffix):
710 return var[:-len(suffix)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500711 return var
712
713def mkdirhier(directory):
714 """Create a directory like 'mkdir -p', but does not complain if
715 directory already exists like os.makedirs
716 """
717
718 try:
719 os.makedirs(directory)
720 except OSError as e:
Brad Bishopc342db32019-05-15 21:57:59 -0400721 if e.errno != errno.EEXIST or not os.path.isdir(directory):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500722 raise e
723
724def movefile(src, dest, newmtime = None, sstat = None):
725 """Moves a file from src to dest, preserving all permissions and
726 attributes; mtime will be preserved even when moving across
727 filesystems. Returns true on success and false on failure. Move is
728 atomic.
729 """
730
731 #print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
732 try:
733 if not sstat:
734 sstat = os.lstat(src)
735 except Exception as e:
736 print("movefile: Stating source file failed...", e)
737 return None
738
739 destexists = 1
740 try:
741 dstat = os.lstat(dest)
742 except:
743 dstat = os.lstat(os.path.dirname(dest))
744 destexists = 0
745
746 if destexists:
747 if stat.S_ISLNK(dstat[stat.ST_MODE]):
748 try:
749 os.unlink(dest)
750 destexists = 0
751 except Exception as e:
752 pass
753
754 if stat.S_ISLNK(sstat[stat.ST_MODE]):
755 try:
756 target = os.readlink(src)
757 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
758 os.unlink(dest)
759 os.symlink(target, dest)
760 #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
761 os.unlink(src)
762 return os.lstat(dest)
763 except Exception as e:
764 print("movefile: failed to properly create symlink:", dest, "->", target, e)
765 return None
766
767 renamefailed = 1
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500768 # os.rename needs to know the dest path ending with file name
769 # so append the file name to a path only if it's a dir specified
770 srcfname = os.path.basename(src)
771 destpath = os.path.join(dest, srcfname) if os.path.isdir(dest) \
772 else dest
773
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500774 if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
775 try:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776 os.rename(src, destpath)
777 renamefailed = 0
778 except Exception as e:
Brad Bishop79641f22019-09-10 07:20:22 -0400779 if e.errno != errno.EXDEV:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500780 # Some random error.
781 print("movefile: Failed to move", src, "to", dest, e)
782 return None
783 # Invalid cross-device-link 'bind' mounted or actually Cross-Device
784
785 if renamefailed:
786 didcopy = 0
787 if stat.S_ISREG(sstat[stat.ST_MODE]):
788 try: # For safety copy then move it over.
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500789 shutil.copyfile(src, destpath + "#new")
790 os.rename(destpath + "#new", destpath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500791 didcopy = 1
792 except Exception as e:
793 print('movefile: copy', src, '->', dest, 'failed.', e)
794 return None
795 else:
796 #we don't yet handle special, so we need to fall back to /bin/mv
797 a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'")
798 if a[0] != 0:
799 print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
800 return None # failure
801 try:
802 if didcopy:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400803 os.lchown(destpath, sstat[stat.ST_UID], sstat[stat.ST_GID])
804 os.chmod(destpath, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500805 os.unlink(src)
806 except Exception as e:
807 print("movefile: Failed to chown/chmod/unlink", dest, e)
808 return None
809
810 if newmtime:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500811 os.utime(destpath, (newmtime, newmtime))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500812 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500813 os.utime(destpath, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500814 newmtime = sstat[stat.ST_MTIME]
815 return newmtime
816
817def copyfile(src, dest, newmtime = None, sstat = None):
818 """
819 Copies a file from src to dest, preserving all permissions and
820 attributes; mtime will be preserved even when moving across
821 filesystems. Returns true on success and false on failure.
822 """
823 #print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
824 try:
825 if not sstat:
826 sstat = os.lstat(src)
827 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600828 logger.warning("copyfile: stat of %s failed (%s)" % (src, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500829 return False
830
831 destexists = 1
832 try:
833 dstat = os.lstat(dest)
834 except:
835 dstat = os.lstat(os.path.dirname(dest))
836 destexists = 0
837
838 if destexists:
839 if stat.S_ISLNK(dstat[stat.ST_MODE]):
840 try:
841 os.unlink(dest)
842 destexists = 0
843 except Exception as e:
844 pass
845
846 if stat.S_ISLNK(sstat[stat.ST_MODE]):
847 try:
848 target = os.readlink(src)
849 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
850 os.unlink(dest)
851 os.symlink(target, dest)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500852 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500853 return os.lstat(dest)
854 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600855 logger.warning("copyfile: failed to create symlink %s to %s (%s)" % (dest, target, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500856 return False
857
858 if stat.S_ISREG(sstat[stat.ST_MODE]):
859 try:
860 srcchown = False
861 if not os.access(src, os.R_OK):
862 # Make sure we can read it
863 srcchown = True
864 os.chmod(src, sstat[stat.ST_MODE] | stat.S_IRUSR)
865
866 # For safety copy then move it over.
867 shutil.copyfile(src, dest + "#new")
868 os.rename(dest + "#new", dest)
869 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600870 logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500871 return False
872 finally:
873 if srcchown:
874 os.chmod(src, sstat[stat.ST_MODE])
875 os.utime(src, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
876
877 else:
878 #we don't yet handle special, so we need to fall back to /bin/mv
879 a = getstatusoutput("/bin/cp -f " + "'" + src + "' '" + dest + "'")
880 if a[0] != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600881 logger.warning("copyfile: failed to copy special file %s to %s (%s)" % (src, dest, a))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500882 return False # failure
883 try:
884 os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
885 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
886 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600887 logger.warning("copyfile: failed to chown/chmod %s (%s)" % (dest, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500888 return False
889
890 if newmtime:
891 os.utime(dest, (newmtime, newmtime))
892 else:
893 os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
894 newmtime = sstat[stat.ST_MTIME]
895 return newmtime
896
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800897def break_hardlinks(src, sstat = None):
898 """
899 Ensures src is the only hardlink to this file. Other hardlinks,
900 if any, are not affected (other than in their st_nlink value, of
901 course). Returns true on success and false on failure.
902
903 """
904 try:
905 if not sstat:
906 sstat = os.lstat(src)
907 except Exception as e:
908 logger.warning("break_hardlinks: stat of %s failed (%s)" % (src, e))
909 return False
910 if sstat[stat.ST_NLINK] == 1:
911 return True
912 return copyfile(src, src, sstat=sstat)
913
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500914def which(path, item, direction = 0, history = False, executable=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500915 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500916 Locate `item` in the list of paths `path` (colon separated string like $PATH).
917 If `direction` is non-zero then the list is reversed.
918 If `history` is True then the list of candidates also returned as result,history.
919 If `executable` is True then the candidate has to be an executable file,
920 otherwise the candidate simply has to exist.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500921 """
922
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500923 if executable:
924 is_candidate = lambda p: os.path.isfile(p) and os.access(p, os.X_OK)
925 else:
926 is_candidate = lambda p: os.path.exists(p)
927
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500928 hist = []
929 paths = (path or "").split(':')
930 if direction != 0:
931 paths.reverse()
932
933 for p in paths:
934 next = os.path.join(p, item)
935 hist.append(next)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500936 if is_candidate(next):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500937 if not os.path.isabs(next):
938 next = os.path.abspath(next)
939 if history:
940 return next, hist
941 return next
942
943 if history:
944 return "", hist
945 return ""
946
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500947@contextmanager
948def umask(new_mask):
949 """
950 Context manager to set the umask to a specific mask, and restore it afterwards.
951 """
952 current_mask = os.umask(new_mask)
953 try:
954 yield
955 finally:
956 os.umask(current_mask)
957
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500958def to_boolean(string, default=None):
959 if not string:
960 return default
961
962 normalized = string.lower()
963 if normalized in ("y", "yes", "1", "true"):
964 return True
965 elif normalized in ("n", "no", "0", "false"):
966 return False
967 else:
968 raise ValueError("Invalid value for to_boolean: %s" % string)
969
970def contains(variable, checkvalues, truevalue, falsevalue, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500971 """Check if a variable contains all the values specified.
972
973 Arguments:
974
975 variable -- the variable name. This will be fetched and expanded (using
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500976 d.getVar(variable)) and then split into a set().
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500977
978 checkvalues -- if this is a string it is split on whitespace into a set(),
979 otherwise coerced directly into a set().
980
981 truevalue -- the value to return if checkvalues is a subset of variable.
982
983 falsevalue -- the value to return if variable is empty or if checkvalues is
984 not a subset of variable.
985
986 d -- the data store.
987 """
988
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500989 val = d.getVar(variable)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500990 if not val:
991 return falsevalue
992 val = set(val.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600993 if isinstance(checkvalues, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500994 checkvalues = set(checkvalues.split())
995 else:
996 checkvalues = set(checkvalues)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500997 if checkvalues.issubset(val):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500998 return truevalue
999 return falsevalue
1000
1001def contains_any(variable, checkvalues, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001002 val = d.getVar(variable)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001003 if not val:
1004 return falsevalue
1005 val = set(val.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001006 if isinstance(checkvalues, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001007 checkvalues = set(checkvalues.split())
1008 else:
1009 checkvalues = set(checkvalues)
1010 if checkvalues & val:
1011 return truevalue
1012 return falsevalue
1013
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001014def filter(variable, checkvalues, d):
1015 """Return all words in the variable that are present in the checkvalues.
1016
1017 Arguments:
1018
1019 variable -- the variable name. This will be fetched and expanded (using
1020 d.getVar(variable)) and then split into a set().
1021
1022 checkvalues -- if this is a string it is split on whitespace into a set(),
1023 otherwise coerced directly into a set().
1024
1025 d -- the data store.
1026 """
1027
1028 val = d.getVar(variable)
1029 if not val:
1030 return ''
1031 val = set(val.split())
1032 if isinstance(checkvalues, str):
1033 checkvalues = set(checkvalues.split())
1034 else:
1035 checkvalues = set(checkvalues)
1036 return ' '.join(sorted(checkvalues & val))
1037
Andrew Geissler82c905d2020-04-13 13:39:40 -05001038
1039def get_referenced_vars(start_expr, d):
1040 """
1041 :return: names of vars referenced in start_expr (recursively), in quasi-BFS order (variables within the same level
1042 are ordered arbitrarily)
1043 """
1044
1045 seen = set()
1046 ret = []
1047
1048 # The first entry in the queue is the unexpanded start expression
1049 queue = collections.deque([start_expr])
1050 # Subsequent entries will be variable names, so we need to track whether or not entry requires getVar
1051 is_first = True
1052
1053 empty_data = bb.data.init()
1054 while queue:
1055 entry = queue.popleft()
1056 if is_first:
1057 # Entry is the start expression - no expansion needed
1058 is_first = False
1059 expression = entry
1060 else:
1061 # This is a variable name - need to get the value
1062 expression = d.getVar(entry, False)
1063 ret.append(entry)
1064
1065 # expandWithRefs is how we actually get the referenced variables in the expression. We call it using an empty
1066 # data store because we only want the variables directly used in the expression. It returns a set, which is what
1067 # dooms us to only ever be "quasi-BFS" rather than full BFS.
1068 new_vars = empty_data.expandWithRefs(expression, None).references - set(seen)
1069
1070 queue.extend(new_vars)
1071 seen.update(new_vars)
1072 return ret
1073
1074
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001075def cpu_count():
1076 return multiprocessing.cpu_count()
1077
1078def nonblockingfd(fd):
1079 fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
1080
1081def process_profilelog(fn, pout = None):
1082 # Either call with a list of filenames and set pout or a filename and optionally pout.
1083 if not pout:
1084 pout = fn + '.processed'
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001085
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001086 with open(pout, 'w') as pout:
1087 import pstats
1088 if isinstance(fn, list):
1089 p = pstats.Stats(*fn, stream=pout)
1090 else:
1091 p = pstats.Stats(fn, stream=pout)
1092 p.sort_stats('time')
1093 p.print_stats()
1094 p.print_callers()
1095 p.sort_stats('cumulative')
1096 p.print_stats()
1097
1098 pout.flush()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001099
1100#
1101# Was present to work around multiprocessing pool bugs in python < 2.7.3
1102#
1103def multiprocessingpool(*args, **kwargs):
1104
1105 import multiprocessing.pool
1106 #import multiprocessing.util
1107 #multiprocessing.util.log_to_stderr(10)
1108 # Deal with a multiprocessing bug where signals to the processes would be delayed until the work
1109 # completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed.
1110 def wrapper(func):
1111 def wrap(self, timeout=None):
1112 return func(self, timeout=timeout if timeout is not None else 1e100)
1113 return wrap
1114 multiprocessing.pool.IMapIterator.next = wrapper(multiprocessing.pool.IMapIterator.next)
1115
1116 return multiprocessing.Pool(*args, **kwargs)
1117
1118def exec_flat_python_func(func, *args, **kwargs):
1119 """Execute a flat python function (defined with def funcname(args):...)"""
1120 # Prepare a small piece of python code which calls the requested function
1121 # To do this we need to prepare two things - a set of variables we can use to pass
1122 # the values of arguments into the calling function, and the list of arguments for
1123 # the function being called
1124 context = {}
1125 funcargs = []
1126 # Handle unnamed arguments
1127 aidx = 1
1128 for arg in args:
1129 argname = 'arg_%s' % aidx
1130 context[argname] = arg
1131 funcargs.append(argname)
1132 aidx += 1
1133 # Handle keyword arguments
1134 context.update(kwargs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001135 funcargs.extend(['%s=%s' % (arg, arg) for arg in kwargs.keys()])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001136 code = 'retval = %s(%s)' % (func, ', '.join(funcargs))
1137 comp = bb.utils.better_compile(code, '<string>', '<string>')
1138 bb.utils.better_exec(comp, context, code, '<string>')
1139 return context['retval']
1140
1141def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
1142 """Edit lines from a recipe or config file and modify one or more
1143 specified variable values set in the file using a specified callback
1144 function. Lines are expected to have trailing newlines.
1145 Parameters:
1146 meta_lines: lines from the file; can be a list or an iterable
1147 (e.g. file pointer)
1148 variables: a list of variable names to look for. Functions
1149 may also be specified, but must be specified with '()' at
1150 the end of the name. Note that the function doesn't have
1151 any intrinsic understanding of _append, _prepend, _remove,
1152 or overrides, so these are considered as part of the name.
1153 These values go into a regular expression, so regular
1154 expression syntax is allowed.
1155 varfunc: callback function called for every variable matching
1156 one of the entries in the variables parameter. The function
1157 should take four arguments:
1158 varname: name of variable matched
1159 origvalue: current value in file
1160 op: the operator (e.g. '+=')
1161 newlines: list of lines up to this point. You can use
1162 this to prepend lines before this variable setting
1163 if you wish.
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001164 and should return a four-element tuple:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001165 newvalue: new value to substitute in, or None to drop
1166 the variable setting entirely. (If the removal
1167 results in two consecutive blank lines, one of the
1168 blank lines will also be dropped).
1169 newop: the operator to use - if you specify None here,
1170 the original operation will be used.
1171 indent: number of spaces to indent multi-line entries,
1172 or -1 to indent up to the level of the assignment
1173 and opening quote, or a string to use as the indent.
1174 minbreak: True to allow the first element of a
1175 multi-line value to continue on the same line as
1176 the assignment, False to indent before the first
1177 element.
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001178 To clarify, if you wish not to change the value, then you
1179 would return like this: return origvalue, None, 0, True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001180 match_overrides: True to match items with _overrides on the end,
1181 False otherwise
1182 Returns a tuple:
1183 updated:
1184 True if changes were made, False otherwise.
1185 newlines:
1186 Lines after processing
1187 """
1188
1189 var_res = {}
1190 if match_overrides:
Brad Bishop19323692019-04-05 15:28:33 -04001191 override_re = r'(_[a-zA-Z0-9-_$(){}]+)?'
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001192 else:
1193 override_re = ''
1194 for var in variables:
1195 if var.endswith('()'):
Brad Bishop19323692019-04-05 15:28:33 -04001196 var_res[var] = re.compile(r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001197 else:
Brad Bishop19323692019-04-05 15:28:33 -04001198 var_res[var] = re.compile(r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001199
1200 updated = False
1201 varset_start = ''
1202 varlines = []
1203 newlines = []
1204 in_var = None
1205 full_value = ''
1206 var_end = ''
1207
1208 def handle_var_end():
1209 prerun_newlines = newlines[:]
1210 op = varset_start[len(in_var):].strip()
1211 (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines)
1212 changed = (prerun_newlines != newlines)
1213
1214 if newvalue is None:
1215 # Drop the value
1216 return True
1217 elif newvalue != full_value or (newop not in [None, op]):
1218 if newop not in [None, op]:
1219 # Callback changed the operator
1220 varset_new = "%s %s" % (in_var, newop)
1221 else:
1222 varset_new = varset_start
1223
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001224 if isinstance(indent, int):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001225 if indent == -1:
1226 indentspc = ' ' * (len(varset_new) + 2)
1227 else:
1228 indentspc = ' ' * indent
1229 else:
1230 indentspc = indent
1231 if in_var.endswith('()'):
1232 # A function definition
1233 if isinstance(newvalue, list):
1234 newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue)))
1235 else:
1236 if not newvalue.startswith('\n'):
1237 newvalue = '\n' + newvalue
1238 if not newvalue.endswith('\n'):
1239 newvalue = newvalue + '\n'
1240 newlines.append('%s {%s}\n' % (varset_new, newvalue))
1241 else:
1242 # Normal variable
1243 if isinstance(newvalue, list):
1244 if not newvalue:
1245 # Empty list -> empty string
1246 newlines.append('%s ""\n' % varset_new)
1247 elif minbreak:
1248 # First item on first line
1249 if len(newvalue) == 1:
1250 newlines.append('%s "%s"\n' % (varset_new, newvalue[0]))
1251 else:
1252 newlines.append('%s "%s \\\n' % (varset_new, newvalue[0]))
1253 for item in newvalue[1:]:
1254 newlines.append('%s%s \\\n' % (indentspc, item))
1255 newlines.append('%s"\n' % indentspc)
1256 else:
1257 # No item on first line
1258 newlines.append('%s " \\\n' % varset_new)
1259 for item in newvalue:
1260 newlines.append('%s%s \\\n' % (indentspc, item))
1261 newlines.append('%s"\n' % indentspc)
1262 else:
1263 newlines.append('%s "%s"\n' % (varset_new, newvalue))
1264 return True
1265 else:
1266 # Put the old lines back where they were
1267 newlines.extend(varlines)
1268 # If newlines was touched by the function, we'll need to return True
1269 return changed
1270
1271 checkspc = False
1272
1273 for line in meta_lines:
1274 if in_var:
1275 value = line.rstrip()
1276 varlines.append(line)
1277 if in_var.endswith('()'):
1278 full_value += '\n' + value
1279 else:
1280 full_value += value[:-1]
1281 if value.endswith(var_end):
1282 if in_var.endswith('()'):
1283 if full_value.count('{') - full_value.count('}') >= 0:
1284 continue
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001285 full_value = full_value[:-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001286 if handle_var_end():
1287 updated = True
1288 checkspc = True
1289 in_var = None
1290 else:
1291 skip = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001292 for (varname, var_re) in var_res.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001293 res = var_re.match(line)
1294 if res:
1295 isfunc = varname.endswith('()')
1296 if isfunc:
1297 splitvalue = line.split('{', 1)
1298 var_end = '}'
1299 else:
1300 var_end = res.groups()[-1]
1301 splitvalue = line.split(var_end, 1)
1302 varset_start = splitvalue[0].rstrip()
1303 value = splitvalue[1].rstrip()
1304 if not isfunc and value.endswith('\\'):
1305 value = value[:-1]
1306 full_value = value
1307 varlines = [line]
1308 in_var = res.group(1)
1309 if isfunc:
1310 in_var += '()'
1311 if value.endswith(var_end):
1312 full_value = full_value[:-1]
1313 if handle_var_end():
1314 updated = True
1315 checkspc = True
1316 in_var = None
1317 skip = True
1318 break
1319 if not skip:
1320 if checkspc:
1321 checkspc = False
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001322 if newlines and newlines[-1] == '\n' and line == '\n':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001323 # Squash blank line if there are two consecutive blanks after a removal
1324 continue
1325 newlines.append(line)
1326 return (updated, newlines)
1327
1328
1329def edit_metadata_file(meta_file, variables, varfunc):
1330 """Edit a recipe or config file and modify one or more specified
1331 variable values set in the file using a specified callback function.
1332 The file is only written to if the value(s) actually change.
1333 This is basically the file version of edit_metadata(), see that
1334 function's description for parameter/usage information.
1335 Returns True if the file was written to, False otherwise.
1336 """
1337 with open(meta_file, 'r') as f:
1338 (updated, newlines) = edit_metadata(f, variables, varfunc)
1339 if updated:
1340 with open(meta_file, 'w') as f:
1341 f.writelines(newlines)
1342 return updated
1343
1344
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001345def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001346 """Edit bblayers.conf, adding and/or removing layers
1347 Parameters:
1348 bblayers_conf: path to bblayers.conf file to edit
1349 add: layer path (or list of layer paths) to add; None or empty
1350 list to add nothing
1351 remove: layer path (or list of layer paths) to remove; None or
1352 empty list to remove nothing
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001353 edit_cb: optional callback function that will be called after
1354 processing adds/removes once per existing entry.
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001355 Returns a tuple:
1356 notadded: list of layers specified to be added but weren't
1357 (because they were already in the list)
1358 notremoved: list of layers that were specified to be removed
1359 but weren't (because they weren't in the list)
1360 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001361
1362 import fnmatch
1363
1364 def remove_trailing_sep(pth):
1365 if pth and pth[-1] == os.sep:
1366 pth = pth[:-1]
1367 return pth
1368
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001369 approved = bb.utils.approved_variables()
1370 def canonicalise_path(pth):
1371 pth = remove_trailing_sep(pth)
1372 if 'HOME' in approved and '~' in pth:
1373 pth = os.path.expanduser(pth)
1374 return pth
1375
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001376 def layerlist_param(value):
1377 if not value:
1378 return []
1379 elif isinstance(value, list):
1380 return [remove_trailing_sep(x) for x in value]
1381 else:
1382 return [remove_trailing_sep(value)]
1383
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001384 addlayers = layerlist_param(add)
1385 removelayers = layerlist_param(remove)
1386
1387 # Need to use a list here because we can't set non-local variables from a callback in python 2.x
1388 bblayercalls = []
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001389 removed = []
1390 plusequals = False
1391 orig_bblayers = []
1392
1393 def handle_bblayers_firstpass(varname, origvalue, op, newlines):
1394 bblayercalls.append(op)
1395 if op == '=':
1396 del orig_bblayers[:]
1397 orig_bblayers.extend([canonicalise_path(x) for x in origvalue.split()])
1398 return (origvalue, None, 2, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001399
1400 def handle_bblayers(varname, origvalue, op, newlines):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001401 updated = False
1402 bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
1403 if removelayers:
1404 for removelayer in removelayers:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001405 for layer in bblayers:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001406 if fnmatch.fnmatch(canonicalise_path(layer), canonicalise_path(removelayer)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001407 updated = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001408 bblayers.remove(layer)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001409 removed.append(removelayer)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001410 break
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001411 if addlayers and not plusequals:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001412 for addlayer in addlayers:
1413 if addlayer not in bblayers:
1414 updated = True
1415 bblayers.append(addlayer)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001416 del addlayers[:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001417
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001418 if edit_cb:
1419 newlist = []
1420 for layer in bblayers:
1421 res = edit_cb(layer, canonicalise_path(layer))
1422 if res != layer:
1423 newlist.append(res)
1424 updated = True
1425 else:
1426 newlist.append(layer)
1427 bblayers = newlist
1428
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001429 if updated:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001430 if op == '+=' and not bblayers:
1431 bblayers = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001432 return (bblayers, None, 2, False)
1433 else:
1434 return (origvalue, None, 2, False)
1435
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001436 with open(bblayers_conf, 'r') as f:
1437 (_, newlines) = edit_metadata(f, ['BBLAYERS'], handle_bblayers_firstpass)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001438
1439 if not bblayercalls:
1440 raise Exception('Unable to find BBLAYERS in %s' % bblayers_conf)
1441
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001442 # Try to do the "smart" thing depending on how the user has laid out
1443 # their bblayers.conf file
1444 if bblayercalls.count('+=') > 1:
1445 plusequals = True
1446
1447 removelayers_canon = [canonicalise_path(layer) for layer in removelayers]
1448 notadded = []
1449 for layer in addlayers:
1450 layer_canon = canonicalise_path(layer)
1451 if layer_canon in orig_bblayers and not layer_canon in removelayers_canon:
1452 notadded.append(layer)
1453 notadded_canon = [canonicalise_path(layer) for layer in notadded]
1454 addlayers[:] = [layer for layer in addlayers if canonicalise_path(layer) not in notadded_canon]
1455
1456 (updated, newlines) = edit_metadata(newlines, ['BBLAYERS'], handle_bblayers)
1457 if addlayers:
1458 # Still need to add these
1459 for addlayer in addlayers:
1460 newlines.append('BBLAYERS += "%s"\n' % addlayer)
1461 updated = True
1462
1463 if updated:
1464 with open(bblayers_conf, 'w') as f:
1465 f.writelines(newlines)
1466
1467 notremoved = list(set(removelayers) - set(removed))
1468
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001469 return (notadded, notremoved)
1470
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001471def get_collection_res(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001472 collections = (d.getVar('BBFILE_COLLECTIONS') or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001473 collection_res = {}
1474 for collection in collections:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001475 collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection) or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001476
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001477 return collection_res
1478
1479
1480def get_file_layer(filename, d, collection_res={}):
1481 """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
1482 if not collection_res:
1483 collection_res = get_collection_res(d)
1484
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001485 def path_to_layer(path):
1486 # Use longest path so we handle nested layers
1487 matchlen = 0
1488 match = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001489 for collection, regex in collection_res.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001490 if len(regex) > matchlen and re.match(regex, path):
1491 matchlen = len(regex)
1492 match = collection
1493 return match
1494
1495 result = None
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001496 bbfiles = (d.getVar('BBFILES_PRIORITIZED') or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001497 bbfilesmatch = False
1498 for bbfilesentry in bbfiles:
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001499 if fnmatch.fnmatchcase(filename, bbfilesentry):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001500 bbfilesmatch = True
1501 result = path_to_layer(bbfilesentry)
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001502 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001503
1504 if not bbfilesmatch:
1505 # Probably a bbclass
1506 result = path_to_layer(filename)
1507
1508 return result
1509
1510
1511# Constant taken from http://linux.die.net/include/linux/prctl.h
1512PR_SET_PDEATHSIG = 1
1513
1514class PrCtlError(Exception):
1515 pass
1516
1517def signal_on_parent_exit(signame):
1518 """
1519 Trigger signame to be sent when the parent process dies
1520 """
1521 signum = getattr(signal, signame)
1522 # http://linux.die.net/man/2/prctl
1523 result = cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG, signum)
1524 if result != 0:
1525 raise PrCtlError('prctl failed with error code %s' % result)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001526
1527#
1528# Manually call the ioprio syscall. We could depend on other libs like psutil
1529# however this gets us enough of what we need to bitbake for now without the
1530# dependency
1531#
1532_unamearch = os.uname()[4]
1533IOPRIO_WHO_PROCESS = 1
1534IOPRIO_CLASS_SHIFT = 13
1535
1536def ioprio_set(who, cls, value):
1537 NR_ioprio_set = None
1538 if _unamearch == "x86_64":
1539 NR_ioprio_set = 251
1540 elif _unamearch[0] == "i" and _unamearch[2:3] == "86":
1541 NR_ioprio_set = 289
Brad Bishop19323692019-04-05 15:28:33 -04001542 elif _unamearch == "aarch64":
1543 NR_ioprio_set = 30
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001544
1545 if NR_ioprio_set:
1546 ioprio = value | (cls << IOPRIO_CLASS_SHIFT)
1547 rc = cdll['libc.so.6'].syscall(NR_ioprio_set, IOPRIO_WHO_PROCESS, who, ioprio)
1548 if rc != 0:
1549 raise ValueError("Unable to set ioprio, syscall returned %s" % rc)
1550 else:
1551 bb.warn("Unable to set IO Prio for arch %s" % _unamearch)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001552
1553def set_process_name(name):
1554 from ctypes import cdll, byref, create_string_buffer
1555 # This is nice to have for debugging, not essential
1556 try:
1557 libc = cdll.LoadLibrary('libc.so.6')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001558 buf = create_string_buffer(bytes(name, 'utf-8'))
1559 libc.prctl(15, byref(buf), 0, 0, 0)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001560 except:
1561 pass
1562
1563# export common proxies variables from datastore to environment
1564def export_proxies(d):
1565 import os
1566
1567 variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001568 'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
1569 'GIT_PROXY_COMMAND']
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001570 exported = False
1571
1572 for v in variables:
1573 if v in os.environ.keys():
1574 exported = True
1575 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001576 v_proxy = d.getVar(v)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001577 if v_proxy is not None:
1578 os.environ[v] = v_proxy
1579 exported = True
1580
1581 return exported
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001582
1583
1584def load_plugins(logger, plugins, pluginpath):
1585 def load_plugin(name):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001586 logger.debug(1, 'Loading plugin %s' % name)
Brad Bishop19323692019-04-05 15:28:33 -04001587 spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
1588 if spec:
1589 return spec.loader.load_module()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001590
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001591 logger.debug(1, 'Loading plugins from %s...' % pluginpath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001592
1593 expanded = (glob.glob(os.path.join(pluginpath, '*' + ext))
1594 for ext in python_extensions)
1595 files = itertools.chain.from_iterable(expanded)
1596 names = set(os.path.splitext(os.path.basename(fn))[0] for fn in files)
1597 for name in names:
1598 if name != '__init__':
1599 plugin = load_plugin(name)
1600 if hasattr(plugin, 'plugin_init'):
1601 obj = plugin.plugin_init(plugins)
1602 plugins.append(obj or plugin)
1603 else:
1604 plugins.append(plugin)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001605
1606
1607class LogCatcher(logging.Handler):
1608 """Logging handler for collecting logged messages so you can check them later"""
1609 def __init__(self):
1610 self.messages = []
1611 logging.Handler.__init__(self, logging.WARNING)
1612 def emit(self, record):
1613 self.messages.append(bb.build.logformatter.format(record))
1614 def contains(self, message):
1615 return (message in self.messages)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001616
1617def is_semver(version):
1618 """
1619 Is the version string following the semver semantic?
1620
1621 https://semver.org/spec/v2.0.0.html
1622 """
1623 regex = re.compile(
1624 r"""
1625 ^
1626 (0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)
1627 (?:-(
1628 (?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)
1629 (?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*
1630 ))?
1631 (?:\+(
1632 [0-9a-zA-Z-]+
1633 (?:\.[0-9a-zA-Z-]+)*
1634 ))?
1635 $
1636 """, re.VERBOSE)
1637
1638 if regex.match(version) is None:
1639 return False
1640
1641 return True