blob: b652a6838a290a48812ac59ef2aafd0abec04203 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake Utility Functions
5"""
6
7# Copyright (C) 2004 Michael Lauer
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22import re, fcntl, os, string, stat, shutil, time
23import sys
24import errno
25import logging
26import bb
27import bb.msg
28import multiprocessing
29import fcntl
Brad Bishop19323692019-04-05 15:28:33 -040030import importlib
31from importlib import machinery
Patrick Williamsc0f7c042017-02-23 20:41:17 -060032import itertools
Patrick Williamsc124f4f2015-09-15 14:41:29 -050033import subprocess
34import glob
35import fnmatch
36import traceback
37import errno
38import signal
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050039import ast
Patrick Williamsc0f7c042017-02-23 20:41:17 -060040import collections
41import copy
42from subprocess import getstatusoutput
Patrick Williamsc124f4f2015-09-15 14:41:29 -050043from contextlib import contextmanager
44from ctypes import cdll
45
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046logger = logging.getLogger("BitBake.Util")
Brad Bishop19323692019-04-05 15:28:33 -040047python_extensions = importlib.machinery.all_suffixes()
Patrick Williamsc0f7c042017-02-23 20:41:17 -060048
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049
50def clean_context():
51 return {
52 "os": os,
53 "bb": bb,
54 "time": time,
55 }
56
57def get_context():
58 return _context
59
60
61def set_context(ctx):
62 _context = ctx
63
64# Context used in better_exec, eval
65_context = clean_context()
66
67class VersionStringException(Exception):
68 """Exception raised when an invalid version specification is found"""
69
70def explode_version(s):
71 r = []
Brad Bishop19323692019-04-05 15:28:33 -040072 alpha_regexp = re.compile(r'^([a-zA-Z]+)(.*)$')
73 numeric_regexp = re.compile(r'^(\d+)(.*)$')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050074 while (s != ''):
75 if s[0] in string.digits:
76 m = numeric_regexp.match(s)
77 r.append((0, int(m.group(1))))
78 s = m.group(2)
79 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -060080 if s[0] in string.ascii_letters:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081 m = alpha_regexp.match(s)
82 r.append((1, m.group(1)))
83 s = m.group(2)
84 continue
85 if s[0] == '~':
86 r.append((-1, s[0]))
87 else:
88 r.append((2, s[0]))
89 s = s[1:]
90 return r
91
92def split_version(s):
93 """Split a version string into its constituent parts (PE, PV, PR)"""
94 s = s.strip(" <>=")
95 e = 0
96 if s.count(':'):
97 e = int(s.split(":")[0])
98 s = s.split(":")[1]
99 r = ""
100 if s.count('-'):
101 r = s.rsplit("-", 1)[1]
102 s = s.rsplit("-", 1)[0]
103 v = s
104 return (e, v, r)
105
106def vercmp_part(a, b):
107 va = explode_version(a)
108 vb = explode_version(b)
109 while True:
110 if va == []:
111 (oa, ca) = (0, None)
112 else:
113 (oa, ca) = va.pop(0)
114 if vb == []:
115 (ob, cb) = (0, None)
116 else:
117 (ob, cb) = vb.pop(0)
118 if (oa, ca) == (0, None) and (ob, cb) == (0, None):
119 return 0
120 if oa < ob:
121 return -1
122 elif oa > ob:
123 return 1
Brad Bishop19323692019-04-05 15:28:33 -0400124 elif ca is None:
125 return -1
126 elif cb is None:
127 return 1
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 elif ca < cb:
129 return -1
130 elif ca > cb:
131 return 1
132
133def vercmp(ta, tb):
134 (ea, va, ra) = ta
135 (eb, vb, rb) = tb
136
137 r = int(ea or 0) - int(eb or 0)
138 if (r == 0):
139 r = vercmp_part(va, vb)
140 if (r == 0):
141 r = vercmp_part(ra, rb)
142 return r
143
144def vercmp_string(a, b):
145 ta = split_version(a)
146 tb = split_version(b)
147 return vercmp(ta, tb)
148
149def vercmp_string_op(a, b, op):
150 """
151 Compare two versions and check if the specified comparison operator matches the result of the comparison.
152 This function is fairly liberal about what operators it will accept since there are a variety of styles
153 depending on the context.
154 """
155 res = vercmp_string(a, b)
156 if op in ('=', '=='):
157 return res == 0
158 elif op == '<=':
159 return res <= 0
160 elif op == '>=':
161 return res >= 0
162 elif op in ('>', '>>'):
163 return res > 0
164 elif op in ('<', '<<'):
165 return res < 0
166 elif op == '!=':
167 return res != 0
168 else:
169 raise VersionStringException('Unsupported comparison operator "%s"' % op)
170
171def explode_deps(s):
172 """
173 Take an RDEPENDS style string of format:
174 "DEPEND1 (optional version) DEPEND2 (optional version) ..."
175 and return a list of dependencies.
176 Version information is ignored.
177 """
178 r = []
179 l = s.split()
180 flag = False
181 for i in l:
182 if i[0] == '(':
183 flag = True
184 #j = []
185 if not flag:
186 r.append(i)
187 #else:
188 # j.append(i)
189 if flag and i.endswith(')'):
190 flag = False
191 # Ignore version
192 #r[-1] += ' ' + ' '.join(j)
193 return r
194
Brad Bishop316dfdd2018-06-25 12:45:53 -0400195def explode_dep_versions2(s, *, sort=True):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196 """
197 Take an RDEPENDS style string of format:
198 "DEPEND1 (optional version) DEPEND2 (optional version) ..."
199 and return a dictionary of dependencies and versions.
200 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600201 r = collections.OrderedDict()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500202 l = s.replace(",", "").split()
203 lastdep = None
204 lastcmp = ""
205 lastver = ""
206 incmp = False
207 inversion = False
208 for i in l:
209 if i[0] == '(':
210 incmp = True
211 i = i[1:].strip()
212 if not i:
213 continue
214
215 if incmp:
216 incmp = False
217 inversion = True
218 # This list is based on behavior and supported comparisons from deb, opkg and rpm.
219 #
220 # Even though =<, <<, ==, !=, =>, and >> may not be supported,
221 # we list each possibly valid item.
222 # The build system is responsible for validation of what it supports.
223 if i.startswith(('<=', '=<', '<<', '==', '!=', '>=', '=>', '>>')):
224 lastcmp = i[0:2]
225 i = i[2:]
226 elif i.startswith(('<', '>', '=')):
227 lastcmp = i[0:1]
228 i = i[1:]
229 else:
230 # This is an unsupported case!
231 raise VersionStringException('Invalid version specification in "(%s" - invalid or missing operator' % i)
232 lastcmp = (i or "")
233 i = ""
234 i.strip()
235 if not i:
236 continue
237
238 if inversion:
239 if i.endswith(')'):
240 i = i[:-1] or ""
241 inversion = False
242 if lastver and i:
243 lastver += " "
244 if i:
245 lastver += i
246 if lastdep not in r:
247 r[lastdep] = []
248 r[lastdep].append(lastcmp + " " + lastver)
249 continue
250
251 #if not inversion:
252 lastdep = i
253 lastver = ""
254 lastcmp = ""
255 if not (i in r and r[i]):
256 r[lastdep] = []
257
Brad Bishop316dfdd2018-06-25 12:45:53 -0400258 if sort:
259 r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260 return r
261
262def explode_dep_versions(s):
263 r = explode_dep_versions2(s)
264 for d in r:
265 if not r[d]:
266 r[d] = None
267 continue
268 if len(r[d]) > 1:
269 bb.warn("explode_dep_versions(): Item %s appeared in dependency string '%s' multiple times with different values. explode_dep_versions cannot cope with this." % (d, s))
270 r[d] = r[d][0]
271 return r
272
273def join_deps(deps, commasep=True):
274 """
275 Take the result from explode_dep_versions and generate a dependency string
276 """
277 result = []
278 for dep in deps:
279 if deps[dep]:
280 if isinstance(deps[dep], list):
281 for v in deps[dep]:
282 result.append(dep + " (" + v + ")")
283 else:
284 result.append(dep + " (" + deps[dep] + ")")
285 else:
286 result.append(dep)
287 if commasep:
288 return ", ".join(result)
289 else:
290 return " ".join(result)
291
292def _print_trace(body, line):
293 """
294 Print the Environment of a Text Body
295 """
296 error = []
297 # print the environment of the method
298 min_line = max(1, line-4)
299 max_line = min(line + 4, len(body))
300 for i in range(min_line, max_line + 1):
301 if line == i:
302 error.append(' *** %.4d:%s' % (i, body[i-1].rstrip()))
303 else:
304 error.append(' %.4d:%s' % (i, body[i-1].rstrip()))
305 return error
306
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500307def better_compile(text, file, realfile, mode = "exec", lineno = 0):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308 """
309 A better compile method. This method
310 will print the offending lines.
311 """
312 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500313 cache = bb.methodpool.compile_cache(text)
314 if cache:
315 return cache
316 # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
317 text2 = "\n" * int(lineno) + text
318 code = compile(text2, realfile, mode)
319 bb.methodpool.compile_cache_add(text, code)
320 return code
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 except Exception as e:
322 error = []
323 # split the text into lines again
324 body = text.split('\n')
Brad Bishop19323692019-04-05 15:28:33 -0400325 error.append("Error in compiling python function in %s, line %s:\n" % (realfile, e.lineno))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500326 if hasattr(e, "lineno"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327 error.append("The code lines resulting in this error were:")
Brad Bishop19323692019-04-05 15:28:33 -0400328 # e.lineno: line's position in reaflile
329 # lineno: function name's "position -1" in realfile
330 # e.lineno - lineno: line's relative position in function
331 error.extend(_print_trace(body, e.lineno - lineno))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500332 else:
333 error.append("The function causing this error was:")
334 for line in body:
335 error.append(line)
336 error.append("%s: %s" % (e.__class__.__name__, str(e)))
337
338 logger.error("\n".join(error))
339
340 e = bb.BBHandledException(e)
341 raise e
342
343def _print_exception(t, value, tb, realfile, text, context):
344 error = []
345 try:
346 exception = traceback.format_exception_only(t, value)
347 error.append('Error executing a python function in %s:\n' % realfile)
348
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500349 # Strip 'us' from the stack (better_exec call) unless that was where the
350 # error came from
351 if tb.tb_next is not None:
352 tb = tb.tb_next
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500353
354 textarray = text.split('\n')
355
356 linefailed = tb.tb_lineno
357
358 tbextract = traceback.extract_tb(tb)
359 tbformat = traceback.format_list(tbextract)
360 error.append("The stack trace of python calls that resulted in this exception/failure was:")
361 error.append("File: '%s', lineno: %s, function: %s" % (tbextract[0][0], tbextract[0][1], tbextract[0][2]))
362 error.extend(_print_trace(textarray, linefailed))
363
364 # See if this is a function we constructed and has calls back into other functions in
365 # "text". If so, try and improve the context of the error by diving down the trace
366 level = 0
367 nexttb = tb.tb_next
368 while nexttb is not None and (level+1) < len(tbextract):
369 error.append("File: '%s', lineno: %s, function: %s" % (tbextract[level+1][0], tbextract[level+1][1], tbextract[level+1][2]))
370 if tbextract[level][0] == tbextract[level+1][0] and tbextract[level+1][2] == tbextract[level][0]:
371 # The code was possibly in the string we compiled ourselves
372 error.extend(_print_trace(textarray, tbextract[level+1][1]))
373 elif tbextract[level+1][0].startswith("/"):
374 # The code looks like it might be in a file, try and load it
375 try:
376 with open(tbextract[level+1][0], "r") as f:
377 text = f.readlines()
378 error.extend(_print_trace(text, tbextract[level+1][1]))
379 except:
380 error.append(tbformat[level+1])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381 else:
382 error.append(tbformat[level+1])
383 nexttb = tb.tb_next
384 level = level + 1
385
386 error.append("Exception: %s" % ''.join(exception))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600387
388 # If the exception is from spwaning a task, let's be helpful and display
389 # the output (which hopefully includes stderr).
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500390 if isinstance(value, subprocess.CalledProcessError) and value.output:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600391 error.append("Subprocess output:")
392 error.append(value.output.decode("utf-8", errors="ignore"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500393 finally:
394 logger.error("\n".join(error))
395
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500396def better_exec(code, context, text = None, realfile = "<code>", pythonexception=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500397 """
398 Similiar to better_compile, better_exec will
399 print the lines that are responsible for the
400 error.
401 """
402 import bb.parse
403 if not text:
404 text = code
405 if not hasattr(code, "co_filename"):
406 code = better_compile(code, realfile, realfile)
407 try:
408 exec(code, get_context(), context)
409 except (bb.BBHandledException, bb.parse.SkipRecipe, bb.build.FuncFailed, bb.data_smart.ExpansionError):
410 # Error already shown so passthrough, no need for traceback
411 raise
412 except Exception as e:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500413 if pythonexception:
414 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415 (t, value, tb) = sys.exc_info()
416 try:
417 _print_exception(t, value, tb, realfile, text, context)
418 except Exception as e:
419 logger.error("Exception handler error: %s" % str(e))
420
421 e = bb.BBHandledException(e)
422 raise e
423
424def simple_exec(code, context):
425 exec(code, get_context(), context)
426
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600427def better_eval(source, locals, extraglobals = None):
428 ctx = get_context()
429 if extraglobals:
430 ctx = copy.copy(ctx)
431 for g in extraglobals:
432 ctx[g] = extraglobals[g]
433 return eval(source, ctx, locals)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500434
435@contextmanager
436def fileslocked(files):
437 """Context manager for locking and unlocking file locks."""
438 locks = []
439 if files:
440 for lockfile in files:
441 locks.append(bb.utils.lockfile(lockfile))
442
443 yield
444
445 for lock in locks:
446 bb.utils.unlockfile(lock)
447
448@contextmanager
449def timeout(seconds):
450 def timeout_handler(signum, frame):
451 pass
452
453 original_handler = signal.signal(signal.SIGALRM, timeout_handler)
454
455 try:
456 signal.alarm(seconds)
457 yield
458 finally:
459 signal.alarm(0)
460 signal.signal(signal.SIGALRM, original_handler)
461
462def lockfile(name, shared=False, retry=True, block=False):
463 """
464 Use the specified file as a lock file, return when the lock has
465 been acquired. Returns a variable to pass to unlockfile().
466 Parameters:
467 retry: True to re-try locking if it fails, False otherwise
468 block: True to block until the lock succeeds, False otherwise
469 The retry and block parameters are kind of equivalent unless you
470 consider the possibility of sending a signal to the process to break
471 out - at which point you want block=True rather than retry=True.
472 """
473 dirname = os.path.dirname(name)
474 mkdirhier(dirname)
475
476 if not os.access(dirname, os.W_OK):
477 logger.error("Unable to acquire lock '%s', directory is not writable",
478 name)
479 sys.exit(1)
480
481 op = fcntl.LOCK_EX
482 if shared:
483 op = fcntl.LOCK_SH
484 if not retry and not block:
485 op = op | fcntl.LOCK_NB
486
487 while True:
488 # If we leave the lockfiles lying around there is no problem
489 # but we should clean up after ourselves. This gives potential
490 # for races though. To work around this, when we acquire the lock
491 # we check the file we locked was still the lock file on disk.
492 # by comparing inode numbers. If they don't match or the lockfile
493 # no longer exists, we start again.
494
495 # This implementation is unfair since the last person to request the
496 # lock is the most likely to win it.
497
498 try:
499 lf = open(name, 'a+')
500 fileno = lf.fileno()
501 fcntl.flock(fileno, op)
502 statinfo = os.fstat(fileno)
503 if os.path.exists(lf.name):
504 statinfo2 = os.stat(lf.name)
505 if statinfo.st_ino == statinfo2.st_ino:
506 return lf
507 lf.close()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800508 except OSError as e:
509 if e.errno == errno.EACCES:
510 logger.error("Unable to acquire lock '%s', %s",
511 e.strerror, name)
512 sys.exit(1)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500513 try:
514 lf.close()
515 except Exception:
516 pass
517 pass
518 if not retry:
519 return None
520
521def unlockfile(lf):
522 """
523 Unlock a file locked using lockfile()
524 """
525 try:
526 # If we had a shared lock, we need to promote to exclusive before
527 # removing the lockfile. Attempt this, ignore failures.
528 fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
529 os.unlink(lf.name)
530 except (IOError, OSError):
531 pass
532 fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
533 lf.close()
534
535def md5_file(filename):
536 """
537 Return the hex string representation of the MD5 checksum of filename.
538 """
Brad Bishop220d5532018-08-14 00:59:39 +0100539 import hashlib, mmap
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500540
541 with open(filename, "rb") as f:
Brad Bishop220d5532018-08-14 00:59:39 +0100542 m = hashlib.md5()
543 try:
544 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
545 for chunk in iter(lambda: mm.read(8192), b''):
546 m.update(chunk)
547 except ValueError:
548 # You can't mmap() an empty file so silence this exception
549 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500550 return m.hexdigest()
551
552def sha256_file(filename):
553 """
554 Return the hex string representation of the 256-bit SHA checksum of
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500555 filename.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500556 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500557 import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500558
559 s = hashlib.sha256()
560 with open(filename, "rb") as f:
561 for line in f:
562 s.update(line)
563 return s.hexdigest()
564
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500565def sha1_file(filename):
566 """
567 Return the hex string representation of the SHA1 checksum of the filename
568 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500569 import hashlib
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500570
571 s = hashlib.sha1()
572 with open(filename, "rb") as f:
573 for line in f:
574 s.update(line)
575 return s.hexdigest()
576
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500577def preserved_envvars_exported():
578 """Variables which are taken from the environment and placed in and exported
579 from the metadata"""
580 return [
581 'BB_TASKHASH',
582 'HOME',
583 'LOGNAME',
584 'PATH',
585 'PWD',
586 'SHELL',
587 'TERM',
588 'USER',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600589 'LC_ALL',
590 'BBSERVER',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500591 ]
592
593def preserved_envvars():
594 """Variables which are taken from the environment and placed in the metadata"""
595 v = [
596 'BBPATH',
597 'BB_PRESERVE_ENV',
598 'BB_ENV_WHITELIST',
599 'BB_ENV_EXTRAWHITE',
600 ]
601 return v + preserved_envvars_exported()
602
603def filter_environment(good_vars):
604 """
605 Create a pristine environment for bitbake. This will remove variables that
606 are not known and may influence the build in a negative way.
607 """
608
609 removed_vars = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600610 for key in list(os.environ):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611 if key in good_vars:
612 continue
613
614 removed_vars[key] = os.environ[key]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615 del os.environ[key]
616
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600617 # If we spawn a python process, we need to have a UTF-8 locale, else python's file
618 # access methods will use ascii. You can't change that mode once the interpreter is
619 # started so we have to ensure a locale is set. Ideally we'd use C.UTF-8 but not all
620 # distros support that and we need to set something.
621 os.environ["LC_ALL"] = "en_US.UTF-8"
622
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500623 if removed_vars:
624 logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
625
626 return removed_vars
627
628def approved_variables():
629 """
630 Determine and return the list of whitelisted variables which are approved
631 to remain in the environment.
632 """
633 if 'BB_PRESERVE_ENV' in os.environ:
634 return os.environ.keys()
635 approved = []
636 if 'BB_ENV_WHITELIST' in os.environ:
637 approved = os.environ['BB_ENV_WHITELIST'].split()
638 approved.extend(['BB_ENV_WHITELIST'])
639 else:
640 approved = preserved_envvars()
641 if 'BB_ENV_EXTRAWHITE' in os.environ:
642 approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split())
643 if 'BB_ENV_EXTRAWHITE' not in approved:
644 approved.extend(['BB_ENV_EXTRAWHITE'])
645 return approved
646
647def clean_environment():
648 """
649 Clean up any spurious environment variables. This will remove any
650 variables the user hasn't chosen to preserve.
651 """
652 if 'BB_PRESERVE_ENV' not in os.environ:
653 good_vars = approved_variables()
654 return filter_environment(good_vars)
655
656 return {}
657
658def empty_environment():
659 """
660 Remove all variables from the environment.
661 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600662 for s in list(os.environ.keys()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500663 os.unsetenv(s)
664 del os.environ[s]
665
666def build_environment(d):
667 """
668 Build an environment from all exported variables.
669 """
670 import bb.data
671 for var in bb.data.keys(d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500672 export = d.getVarFlag(var, "export", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500673 if export:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500674 os.environ[var] = d.getVar(var) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500675
676def _check_unsafe_delete_path(path):
677 """
678 Basic safeguard against recursively deleting something we shouldn't. If it returns True,
679 the caller should raise an exception with an appropriate message.
680 NOTE: This is NOT meant to be a security mechanism - just a guard against silly mistakes
681 with potentially disastrous results.
682 """
683 extra = ''
684 # HOME might not be /home/something, so in case we can get it, check against it
685 homedir = os.environ.get('HOME', '')
686 if homedir:
687 extra = '|%s' % homedir
688 if re.match('(/|//|/home|/home/[^/]*%s)$' % extra, os.path.abspath(path)):
689 return True
690 return False
691
692def remove(path, recurse=False):
693 """Equivalent to rm -f or rm -rf"""
694 if not path:
695 return
696 if recurse:
697 for name in glob.glob(path):
698 if _check_unsafe_delete_path(path):
699 raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
700 # shutil.rmtree(name) would be ideal but its too slow
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500701 subprocess.check_call(['rm', '-rf'] + glob.glob(path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500702 return
703 for name in glob.glob(path):
704 try:
705 os.unlink(name)
706 except OSError as exc:
707 if exc.errno != errno.ENOENT:
708 raise
709
710def prunedir(topdir):
711 # Delete everything reachable from the directory named in 'topdir'.
712 # CAUTION: This is dangerous!
713 if _check_unsafe_delete_path(topdir):
714 raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
Brad Bishop19323692019-04-05 15:28:33 -0400715 remove(topdir, recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500716
717#
718# Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var)
719# but thats possibly insane and suffixes is probably going to be small
720#
721def prune_suffix(var, suffixes, d):
722 # See if var ends with any of the suffixes listed and
723 # remove it if found
724 for suffix in suffixes:
725 if var.endswith(suffix):
726 return var.replace(suffix, "")
727 return var
728
729def mkdirhier(directory):
730 """Create a directory like 'mkdir -p', but does not complain if
731 directory already exists like os.makedirs
732 """
733
734 try:
735 os.makedirs(directory)
736 except OSError as e:
737 if e.errno != errno.EEXIST:
738 raise e
739
740def movefile(src, dest, newmtime = None, sstat = None):
741 """Moves a file from src to dest, preserving all permissions and
742 attributes; mtime will be preserved even when moving across
743 filesystems. Returns true on success and false on failure. Move is
744 atomic.
745 """
746
747 #print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
748 try:
749 if not sstat:
750 sstat = os.lstat(src)
751 except Exception as e:
752 print("movefile: Stating source file failed...", e)
753 return None
754
755 destexists = 1
756 try:
757 dstat = os.lstat(dest)
758 except:
759 dstat = os.lstat(os.path.dirname(dest))
760 destexists = 0
761
762 if destexists:
763 if stat.S_ISLNK(dstat[stat.ST_MODE]):
764 try:
765 os.unlink(dest)
766 destexists = 0
767 except Exception as e:
768 pass
769
770 if stat.S_ISLNK(sstat[stat.ST_MODE]):
771 try:
772 target = os.readlink(src)
773 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
774 os.unlink(dest)
775 os.symlink(target, dest)
776 #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
777 os.unlink(src)
778 return os.lstat(dest)
779 except Exception as e:
780 print("movefile: failed to properly create symlink:", dest, "->", target, e)
781 return None
782
783 renamefailed = 1
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500784 # os.rename needs to know the dest path ending with file name
785 # so append the file name to a path only if it's a dir specified
786 srcfname = os.path.basename(src)
787 destpath = os.path.join(dest, srcfname) if os.path.isdir(dest) \
788 else dest
789
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500790 if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
791 try:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500792 os.rename(src, destpath)
793 renamefailed = 0
794 except Exception as e:
795 if e[0] != errno.EXDEV:
796 # Some random error.
797 print("movefile: Failed to move", src, "to", dest, e)
798 return None
799 # Invalid cross-device-link 'bind' mounted or actually Cross-Device
800
801 if renamefailed:
802 didcopy = 0
803 if stat.S_ISREG(sstat[stat.ST_MODE]):
804 try: # For safety copy then move it over.
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500805 shutil.copyfile(src, destpath + "#new")
806 os.rename(destpath + "#new", destpath)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500807 didcopy = 1
808 except Exception as e:
809 print('movefile: copy', src, '->', dest, 'failed.', e)
810 return None
811 else:
812 #we don't yet handle special, so we need to fall back to /bin/mv
813 a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'")
814 if a[0] != 0:
815 print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
816 return None # failure
817 try:
818 if didcopy:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400819 os.lchown(destpath, sstat[stat.ST_UID], sstat[stat.ST_GID])
820 os.chmod(destpath, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500821 os.unlink(src)
822 except Exception as e:
823 print("movefile: Failed to chown/chmod/unlink", dest, e)
824 return None
825
826 if newmtime:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500827 os.utime(destpath, (newmtime, newmtime))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500828 else:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500829 os.utime(destpath, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500830 newmtime = sstat[stat.ST_MTIME]
831 return newmtime
832
833def copyfile(src, dest, newmtime = None, sstat = None):
834 """
835 Copies a file from src to dest, preserving all permissions and
836 attributes; mtime will be preserved even when moving across
837 filesystems. Returns true on success and false on failure.
838 """
839 #print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
840 try:
841 if not sstat:
842 sstat = os.lstat(src)
843 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600844 logger.warning("copyfile: stat of %s failed (%s)" % (src, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500845 return False
846
847 destexists = 1
848 try:
849 dstat = os.lstat(dest)
850 except:
851 dstat = os.lstat(os.path.dirname(dest))
852 destexists = 0
853
854 if destexists:
855 if stat.S_ISLNK(dstat[stat.ST_MODE]):
856 try:
857 os.unlink(dest)
858 destexists = 0
859 except Exception as e:
860 pass
861
862 if stat.S_ISLNK(sstat[stat.ST_MODE]):
863 try:
864 target = os.readlink(src)
865 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
866 os.unlink(dest)
867 os.symlink(target, dest)
868 #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
869 return os.lstat(dest)
870 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600871 logger.warning("copyfile: failed to create symlink %s to %s (%s)" % (dest, target, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500872 return False
873
874 if stat.S_ISREG(sstat[stat.ST_MODE]):
875 try:
876 srcchown = False
877 if not os.access(src, os.R_OK):
878 # Make sure we can read it
879 srcchown = True
880 os.chmod(src, sstat[stat.ST_MODE] | stat.S_IRUSR)
881
882 # For safety copy then move it over.
883 shutil.copyfile(src, dest + "#new")
884 os.rename(dest + "#new", dest)
885 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600886 logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500887 return False
888 finally:
889 if srcchown:
890 os.chmod(src, sstat[stat.ST_MODE])
891 os.utime(src, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
892
893 else:
894 #we don't yet handle special, so we need to fall back to /bin/mv
895 a = getstatusoutput("/bin/cp -f " + "'" + src + "' '" + dest + "'")
896 if a[0] != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600897 logger.warning("copyfile: failed to copy special file %s to %s (%s)" % (src, dest, a))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500898 return False # failure
899 try:
900 os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
901 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
902 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600903 logger.warning("copyfile: failed to chown/chmod %s (%s)" % (dest, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500904 return False
905
906 if newmtime:
907 os.utime(dest, (newmtime, newmtime))
908 else:
909 os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
910 newmtime = sstat[stat.ST_MTIME]
911 return newmtime
912
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800913def break_hardlinks(src, sstat = None):
914 """
915 Ensures src is the only hardlink to this file. Other hardlinks,
916 if any, are not affected (other than in their st_nlink value, of
917 course). Returns true on success and false on failure.
918
919 """
920 try:
921 if not sstat:
922 sstat = os.lstat(src)
923 except Exception as e:
924 logger.warning("break_hardlinks: stat of %s failed (%s)" % (src, e))
925 return False
926 if sstat[stat.ST_NLINK] == 1:
927 return True
928 return copyfile(src, src, sstat=sstat)
929
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500930def which(path, item, direction = 0, history = False, executable=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500931 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500932 Locate `item` in the list of paths `path` (colon separated string like $PATH).
933 If `direction` is non-zero then the list is reversed.
934 If `history` is True then the list of candidates also returned as result,history.
935 If `executable` is True then the candidate has to be an executable file,
936 otherwise the candidate simply has to exist.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500937 """
938
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500939 if executable:
940 is_candidate = lambda p: os.path.isfile(p) and os.access(p, os.X_OK)
941 else:
942 is_candidate = lambda p: os.path.exists(p)
943
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500944 hist = []
945 paths = (path or "").split(':')
946 if direction != 0:
947 paths.reverse()
948
949 for p in paths:
950 next = os.path.join(p, item)
951 hist.append(next)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500952 if is_candidate(next):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500953 if not os.path.isabs(next):
954 next = os.path.abspath(next)
955 if history:
956 return next, hist
957 return next
958
959 if history:
960 return "", hist
961 return ""
962
963def to_boolean(string, default=None):
964 if not string:
965 return default
966
967 normalized = string.lower()
968 if normalized in ("y", "yes", "1", "true"):
969 return True
970 elif normalized in ("n", "no", "0", "false"):
971 return False
972 else:
973 raise ValueError("Invalid value for to_boolean: %s" % string)
974
975def contains(variable, checkvalues, truevalue, falsevalue, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500976 """Check if a variable contains all the values specified.
977
978 Arguments:
979
980 variable -- the variable name. This will be fetched and expanded (using
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500981 d.getVar(variable)) and then split into a set().
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500982
983 checkvalues -- if this is a string it is split on whitespace into a set(),
984 otherwise coerced directly into a set().
985
986 truevalue -- the value to return if checkvalues is a subset of variable.
987
988 falsevalue -- the value to return if variable is empty or if checkvalues is
989 not a subset of variable.
990
991 d -- the data store.
992 """
993
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500994 val = d.getVar(variable)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500995 if not val:
996 return falsevalue
997 val = set(val.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600998 if isinstance(checkvalues, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500999 checkvalues = set(checkvalues.split())
1000 else:
1001 checkvalues = set(checkvalues)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001002 if checkvalues.issubset(val):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001003 return truevalue
1004 return falsevalue
1005
1006def contains_any(variable, checkvalues, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001007 val = d.getVar(variable)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001008 if not val:
1009 return falsevalue
1010 val = set(val.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001011 if isinstance(checkvalues, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001012 checkvalues = set(checkvalues.split())
1013 else:
1014 checkvalues = set(checkvalues)
1015 if checkvalues & val:
1016 return truevalue
1017 return falsevalue
1018
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001019def filter(variable, checkvalues, d):
1020 """Return all words in the variable that are present in the checkvalues.
1021
1022 Arguments:
1023
1024 variable -- the variable name. This will be fetched and expanded (using
1025 d.getVar(variable)) and then split into a set().
1026
1027 checkvalues -- if this is a string it is split on whitespace into a set(),
1028 otherwise coerced directly into a set().
1029
1030 d -- the data store.
1031 """
1032
1033 val = d.getVar(variable)
1034 if not val:
1035 return ''
1036 val = set(val.split())
1037 if isinstance(checkvalues, str):
1038 checkvalues = set(checkvalues.split())
1039 else:
1040 checkvalues = set(checkvalues)
1041 return ' '.join(sorted(checkvalues & val))
1042
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001043def cpu_count():
1044 return multiprocessing.cpu_count()
1045
1046def nonblockingfd(fd):
1047 fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
1048
1049def process_profilelog(fn, pout = None):
1050 # Either call with a list of filenames and set pout or a filename and optionally pout.
1051 if not pout:
1052 pout = fn + '.processed'
1053 pout = open(pout, 'w')
1054
1055 import pstats
1056 if isinstance(fn, list):
1057 p = pstats.Stats(*fn, stream=pout)
1058 else:
1059 p = pstats.Stats(fn, stream=pout)
1060 p.sort_stats('time')
1061 p.print_stats()
1062 p.print_callers()
1063 p.sort_stats('cumulative')
1064 p.print_stats()
1065
1066 pout.flush()
1067 pout.close()
1068
1069#
1070# Was present to work around multiprocessing pool bugs in python < 2.7.3
1071#
1072def multiprocessingpool(*args, **kwargs):
1073
1074 import multiprocessing.pool
1075 #import multiprocessing.util
1076 #multiprocessing.util.log_to_stderr(10)
1077 # Deal with a multiprocessing bug where signals to the processes would be delayed until the work
1078 # completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed.
1079 def wrapper(func):
1080 def wrap(self, timeout=None):
1081 return func(self, timeout=timeout if timeout is not None else 1e100)
1082 return wrap
1083 multiprocessing.pool.IMapIterator.next = wrapper(multiprocessing.pool.IMapIterator.next)
1084
1085 return multiprocessing.Pool(*args, **kwargs)
1086
1087def exec_flat_python_func(func, *args, **kwargs):
1088 """Execute a flat python function (defined with def funcname(args):...)"""
1089 # Prepare a small piece of python code which calls the requested function
1090 # To do this we need to prepare two things - a set of variables we can use to pass
1091 # the values of arguments into the calling function, and the list of arguments for
1092 # the function being called
1093 context = {}
1094 funcargs = []
1095 # Handle unnamed arguments
1096 aidx = 1
1097 for arg in args:
1098 argname = 'arg_%s' % aidx
1099 context[argname] = arg
1100 funcargs.append(argname)
1101 aidx += 1
1102 # Handle keyword arguments
1103 context.update(kwargs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001104 funcargs.extend(['%s=%s' % (arg, arg) for arg in kwargs.keys()])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001105 code = 'retval = %s(%s)' % (func, ', '.join(funcargs))
1106 comp = bb.utils.better_compile(code, '<string>', '<string>')
1107 bb.utils.better_exec(comp, context, code, '<string>')
1108 return context['retval']
1109
1110def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
1111 """Edit lines from a recipe or config file and modify one or more
1112 specified variable values set in the file using a specified callback
1113 function. Lines are expected to have trailing newlines.
1114 Parameters:
1115 meta_lines: lines from the file; can be a list or an iterable
1116 (e.g. file pointer)
1117 variables: a list of variable names to look for. Functions
1118 may also be specified, but must be specified with '()' at
1119 the end of the name. Note that the function doesn't have
1120 any intrinsic understanding of _append, _prepend, _remove,
1121 or overrides, so these are considered as part of the name.
1122 These values go into a regular expression, so regular
1123 expression syntax is allowed.
1124 varfunc: callback function called for every variable matching
1125 one of the entries in the variables parameter. The function
1126 should take four arguments:
1127 varname: name of variable matched
1128 origvalue: current value in file
1129 op: the operator (e.g. '+=')
1130 newlines: list of lines up to this point. You can use
1131 this to prepend lines before this variable setting
1132 if you wish.
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001133 and should return a four-element tuple:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001134 newvalue: new value to substitute in, or None to drop
1135 the variable setting entirely. (If the removal
1136 results in two consecutive blank lines, one of the
1137 blank lines will also be dropped).
1138 newop: the operator to use - if you specify None here,
1139 the original operation will be used.
1140 indent: number of spaces to indent multi-line entries,
1141 or -1 to indent up to the level of the assignment
1142 and opening quote, or a string to use as the indent.
1143 minbreak: True to allow the first element of a
1144 multi-line value to continue on the same line as
1145 the assignment, False to indent before the first
1146 element.
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001147 To clarify, if you wish not to change the value, then you
1148 would return like this: return origvalue, None, 0, True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001149 match_overrides: True to match items with _overrides on the end,
1150 False otherwise
1151 Returns a tuple:
1152 updated:
1153 True if changes were made, False otherwise.
1154 newlines:
1155 Lines after processing
1156 """
1157
1158 var_res = {}
1159 if match_overrides:
Brad Bishop19323692019-04-05 15:28:33 -04001160 override_re = r'(_[a-zA-Z0-9-_$(){}]+)?'
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001161 else:
1162 override_re = ''
1163 for var in variables:
1164 if var.endswith('()'):
Brad Bishop19323692019-04-05 15:28:33 -04001165 var_res[var] = re.compile(r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001166 else:
Brad Bishop19323692019-04-05 15:28:33 -04001167 var_res[var] = re.compile(r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001168
1169 updated = False
1170 varset_start = ''
1171 varlines = []
1172 newlines = []
1173 in_var = None
1174 full_value = ''
1175 var_end = ''
1176
1177 def handle_var_end():
1178 prerun_newlines = newlines[:]
1179 op = varset_start[len(in_var):].strip()
1180 (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines)
1181 changed = (prerun_newlines != newlines)
1182
1183 if newvalue is None:
1184 # Drop the value
1185 return True
1186 elif newvalue != full_value or (newop not in [None, op]):
1187 if newop not in [None, op]:
1188 # Callback changed the operator
1189 varset_new = "%s %s" % (in_var, newop)
1190 else:
1191 varset_new = varset_start
1192
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001193 if isinstance(indent, int):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001194 if indent == -1:
1195 indentspc = ' ' * (len(varset_new) + 2)
1196 else:
1197 indentspc = ' ' * indent
1198 else:
1199 indentspc = indent
1200 if in_var.endswith('()'):
1201 # A function definition
1202 if isinstance(newvalue, list):
1203 newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue)))
1204 else:
1205 if not newvalue.startswith('\n'):
1206 newvalue = '\n' + newvalue
1207 if not newvalue.endswith('\n'):
1208 newvalue = newvalue + '\n'
1209 newlines.append('%s {%s}\n' % (varset_new, newvalue))
1210 else:
1211 # Normal variable
1212 if isinstance(newvalue, list):
1213 if not newvalue:
1214 # Empty list -> empty string
1215 newlines.append('%s ""\n' % varset_new)
1216 elif minbreak:
1217 # First item on first line
1218 if len(newvalue) == 1:
1219 newlines.append('%s "%s"\n' % (varset_new, newvalue[0]))
1220 else:
1221 newlines.append('%s "%s \\\n' % (varset_new, newvalue[0]))
1222 for item in newvalue[1:]:
1223 newlines.append('%s%s \\\n' % (indentspc, item))
1224 newlines.append('%s"\n' % indentspc)
1225 else:
1226 # No item on first line
1227 newlines.append('%s " \\\n' % varset_new)
1228 for item in newvalue:
1229 newlines.append('%s%s \\\n' % (indentspc, item))
1230 newlines.append('%s"\n' % indentspc)
1231 else:
1232 newlines.append('%s "%s"\n' % (varset_new, newvalue))
1233 return True
1234 else:
1235 # Put the old lines back where they were
1236 newlines.extend(varlines)
1237 # If newlines was touched by the function, we'll need to return True
1238 return changed
1239
1240 checkspc = False
1241
1242 for line in meta_lines:
1243 if in_var:
1244 value = line.rstrip()
1245 varlines.append(line)
1246 if in_var.endswith('()'):
1247 full_value += '\n' + value
1248 else:
1249 full_value += value[:-1]
1250 if value.endswith(var_end):
1251 if in_var.endswith('()'):
1252 if full_value.count('{') - full_value.count('}') >= 0:
1253 continue
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001254 full_value = full_value[:-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001255 if handle_var_end():
1256 updated = True
1257 checkspc = True
1258 in_var = None
1259 else:
1260 skip = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001261 for (varname, var_re) in var_res.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001262 res = var_re.match(line)
1263 if res:
1264 isfunc = varname.endswith('()')
1265 if isfunc:
1266 splitvalue = line.split('{', 1)
1267 var_end = '}'
1268 else:
1269 var_end = res.groups()[-1]
1270 splitvalue = line.split(var_end, 1)
1271 varset_start = splitvalue[0].rstrip()
1272 value = splitvalue[1].rstrip()
1273 if not isfunc and value.endswith('\\'):
1274 value = value[:-1]
1275 full_value = value
1276 varlines = [line]
1277 in_var = res.group(1)
1278 if isfunc:
1279 in_var += '()'
1280 if value.endswith(var_end):
1281 full_value = full_value[:-1]
1282 if handle_var_end():
1283 updated = True
1284 checkspc = True
1285 in_var = None
1286 skip = True
1287 break
1288 if not skip:
1289 if checkspc:
1290 checkspc = False
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001291 if newlines and newlines[-1] == '\n' and line == '\n':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001292 # Squash blank line if there are two consecutive blanks after a removal
1293 continue
1294 newlines.append(line)
1295 return (updated, newlines)
1296
1297
1298def edit_metadata_file(meta_file, variables, varfunc):
1299 """Edit a recipe or config file and modify one or more specified
1300 variable values set in the file using a specified callback function.
1301 The file is only written to if the value(s) actually change.
1302 This is basically the file version of edit_metadata(), see that
1303 function's description for parameter/usage information.
1304 Returns True if the file was written to, False otherwise.
1305 """
1306 with open(meta_file, 'r') as f:
1307 (updated, newlines) = edit_metadata(f, variables, varfunc)
1308 if updated:
1309 with open(meta_file, 'w') as f:
1310 f.writelines(newlines)
1311 return updated
1312
1313
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001314def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001315 """Edit bblayers.conf, adding and/or removing layers
1316 Parameters:
1317 bblayers_conf: path to bblayers.conf file to edit
1318 add: layer path (or list of layer paths) to add; None or empty
1319 list to add nothing
1320 remove: layer path (or list of layer paths) to remove; None or
1321 empty list to remove nothing
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001322 edit_cb: optional callback function that will be called after
1323 processing adds/removes once per existing entry.
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001324 Returns a tuple:
1325 notadded: list of layers specified to be added but weren't
1326 (because they were already in the list)
1327 notremoved: list of layers that were specified to be removed
1328 but weren't (because they weren't in the list)
1329 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001330
1331 import fnmatch
1332
1333 def remove_trailing_sep(pth):
1334 if pth and pth[-1] == os.sep:
1335 pth = pth[:-1]
1336 return pth
1337
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001338 approved = bb.utils.approved_variables()
1339 def canonicalise_path(pth):
1340 pth = remove_trailing_sep(pth)
1341 if 'HOME' in approved and '~' in pth:
1342 pth = os.path.expanduser(pth)
1343 return pth
1344
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001345 def layerlist_param(value):
1346 if not value:
1347 return []
1348 elif isinstance(value, list):
1349 return [remove_trailing_sep(x) for x in value]
1350 else:
1351 return [remove_trailing_sep(value)]
1352
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001353 addlayers = layerlist_param(add)
1354 removelayers = layerlist_param(remove)
1355
1356 # Need to use a list here because we can't set non-local variables from a callback in python 2.x
1357 bblayercalls = []
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001358 removed = []
1359 plusequals = False
1360 orig_bblayers = []
1361
1362 def handle_bblayers_firstpass(varname, origvalue, op, newlines):
1363 bblayercalls.append(op)
1364 if op == '=':
1365 del orig_bblayers[:]
1366 orig_bblayers.extend([canonicalise_path(x) for x in origvalue.split()])
1367 return (origvalue, None, 2, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001368
1369 def handle_bblayers(varname, origvalue, op, newlines):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001370 updated = False
1371 bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
1372 if removelayers:
1373 for removelayer in removelayers:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001374 for layer in bblayers:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001375 if fnmatch.fnmatch(canonicalise_path(layer), canonicalise_path(removelayer)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001376 updated = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001377 bblayers.remove(layer)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001378 removed.append(removelayer)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001379 break
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001380 if addlayers and not plusequals:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001381 for addlayer in addlayers:
1382 if addlayer not in bblayers:
1383 updated = True
1384 bblayers.append(addlayer)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001385 del addlayers[:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001386
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001387 if edit_cb:
1388 newlist = []
1389 for layer in bblayers:
1390 res = edit_cb(layer, canonicalise_path(layer))
1391 if res != layer:
1392 newlist.append(res)
1393 updated = True
1394 else:
1395 newlist.append(layer)
1396 bblayers = newlist
1397
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001398 if updated:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001399 if op == '+=' and not bblayers:
1400 bblayers = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001401 return (bblayers, None, 2, False)
1402 else:
1403 return (origvalue, None, 2, False)
1404
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001405 with open(bblayers_conf, 'r') as f:
1406 (_, newlines) = edit_metadata(f, ['BBLAYERS'], handle_bblayers_firstpass)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001407
1408 if not bblayercalls:
1409 raise Exception('Unable to find BBLAYERS in %s' % bblayers_conf)
1410
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001411 # Try to do the "smart" thing depending on how the user has laid out
1412 # their bblayers.conf file
1413 if bblayercalls.count('+=') > 1:
1414 plusequals = True
1415
1416 removelayers_canon = [canonicalise_path(layer) for layer in removelayers]
1417 notadded = []
1418 for layer in addlayers:
1419 layer_canon = canonicalise_path(layer)
1420 if layer_canon in orig_bblayers and not layer_canon in removelayers_canon:
1421 notadded.append(layer)
1422 notadded_canon = [canonicalise_path(layer) for layer in notadded]
1423 addlayers[:] = [layer for layer in addlayers if canonicalise_path(layer) not in notadded_canon]
1424
1425 (updated, newlines) = edit_metadata(newlines, ['BBLAYERS'], handle_bblayers)
1426 if addlayers:
1427 # Still need to add these
1428 for addlayer in addlayers:
1429 newlines.append('BBLAYERS += "%s"\n' % addlayer)
1430 updated = True
1431
1432 if updated:
1433 with open(bblayers_conf, 'w') as f:
1434 f.writelines(newlines)
1435
1436 notremoved = list(set(removelayers) - set(removed))
1437
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001438 return (notadded, notremoved)
1439
1440
1441def get_file_layer(filename, d):
1442 """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001443 collections = (d.getVar('BBFILE_COLLECTIONS') or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001444 collection_res = {}
1445 for collection in collections:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001446 collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection) or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001447
1448 def path_to_layer(path):
1449 # Use longest path so we handle nested layers
1450 matchlen = 0
1451 match = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001452 for collection, regex in collection_res.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001453 if len(regex) > matchlen and re.match(regex, path):
1454 matchlen = len(regex)
1455 match = collection
1456 return match
1457
1458 result = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001459 bbfiles = (d.getVar('BBFILES') or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001460 bbfilesmatch = False
1461 for bbfilesentry in bbfiles:
1462 if fnmatch.fnmatch(filename, bbfilesentry):
1463 bbfilesmatch = True
1464 result = path_to_layer(bbfilesentry)
1465
1466 if not bbfilesmatch:
1467 # Probably a bbclass
1468 result = path_to_layer(filename)
1469
1470 return result
1471
1472
1473# Constant taken from http://linux.die.net/include/linux/prctl.h
1474PR_SET_PDEATHSIG = 1
1475
1476class PrCtlError(Exception):
1477 pass
1478
1479def signal_on_parent_exit(signame):
1480 """
1481 Trigger signame to be sent when the parent process dies
1482 """
1483 signum = getattr(signal, signame)
1484 # http://linux.die.net/man/2/prctl
1485 result = cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG, signum)
1486 if result != 0:
1487 raise PrCtlError('prctl failed with error code %s' % result)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001488
1489#
1490# Manually call the ioprio syscall. We could depend on other libs like psutil
1491# however this gets us enough of what we need to bitbake for now without the
1492# dependency
1493#
1494_unamearch = os.uname()[4]
1495IOPRIO_WHO_PROCESS = 1
1496IOPRIO_CLASS_SHIFT = 13
1497
1498def ioprio_set(who, cls, value):
1499 NR_ioprio_set = None
1500 if _unamearch == "x86_64":
1501 NR_ioprio_set = 251
1502 elif _unamearch[0] == "i" and _unamearch[2:3] == "86":
1503 NR_ioprio_set = 289
Brad Bishop19323692019-04-05 15:28:33 -04001504 elif _unamearch == "aarch64":
1505 NR_ioprio_set = 30
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001506
1507 if NR_ioprio_set:
1508 ioprio = value | (cls << IOPRIO_CLASS_SHIFT)
1509 rc = cdll['libc.so.6'].syscall(NR_ioprio_set, IOPRIO_WHO_PROCESS, who, ioprio)
1510 if rc != 0:
1511 raise ValueError("Unable to set ioprio, syscall returned %s" % rc)
1512 else:
1513 bb.warn("Unable to set IO Prio for arch %s" % _unamearch)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001514
1515def set_process_name(name):
1516 from ctypes import cdll, byref, create_string_buffer
1517 # This is nice to have for debugging, not essential
1518 try:
1519 libc = cdll.LoadLibrary('libc.so.6')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001520 buf = create_string_buffer(bytes(name, 'utf-8'))
1521 libc.prctl(15, byref(buf), 0, 0, 0)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001522 except:
1523 pass
1524
1525# export common proxies variables from datastore to environment
1526def export_proxies(d):
1527 import os
1528
1529 variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001530 'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
1531 'GIT_PROXY_COMMAND']
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001532 exported = False
1533
1534 for v in variables:
1535 if v in os.environ.keys():
1536 exported = True
1537 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001538 v_proxy = d.getVar(v)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001539 if v_proxy is not None:
1540 os.environ[v] = v_proxy
1541 exported = True
1542
1543 return exported
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001544
1545
1546def load_plugins(logger, plugins, pluginpath):
1547 def load_plugin(name):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001548 logger.debug(1, 'Loading plugin %s' % name)
Brad Bishop19323692019-04-05 15:28:33 -04001549 spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
1550 if spec:
1551 return spec.loader.load_module()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001552
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001553 logger.debug(1, 'Loading plugins from %s...' % pluginpath)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001554
1555 expanded = (glob.glob(os.path.join(pluginpath, '*' + ext))
1556 for ext in python_extensions)
1557 files = itertools.chain.from_iterable(expanded)
1558 names = set(os.path.splitext(os.path.basename(fn))[0] for fn in files)
1559 for name in names:
1560 if name != '__init__':
1561 plugin = load_plugin(name)
1562 if hasattr(plugin, 'plugin_init'):
1563 obj = plugin.plugin_init(plugins)
1564 plugins.append(obj or plugin)
1565 else:
1566 plugins.append(plugin)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001567
1568
1569class LogCatcher(logging.Handler):
1570 """Logging handler for collecting logged messages so you can check them later"""
1571 def __init__(self):
1572 self.messages = []
1573 logging.Handler.__init__(self, logging.WARNING)
1574 def emit(self, record):
1575 self.messages.append(bb.build.logformatter.format(record))
1576 def contains(self, message):
1577 return (message in self.messages)