blob: 16fc9db2575f58130be3281bb1552329a085652d [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake Utility Functions
5"""
6
7# Copyright (C) 2004 Michael Lauer
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22import re, fcntl, os, string, stat, shutil, time
23import sys
24import errno
25import logging
26import bb
27import bb.msg
28import multiprocessing
29import fcntl
Patrick Williamsc0f7c042017-02-23 20:41:17 -060030import imp
31import itertools
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032import subprocess
33import glob
34import fnmatch
35import traceback
36import errno
37import signal
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050038import ast
Patrick Williamsc0f7c042017-02-23 20:41:17 -060039import collections
40import copy
41from subprocess import getstatusoutput
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042from contextlib import contextmanager
43from ctypes import cdll
44
Patrick Williamsc124f4f2015-09-15 14:41:29 -050045logger = logging.getLogger("BitBake.Util")
Patrick Williamsc0f7c042017-02-23 20:41:17 -060046python_extensions = [e for e, _, _ in imp.get_suffixes()]
47
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048
49def clean_context():
50 return {
51 "os": os,
52 "bb": bb,
53 "time": time,
54 }
55
56def get_context():
57 return _context
58
59
60def set_context(ctx):
61 _context = ctx
62
63# Context used in better_exec, eval
64_context = clean_context()
65
66class VersionStringException(Exception):
67 """Exception raised when an invalid version specification is found"""
68
69def explode_version(s):
70 r = []
71 alpha_regexp = re.compile('^([a-zA-Z]+)(.*)$')
72 numeric_regexp = re.compile('^(\d+)(.*)$')
73 while (s != ''):
74 if s[0] in string.digits:
75 m = numeric_regexp.match(s)
76 r.append((0, int(m.group(1))))
77 s = m.group(2)
78 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -060079 if s[0] in string.ascii_letters:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050080 m = alpha_regexp.match(s)
81 r.append((1, m.group(1)))
82 s = m.group(2)
83 continue
84 if s[0] == '~':
85 r.append((-1, s[0]))
86 else:
87 r.append((2, s[0]))
88 s = s[1:]
89 return r
90
91def split_version(s):
92 """Split a version string into its constituent parts (PE, PV, PR)"""
93 s = s.strip(" <>=")
94 e = 0
95 if s.count(':'):
96 e = int(s.split(":")[0])
97 s = s.split(":")[1]
98 r = ""
99 if s.count('-'):
100 r = s.rsplit("-", 1)[1]
101 s = s.rsplit("-", 1)[0]
102 v = s
103 return (e, v, r)
104
105def vercmp_part(a, b):
106 va = explode_version(a)
107 vb = explode_version(b)
108 while True:
109 if va == []:
110 (oa, ca) = (0, None)
111 else:
112 (oa, ca) = va.pop(0)
113 if vb == []:
114 (ob, cb) = (0, None)
115 else:
116 (ob, cb) = vb.pop(0)
117 if (oa, ca) == (0, None) and (ob, cb) == (0, None):
118 return 0
119 if oa < ob:
120 return -1
121 elif oa > ob:
122 return 1
123 elif ca < cb:
124 return -1
125 elif ca > cb:
126 return 1
127
128def vercmp(ta, tb):
129 (ea, va, ra) = ta
130 (eb, vb, rb) = tb
131
132 r = int(ea or 0) - int(eb or 0)
133 if (r == 0):
134 r = vercmp_part(va, vb)
135 if (r == 0):
136 r = vercmp_part(ra, rb)
137 return r
138
139def vercmp_string(a, b):
140 ta = split_version(a)
141 tb = split_version(b)
142 return vercmp(ta, tb)
143
144def vercmp_string_op(a, b, op):
145 """
146 Compare two versions and check if the specified comparison operator matches the result of the comparison.
147 This function is fairly liberal about what operators it will accept since there are a variety of styles
148 depending on the context.
149 """
150 res = vercmp_string(a, b)
151 if op in ('=', '=='):
152 return res == 0
153 elif op == '<=':
154 return res <= 0
155 elif op == '>=':
156 return res >= 0
157 elif op in ('>', '>>'):
158 return res > 0
159 elif op in ('<', '<<'):
160 return res < 0
161 elif op == '!=':
162 return res != 0
163 else:
164 raise VersionStringException('Unsupported comparison operator "%s"' % op)
165
166def explode_deps(s):
167 """
168 Take an RDEPENDS style string of format:
169 "DEPEND1 (optional version) DEPEND2 (optional version) ..."
170 and return a list of dependencies.
171 Version information is ignored.
172 """
173 r = []
174 l = s.split()
175 flag = False
176 for i in l:
177 if i[0] == '(':
178 flag = True
179 #j = []
180 if not flag:
181 r.append(i)
182 #else:
183 # j.append(i)
184 if flag and i.endswith(')'):
185 flag = False
186 # Ignore version
187 #r[-1] += ' ' + ' '.join(j)
188 return r
189
190def explode_dep_versions2(s):
191 """
192 Take an RDEPENDS style string of format:
193 "DEPEND1 (optional version) DEPEND2 (optional version) ..."
194 and return a dictionary of dependencies and versions.
195 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600196 r = collections.OrderedDict()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 l = s.replace(",", "").split()
198 lastdep = None
199 lastcmp = ""
200 lastver = ""
201 incmp = False
202 inversion = False
203 for i in l:
204 if i[0] == '(':
205 incmp = True
206 i = i[1:].strip()
207 if not i:
208 continue
209
210 if incmp:
211 incmp = False
212 inversion = True
213 # This list is based on behavior and supported comparisons from deb, opkg and rpm.
214 #
215 # Even though =<, <<, ==, !=, =>, and >> may not be supported,
216 # we list each possibly valid item.
217 # The build system is responsible for validation of what it supports.
218 if i.startswith(('<=', '=<', '<<', '==', '!=', '>=', '=>', '>>')):
219 lastcmp = i[0:2]
220 i = i[2:]
221 elif i.startswith(('<', '>', '=')):
222 lastcmp = i[0:1]
223 i = i[1:]
224 else:
225 # This is an unsupported case!
226 raise VersionStringException('Invalid version specification in "(%s" - invalid or missing operator' % i)
227 lastcmp = (i or "")
228 i = ""
229 i.strip()
230 if not i:
231 continue
232
233 if inversion:
234 if i.endswith(')'):
235 i = i[:-1] or ""
236 inversion = False
237 if lastver and i:
238 lastver += " "
239 if i:
240 lastver += i
241 if lastdep not in r:
242 r[lastdep] = []
243 r[lastdep].append(lastcmp + " " + lastver)
244 continue
245
246 #if not inversion:
247 lastdep = i
248 lastver = ""
249 lastcmp = ""
250 if not (i in r and r[i]):
251 r[lastdep] = []
252
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600253 r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 return r
255
256def explode_dep_versions(s):
257 r = explode_dep_versions2(s)
258 for d in r:
259 if not r[d]:
260 r[d] = None
261 continue
262 if len(r[d]) > 1:
263 bb.warn("explode_dep_versions(): Item %s appeared in dependency string '%s' multiple times with different values. explode_dep_versions cannot cope with this." % (d, s))
264 r[d] = r[d][0]
265 return r
266
267def join_deps(deps, commasep=True):
268 """
269 Take the result from explode_dep_versions and generate a dependency string
270 """
271 result = []
272 for dep in deps:
273 if deps[dep]:
274 if isinstance(deps[dep], list):
275 for v in deps[dep]:
276 result.append(dep + " (" + v + ")")
277 else:
278 result.append(dep + " (" + deps[dep] + ")")
279 else:
280 result.append(dep)
281 if commasep:
282 return ", ".join(result)
283 else:
284 return " ".join(result)
285
286def _print_trace(body, line):
287 """
288 Print the Environment of a Text Body
289 """
290 error = []
291 # print the environment of the method
292 min_line = max(1, line-4)
293 max_line = min(line + 4, len(body))
294 for i in range(min_line, max_line + 1):
295 if line == i:
296 error.append(' *** %.4d:%s' % (i, body[i-1].rstrip()))
297 else:
298 error.append(' %.4d:%s' % (i, body[i-1].rstrip()))
299 return error
300
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500301def better_compile(text, file, realfile, mode = "exec", lineno = 0):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302 """
303 A better compile method. This method
304 will print the offending lines.
305 """
306 try:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500307 cache = bb.methodpool.compile_cache(text)
308 if cache:
309 return cache
310 # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
311 text2 = "\n" * int(lineno) + text
312 code = compile(text2, realfile, mode)
313 bb.methodpool.compile_cache_add(text, code)
314 return code
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315 except Exception as e:
316 error = []
317 # split the text into lines again
318 body = text.split('\n')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500319 error.append("Error in compiling python function in %s, line %s:\n" % (realfile, lineno))
320 if hasattr(e, "lineno"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 error.append("The code lines resulting in this error were:")
322 error.extend(_print_trace(body, e.lineno))
323 else:
324 error.append("The function causing this error was:")
325 for line in body:
326 error.append(line)
327 error.append("%s: %s" % (e.__class__.__name__, str(e)))
328
329 logger.error("\n".join(error))
330
331 e = bb.BBHandledException(e)
332 raise e
333
334def _print_exception(t, value, tb, realfile, text, context):
335 error = []
336 try:
337 exception = traceback.format_exception_only(t, value)
338 error.append('Error executing a python function in %s:\n' % realfile)
339
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500340 # Strip 'us' from the stack (better_exec call) unless that was where the
341 # error came from
342 if tb.tb_next is not None:
343 tb = tb.tb_next
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344
345 textarray = text.split('\n')
346
347 linefailed = tb.tb_lineno
348
349 tbextract = traceback.extract_tb(tb)
350 tbformat = traceback.format_list(tbextract)
351 error.append("The stack trace of python calls that resulted in this exception/failure was:")
352 error.append("File: '%s', lineno: %s, function: %s" % (tbextract[0][0], tbextract[0][1], tbextract[0][2]))
353 error.extend(_print_trace(textarray, linefailed))
354
355 # See if this is a function we constructed and has calls back into other functions in
356 # "text". If so, try and improve the context of the error by diving down the trace
357 level = 0
358 nexttb = tb.tb_next
359 while nexttb is not None and (level+1) < len(tbextract):
360 error.append("File: '%s', lineno: %s, function: %s" % (tbextract[level+1][0], tbextract[level+1][1], tbextract[level+1][2]))
361 if tbextract[level][0] == tbextract[level+1][0] and tbextract[level+1][2] == tbextract[level][0]:
362 # The code was possibly in the string we compiled ourselves
363 error.extend(_print_trace(textarray, tbextract[level+1][1]))
364 elif tbextract[level+1][0].startswith("/"):
365 # The code looks like it might be in a file, try and load it
366 try:
367 with open(tbextract[level+1][0], "r") as f:
368 text = f.readlines()
369 error.extend(_print_trace(text, tbextract[level+1][1]))
370 except:
371 error.append(tbformat[level+1])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500372 else:
373 error.append(tbformat[level+1])
374 nexttb = tb.tb_next
375 level = level + 1
376
377 error.append("Exception: %s" % ''.join(exception))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600378
379 # If the exception is from spwaning a task, let's be helpful and display
380 # the output (which hopefully includes stderr).
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500381 if isinstance(value, subprocess.CalledProcessError) and value.output:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600382 error.append("Subprocess output:")
383 error.append(value.output.decode("utf-8", errors="ignore"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500384 finally:
385 logger.error("\n".join(error))
386
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500387def better_exec(code, context, text = None, realfile = "<code>", pythonexception=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500388 """
389 Similiar to better_compile, better_exec will
390 print the lines that are responsible for the
391 error.
392 """
393 import bb.parse
394 if not text:
395 text = code
396 if not hasattr(code, "co_filename"):
397 code = better_compile(code, realfile, realfile)
398 try:
399 exec(code, get_context(), context)
400 except (bb.BBHandledException, bb.parse.SkipRecipe, bb.build.FuncFailed, bb.data_smart.ExpansionError):
401 # Error already shown so passthrough, no need for traceback
402 raise
403 except Exception as e:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500404 if pythonexception:
405 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500406 (t, value, tb) = sys.exc_info()
407 try:
408 _print_exception(t, value, tb, realfile, text, context)
409 except Exception as e:
410 logger.error("Exception handler error: %s" % str(e))
411
412 e = bb.BBHandledException(e)
413 raise e
414
415def simple_exec(code, context):
416 exec(code, get_context(), context)
417
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600418def better_eval(source, locals, extraglobals = None):
419 ctx = get_context()
420 if extraglobals:
421 ctx = copy.copy(ctx)
422 for g in extraglobals:
423 ctx[g] = extraglobals[g]
424 return eval(source, ctx, locals)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500425
426@contextmanager
427def fileslocked(files):
428 """Context manager for locking and unlocking file locks."""
429 locks = []
430 if files:
431 for lockfile in files:
432 locks.append(bb.utils.lockfile(lockfile))
433
434 yield
435
436 for lock in locks:
437 bb.utils.unlockfile(lock)
438
439@contextmanager
440def timeout(seconds):
441 def timeout_handler(signum, frame):
442 pass
443
444 original_handler = signal.signal(signal.SIGALRM, timeout_handler)
445
446 try:
447 signal.alarm(seconds)
448 yield
449 finally:
450 signal.alarm(0)
451 signal.signal(signal.SIGALRM, original_handler)
452
453def lockfile(name, shared=False, retry=True, block=False):
454 """
455 Use the specified file as a lock file, return when the lock has
456 been acquired. Returns a variable to pass to unlockfile().
457 Parameters:
458 retry: True to re-try locking if it fails, False otherwise
459 block: True to block until the lock succeeds, False otherwise
460 The retry and block parameters are kind of equivalent unless you
461 consider the possibility of sending a signal to the process to break
462 out - at which point you want block=True rather than retry=True.
463 """
464 dirname = os.path.dirname(name)
465 mkdirhier(dirname)
466
467 if not os.access(dirname, os.W_OK):
468 logger.error("Unable to acquire lock '%s', directory is not writable",
469 name)
470 sys.exit(1)
471
472 op = fcntl.LOCK_EX
473 if shared:
474 op = fcntl.LOCK_SH
475 if not retry and not block:
476 op = op | fcntl.LOCK_NB
477
478 while True:
479 # If we leave the lockfiles lying around there is no problem
480 # but we should clean up after ourselves. This gives potential
481 # for races though. To work around this, when we acquire the lock
482 # we check the file we locked was still the lock file on disk.
483 # by comparing inode numbers. If they don't match or the lockfile
484 # no longer exists, we start again.
485
486 # This implementation is unfair since the last person to request the
487 # lock is the most likely to win it.
488
489 try:
490 lf = open(name, 'a+')
491 fileno = lf.fileno()
492 fcntl.flock(fileno, op)
493 statinfo = os.fstat(fileno)
494 if os.path.exists(lf.name):
495 statinfo2 = os.stat(lf.name)
496 if statinfo.st_ino == statinfo2.st_ino:
497 return lf
498 lf.close()
499 except Exception:
500 try:
501 lf.close()
502 except Exception:
503 pass
504 pass
505 if not retry:
506 return None
507
508def unlockfile(lf):
509 """
510 Unlock a file locked using lockfile()
511 """
512 try:
513 # If we had a shared lock, we need to promote to exclusive before
514 # removing the lockfile. Attempt this, ignore failures.
515 fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
516 os.unlink(lf.name)
517 except (IOError, OSError):
518 pass
519 fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
520 lf.close()
521
522def md5_file(filename):
523 """
524 Return the hex string representation of the MD5 checksum of filename.
525 """
526 try:
527 import hashlib
528 m = hashlib.md5()
529 except ImportError:
530 import md5
531 m = md5.new()
532
533 with open(filename, "rb") as f:
534 for line in f:
535 m.update(line)
536 return m.hexdigest()
537
538def sha256_file(filename):
539 """
540 Return the hex string representation of the 256-bit SHA checksum of
541 filename. On Python 2.4 this will return None, so callers will need to
542 handle that by either skipping SHA checks, or running a standalone sha256sum
543 binary.
544 """
545 try:
546 import hashlib
547 except ImportError:
548 return None
549
550 s = hashlib.sha256()
551 with open(filename, "rb") as f:
552 for line in f:
553 s.update(line)
554 return s.hexdigest()
555
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500556def sha1_file(filename):
557 """
558 Return the hex string representation of the SHA1 checksum of the filename
559 """
560 try:
561 import hashlib
562 except ImportError:
563 return None
564
565 s = hashlib.sha1()
566 with open(filename, "rb") as f:
567 for line in f:
568 s.update(line)
569 return s.hexdigest()
570
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500571def preserved_envvars_exported():
572 """Variables which are taken from the environment and placed in and exported
573 from the metadata"""
574 return [
575 'BB_TASKHASH',
576 'HOME',
577 'LOGNAME',
578 'PATH',
579 'PWD',
580 'SHELL',
581 'TERM',
582 'USER',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600583 'LC_ALL',
584 'BBSERVER',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500585 ]
586
587def preserved_envvars():
588 """Variables which are taken from the environment and placed in the metadata"""
589 v = [
590 'BBPATH',
591 'BB_PRESERVE_ENV',
592 'BB_ENV_WHITELIST',
593 'BB_ENV_EXTRAWHITE',
594 ]
595 return v + preserved_envvars_exported()
596
597def filter_environment(good_vars):
598 """
599 Create a pristine environment for bitbake. This will remove variables that
600 are not known and may influence the build in a negative way.
601 """
602
603 removed_vars = {}
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600604 for key in list(os.environ):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500605 if key in good_vars:
606 continue
607
608 removed_vars[key] = os.environ[key]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500609 del os.environ[key]
610
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600611 # If we spawn a python process, we need to have a UTF-8 locale, else python's file
612 # access methods will use ascii. You can't change that mode once the interpreter is
613 # started so we have to ensure a locale is set. Ideally we'd use C.UTF-8 but not all
614 # distros support that and we need to set something.
615 os.environ["LC_ALL"] = "en_US.UTF-8"
616
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500617 if removed_vars:
618 logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
619
620 return removed_vars
621
622def approved_variables():
623 """
624 Determine and return the list of whitelisted variables which are approved
625 to remain in the environment.
626 """
627 if 'BB_PRESERVE_ENV' in os.environ:
628 return os.environ.keys()
629 approved = []
630 if 'BB_ENV_WHITELIST' in os.environ:
631 approved = os.environ['BB_ENV_WHITELIST'].split()
632 approved.extend(['BB_ENV_WHITELIST'])
633 else:
634 approved = preserved_envvars()
635 if 'BB_ENV_EXTRAWHITE' in os.environ:
636 approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split())
637 if 'BB_ENV_EXTRAWHITE' not in approved:
638 approved.extend(['BB_ENV_EXTRAWHITE'])
639 return approved
640
641def clean_environment():
642 """
643 Clean up any spurious environment variables. This will remove any
644 variables the user hasn't chosen to preserve.
645 """
646 if 'BB_PRESERVE_ENV' not in os.environ:
647 good_vars = approved_variables()
648 return filter_environment(good_vars)
649
650 return {}
651
652def empty_environment():
653 """
654 Remove all variables from the environment.
655 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600656 for s in list(os.environ.keys()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500657 os.unsetenv(s)
658 del os.environ[s]
659
660def build_environment(d):
661 """
662 Build an environment from all exported variables.
663 """
664 import bb.data
665 for var in bb.data.keys(d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500666 export = d.getVarFlag(var, "export", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500667 if export:
668 os.environ[var] = d.getVar(var, True) or ""
669
670def _check_unsafe_delete_path(path):
671 """
672 Basic safeguard against recursively deleting something we shouldn't. If it returns True,
673 the caller should raise an exception with an appropriate message.
674 NOTE: This is NOT meant to be a security mechanism - just a guard against silly mistakes
675 with potentially disastrous results.
676 """
677 extra = ''
678 # HOME might not be /home/something, so in case we can get it, check against it
679 homedir = os.environ.get('HOME', '')
680 if homedir:
681 extra = '|%s' % homedir
682 if re.match('(/|//|/home|/home/[^/]*%s)$' % extra, os.path.abspath(path)):
683 return True
684 return False
685
686def remove(path, recurse=False):
687 """Equivalent to rm -f or rm -rf"""
688 if not path:
689 return
690 if recurse:
691 for name in glob.glob(path):
692 if _check_unsafe_delete_path(path):
693 raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
694 # shutil.rmtree(name) would be ideal but its too slow
695 subprocess.call(['rm', '-rf'] + glob.glob(path))
696 return
697 for name in glob.glob(path):
698 try:
699 os.unlink(name)
700 except OSError as exc:
701 if exc.errno != errno.ENOENT:
702 raise
703
704def prunedir(topdir):
705 # Delete everything reachable from the directory named in 'topdir'.
706 # CAUTION: This is dangerous!
707 if _check_unsafe_delete_path(topdir):
708 raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
709 for root, dirs, files in os.walk(topdir, topdown = False):
710 for name in files:
711 os.remove(os.path.join(root, name))
712 for name in dirs:
713 if os.path.islink(os.path.join(root, name)):
714 os.remove(os.path.join(root, name))
715 else:
716 os.rmdir(os.path.join(root, name))
717 os.rmdir(topdir)
718
719#
720# Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var)
721# but thats possibly insane and suffixes is probably going to be small
722#
723def prune_suffix(var, suffixes, d):
724 # See if var ends with any of the suffixes listed and
725 # remove it if found
726 for suffix in suffixes:
727 if var.endswith(suffix):
728 return var.replace(suffix, "")
729 return var
730
731def mkdirhier(directory):
732 """Create a directory like 'mkdir -p', but does not complain if
733 directory already exists like os.makedirs
734 """
735
736 try:
737 os.makedirs(directory)
738 except OSError as e:
739 if e.errno != errno.EEXIST:
740 raise e
741
742def movefile(src, dest, newmtime = None, sstat = None):
743 """Moves a file from src to dest, preserving all permissions and
744 attributes; mtime will be preserved even when moving across
745 filesystems. Returns true on success and false on failure. Move is
746 atomic.
747 """
748
749 #print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
750 try:
751 if not sstat:
752 sstat = os.lstat(src)
753 except Exception as e:
754 print("movefile: Stating source file failed...", e)
755 return None
756
757 destexists = 1
758 try:
759 dstat = os.lstat(dest)
760 except:
761 dstat = os.lstat(os.path.dirname(dest))
762 destexists = 0
763
764 if destexists:
765 if stat.S_ISLNK(dstat[stat.ST_MODE]):
766 try:
767 os.unlink(dest)
768 destexists = 0
769 except Exception as e:
770 pass
771
772 if stat.S_ISLNK(sstat[stat.ST_MODE]):
773 try:
774 target = os.readlink(src)
775 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
776 os.unlink(dest)
777 os.symlink(target, dest)
778 #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
779 os.unlink(src)
780 return os.lstat(dest)
781 except Exception as e:
782 print("movefile: failed to properly create symlink:", dest, "->", target, e)
783 return None
784
785 renamefailed = 1
786 if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
787 try:
788 # os.rename needs to know the dest path ending with file name
789 # so append the file name to a path only if it's a dir specified
790 srcfname = os.path.basename(src)
791 destpath = os.path.join(dest, srcfname) if os.path.isdir(dest) \
792 else dest
793 os.rename(src, destpath)
794 renamefailed = 0
795 except Exception as e:
796 if e[0] != errno.EXDEV:
797 # Some random error.
798 print("movefile: Failed to move", src, "to", dest, e)
799 return None
800 # Invalid cross-device-link 'bind' mounted or actually Cross-Device
801
802 if renamefailed:
803 didcopy = 0
804 if stat.S_ISREG(sstat[stat.ST_MODE]):
805 try: # For safety copy then move it over.
806 shutil.copyfile(src, dest + "#new")
807 os.rename(dest + "#new", dest)
808 didcopy = 1
809 except Exception as e:
810 print('movefile: copy', src, '->', dest, 'failed.', e)
811 return None
812 else:
813 #we don't yet handle special, so we need to fall back to /bin/mv
814 a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'")
815 if a[0] != 0:
816 print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
817 return None # failure
818 try:
819 if didcopy:
820 os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
821 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
822 os.unlink(src)
823 except Exception as e:
824 print("movefile: Failed to chown/chmod/unlink", dest, e)
825 return None
826
827 if newmtime:
828 os.utime(dest, (newmtime, newmtime))
829 else:
830 os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
831 newmtime = sstat[stat.ST_MTIME]
832 return newmtime
833
834def copyfile(src, dest, newmtime = None, sstat = None):
835 """
836 Copies a file from src to dest, preserving all permissions and
837 attributes; mtime will be preserved even when moving across
838 filesystems. Returns true on success and false on failure.
839 """
840 #print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
841 try:
842 if not sstat:
843 sstat = os.lstat(src)
844 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600845 logger.warning("copyfile: stat of %s failed (%s)" % (src, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500846 return False
847
848 destexists = 1
849 try:
850 dstat = os.lstat(dest)
851 except:
852 dstat = os.lstat(os.path.dirname(dest))
853 destexists = 0
854
855 if destexists:
856 if stat.S_ISLNK(dstat[stat.ST_MODE]):
857 try:
858 os.unlink(dest)
859 destexists = 0
860 except Exception as e:
861 pass
862
863 if stat.S_ISLNK(sstat[stat.ST_MODE]):
864 try:
865 target = os.readlink(src)
866 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
867 os.unlink(dest)
868 os.symlink(target, dest)
869 #os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
870 return os.lstat(dest)
871 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600872 logger.warning("copyfile: failed to create symlink %s to %s (%s)" % (dest, target, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500873 return False
874
875 if stat.S_ISREG(sstat[stat.ST_MODE]):
876 try:
877 srcchown = False
878 if not os.access(src, os.R_OK):
879 # Make sure we can read it
880 srcchown = True
881 os.chmod(src, sstat[stat.ST_MODE] | stat.S_IRUSR)
882
883 # For safety copy then move it over.
884 shutil.copyfile(src, dest + "#new")
885 os.rename(dest + "#new", dest)
886 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600887 logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500888 return False
889 finally:
890 if srcchown:
891 os.chmod(src, sstat[stat.ST_MODE])
892 os.utime(src, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
893
894 else:
895 #we don't yet handle special, so we need to fall back to /bin/mv
896 a = getstatusoutput("/bin/cp -f " + "'" + src + "' '" + dest + "'")
897 if a[0] != 0:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600898 logger.warning("copyfile: failed to copy special file %s to %s (%s)" % (src, dest, a))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500899 return False # failure
900 try:
901 os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
902 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
903 except Exception as e:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600904 logger.warning("copyfile: failed to chown/chmod %s (%s)" % (dest, e))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500905 return False
906
907 if newmtime:
908 os.utime(dest, (newmtime, newmtime))
909 else:
910 os.utime(dest, (sstat[stat.ST_ATIME], sstat[stat.ST_MTIME]))
911 newmtime = sstat[stat.ST_MTIME]
912 return newmtime
913
914def which(path, item, direction = 0, history = False):
915 """
916 Locate a file in a PATH
917 """
918
919 hist = []
920 paths = (path or "").split(':')
921 if direction != 0:
922 paths.reverse()
923
924 for p in paths:
925 next = os.path.join(p, item)
926 hist.append(next)
927 if os.path.exists(next):
928 if not os.path.isabs(next):
929 next = os.path.abspath(next)
930 if history:
931 return next, hist
932 return next
933
934 if history:
935 return "", hist
936 return ""
937
938def to_boolean(string, default=None):
939 if not string:
940 return default
941
942 normalized = string.lower()
943 if normalized in ("y", "yes", "1", "true"):
944 return True
945 elif normalized in ("n", "no", "0", "false"):
946 return False
947 else:
948 raise ValueError("Invalid value for to_boolean: %s" % string)
949
950def contains(variable, checkvalues, truevalue, falsevalue, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500951 """Check if a variable contains all the values specified.
952
953 Arguments:
954
955 variable -- the variable name. This will be fetched and expanded (using
956 d.getVar(variable, True)) and then split into a set().
957
958 checkvalues -- if this is a string it is split on whitespace into a set(),
959 otherwise coerced directly into a set().
960
961 truevalue -- the value to return if checkvalues is a subset of variable.
962
963 falsevalue -- the value to return if variable is empty or if checkvalues is
964 not a subset of variable.
965
966 d -- the data store.
967 """
968
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500969 val = d.getVar(variable, True)
970 if not val:
971 return falsevalue
972 val = set(val.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600973 if isinstance(checkvalues, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500974 checkvalues = set(checkvalues.split())
975 else:
976 checkvalues = set(checkvalues)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500977 if checkvalues.issubset(val):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500978 return truevalue
979 return falsevalue
980
981def contains_any(variable, checkvalues, truevalue, falsevalue, d):
982 val = d.getVar(variable, True)
983 if not val:
984 return falsevalue
985 val = set(val.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600986 if isinstance(checkvalues, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500987 checkvalues = set(checkvalues.split())
988 else:
989 checkvalues = set(checkvalues)
990 if checkvalues & val:
991 return truevalue
992 return falsevalue
993
994def cpu_count():
995 return multiprocessing.cpu_count()
996
997def nonblockingfd(fd):
998 fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
999
1000def process_profilelog(fn, pout = None):
1001 # Either call with a list of filenames and set pout or a filename and optionally pout.
1002 if not pout:
1003 pout = fn + '.processed'
1004 pout = open(pout, 'w')
1005
1006 import pstats
1007 if isinstance(fn, list):
1008 p = pstats.Stats(*fn, stream=pout)
1009 else:
1010 p = pstats.Stats(fn, stream=pout)
1011 p.sort_stats('time')
1012 p.print_stats()
1013 p.print_callers()
1014 p.sort_stats('cumulative')
1015 p.print_stats()
1016
1017 pout.flush()
1018 pout.close()
1019
1020#
1021# Was present to work around multiprocessing pool bugs in python < 2.7.3
1022#
1023def multiprocessingpool(*args, **kwargs):
1024
1025 import multiprocessing.pool
1026 #import multiprocessing.util
1027 #multiprocessing.util.log_to_stderr(10)
1028 # Deal with a multiprocessing bug where signals to the processes would be delayed until the work
1029 # completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed.
1030 def wrapper(func):
1031 def wrap(self, timeout=None):
1032 return func(self, timeout=timeout if timeout is not None else 1e100)
1033 return wrap
1034 multiprocessing.pool.IMapIterator.next = wrapper(multiprocessing.pool.IMapIterator.next)
1035
1036 return multiprocessing.Pool(*args, **kwargs)
1037
1038def exec_flat_python_func(func, *args, **kwargs):
1039 """Execute a flat python function (defined with def funcname(args):...)"""
1040 # Prepare a small piece of python code which calls the requested function
1041 # To do this we need to prepare two things - a set of variables we can use to pass
1042 # the values of arguments into the calling function, and the list of arguments for
1043 # the function being called
1044 context = {}
1045 funcargs = []
1046 # Handle unnamed arguments
1047 aidx = 1
1048 for arg in args:
1049 argname = 'arg_%s' % aidx
1050 context[argname] = arg
1051 funcargs.append(argname)
1052 aidx += 1
1053 # Handle keyword arguments
1054 context.update(kwargs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001055 funcargs.extend(['%s=%s' % (arg, arg) for arg in kwargs.keys()])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001056 code = 'retval = %s(%s)' % (func, ', '.join(funcargs))
1057 comp = bb.utils.better_compile(code, '<string>', '<string>')
1058 bb.utils.better_exec(comp, context, code, '<string>')
1059 return context['retval']
1060
1061def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
1062 """Edit lines from a recipe or config file and modify one or more
1063 specified variable values set in the file using a specified callback
1064 function. Lines are expected to have trailing newlines.
1065 Parameters:
1066 meta_lines: lines from the file; can be a list or an iterable
1067 (e.g. file pointer)
1068 variables: a list of variable names to look for. Functions
1069 may also be specified, but must be specified with '()' at
1070 the end of the name. Note that the function doesn't have
1071 any intrinsic understanding of _append, _prepend, _remove,
1072 or overrides, so these are considered as part of the name.
1073 These values go into a regular expression, so regular
1074 expression syntax is allowed.
1075 varfunc: callback function called for every variable matching
1076 one of the entries in the variables parameter. The function
1077 should take four arguments:
1078 varname: name of variable matched
1079 origvalue: current value in file
1080 op: the operator (e.g. '+=')
1081 newlines: list of lines up to this point. You can use
1082 this to prepend lines before this variable setting
1083 if you wish.
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001084 and should return a four-element tuple:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001085 newvalue: new value to substitute in, or None to drop
1086 the variable setting entirely. (If the removal
1087 results in two consecutive blank lines, one of the
1088 blank lines will also be dropped).
1089 newop: the operator to use - if you specify None here,
1090 the original operation will be used.
1091 indent: number of spaces to indent multi-line entries,
1092 or -1 to indent up to the level of the assignment
1093 and opening quote, or a string to use as the indent.
1094 minbreak: True to allow the first element of a
1095 multi-line value to continue on the same line as
1096 the assignment, False to indent before the first
1097 element.
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001098 To clarify, if you wish not to change the value, then you
1099 would return like this: return origvalue, None, 0, True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001100 match_overrides: True to match items with _overrides on the end,
1101 False otherwise
1102 Returns a tuple:
1103 updated:
1104 True if changes were made, False otherwise.
1105 newlines:
1106 Lines after processing
1107 """
1108
1109 var_res = {}
1110 if match_overrides:
1111 override_re = '(_[a-zA-Z0-9-_$(){}]+)?'
1112 else:
1113 override_re = ''
1114 for var in variables:
1115 if var.endswith('()'):
1116 var_res[var] = re.compile('^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
1117 else:
1118 var_res[var] = re.compile('^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
1119
1120 updated = False
1121 varset_start = ''
1122 varlines = []
1123 newlines = []
1124 in_var = None
1125 full_value = ''
1126 var_end = ''
1127
1128 def handle_var_end():
1129 prerun_newlines = newlines[:]
1130 op = varset_start[len(in_var):].strip()
1131 (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines)
1132 changed = (prerun_newlines != newlines)
1133
1134 if newvalue is None:
1135 # Drop the value
1136 return True
1137 elif newvalue != full_value or (newop not in [None, op]):
1138 if newop not in [None, op]:
1139 # Callback changed the operator
1140 varset_new = "%s %s" % (in_var, newop)
1141 else:
1142 varset_new = varset_start
1143
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001144 if isinstance(indent, int):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001145 if indent == -1:
1146 indentspc = ' ' * (len(varset_new) + 2)
1147 else:
1148 indentspc = ' ' * indent
1149 else:
1150 indentspc = indent
1151 if in_var.endswith('()'):
1152 # A function definition
1153 if isinstance(newvalue, list):
1154 newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue)))
1155 else:
1156 if not newvalue.startswith('\n'):
1157 newvalue = '\n' + newvalue
1158 if not newvalue.endswith('\n'):
1159 newvalue = newvalue + '\n'
1160 newlines.append('%s {%s}\n' % (varset_new, newvalue))
1161 else:
1162 # Normal variable
1163 if isinstance(newvalue, list):
1164 if not newvalue:
1165 # Empty list -> empty string
1166 newlines.append('%s ""\n' % varset_new)
1167 elif minbreak:
1168 # First item on first line
1169 if len(newvalue) == 1:
1170 newlines.append('%s "%s"\n' % (varset_new, newvalue[0]))
1171 else:
1172 newlines.append('%s "%s \\\n' % (varset_new, newvalue[0]))
1173 for item in newvalue[1:]:
1174 newlines.append('%s%s \\\n' % (indentspc, item))
1175 newlines.append('%s"\n' % indentspc)
1176 else:
1177 # No item on first line
1178 newlines.append('%s " \\\n' % varset_new)
1179 for item in newvalue:
1180 newlines.append('%s%s \\\n' % (indentspc, item))
1181 newlines.append('%s"\n' % indentspc)
1182 else:
1183 newlines.append('%s "%s"\n' % (varset_new, newvalue))
1184 return True
1185 else:
1186 # Put the old lines back where they were
1187 newlines.extend(varlines)
1188 # If newlines was touched by the function, we'll need to return True
1189 return changed
1190
1191 checkspc = False
1192
1193 for line in meta_lines:
1194 if in_var:
1195 value = line.rstrip()
1196 varlines.append(line)
1197 if in_var.endswith('()'):
1198 full_value += '\n' + value
1199 else:
1200 full_value += value[:-1]
1201 if value.endswith(var_end):
1202 if in_var.endswith('()'):
1203 if full_value.count('{') - full_value.count('}') >= 0:
1204 continue
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001205 full_value = full_value[:-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001206 if handle_var_end():
1207 updated = True
1208 checkspc = True
1209 in_var = None
1210 else:
1211 skip = False
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001212 for (varname, var_re) in var_res.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001213 res = var_re.match(line)
1214 if res:
1215 isfunc = varname.endswith('()')
1216 if isfunc:
1217 splitvalue = line.split('{', 1)
1218 var_end = '}'
1219 else:
1220 var_end = res.groups()[-1]
1221 splitvalue = line.split(var_end, 1)
1222 varset_start = splitvalue[0].rstrip()
1223 value = splitvalue[1].rstrip()
1224 if not isfunc and value.endswith('\\'):
1225 value = value[:-1]
1226 full_value = value
1227 varlines = [line]
1228 in_var = res.group(1)
1229 if isfunc:
1230 in_var += '()'
1231 if value.endswith(var_end):
1232 full_value = full_value[:-1]
1233 if handle_var_end():
1234 updated = True
1235 checkspc = True
1236 in_var = None
1237 skip = True
1238 break
1239 if not skip:
1240 if checkspc:
1241 checkspc = False
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001242 if newlines and newlines[-1] == '\n' and line == '\n':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001243 # Squash blank line if there are two consecutive blanks after a removal
1244 continue
1245 newlines.append(line)
1246 return (updated, newlines)
1247
1248
1249def edit_metadata_file(meta_file, variables, varfunc):
1250 """Edit a recipe or config file and modify one or more specified
1251 variable values set in the file using a specified callback function.
1252 The file is only written to if the value(s) actually change.
1253 This is basically the file version of edit_metadata(), see that
1254 function's description for parameter/usage information.
1255 Returns True if the file was written to, False otherwise.
1256 """
1257 with open(meta_file, 'r') as f:
1258 (updated, newlines) = edit_metadata(f, variables, varfunc)
1259 if updated:
1260 with open(meta_file, 'w') as f:
1261 f.writelines(newlines)
1262 return updated
1263
1264
1265def edit_bblayers_conf(bblayers_conf, add, remove):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001266 """Edit bblayers.conf, adding and/or removing layers
1267 Parameters:
1268 bblayers_conf: path to bblayers.conf file to edit
1269 add: layer path (or list of layer paths) to add; None or empty
1270 list to add nothing
1271 remove: layer path (or list of layer paths) to remove; None or
1272 empty list to remove nothing
1273 Returns a tuple:
1274 notadded: list of layers specified to be added but weren't
1275 (because they were already in the list)
1276 notremoved: list of layers that were specified to be removed
1277 but weren't (because they weren't in the list)
1278 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001279
1280 import fnmatch
1281
1282 def remove_trailing_sep(pth):
1283 if pth and pth[-1] == os.sep:
1284 pth = pth[:-1]
1285 return pth
1286
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001287 approved = bb.utils.approved_variables()
1288 def canonicalise_path(pth):
1289 pth = remove_trailing_sep(pth)
1290 if 'HOME' in approved and '~' in pth:
1291 pth = os.path.expanduser(pth)
1292 return pth
1293
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001294 def layerlist_param(value):
1295 if not value:
1296 return []
1297 elif isinstance(value, list):
1298 return [remove_trailing_sep(x) for x in value]
1299 else:
1300 return [remove_trailing_sep(value)]
1301
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001302 addlayers = layerlist_param(add)
1303 removelayers = layerlist_param(remove)
1304
1305 # Need to use a list here because we can't set non-local variables from a callback in python 2.x
1306 bblayercalls = []
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001307 removed = []
1308 plusequals = False
1309 orig_bblayers = []
1310
1311 def handle_bblayers_firstpass(varname, origvalue, op, newlines):
1312 bblayercalls.append(op)
1313 if op == '=':
1314 del orig_bblayers[:]
1315 orig_bblayers.extend([canonicalise_path(x) for x in origvalue.split()])
1316 return (origvalue, None, 2, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001317
1318 def handle_bblayers(varname, origvalue, op, newlines):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001319 updated = False
1320 bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
1321 if removelayers:
1322 for removelayer in removelayers:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001323 for layer in bblayers:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001324 if fnmatch.fnmatch(canonicalise_path(layer), canonicalise_path(removelayer)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001325 updated = True
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001326 bblayers.remove(layer)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001327 removed.append(removelayer)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001328 break
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001329 if addlayers and not plusequals:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001330 for addlayer in addlayers:
1331 if addlayer not in bblayers:
1332 updated = True
1333 bblayers.append(addlayer)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001334 del addlayers[:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001335
1336 if updated:
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001337 if op == '+=' and not bblayers:
1338 bblayers = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001339 return (bblayers, None, 2, False)
1340 else:
1341 return (origvalue, None, 2, False)
1342
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001343 with open(bblayers_conf, 'r') as f:
1344 (_, newlines) = edit_metadata(f, ['BBLAYERS'], handle_bblayers_firstpass)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001345
1346 if not bblayercalls:
1347 raise Exception('Unable to find BBLAYERS in %s' % bblayers_conf)
1348
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001349 # Try to do the "smart" thing depending on how the user has laid out
1350 # their bblayers.conf file
1351 if bblayercalls.count('+=') > 1:
1352 plusequals = True
1353
1354 removelayers_canon = [canonicalise_path(layer) for layer in removelayers]
1355 notadded = []
1356 for layer in addlayers:
1357 layer_canon = canonicalise_path(layer)
1358 if layer_canon in orig_bblayers and not layer_canon in removelayers_canon:
1359 notadded.append(layer)
1360 notadded_canon = [canonicalise_path(layer) for layer in notadded]
1361 addlayers[:] = [layer for layer in addlayers if canonicalise_path(layer) not in notadded_canon]
1362
1363 (updated, newlines) = edit_metadata(newlines, ['BBLAYERS'], handle_bblayers)
1364 if addlayers:
1365 # Still need to add these
1366 for addlayer in addlayers:
1367 newlines.append('BBLAYERS += "%s"\n' % addlayer)
1368 updated = True
1369
1370 if updated:
1371 with open(bblayers_conf, 'w') as f:
1372 f.writelines(newlines)
1373
1374 notremoved = list(set(removelayers) - set(removed))
1375
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001376 return (notadded, notremoved)
1377
1378
1379def get_file_layer(filename, d):
1380 """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
1381 collections = (d.getVar('BBFILE_COLLECTIONS', True) or '').split()
1382 collection_res = {}
1383 for collection in collections:
1384 collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection, True) or ''
1385
1386 def path_to_layer(path):
1387 # Use longest path so we handle nested layers
1388 matchlen = 0
1389 match = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001390 for collection, regex in collection_res.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001391 if len(regex) > matchlen and re.match(regex, path):
1392 matchlen = len(regex)
1393 match = collection
1394 return match
1395
1396 result = None
1397 bbfiles = (d.getVar('BBFILES', True) or '').split()
1398 bbfilesmatch = False
1399 for bbfilesentry in bbfiles:
1400 if fnmatch.fnmatch(filename, bbfilesentry):
1401 bbfilesmatch = True
1402 result = path_to_layer(bbfilesentry)
1403
1404 if not bbfilesmatch:
1405 # Probably a bbclass
1406 result = path_to_layer(filename)
1407
1408 return result
1409
1410
1411# Constant taken from http://linux.die.net/include/linux/prctl.h
1412PR_SET_PDEATHSIG = 1
1413
1414class PrCtlError(Exception):
1415 pass
1416
1417def signal_on_parent_exit(signame):
1418 """
1419 Trigger signame to be sent when the parent process dies
1420 """
1421 signum = getattr(signal, signame)
1422 # http://linux.die.net/man/2/prctl
1423 result = cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG, signum)
1424 if result != 0:
1425 raise PrCtlError('prctl failed with error code %s' % result)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001426
1427#
1428# Manually call the ioprio syscall. We could depend on other libs like psutil
1429# however this gets us enough of what we need to bitbake for now without the
1430# dependency
1431#
1432_unamearch = os.uname()[4]
1433IOPRIO_WHO_PROCESS = 1
1434IOPRIO_CLASS_SHIFT = 13
1435
1436def ioprio_set(who, cls, value):
1437 NR_ioprio_set = None
1438 if _unamearch == "x86_64":
1439 NR_ioprio_set = 251
1440 elif _unamearch[0] == "i" and _unamearch[2:3] == "86":
1441 NR_ioprio_set = 289
1442
1443 if NR_ioprio_set:
1444 ioprio = value | (cls << IOPRIO_CLASS_SHIFT)
1445 rc = cdll['libc.so.6'].syscall(NR_ioprio_set, IOPRIO_WHO_PROCESS, who, ioprio)
1446 if rc != 0:
1447 raise ValueError("Unable to set ioprio, syscall returned %s" % rc)
1448 else:
1449 bb.warn("Unable to set IO Prio for arch %s" % _unamearch)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001450
1451def set_process_name(name):
1452 from ctypes import cdll, byref, create_string_buffer
1453 # This is nice to have for debugging, not essential
1454 try:
1455 libc = cdll.LoadLibrary('libc.so.6')
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001456 buf = create_string_buffer(bytes(name, 'utf-8'))
1457 libc.prctl(15, byref(buf), 0, 0, 0)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001458 except:
1459 pass
1460
1461# export common proxies variables from datastore to environment
1462def export_proxies(d):
1463 import os
1464
1465 variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001466 'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
1467 'GIT_PROXY_COMMAND']
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001468 exported = False
1469
1470 for v in variables:
1471 if v in os.environ.keys():
1472 exported = True
1473 else:
1474 v_proxy = d.getVar(v, True)
1475 if v_proxy is not None:
1476 os.environ[v] = v_proxy
1477 exported = True
1478
1479 return exported
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001480
1481
1482def load_plugins(logger, plugins, pluginpath):
1483 def load_plugin(name):
1484 logger.debug('Loading plugin %s' % name)
1485 fp, pathname, description = imp.find_module(name, [pluginpath])
1486 try:
1487 return imp.load_module(name, fp, pathname, description)
1488 finally:
1489 if fp:
1490 fp.close()
1491
1492 logger.debug('Loading plugins from %s...' % pluginpath)
1493
1494 expanded = (glob.glob(os.path.join(pluginpath, '*' + ext))
1495 for ext in python_extensions)
1496 files = itertools.chain.from_iterable(expanded)
1497 names = set(os.path.splitext(os.path.basename(fn))[0] for fn in files)
1498 for name in names:
1499 if name != '__init__':
1500 plugin = load_plugin(name)
1501 if hasattr(plugin, 'plugin_init'):
1502 obj = plugin.plugin_init(plugins)
1503 plugins.append(obj or plugin)
1504 else:
1505 plugins.append(plugin)