blob: cedd053d3623f3797b1f09c4f12df19eb4ec31e7 [file] [log] [blame]
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001import subprocess
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08002import multiprocessing
3import traceback
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004
5def read_file(filename):
6 try:
7 f = open( filename, "r" )
8 except IOError as reason:
9 return "" # WARNING: can't raise an error now because of the new RDEPENDS handling. This is a bit ugly. :M:
10 else:
11 data = f.read().strip()
12 f.close()
13 return data
14 return None
15
16def ifelse(condition, iftrue = True, iffalse = False):
17 if condition:
18 return iftrue
19 else:
20 return iffalse
21
22def conditional(variable, checkvalue, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050023 if d.getVar(variable) == checkvalue:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024 return truevalue
25 else:
26 return falsevalue
27
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080028def vartrue(var, iftrue, iffalse, d):
29 import oe.types
30 if oe.types.boolean(d.getVar(var)):
31 return iftrue
32 else:
33 return iffalse
34
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035def less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050036 if float(d.getVar(variable)) <= float(checkvalue):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 return truevalue
38 else:
39 return falsevalue
40
41def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050042 result = bb.utils.vercmp_string(d.getVar(variable), checkvalue)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050043 if result <= 0:
44 return truevalue
45 else:
46 return falsevalue
47
48def both_contain(variable1, variable2, checkvalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050049 val1 = d.getVar(variable1)
50 val2 = d.getVar(variable2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050051 val1 = set(val1.split())
52 val2 = set(val2.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -060053 if isinstance(checkvalue, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054 checkvalue = set(checkvalue.split())
55 else:
56 checkvalue = set(checkvalue)
57 if checkvalue.issubset(val1) and checkvalue.issubset(val2):
58 return " ".join(checkvalue)
59 else:
60 return ""
61
62def set_intersect(variable1, variable2, d):
63 """
64 Expand both variables, interpret them as lists of strings, and return the
65 intersection as a flattened string.
66
67 For example:
68 s1 = "a b c"
69 s2 = "b c d"
70 s3 = set_intersect(s1, s2)
71 => s3 = "b c"
72 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -050073 val1 = set(d.getVar(variable1).split())
74 val2 = set(d.getVar(variable2).split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075 return " ".join(val1 & val2)
76
77def prune_suffix(var, suffixes, d):
78 # See if var ends with any of the suffixes listed and
79 # remove it if found
80 for suffix in suffixes:
81 if var.endswith(suffix):
82 var = var.replace(suffix, "")
83
Brad Bishop6e60e8b2018-02-01 10:27:11 -050084 prefix = d.getVar("MLPREFIX")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085 if prefix and var.startswith(prefix):
86 var = var.replace(prefix, "")
87
88 return var
89
90def str_filter(f, str, d):
91 from re import match
Patrick Williamsc0f7c042017-02-23 20:41:17 -060092 return " ".join([x for x in str.split() if match(f, x, 0)])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050093
94def str_filter_out(f, str, d):
95 from re import match
Patrick Williamsc0f7c042017-02-23 20:41:17 -060096 return " ".join([x for x in str.split() if not match(f, x, 0)])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050097
Brad Bishop6e60e8b2018-02-01 10:27:11 -050098def build_depends_string(depends, task):
99 """Append a taskname to a string of dependencies as used by the [depends] flag"""
100 return " ".join(dep + ":" + task for dep in depends.split())
101
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102def inherits(d, *classes):
103 """Return True if the metadata inherits any of the specified classes"""
104 return any(bb.data.inherits_class(cls, d) for cls in classes)
105
106def features_backfill(var,d):
107 # This construct allows the addition of new features to variable specified
108 # as var
109 # Example for var = "DISTRO_FEATURES"
110 # This construct allows the addition of new features to DISTRO_FEATURES
111 # that if not present would disable existing functionality, without
112 # disturbing distributions that have already set DISTRO_FEATURES.
113 # Distributions wanting to elide a value in DISTRO_FEATURES_BACKFILL should
114 # add the feature to DISTRO_FEATURES_BACKFILL_CONSIDERED
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500115 features = (d.getVar(var) or "").split()
116 backfill = (d.getVar(var+"_BACKFILL") or "").split()
117 considered = (d.getVar(var+"_BACKFILL_CONSIDERED") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500118
119 addfeatures = []
120 for feature in backfill:
121 if feature not in features and feature not in considered:
122 addfeatures.append(feature)
123
124 if addfeatures:
125 d.appendVar(var, " " + " ".join(addfeatures))
126
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500127def all_distro_features(d, features, truevalue="1", falsevalue=""):
128 """
129 Returns truevalue if *all* given features are set in DISTRO_FEATURES,
130 else falsevalue. The features can be given as single string or anything
131 that can be turned into a set.
132
133 This is a shorter, more flexible version of
134 bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d).
135
136 Without explicit true/false values it can be used directly where
137 Python expects a boolean:
138 if oe.utils.all_distro_features(d, "foo bar"):
139 bb.fatal("foo and bar are mutually exclusive DISTRO_FEATURES")
140
141 With just a truevalue, it can be used to include files that are meant to be
142 used only when requested via DISTRO_FEATURES:
143 require ${@ oe.utils.all_distro_features(d, "foo bar", "foo-and-bar.inc")
144 """
145 return bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d)
146
147def any_distro_features(d, features, truevalue="1", falsevalue=""):
148 """
149 Returns truevalue if at least *one* of the given features is set in DISTRO_FEATURES,
150 else falsevalue. The features can be given as single string or anything
151 that can be turned into a set.
152
153 This is a shorter, more flexible version of
154 bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d).
155
156 Without explicit true/false values it can be used directly where
157 Python expects a boolean:
158 if not oe.utils.any_distro_features(d, "foo bar"):
159 bb.fatal("foo, bar or both must be set in DISTRO_FEATURES")
160
161 With just a truevalue, it can be used to include files that are meant to be
162 used only when requested via DISTRO_FEATURES:
163 require ${@ oe.utils.any_distro_features(d, "foo bar", "foo-or-bar.inc")
164
165 """
166 return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167
Brad Bishop316dfdd2018-06-25 12:45:53 -0400168def parallel_make(d):
169 """
170 Return the integer value for the number of parallel threads to use when
171 building, scraped out of PARALLEL_MAKE. If no parallelization option is
172 found, returns None
173
174 e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
175 """
176 pm = (d.getVar('PARALLEL_MAKE') or '').split()
177 # look for '-j' and throw other options (e.g. '-l') away
178 while pm:
179 opt = pm.pop(0)
180 if opt == '-j':
181 v = pm.pop(0)
182 elif opt.startswith('-j'):
183 v = opt[2:].strip()
184 else:
185 continue
186
187 return int(v)
188
189 return None
190
191def parallel_make_argument(d, fmt, limit=None):
192 """
193 Helper utility to construct a parallel make argument from the number of
194 parallel threads specified in PARALLEL_MAKE.
195
196 Returns the input format string `fmt` where a single '%d' will be expanded
197 with the number of parallel threads to use. If `limit` is specified, the
198 number of parallel threads will be no larger than it. If no parallelization
199 option is found in PARALLEL_MAKE, returns an empty string
200
201 e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
202 "-n 10"
203 """
204 v = parallel_make(d)
205 if v:
206 if limit:
207 v = min(limit, v)
208 return fmt % v
209 return ''
210
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500211def packages_filter_out_system(d):
212 """
213 Return a list of packages from PACKAGES with the "system" packages such as
214 PN-dbg PN-doc PN-locale-eb-gb removed.
215 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500216 pn = d.getVar('PN')
Brad Bishop19323692019-04-05 15:28:33 -0400217 blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev', '-src')]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218 localepkg = pn + "-locale-"
219 pkgs = []
220
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500221 for pkg in d.getVar('PACKAGES').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500222 if pkg not in blacklist and localepkg not in pkg:
223 pkgs.append(pkg)
224 return pkgs
225
226def getstatusoutput(cmd):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500227 return subprocess.getstatusoutput(cmd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228
229
230def trim_version(version, num_parts=2):
231 """
232 Return just the first <num_parts> of <version>, split by periods. For
233 example, trim_version("1.2.3", 2) will return "1.2".
234 """
235 if type(version) is not str:
236 raise TypeError("Version should be a string")
237 if num_parts < 1:
238 raise ValueError("Cannot split to parts < 1")
239
240 parts = version.split(".")
241 trimmed = ".".join(parts[:num_parts])
242 return trimmed
243
244def cpu_count():
245 import multiprocessing
246 return multiprocessing.cpu_count()
247
248def execute_pre_post_process(d, cmds):
249 if cmds is None:
250 return
251
252 for cmd in cmds.strip().split(';'):
253 cmd = cmd.strip()
254 if cmd != '':
255 bb.note("Executing %s ..." % cmd)
256 bb.build.exec_func(cmd, d)
257
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800258# For each item in items, call the function 'target' with item as the first
259# argument, extraargs as the other arguments and handle any exceptions in the
260# parent thread
261def multiprocess_launch(target, items, d, extraargs=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500262
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800263 class ProcessLaunch(multiprocessing.Process):
264 def __init__(self, *args, **kwargs):
265 multiprocessing.Process.__init__(self, *args, **kwargs)
266 self._pconn, self._cconn = multiprocessing.Pipe()
267 self._exception = None
268 self._result = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800270 def run(self):
271 try:
272 ret = self._target(*self._args, **self._kwargs)
273 self._cconn.send((None, ret))
274 except Exception as e:
275 tb = traceback.format_exc()
276 self._cconn.send((e, tb))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500277
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800278 def update(self):
279 if self._pconn.poll():
280 (e, tb) = self._pconn.recv()
281 if e is not None:
282 self._exception = (e, tb)
283 else:
284 self._result = tb
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500285
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800286 @property
287 def exception(self):
288 self.update()
289 return self._exception
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500290
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800291 @property
292 def result(self):
293 self.update()
294 return self._result
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800296 max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
297 launched = []
298 errors = []
299 results = []
300 items = list(items)
301 while (items and not errors) or launched:
302 if not errors and items and len(launched) < max_process:
303 args = (items.pop(),)
304 if extraargs is not None:
305 args = args + extraargs
306 p = ProcessLaunch(target=target, args=args)
307 p.start()
308 launched.append(p)
309 for q in launched:
Brad Bishop19323692019-04-05 15:28:33 -0400310 # Have to manually call update() to avoid deadlocks. The pipe can be full and
311 # transfer stalled until we try and read the results object but the subprocess won't exit
312 # as it still has data to write (https://bugs.python.org/issue8426)
313 q.update()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800314 # The finished processes are joined when calling is_alive()
315 if not q.is_alive():
316 if q.exception:
317 errors.append(q.exception)
318 if q.result:
319 results.append(q.result)
320 launched.remove(q)
321 # Paranoia doesn't hurt
322 for p in launched:
323 p.join()
324 if errors:
325 msg = ""
326 for (e, tb) in errors:
327 msg = msg + str(e) + ": " + str(tb) + "\n"
328 bb.fatal("Fatal errors occurred in subprocesses:\n%s" % msg)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500329 return results
330
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331def squashspaces(string):
332 import re
Brad Bishop19323692019-04-05 15:28:33 -0400333 return re.sub(r"\s+", " ", string).strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500335def format_pkg_list(pkg_dict, ret_format=None):
336 output = []
337
338 if ret_format == "arch":
339 for pkg in sorted(pkg_dict):
340 output.append("%s %s" % (pkg, pkg_dict[pkg]["arch"]))
341 elif ret_format == "file":
342 for pkg in sorted(pkg_dict):
343 output.append("%s %s %s" % (pkg, pkg_dict[pkg]["filename"], pkg_dict[pkg]["arch"]))
344 elif ret_format == "ver":
345 for pkg in sorted(pkg_dict):
346 output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"]))
347 elif ret_format == "deps":
348 for pkg in sorted(pkg_dict):
349 for dep in pkg_dict[pkg]["deps"]:
350 output.append("%s|%s" % (pkg, dep))
351 else:
352 for pkg in sorted(pkg_dict):
353 output.append(pkg)
354
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800355 output_str = '\n'.join(output)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500356
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800357 if output_str:
358 # make sure last line is newline terminated
359 output_str += '\n'
360
361 return output_str
362
363def host_gcc_version(d, taskcontextonly=False):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500364 import re, subprocess
365
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800366 if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
367 return
368
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500369 compiler = d.getVar("BUILD_CC")
Brad Bishop19323692019-04-05 15:28:33 -0400370 # Get rid of ccache since it is not present when parsing.
371 if compiler.startswith('ccache '):
372 compiler = compiler[7:]
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500373 try:
374 env = os.environ.copy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500375 env["PATH"] = d.getVar("PATH")
Brad Bishop19323692019-04-05 15:28:33 -0400376 output = subprocess.check_output("%s --version" % compiler, \
377 shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500378 except subprocess.CalledProcessError as e:
379 bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
380
Brad Bishop19323692019-04-05 15:28:33 -0400381 match = re.match(r".* (\d\.\d)\.\d.*", output.split('\n')[0])
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500382 if not match:
383 bb.fatal("Can't get compiler version from %s --version output" % compiler)
384
385 version = match.group(1)
386 return "-%s" % version if version in ("4.8", "4.9") else ""
387
Brad Bishop316dfdd2018-06-25 12:45:53 -0400388
389def get_multilib_datastore(variant, d):
390 localdata = bb.data.createCopy(d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800391 if variant:
392 overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant
393 localdata.setVar("OVERRIDES", overrides)
394 localdata.setVar("MLPREFIX", variant + "-")
395 else:
396 origdefault = localdata.getVar("DEFAULTTUNE_MULTILIB_ORIGINAL")
397 if origdefault:
398 localdata.setVar("DEFAULTTUNE", origdefault)
399 overrides = localdata.getVar("OVERRIDES", False).split(":")
400 overrides = ":".join([x for x in overrides if not x.startswith("virtclass-multilib-")])
401 localdata.setVar("OVERRIDES", overrides)
402 localdata.setVar("MLPREFIX", "")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400403 return localdata
404
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500405#
406# Python 2.7 doesn't have threaded pools (just multiprocessing)
407# so implement a version here
408#
409
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600410from queue import Queue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500411from threading import Thread
412
413class ThreadedWorker(Thread):
414 """Thread executing tasks from a given tasks queue"""
415 def __init__(self, tasks, worker_init, worker_end):
416 Thread.__init__(self)
417 self.tasks = tasks
418 self.daemon = True
419
420 self.worker_init = worker_init
421 self.worker_end = worker_end
422
423 def run(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600424 from queue import Empty
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500425
426 if self.worker_init is not None:
427 self.worker_init(self)
428
429 while True:
430 try:
431 func, args, kargs = self.tasks.get(block=False)
432 except Empty:
433 if self.worker_end is not None:
434 self.worker_end(self)
435 break
436
437 try:
438 func(self, *args, **kargs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600439 except Exception as e:
440 print(e)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500441 finally:
442 self.tasks.task_done()
443
444class ThreadedPool:
445 """Pool of threads consuming tasks from a queue"""
446 def __init__(self, num_workers, num_tasks, worker_init=None,
447 worker_end=None):
448 self.tasks = Queue(num_tasks)
449 self.workers = []
450
451 for _ in range(num_workers):
452 worker = ThreadedWorker(self.tasks, worker_init, worker_end)
453 self.workers.append(worker)
454
455 def start(self):
456 for worker in self.workers:
457 worker.start()
458
459 def add_task(self, func, *args, **kargs):
460 """Add a task to the queue"""
461 self.tasks.put((func, args, kargs))
462
463 def wait_completion(self):
464 """Wait for completion of all the tasks in the queue"""
465 self.tasks.join()
466 for worker in self.workers:
467 worker.join()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500468
469def write_ld_so_conf(d):
470 # Some utils like prelink may not have the correct target library paths
471 # so write an ld.so.conf to help them
472 ldsoconf = d.expand("${STAGING_DIR_TARGET}${sysconfdir}/ld.so.conf")
473 if os.path.exists(ldsoconf):
474 bb.utils.remove(ldsoconf)
475 bb.utils.mkdirhier(os.path.dirname(ldsoconf))
476 with open(ldsoconf, "w") as f:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500477 f.write(d.getVar("base_libdir") + '\n')
478 f.write(d.getVar("libdir") + '\n')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600479
480class ImageQAFailed(bb.build.FuncFailed):
481 def __init__(self, description, name=None, logfile=None):
482 self.description = description
483 self.name = name
484 self.logfile=logfile
485
486 def __str__(self):
487 msg = 'Function failed: %s' % self.name
488 if self.description:
489 msg = msg + ' (%s)' % self.description
490
491 return msg
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800492
Brad Bishop19323692019-04-05 15:28:33 -0400493def sh_quote(string):
494 import shlex
495 return shlex.quote(string)