blob: 652b2be145e63627555c96b4129acc27fe5bbe51 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
Brad Bishop6e60e8b2018-02-01 10:27:11 -05005import subprocess
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08006import multiprocessing
7import traceback
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008
9def read_file(filename):
10 try:
11 f = open( filename, "r" )
12 except IOError as reason:
13 return "" # WARNING: can't raise an error now because of the new RDEPENDS handling. This is a bit ugly. :M:
14 else:
15 data = f.read().strip()
16 f.close()
17 return data
18 return None
19
20def ifelse(condition, iftrue = True, iffalse = False):
21 if condition:
22 return iftrue
23 else:
24 return iffalse
25
26def conditional(variable, checkvalue, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050027 if d.getVar(variable) == checkvalue:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028 return truevalue
29 else:
30 return falsevalue
31
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080032def vartrue(var, iftrue, iffalse, d):
33 import oe.types
34 if oe.types.boolean(d.getVar(var)):
35 return iftrue
36 else:
37 return iffalse
38
Patrick Williamsc124f4f2015-09-15 14:41:29 -050039def less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050040 if float(d.getVar(variable)) <= float(checkvalue):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041 return truevalue
42 else:
43 return falsevalue
44
45def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050046 result = bb.utils.vercmp_string(d.getVar(variable), checkvalue)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047 if result <= 0:
48 return truevalue
49 else:
50 return falsevalue
51
52def both_contain(variable1, variable2, checkvalue, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050053 val1 = d.getVar(variable1)
54 val2 = d.getVar(variable2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055 val1 = set(val1.split())
56 val2 = set(val2.split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -060057 if isinstance(checkvalue, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050058 checkvalue = set(checkvalue.split())
59 else:
60 checkvalue = set(checkvalue)
61 if checkvalue.issubset(val1) and checkvalue.issubset(val2):
62 return " ".join(checkvalue)
63 else:
64 return ""
65
66def set_intersect(variable1, variable2, d):
67 """
68 Expand both variables, interpret them as lists of strings, and return the
69 intersection as a flattened string.
70
71 For example:
72 s1 = "a b c"
73 s2 = "b c d"
74 s3 = set_intersect(s1, s2)
75 => s3 = "b c"
76 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -050077 val1 = set(d.getVar(variable1).split())
78 val2 = set(d.getVar(variable2).split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079 return " ".join(val1 & val2)
80
81def prune_suffix(var, suffixes, d):
82 # See if var ends with any of the suffixes listed and
83 # remove it if found
84 for suffix in suffixes:
Brad Bishopd89cb5f2019-04-10 09:02:41 -040085 if suffix and var.endswith(suffix):
86 var = var[:-len(suffix)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087
Brad Bishop6e60e8b2018-02-01 10:27:11 -050088 prefix = d.getVar("MLPREFIX")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050089 if prefix and var.startswith(prefix):
Brad Bishopd89cb5f2019-04-10 09:02:41 -040090 var = var[len(prefix):]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050091
92 return var
93
94def str_filter(f, str, d):
95 from re import match
Patrick Williamsc0f7c042017-02-23 20:41:17 -060096 return " ".join([x for x in str.split() if match(f, x, 0)])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050097
98def str_filter_out(f, str, d):
99 from re import match
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600100 return " ".join([x for x in str.split() if not match(f, x, 0)])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500101
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500102def build_depends_string(depends, task):
103 """Append a taskname to a string of dependencies as used by the [depends] flag"""
104 return " ".join(dep + ":" + task for dep in depends.split())
105
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500106def inherits(d, *classes):
107 """Return True if the metadata inherits any of the specified classes"""
108 return any(bb.data.inherits_class(cls, d) for cls in classes)
109
110def features_backfill(var,d):
111 # This construct allows the addition of new features to variable specified
112 # as var
113 # Example for var = "DISTRO_FEATURES"
114 # This construct allows the addition of new features to DISTRO_FEATURES
115 # that if not present would disable existing functionality, without
116 # disturbing distributions that have already set DISTRO_FEATURES.
117 # Distributions wanting to elide a value in DISTRO_FEATURES_BACKFILL should
118 # add the feature to DISTRO_FEATURES_BACKFILL_CONSIDERED
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500119 features = (d.getVar(var) or "").split()
120 backfill = (d.getVar(var+"_BACKFILL") or "").split()
121 considered = (d.getVar(var+"_BACKFILL_CONSIDERED") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122
123 addfeatures = []
124 for feature in backfill:
125 if feature not in features and feature not in considered:
126 addfeatures.append(feature)
127
128 if addfeatures:
129 d.appendVar(var, " " + " ".join(addfeatures))
130
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500131def all_distro_features(d, features, truevalue="1", falsevalue=""):
132 """
133 Returns truevalue if *all* given features are set in DISTRO_FEATURES,
134 else falsevalue. The features can be given as single string or anything
135 that can be turned into a set.
136
137 This is a shorter, more flexible version of
138 bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d).
139
140 Without explicit true/false values it can be used directly where
141 Python expects a boolean:
142 if oe.utils.all_distro_features(d, "foo bar"):
143 bb.fatal("foo and bar are mutually exclusive DISTRO_FEATURES")
144
145 With just a truevalue, it can be used to include files that are meant to be
146 used only when requested via DISTRO_FEATURES:
147 require ${@ oe.utils.all_distro_features(d, "foo bar", "foo-and-bar.inc")
148 """
149 return bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d)
150
151def any_distro_features(d, features, truevalue="1", falsevalue=""):
152 """
153 Returns truevalue if at least *one* of the given features is set in DISTRO_FEATURES,
154 else falsevalue. The features can be given as single string or anything
155 that can be turned into a set.
156
157 This is a shorter, more flexible version of
158 bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d).
159
160 Without explicit true/false values it can be used directly where
161 Python expects a boolean:
162 if not oe.utils.any_distro_features(d, "foo bar"):
163 bb.fatal("foo, bar or both must be set in DISTRO_FEATURES")
164
165 With just a truevalue, it can be used to include files that are meant to be
166 used only when requested via DISTRO_FEATURES:
167 require ${@ oe.utils.any_distro_features(d, "foo bar", "foo-or-bar.inc")
168
169 """
170 return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500171
Brad Bishop316dfdd2018-06-25 12:45:53 -0400172def parallel_make(d):
173 """
174 Return the integer value for the number of parallel threads to use when
175 building, scraped out of PARALLEL_MAKE. If no parallelization option is
176 found, returns None
177
178 e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
179 """
180 pm = (d.getVar('PARALLEL_MAKE') or '').split()
181 # look for '-j' and throw other options (e.g. '-l') away
182 while pm:
183 opt = pm.pop(0)
184 if opt == '-j':
185 v = pm.pop(0)
186 elif opt.startswith('-j'):
187 v = opt[2:].strip()
188 else:
189 continue
190
191 return int(v)
192
193 return None
194
195def parallel_make_argument(d, fmt, limit=None):
196 """
197 Helper utility to construct a parallel make argument from the number of
198 parallel threads specified in PARALLEL_MAKE.
199
200 Returns the input format string `fmt` where a single '%d' will be expanded
201 with the number of parallel threads to use. If `limit` is specified, the
202 number of parallel threads will be no larger than it. If no parallelization
203 option is found in PARALLEL_MAKE, returns an empty string
204
205 e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
206 "-n 10"
207 """
208 v = parallel_make(d)
209 if v:
210 if limit:
211 v = min(limit, v)
212 return fmt % v
213 return ''
214
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215def packages_filter_out_system(d):
216 """
217 Return a list of packages from PACKAGES with the "system" packages such as
218 PN-dbg PN-doc PN-locale-eb-gb removed.
219 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500220 pn = d.getVar('PN')
Brad Bishop19323692019-04-05 15:28:33 -0400221 blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev', '-src')]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500222 localepkg = pn + "-locale-"
223 pkgs = []
224
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500225 for pkg in d.getVar('PACKAGES').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226 if pkg not in blacklist and localepkg not in pkg:
227 pkgs.append(pkg)
228 return pkgs
229
230def getstatusoutput(cmd):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500231 return subprocess.getstatusoutput(cmd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500232
233
234def trim_version(version, num_parts=2):
235 """
236 Return just the first <num_parts> of <version>, split by periods. For
237 example, trim_version("1.2.3", 2) will return "1.2".
238 """
239 if type(version) is not str:
240 raise TypeError("Version should be a string")
241 if num_parts < 1:
242 raise ValueError("Cannot split to parts < 1")
243
244 parts = version.split(".")
245 trimmed = ".".join(parts[:num_parts])
246 return trimmed
247
248def cpu_count():
249 import multiprocessing
250 return multiprocessing.cpu_count()
251
252def execute_pre_post_process(d, cmds):
253 if cmds is None:
254 return
255
256 for cmd in cmds.strip().split(';'):
257 cmd = cmd.strip()
258 if cmd != '':
259 bb.note("Executing %s ..." % cmd)
260 bb.build.exec_func(cmd, d)
261
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800262# For each item in items, call the function 'target' with item as the first
263# argument, extraargs as the other arguments and handle any exceptions in the
264# parent thread
265def multiprocess_launch(target, items, d, extraargs=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500266
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800267 class ProcessLaunch(multiprocessing.Process):
268 def __init__(self, *args, **kwargs):
269 multiprocessing.Process.__init__(self, *args, **kwargs)
270 self._pconn, self._cconn = multiprocessing.Pipe()
271 self._exception = None
272 self._result = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800274 def run(self):
275 try:
276 ret = self._target(*self._args, **self._kwargs)
277 self._cconn.send((None, ret))
278 except Exception as e:
279 tb = traceback.format_exc()
280 self._cconn.send((e, tb))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500281
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800282 def update(self):
283 if self._pconn.poll():
284 (e, tb) = self._pconn.recv()
285 if e is not None:
286 self._exception = (e, tb)
287 else:
288 self._result = tb
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500289
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800290 @property
291 def exception(self):
292 self.update()
293 return self._exception
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500294
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800295 @property
296 def result(self):
297 self.update()
298 return self._result
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500299
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800300 max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
301 launched = []
302 errors = []
303 results = []
304 items = list(items)
305 while (items and not errors) or launched:
306 if not errors and items and len(launched) < max_process:
307 args = (items.pop(),)
308 if extraargs is not None:
309 args = args + extraargs
310 p = ProcessLaunch(target=target, args=args)
311 p.start()
312 launched.append(p)
313 for q in launched:
Brad Bishop19323692019-04-05 15:28:33 -0400314 # Have to manually call update() to avoid deadlocks. The pipe can be full and
315 # transfer stalled until we try and read the results object but the subprocess won't exit
316 # as it still has data to write (https://bugs.python.org/issue8426)
317 q.update()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800318 # The finished processes are joined when calling is_alive()
319 if not q.is_alive():
320 if q.exception:
321 errors.append(q.exception)
322 if q.result:
323 results.append(q.result)
324 launched.remove(q)
325 # Paranoia doesn't hurt
326 for p in launched:
327 p.join()
328 if errors:
329 msg = ""
330 for (e, tb) in errors:
Brad Bishopc342db32019-05-15 21:57:59 -0400331 if isinstance(e, subprocess.CalledProcessError) and e.output:
332 msg = msg + str(e) + "\n"
333 msg = msg + "Subprocess output:"
334 msg = msg + e.output.decode("utf-8", errors="ignore")
335 else:
336 msg = msg + str(e) + ": " + str(tb) + "\n"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800337 bb.fatal("Fatal errors occurred in subprocesses:\n%s" % msg)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500338 return results
339
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500340def squashspaces(string):
341 import re
Brad Bishop19323692019-04-05 15:28:33 -0400342 return re.sub(r"\s+", " ", string).strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500344def format_pkg_list(pkg_dict, ret_format=None):
345 output = []
346
347 if ret_format == "arch":
348 for pkg in sorted(pkg_dict):
349 output.append("%s %s" % (pkg, pkg_dict[pkg]["arch"]))
350 elif ret_format == "file":
351 for pkg in sorted(pkg_dict):
352 output.append("%s %s %s" % (pkg, pkg_dict[pkg]["filename"], pkg_dict[pkg]["arch"]))
353 elif ret_format == "ver":
354 for pkg in sorted(pkg_dict):
355 output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"]))
356 elif ret_format == "deps":
357 for pkg in sorted(pkg_dict):
358 for dep in pkg_dict[pkg]["deps"]:
359 output.append("%s|%s" % (pkg, dep))
360 else:
361 for pkg in sorted(pkg_dict):
362 output.append(pkg)
363
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800364 output_str = '\n'.join(output)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500365
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800366 if output_str:
367 # make sure last line is newline terminated
368 output_str += '\n'
369
370 return output_str
371
372def host_gcc_version(d, taskcontextonly=False):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500373 import re, subprocess
374
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800375 if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
376 return
377
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500378 compiler = d.getVar("BUILD_CC")
Brad Bishop19323692019-04-05 15:28:33 -0400379 # Get rid of ccache since it is not present when parsing.
380 if compiler.startswith('ccache '):
381 compiler = compiler[7:]
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500382 try:
383 env = os.environ.copy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500384 env["PATH"] = d.getVar("PATH")
Brad Bishop19323692019-04-05 15:28:33 -0400385 output = subprocess.check_output("%s --version" % compiler, \
386 shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500387 except subprocess.CalledProcessError as e:
388 bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
389
Brad Bishop19323692019-04-05 15:28:33 -0400390 match = re.match(r".* (\d\.\d)\.\d.*", output.split('\n')[0])
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500391 if not match:
392 bb.fatal("Can't get compiler version from %s --version output" % compiler)
393
394 version = match.group(1)
395 return "-%s" % version if version in ("4.8", "4.9") else ""
396
Brad Bishop316dfdd2018-06-25 12:45:53 -0400397
398def get_multilib_datastore(variant, d):
399 localdata = bb.data.createCopy(d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800400 if variant:
401 overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant
402 localdata.setVar("OVERRIDES", overrides)
403 localdata.setVar("MLPREFIX", variant + "-")
404 else:
405 origdefault = localdata.getVar("DEFAULTTUNE_MULTILIB_ORIGINAL")
406 if origdefault:
407 localdata.setVar("DEFAULTTUNE", origdefault)
408 overrides = localdata.getVar("OVERRIDES", False).split(":")
409 overrides = ":".join([x for x in overrides if not x.startswith("virtclass-multilib-")])
410 localdata.setVar("OVERRIDES", overrides)
411 localdata.setVar("MLPREFIX", "")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400412 return localdata
413
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500414#
415# Python 2.7 doesn't have threaded pools (just multiprocessing)
416# so implement a version here
417#
418
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600419from queue import Queue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500420from threading import Thread
421
422class ThreadedWorker(Thread):
423 """Thread executing tasks from a given tasks queue"""
424 def __init__(self, tasks, worker_init, worker_end):
425 Thread.__init__(self)
426 self.tasks = tasks
427 self.daemon = True
428
429 self.worker_init = worker_init
430 self.worker_end = worker_end
431
432 def run(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600433 from queue import Empty
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500434
435 if self.worker_init is not None:
436 self.worker_init(self)
437
438 while True:
439 try:
440 func, args, kargs = self.tasks.get(block=False)
441 except Empty:
442 if self.worker_end is not None:
443 self.worker_end(self)
444 break
445
446 try:
447 func(self, *args, **kargs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600448 except Exception as e:
449 print(e)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500450 finally:
451 self.tasks.task_done()
452
453class ThreadedPool:
454 """Pool of threads consuming tasks from a queue"""
455 def __init__(self, num_workers, num_tasks, worker_init=None,
456 worker_end=None):
457 self.tasks = Queue(num_tasks)
458 self.workers = []
459
460 for _ in range(num_workers):
461 worker = ThreadedWorker(self.tasks, worker_init, worker_end)
462 self.workers.append(worker)
463
464 def start(self):
465 for worker in self.workers:
466 worker.start()
467
468 def add_task(self, func, *args, **kargs):
469 """Add a task to the queue"""
470 self.tasks.put((func, args, kargs))
471
472 def wait_completion(self):
473 """Wait for completion of all the tasks in the queue"""
474 self.tasks.join()
475 for worker in self.workers:
476 worker.join()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500477
478def write_ld_so_conf(d):
479 # Some utils like prelink may not have the correct target library paths
480 # so write an ld.so.conf to help them
481 ldsoconf = d.expand("${STAGING_DIR_TARGET}${sysconfdir}/ld.so.conf")
482 if os.path.exists(ldsoconf):
483 bb.utils.remove(ldsoconf)
484 bb.utils.mkdirhier(os.path.dirname(ldsoconf))
485 with open(ldsoconf, "w") as f:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500486 f.write(d.getVar("base_libdir") + '\n')
487 f.write(d.getVar("libdir") + '\n')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600488
Brad Bishop08902b02019-08-20 09:16:51 -0400489class ImageQAFailed(Exception):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600490 def __init__(self, description, name=None, logfile=None):
491 self.description = description
492 self.name = name
493 self.logfile=logfile
494
495 def __str__(self):
496 msg = 'Function failed: %s' % self.name
497 if self.description:
498 msg = msg + ' (%s)' % self.description
499
500 return msg
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800501
Brad Bishop19323692019-04-05 15:28:33 -0400502def sh_quote(string):
503 import shlex
504 return shlex.quote(string)