blob: 0d75d0c1a9f22af03a27abbd5b2ffba7bdde639b [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake 'Data' implementations
3
4Functions for interacting with the data structure used by the
5BitBake build tools.
6
7The expandKeys and update_data are the most expensive
8operations. At night the cookie monster came by and
9suggested 'give me cookies on setting the variables and
10things will work out'. Taking this suggestion into account
11applying the skills from the not yet passed 'Entwurf und
12Analyse von Algorithmen' lecture and the cookie
13monster seems to be right. We will track setVar more carefully
14to have faster update_data and expandKeys operations.
15
16This is a trade-off between speed and memory again but
17the speed is more critical here.
18"""
19
20# Copyright (C) 2003, 2004 Chris Larson
21# Copyright (C) 2005 Holger Hans Peter Freyther
22#
Brad Bishopc342db32019-05-15 21:57:59 -040023# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024#
25# Based on functions from the base bb module, Copyright 2003 Holger Schurig
26
27import sys, os, re
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080028import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029if sys.argv[0][-5:] == "pydoc":
30 path = os.path.dirname(os.path.dirname(sys.argv[1]))
31else:
32 path = os.path.dirname(os.path.dirname(sys.argv[0]))
33sys.path.insert(0, path)
34from itertools import groupby
35
36from bb import data_smart
37from bb import codeparser
38import bb
39
40logger = data_smart.logger
41_dict_type = data_smart.DataSmart
42
43def init():
44 """Return a new object representing the Bitbake data"""
45 return _dict_type()
46
47def init_db(parent = None):
48 """Return a new object representing the Bitbake data,
49 optionally based on an existing object"""
50 if parent is not None:
51 return parent.createCopy()
52 else:
53 return _dict_type()
54
55def createCopy(source):
56 """Link the source set to the destination
57 If one does not find the value in the destination set,
58 search will go on to the source set to get the value.
59 Value from source are copy-on-write. i.e. any try to
60 modify one of them will end up putting the modified value
61 in the destination set.
62 """
63 return source.createCopy()
64
65def initVar(var, d):
66 """Non-destructive var init for data structure"""
67 d.initVar(var)
68
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069def keys(d):
70 """Return a list of keys in d"""
71 return d.keys()
72
73
74__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
75__expand_python_regexp__ = re.compile(r"\${@.+?}")
76
77def expand(s, d, varname = None):
78 """Variable expansion using the data store"""
79 return d.expand(s, varname)
80
81def expandKeys(alterdata, readdata = None):
82 if readdata == None:
83 readdata = alterdata
84
85 todolist = {}
86 for key in alterdata:
87 if not '${' in key:
88 continue
89
90 ekey = expand(key, readdata)
91 if key == ekey:
92 continue
93 todolist[key] = ekey
94
95 # These two for loops are split for performance to maximise the
96 # usefulness of the expand cache
97 for key in sorted(todolist):
98 ekey = todolist[key]
99 newval = alterdata.getVar(ekey, False)
100 if newval is not None:
101 val = alterdata.getVar(key, False)
102 if val is not None:
103 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval))
104 alterdata.renameVar(key, ekey)
105
106def inheritFromOS(d, savedenv, permitted):
107 """Inherit variables from the initial environment."""
108 exportlist = bb.utils.preserved_envvars_exported()
109 for s in savedenv.keys():
110 if s in permitted:
111 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500112 d.setVar(s, savedenv.getVar(s), op = 'from env')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500113 if s in exportlist:
114 d.setVarFlag(s, "export", True, op = 'auto env export')
115 except TypeError:
116 pass
117
118def emit_var(var, o=sys.__stdout__, d = init(), all=False):
119 """Emit a variable to be sourced by a shell."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600120 func = d.getVarFlag(var, "func", False)
121 if d.getVarFlag(var, 'python', False) and func:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122 return False
123
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500124 export = d.getVarFlag(var, "export", False)
125 unexport = d.getVarFlag(var, "unexport", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 if not all and not export and not unexport and not func:
127 return False
128
129 try:
130 if all:
131 oval = d.getVar(var, False)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500132 val = d.getVar(var)
Brad Bishop08902b02019-08-20 09:16:51 -0400133 except (KeyboardInterrupt):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500134 raise
135 except Exception as exc:
136 o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
137 return False
138
139 if all:
140 d.varhistory.emit(var, oval, val, o, d)
141
142 if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
143 return False
144
145 varExpanded = d.expand(var)
146
147 if unexport:
148 o.write('unset %s\n' % varExpanded)
149 return False
150
151 if val is None:
152 return False
153
154 val = str(val)
155
156 if varExpanded.startswith("BASH_FUNC_"):
157 varExpanded = varExpanded[10:-2]
158 val = val[3:] # Strip off "() "
159 o.write("%s() %s\n" % (varExpanded, val))
160 o.write("export -f %s\n" % (varExpanded))
161 return True
162
163 if func:
164 # NOTE: should probably check for unbalanced {} within the var
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500165 val = val.rstrip('\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500166 o.write("%s() {\n%s\n}\n" % (varExpanded, val))
167 return 1
168
169 if export:
170 o.write('export ')
171
172 # if we're going to output this within doublequotes,
173 # to a shell, we need to escape the quotes in the var
174 alter = re.sub('"', '\\"', val)
175 alter = re.sub('\n', ' \\\n', alter)
176 alter = re.sub('\\$', '\\\\$', alter)
177 o.write('%s="%s"\n' % (varExpanded, alter))
178 return False
179
180def emit_env(o=sys.__stdout__, d = init(), all=False):
181 """Emits all items in the data store in a format such that it can be sourced by a shell."""
182
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500183 isfunc = lambda key: bool(d.getVarFlag(key, "func", False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500184 keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
185 grouped = groupby(keys, isfunc)
186 for isfunc, keys in grouped:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500187 for key in sorted(keys):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500188 emit_var(key, o, d, all and not isfunc) and o.write('\n')
189
190def exported_keys(d):
191 return (key for key in d.keys() if not key.startswith('__') and
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500192 d.getVarFlag(key, 'export', False) and
193 not d.getVarFlag(key, 'unexport', False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500194
195def exported_vars(d):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500196 k = list(exported_keys(d))
197 for key in k:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500198 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500199 value = d.getVar(key)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500200 except Exception as err:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500201 bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE"), key, err))
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500202 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203
204 if value is not None:
205 yield key, str(value)
206
207def emit_func(func, o=sys.__stdout__, d = init()):
208 """Emits all items in the data store in a format such that it can be sourced by a shell."""
209
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500210 keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500211 for key in sorted(keys):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500212 emit_var(key, o, d, False)
213
214 o.write('\n')
215 emit_var(func, o, d, False) and o.write('\n')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500216 newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
217 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218 seen = set()
219 while newdeps:
220 deps = newdeps
221 seen |= deps
222 newdeps = set()
223 for dep in deps:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500224 if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225 emit_var(dep, o, d, False) and o.write('\n')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500226 newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep))
227 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228 newdeps -= seen
229
230_functionfmt = """
231def {function}(d):
232{body}"""
233
234def emit_func_python(func, o=sys.__stdout__, d = init()):
235 """Emits all items in the data store in a format such that it can be sourced by a shell."""
236
237 def write_func(func, o, call = False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500238 body = d.getVar(func, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239 if not body.startswith("def"):
240 body = _functionfmt.format(function=func, body=body)
241
242 o.write(body.strip() + "\n\n")
243 if call:
244 o.write(func + "(d)" + "\n\n")
245
246 write_func(func, o, True)
247 pp = bb.codeparser.PythonParser(func, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500248 pp.parse_python(d.getVar(func, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500249 newdeps = pp.execs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500250 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500251 seen = set()
252 while newdeps:
253 deps = newdeps
254 seen |= deps
255 newdeps = set()
256 for dep in deps:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500257 if d.getVarFlag(dep, "func", False) and d.getVarFlag(dep, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500258 write_func(dep, o)
259 pp = bb.codeparser.PythonParser(dep, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500260 pp.parse_python(d.getVar(dep, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 newdeps |= pp.execs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 newdeps -= seen
264
265def update_data(d):
266 """Performs final steps upon the datastore, including application of overrides"""
267 d.finalize(parent = True)
268
269def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
270 deps = set()
271 try:
272 if key[-1] == ']':
273 vf = key[:-1].split('[')
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800274 value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500275 deps |= parser.references
276 deps = deps | (keys & parser.execs)
277 return deps, value
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600278 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500279 vardeps = varflags.get("vardeps")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500280
281 def handle_contains(value, contains, d):
282 newvalue = ""
283 for k in sorted(contains):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500284 l = (d.getVar(k) or "").split()
285 for item in sorted(contains[k]):
286 for word in item.split():
287 if not word in l:
288 newvalue += "\n%s{%s} = Unset" % (k, item)
289 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500290 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500291 newvalue += "\n%s{%s} = Set" % (k, item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500292 if not newvalue:
293 return value
294 if not value:
295 return newvalue
296 return value + newvalue
297
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800298 def handle_remove(value, deps, removes, d):
299 for r in sorted(removes):
300 r2 = d.expandWithRefs(r, None)
301 value += "\n_remove of %s" % r
302 deps |= r2.references
303 deps = deps | (keys & r2.execs)
304 return value
305
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500306 if "vardepvalue" in varflags:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800307 value = varflags.get("vardepvalue")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500308 elif varflags.get("func"):
309 if varflags.get("python"):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800310 value = d.getVarFlag(key, "_content", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500311 parser = bb.codeparser.PythonParser(key, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500312 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500313 deps = deps | parser.references
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500314 deps = deps | (keys & parser.execs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315 value = handle_contains(value, parser.contains, d)
316 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800317 value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318 parser = bb.codeparser.ShellParser(key, logger)
319 parser.parse_shell(parsedvar.value)
320 deps = deps | shelldeps
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500321 deps = deps | parsedvar.references
322 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
323 value = handle_contains(value, parsedvar.contains, d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800324 if hasattr(parsedvar, "removes"):
325 value = handle_remove(value, deps, parsedvar.removes, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500326 if vardeps is None:
327 parser.log.flush()
328 if "prefuncs" in varflags:
329 deps = deps | set(varflags["prefuncs"].split())
330 if "postfuncs" in varflags:
331 deps = deps | set(varflags["postfuncs"].split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600332 if "exports" in varflags:
333 deps = deps | set(varflags["exports"].split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800335 value, parser = d.getVarFlag(key, "_content", False, retparser=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500336 deps |= parser.references
337 deps = deps | (keys & parser.execs)
338 value = handle_contains(value, parser.contains, d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800339 if hasattr(parser, "removes"):
340 value = handle_remove(value, deps, parser.removes, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500341
342 if "vardepvalueexclude" in varflags:
343 exclude = varflags.get("vardepvalueexclude")
344 for excl in exclude.split('|'):
345 if excl:
346 value = value.replace(excl, '')
347
348 # Add varflags, assuming an exclusion list is set
349 if varflagsexcl:
350 varfdeps = []
351 for f in varflags:
352 if f not in varflagsexcl:
353 varfdeps.append('%s[%s]' % (key, f))
354 if varfdeps:
355 deps |= set(varfdeps)
356
357 deps |= set((vardeps or "").split())
358 deps -= set(varflags.get("vardepsexclude", "").split())
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500359 except bb.parse.SkipRecipe:
360 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500361 except Exception as e:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500362 bb.warn("Exception during build_dependencies for %s" % key)
363 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500364 return deps, value
365 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
366 #d.setVarFlag(key, "vardeps", deps)
367
368def generate_dependencies(d):
369
370 keys = set(key for key in d if not key.startswith("__"))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500371 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500372 varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500373
374 deps = {}
375 values = {}
376
377 tasklist = d.getVar('__BBTASKS', False) or []
378 for task in tasklist:
379 deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
380 newdeps = deps[task]
381 seen = set()
382 while newdeps:
383 nextdeps = newdeps
384 seen |= nextdeps
385 newdeps = set()
386 for dep in nextdeps:
387 if dep not in deps:
388 deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
389 newdeps |= deps[dep]
390 newdeps -= seen
391 #print "For %s: %s" % (task, str(deps[task]))
392 return tasklist, deps, values
393
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800394def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
395 taskdeps = {}
396 basehash = {}
397
398 for task in tasklist:
399 data = lookupcache[task]
400
401 if data is None:
402 bb.error("Task %s from %s seems to be empty?!" % (task, fn))
403 data = ''
404
405 gendeps[task] -= whitelist
406 newdeps = gendeps[task]
407 seen = set()
408 while newdeps:
409 nextdeps = newdeps
410 seen |= nextdeps
411 newdeps = set()
412 for dep in nextdeps:
413 if dep in whitelist:
414 continue
415 gendeps[dep] -= whitelist
416 newdeps |= gendeps[dep]
417 newdeps -= seen
418
419 alldeps = sorted(seen)
420 for dep in alldeps:
421 data = data + dep
422 var = lookupcache[dep]
423 if var is not None:
424 data = data + str(var)
Brad Bishop08902b02019-08-20 09:16:51 -0400425 k = fn + ":" + task
Brad Bishop19323692019-04-05 15:28:33 -0400426 basehash[k] = hashlib.sha256(data.encode("utf-8")).hexdigest()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800427 taskdeps[task] = alldeps
428
429 return taskdeps, basehash
430
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500431def inherits_class(klass, d):
432 val = d.getVar('__inherit_cache', False) or []
433 needle = os.path.join('classes', '%s.bbclass' % klass)
434 for v in val:
435 if v.endswith(needle):
436 return True
437 return False