blob: 9d18b1e2bf8ac17f4567acefed10d9693cfa7b68 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake 'Data' implementations
3
4Functions for interacting with the data structure used by the
5BitBake build tools.
6
7The expandKeys and update_data are the most expensive
8operations. At night the cookie monster came by and
9suggested 'give me cookies on setting the variables and
10things will work out'. Taking this suggestion into account
11applying the skills from the not yet passed 'Entwurf und
12Analyse von Algorithmen' lecture and the cookie
13monster seems to be right. We will track setVar more carefully
14to have faster update_data and expandKeys operations.
15
16This is a trade-off between speed and memory again but
17the speed is more critical here.
18"""
19
20# Copyright (C) 2003, 2004 Chris Larson
21# Copyright (C) 2005 Holger Hans Peter Freyther
22#
Brad Bishopc342db32019-05-15 21:57:59 -040023# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024#
25# Based on functions from the base bb module, Copyright 2003 Holger Schurig
26
27import sys, os, re
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080028import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029if sys.argv[0][-5:] == "pydoc":
30 path = os.path.dirname(os.path.dirname(sys.argv[1]))
31else:
32 path = os.path.dirname(os.path.dirname(sys.argv[0]))
33sys.path.insert(0, path)
34from itertools import groupby
35
36from bb import data_smart
37from bb import codeparser
38import bb
39
40logger = data_smart.logger
41_dict_type = data_smart.DataSmart
42
43def init():
44 """Return a new object representing the Bitbake data"""
45 return _dict_type()
46
47def init_db(parent = None):
48 """Return a new object representing the Bitbake data,
49 optionally based on an existing object"""
50 if parent is not None:
51 return parent.createCopy()
52 else:
53 return _dict_type()
54
55def createCopy(source):
56 """Link the source set to the destination
57 If one does not find the value in the destination set,
58 search will go on to the source set to get the value.
59 Value from source are copy-on-write. i.e. any try to
60 modify one of them will end up putting the modified value
61 in the destination set.
62 """
63 return source.createCopy()
64
65def initVar(var, d):
66 """Non-destructive var init for data structure"""
67 d.initVar(var)
68
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069def keys(d):
70 """Return a list of keys in d"""
71 return d.keys()
72
73
74__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
75__expand_python_regexp__ = re.compile(r"\${@.+?}")
76
77def expand(s, d, varname = None):
78 """Variable expansion using the data store"""
79 return d.expand(s, varname)
80
81def expandKeys(alterdata, readdata = None):
Andrew Geissler82c905d2020-04-13 13:39:40 -050082 if readdata is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050083 readdata = alterdata
84
85 todolist = {}
86 for key in alterdata:
87 if not '${' in key:
88 continue
89
90 ekey = expand(key, readdata)
91 if key == ekey:
92 continue
93 todolist[key] = ekey
94
95 # These two for loops are split for performance to maximise the
96 # usefulness of the expand cache
97 for key in sorted(todolist):
98 ekey = todolist[key]
99 newval = alterdata.getVar(ekey, False)
100 if newval is not None:
101 val = alterdata.getVar(key, False)
102 if val is not None:
103 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval))
104 alterdata.renameVar(key, ekey)
105
106def inheritFromOS(d, savedenv, permitted):
107 """Inherit variables from the initial environment."""
108 exportlist = bb.utils.preserved_envvars_exported()
109 for s in savedenv.keys():
110 if s in permitted:
111 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500112 d.setVar(s, savedenv.getVar(s), op = 'from env')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500113 if s in exportlist:
114 d.setVarFlag(s, "export", True, op = 'auto env export')
115 except TypeError:
116 pass
117
118def emit_var(var, o=sys.__stdout__, d = init(), all=False):
119 """Emit a variable to be sourced by a shell."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600120 func = d.getVarFlag(var, "func", False)
121 if d.getVarFlag(var, 'python', False) and func:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122 return False
123
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500124 export = d.getVarFlag(var, "export", False)
125 unexport = d.getVarFlag(var, "unexport", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 if not all and not export and not unexport and not func:
127 return False
128
129 try:
130 if all:
131 oval = d.getVar(var, False)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500132 val = d.getVar(var)
Brad Bishop08902b02019-08-20 09:16:51 -0400133 except (KeyboardInterrupt):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500134 raise
135 except Exception as exc:
136 o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
137 return False
138
139 if all:
140 d.varhistory.emit(var, oval, val, o, d)
141
142 if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
143 return False
144
145 varExpanded = d.expand(var)
146
147 if unexport:
148 o.write('unset %s\n' % varExpanded)
149 return False
150
151 if val is None:
152 return False
153
154 val = str(val)
155
156 if varExpanded.startswith("BASH_FUNC_"):
157 varExpanded = varExpanded[10:-2]
158 val = val[3:] # Strip off "() "
159 o.write("%s() %s\n" % (varExpanded, val))
160 o.write("export -f %s\n" % (varExpanded))
161 return True
162
163 if func:
Andrew Geissler635e0e42020-08-21 15:58:33 -0500164 # Write a comment indicating where the shell function came from (line number and filename) to make it easier
165 # for the user to diagnose task failures. This comment is also used by build.py to determine the metadata
166 # location of shell functions.
167 o.write("# line: {0}, file: {1}\n".format(
168 d.getVarFlag(var, "lineno", False),
169 d.getVarFlag(var, "filename", False)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 # NOTE: should probably check for unbalanced {} within the var
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500171 val = val.rstrip('\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500172 o.write("%s() {\n%s\n}\n" % (varExpanded, val))
173 return 1
174
175 if export:
176 o.write('export ')
177
178 # if we're going to output this within doublequotes,
179 # to a shell, we need to escape the quotes in the var
180 alter = re.sub('"', '\\"', val)
181 alter = re.sub('\n', ' \\\n', alter)
182 alter = re.sub('\\$', '\\\\$', alter)
183 o.write('%s="%s"\n' % (varExpanded, alter))
184 return False
185
186def emit_env(o=sys.__stdout__, d = init(), all=False):
187 """Emits all items in the data store in a format such that it can be sourced by a shell."""
188
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500189 isfunc = lambda key: bool(d.getVarFlag(key, "func", False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500190 keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
191 grouped = groupby(keys, isfunc)
192 for isfunc, keys in grouped:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500193 for key in sorted(keys):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500194 emit_var(key, o, d, all and not isfunc) and o.write('\n')
195
196def exported_keys(d):
197 return (key for key in d.keys() if not key.startswith('__') and
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500198 d.getVarFlag(key, 'export', False) and
199 not d.getVarFlag(key, 'unexport', False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500200
201def exported_vars(d):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500202 k = list(exported_keys(d))
203 for key in k:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500205 value = d.getVar(key)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500206 except Exception as err:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500207 bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE"), key, err))
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500208 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500209
210 if value is not None:
211 yield key, str(value)
212
213def emit_func(func, o=sys.__stdout__, d = init()):
214 """Emits all items in the data store in a format such that it can be sourced by a shell."""
215
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500216 keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500217 for key in sorted(keys):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218 emit_var(key, o, d, False)
219
220 o.write('\n')
221 emit_var(func, o, d, False) and o.write('\n')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500222 newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
223 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500224 seen = set()
225 while newdeps:
226 deps = newdeps
227 seen |= deps
228 newdeps = set()
Patrick Williams93c203f2021-10-06 16:15:23 -0500229 for dep in sorted(deps):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500230 if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231 emit_var(dep, o, d, False) and o.write('\n')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500232 newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep))
233 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234 newdeps -= seen
235
236_functionfmt = """
237def {function}(d):
238{body}"""
239
240def emit_func_python(func, o=sys.__stdout__, d = init()):
241 """Emits all items in the data store in a format such that it can be sourced by a shell."""
242
243 def write_func(func, o, call = False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500244 body = d.getVar(func, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500245 if not body.startswith("def"):
246 body = _functionfmt.format(function=func, body=body)
247
248 o.write(body.strip() + "\n\n")
249 if call:
250 o.write(func + "(d)" + "\n\n")
251
252 write_func(func, o, True)
253 pp = bb.codeparser.PythonParser(func, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500254 pp.parse_python(d.getVar(func, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 newdeps = pp.execs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500256 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 seen = set()
258 while newdeps:
259 deps = newdeps
260 seen |= deps
261 newdeps = set()
262 for dep in deps:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500263 if d.getVarFlag(dep, "func", False) and d.getVarFlag(dep, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264 write_func(dep, o)
265 pp = bb.codeparser.PythonParser(dep, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500266 pp.parse_python(d.getVar(dep, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500267 newdeps |= pp.execs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500268 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269 newdeps -= seen
270
271def update_data(d):
272 """Performs final steps upon the datastore, including application of overrides"""
273 d.finalize(parent = True)
274
275def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
276 deps = set()
277 try:
278 if key[-1] == ']':
279 vf = key[:-1].split('[')
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800280 value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500281 deps |= parser.references
282 deps = deps | (keys & parser.execs)
283 return deps, value
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600284 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500285 vardeps = varflags.get("vardeps")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500286
287 def handle_contains(value, contains, d):
288 newvalue = ""
289 for k in sorted(contains):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500290 l = (d.getVar(k) or "").split()
291 for item in sorted(contains[k]):
292 for word in item.split():
293 if not word in l:
294 newvalue += "\n%s{%s} = Unset" % (k, item)
295 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500297 newvalue += "\n%s{%s} = Set" % (k, item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298 if not newvalue:
299 return value
300 if not value:
301 return newvalue
302 return value + newvalue
303
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800304 def handle_remove(value, deps, removes, d):
305 for r in sorted(removes):
306 r2 = d.expandWithRefs(r, None)
307 value += "\n_remove of %s" % r
308 deps |= r2.references
309 deps = deps | (keys & r2.execs)
310 return value
311
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500312 if "vardepvalue" in varflags:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800313 value = varflags.get("vardepvalue")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314 elif varflags.get("func"):
315 if varflags.get("python"):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800316 value = d.getVarFlag(key, "_content", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317 parser = bb.codeparser.PythonParser(key, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500318 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500319 deps = deps | parser.references
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500320 deps = deps | (keys & parser.execs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 value = handle_contains(value, parser.contains, d)
322 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800323 value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324 parser = bb.codeparser.ShellParser(key, logger)
325 parser.parse_shell(parsedvar.value)
326 deps = deps | shelldeps
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500327 deps = deps | parsedvar.references
328 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
329 value = handle_contains(value, parsedvar.contains, d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800330 if hasattr(parsedvar, "removes"):
331 value = handle_remove(value, deps, parsedvar.removes, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500332 if vardeps is None:
333 parser.log.flush()
334 if "prefuncs" in varflags:
335 deps = deps | set(varflags["prefuncs"].split())
336 if "postfuncs" in varflags:
337 deps = deps | set(varflags["postfuncs"].split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600338 if "exports" in varflags:
339 deps = deps | set(varflags["exports"].split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500340 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800341 value, parser = d.getVarFlag(key, "_content", False, retparser=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500342 deps |= parser.references
343 deps = deps | (keys & parser.execs)
344 value = handle_contains(value, parser.contains, d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800345 if hasattr(parser, "removes"):
346 value = handle_remove(value, deps, parser.removes, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347
348 if "vardepvalueexclude" in varflags:
349 exclude = varflags.get("vardepvalueexclude")
350 for excl in exclude.split('|'):
351 if excl:
352 value = value.replace(excl, '')
353
354 # Add varflags, assuming an exclusion list is set
355 if varflagsexcl:
356 varfdeps = []
357 for f in varflags:
358 if f not in varflagsexcl:
359 varfdeps.append('%s[%s]' % (key, f))
360 if varfdeps:
361 deps |= set(varfdeps)
362
363 deps |= set((vardeps or "").split())
364 deps -= set(varflags.get("vardepsexclude", "").split())
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500365 except bb.parse.SkipRecipe:
366 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500367 except Exception as e:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500368 bb.warn("Exception during build_dependencies for %s" % key)
369 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500370 return deps, value
371 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
372 #d.setVarFlag(key, "vardeps", deps)
373
Andrew Geissler82c905d2020-04-13 13:39:40 -0500374def generate_dependencies(d, whitelist):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500375
376 keys = set(key for key in d if not key.startswith("__"))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500377 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500378 varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500379
380 deps = {}
381 values = {}
382
383 tasklist = d.getVar('__BBTASKS', False) or []
384 for task in tasklist:
385 deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
386 newdeps = deps[task]
387 seen = set()
388 while newdeps:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500389 nextdeps = newdeps - whitelist
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500390 seen |= nextdeps
391 newdeps = set()
392 for dep in nextdeps:
393 if dep not in deps:
394 deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
395 newdeps |= deps[dep]
396 newdeps -= seen
397 #print "For %s: %s" % (task, str(deps[task]))
398 return tasklist, deps, values
399
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800400def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
401 taskdeps = {}
402 basehash = {}
403
404 for task in tasklist:
405 data = lookupcache[task]
406
407 if data is None:
408 bb.error("Task %s from %s seems to be empty?!" % (task, fn))
409 data = ''
410
411 gendeps[task] -= whitelist
412 newdeps = gendeps[task]
413 seen = set()
414 while newdeps:
415 nextdeps = newdeps
416 seen |= nextdeps
417 newdeps = set()
418 for dep in nextdeps:
419 if dep in whitelist:
420 continue
421 gendeps[dep] -= whitelist
422 newdeps |= gendeps[dep]
423 newdeps -= seen
424
425 alldeps = sorted(seen)
426 for dep in alldeps:
427 data = data + dep
428 var = lookupcache[dep]
429 if var is not None:
430 data = data + str(var)
Brad Bishop08902b02019-08-20 09:16:51 -0400431 k = fn + ":" + task
Brad Bishop19323692019-04-05 15:28:33 -0400432 basehash[k] = hashlib.sha256(data.encode("utf-8")).hexdigest()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800433 taskdeps[task] = alldeps
434
435 return taskdeps, basehash
436
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500437def inherits_class(klass, d):
438 val = d.getVar('__inherit_cache', False) or []
439 needle = os.path.join('classes', '%s.bbclass' % klass)
440 for v in val:
441 if v.endswith(needle):
442 return True
443 return False