blob: dbc6dea68d8d9d9574d75f2560ed4e4c6552c18c [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Data' implementations
5
6Functions for interacting with the data structure used by the
7BitBake build tools.
8
9The expandKeys and update_data are the most expensive
10operations. At night the cookie monster came by and
11suggested 'give me cookies on setting the variables and
12things will work out'. Taking this suggestion into account
13applying the skills from the not yet passed 'Entwurf und
14Analyse von Algorithmen' lecture and the cookie
15monster seems to be right. We will track setVar more carefully
16to have faster update_data and expandKeys operations.
17
18This is a trade-off between speed and memory again but
19the speed is more critical here.
20"""
21
22# Copyright (C) 2003, 2004 Chris Larson
23# Copyright (C) 2005 Holger Hans Peter Freyther
24#
25# This program is free software; you can redistribute it and/or modify
26# it under the terms of the GNU General Public License version 2 as
27# published by the Free Software Foundation.
28#
29# This program is distributed in the hope that it will be useful,
30# but WITHOUT ANY WARRANTY; without even the implied warranty of
31# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32# GNU General Public License for more details.
33#
34# You should have received a copy of the GNU General Public License along
35# with this program; if not, write to the Free Software Foundation, Inc.,
36# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
37#
38# Based on functions from the base bb module, Copyright 2003 Holger Schurig
39
40import sys, os, re
41if sys.argv[0][-5:] == "pydoc":
42 path = os.path.dirname(os.path.dirname(sys.argv[1]))
43else:
44 path = os.path.dirname(os.path.dirname(sys.argv[0]))
45sys.path.insert(0, path)
46from itertools import groupby
47
48from bb import data_smart
49from bb import codeparser
50import bb
51
52logger = data_smart.logger
53_dict_type = data_smart.DataSmart
54
55def init():
56 """Return a new object representing the Bitbake data"""
57 return _dict_type()
58
59def init_db(parent = None):
60 """Return a new object representing the Bitbake data,
61 optionally based on an existing object"""
62 if parent is not None:
63 return parent.createCopy()
64 else:
65 return _dict_type()
66
67def createCopy(source):
68 """Link the source set to the destination
69 If one does not find the value in the destination set,
70 search will go on to the source set to get the value.
71 Value from source are copy-on-write. i.e. any try to
72 modify one of them will end up putting the modified value
73 in the destination set.
74 """
75 return source.createCopy()
76
77def initVar(var, d):
78 """Non-destructive var init for data structure"""
79 d.initVar(var)
80
81
82def setVar(var, value, d):
83 """Set a variable to a given value"""
84 d.setVar(var, value)
85
86
87def getVar(var, d, exp = False):
88 """Gets the value of a variable"""
89 return d.getVar(var, exp)
90
91
92def renameVar(key, newkey, d):
93 """Renames a variable from key to newkey"""
94 d.renameVar(key, newkey)
95
96def delVar(var, d):
97 """Removes a variable from the data set"""
98 d.delVar(var)
99
100def appendVar(var, value, d):
101 """Append additional value to a variable"""
102 d.appendVar(var, value)
103
104def setVarFlag(var, flag, flagvalue, d):
105 """Set a flag for a given variable to a given value"""
106 d.setVarFlag(var, flag, flagvalue)
107
108def getVarFlag(var, flag, d):
109 """Gets given flag from given var"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500110 return d.getVarFlag(var, flag, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500111
112def delVarFlag(var, flag, d):
113 """Removes a given flag from the variable's flags"""
114 d.delVarFlag(var, flag)
115
116def setVarFlags(var, flags, d):
117 """Set the flags for a given variable
118
119 Note:
120 setVarFlags will not clear previous
121 flags. Think of this method as
122 addVarFlags
123 """
124 d.setVarFlags(var, flags)
125
126def getVarFlags(var, d):
127 """Gets a variable's flags"""
128 return d.getVarFlags(var)
129
130def delVarFlags(var, d):
131 """Removes a variable's flags"""
132 d.delVarFlags(var)
133
134def keys(d):
135 """Return a list of keys in d"""
136 return d.keys()
137
138
139__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
140__expand_python_regexp__ = re.compile(r"\${@.+?}")
141
142def expand(s, d, varname = None):
143 """Variable expansion using the data store"""
144 return d.expand(s, varname)
145
146def expandKeys(alterdata, readdata = None):
147 if readdata == None:
148 readdata = alterdata
149
150 todolist = {}
151 for key in alterdata:
152 if not '${' in key:
153 continue
154
155 ekey = expand(key, readdata)
156 if key == ekey:
157 continue
158 todolist[key] = ekey
159
160 # These two for loops are split for performance to maximise the
161 # usefulness of the expand cache
162 for key in sorted(todolist):
163 ekey = todolist[key]
164 newval = alterdata.getVar(ekey, False)
165 if newval is not None:
166 val = alterdata.getVar(key, False)
167 if val is not None:
168 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval))
169 alterdata.renameVar(key, ekey)
170
171def inheritFromOS(d, savedenv, permitted):
172 """Inherit variables from the initial environment."""
173 exportlist = bb.utils.preserved_envvars_exported()
174 for s in savedenv.keys():
175 if s in permitted:
176 try:
177 d.setVar(s, savedenv.getVar(s, True), op = 'from env')
178 if s in exportlist:
179 d.setVarFlag(s, "export", True, op = 'auto env export')
180 except TypeError:
181 pass
182
183def emit_var(var, o=sys.__stdout__, d = init(), all=False):
184 """Emit a variable to be sourced by a shell."""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500185 if d.getVarFlag(var, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500186 return False
187
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500188 export = d.getVarFlag(var, "export", False)
189 unexport = d.getVarFlag(var, "unexport", False)
190 func = d.getVarFlag(var, "func", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500191 if not all and not export and not unexport and not func:
192 return False
193
194 try:
195 if all:
196 oval = d.getVar(var, False)
197 val = d.getVar(var, True)
198 except (KeyboardInterrupt, bb.build.FuncFailed):
199 raise
200 except Exception as exc:
201 o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
202 return False
203
204 if all:
205 d.varhistory.emit(var, oval, val, o, d)
206
207 if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
208 return False
209
210 varExpanded = d.expand(var)
211
212 if unexport:
213 o.write('unset %s\n' % varExpanded)
214 return False
215
216 if val is None:
217 return False
218
219 val = str(val)
220
221 if varExpanded.startswith("BASH_FUNC_"):
222 varExpanded = varExpanded[10:-2]
223 val = val[3:] # Strip off "() "
224 o.write("%s() %s\n" % (varExpanded, val))
225 o.write("export -f %s\n" % (varExpanded))
226 return True
227
228 if func:
229 # NOTE: should probably check for unbalanced {} within the var
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500230 val = val.rstrip('\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231 o.write("%s() {\n%s\n}\n" % (varExpanded, val))
232 return 1
233
234 if export:
235 o.write('export ')
236
237 # if we're going to output this within doublequotes,
238 # to a shell, we need to escape the quotes in the var
239 alter = re.sub('"', '\\"', val)
240 alter = re.sub('\n', ' \\\n', alter)
241 alter = re.sub('\\$', '\\\\$', alter)
242 o.write('%s="%s"\n' % (varExpanded, alter))
243 return False
244
245def emit_env(o=sys.__stdout__, d = init(), all=False):
246 """Emits all items in the data store in a format such that it can be sourced by a shell."""
247
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500248 isfunc = lambda key: bool(d.getVarFlag(key, "func", False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500249 keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
250 grouped = groupby(keys, isfunc)
251 for isfunc, keys in grouped:
252 for key in keys:
253 emit_var(key, o, d, all and not isfunc) and o.write('\n')
254
255def exported_keys(d):
256 return (key for key in d.keys() if not key.startswith('__') and
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500257 d.getVarFlag(key, 'export', False) and
258 not d.getVarFlag(key, 'unexport', False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259
260def exported_vars(d):
261 for key in exported_keys(d):
262 try:
263 value = d.getVar(key, True)
264 except Exception:
265 pass
266
267 if value is not None:
268 yield key, str(value)
269
270def emit_func(func, o=sys.__stdout__, d = init()):
271 """Emits all items in the data store in a format such that it can be sourced by a shell."""
272
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500273 keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 for key in keys:
275 emit_var(key, o, d, False)
276
277 o.write('\n')
278 emit_var(func, o, d, False) and o.write('\n')
279 newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
280 newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
281 seen = set()
282 while newdeps:
283 deps = newdeps
284 seen |= deps
285 newdeps = set()
286 for dep in deps:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500287 if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500288 emit_var(dep, o, d, False) and o.write('\n')
289 newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep, True))
290 newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
291 newdeps -= seen
292
293_functionfmt = """
294def {function}(d):
295{body}"""
296
297def emit_func_python(func, o=sys.__stdout__, d = init()):
298 """Emits all items in the data store in a format such that it can be sourced by a shell."""
299
300 def write_func(func, o, call = False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500301 body = d.getVar(func, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302 if not body.startswith("def"):
303 body = _functionfmt.format(function=func, body=body)
304
305 o.write(body.strip() + "\n\n")
306 if call:
307 o.write(func + "(d)" + "\n\n")
308
309 write_func(func, o, True)
310 pp = bb.codeparser.PythonParser(func, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500311 pp.parse_python(d.getVar(func, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500312 newdeps = pp.execs
313 newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
314 seen = set()
315 while newdeps:
316 deps = newdeps
317 seen |= deps
318 newdeps = set()
319 for dep in deps:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500320 if d.getVarFlag(dep, "func", False) and d.getVarFlag(dep, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 write_func(dep, o)
322 pp = bb.codeparser.PythonParser(dep, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500323 pp.parse_python(d.getVar(dep, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324 newdeps |= pp.execs
325 newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
326 newdeps -= seen
327
328def update_data(d):
329 """Performs final steps upon the datastore, including application of overrides"""
330 d.finalize(parent = True)
331
332def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
333 deps = set()
334 try:
335 if key[-1] == ']':
336 vf = key[:-1].split('[')
337 value = d.getVarFlag(vf[0], vf[1], False)
338 parser = d.expandWithRefs(value, key)
339 deps |= parser.references
340 deps = deps | (keys & parser.execs)
341 return deps, value
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500342 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "vardepvalueexclude", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 vardeps = varflags.get("vardeps")
344 value = d.getVar(key, False)
345
346 def handle_contains(value, contains, d):
347 newvalue = ""
348 for k in sorted(contains):
349 l = (d.getVar(k, True) or "").split()
350 for word in sorted(contains[k]):
351 if word in l:
352 newvalue += "\n%s{%s} = Set" % (k, word)
353 else:
354 newvalue += "\n%s{%s} = Unset" % (k, word)
355 if not newvalue:
356 return value
357 if not value:
358 return newvalue
359 return value + newvalue
360
361 if "vardepvalue" in varflags:
362 value = varflags.get("vardepvalue")
363 elif varflags.get("func"):
364 if varflags.get("python"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500365 parser = bb.codeparser.PythonParser(key, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500366 if value and "\t" in value:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500367 logger.warn("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True)))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500368 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369 deps = deps | parser.references
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500370 deps = deps | (keys & parser.execs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500371 value = handle_contains(value, parser.contains, d)
372 else:
373 parsedvar = d.expandWithRefs(value, key)
374 parser = bb.codeparser.ShellParser(key, logger)
375 parser.parse_shell(parsedvar.value)
376 deps = deps | shelldeps
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500377 deps = deps | parsedvar.references
378 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
379 value = handle_contains(value, parsedvar.contains, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500380 if vardeps is None:
381 parser.log.flush()
382 if "prefuncs" in varflags:
383 deps = deps | set(varflags["prefuncs"].split())
384 if "postfuncs" in varflags:
385 deps = deps | set(varflags["postfuncs"].split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386 else:
387 parser = d.expandWithRefs(value, key)
388 deps |= parser.references
389 deps = deps | (keys & parser.execs)
390 value = handle_contains(value, parser.contains, d)
391
392 if "vardepvalueexclude" in varflags:
393 exclude = varflags.get("vardepvalueexclude")
394 for excl in exclude.split('|'):
395 if excl:
396 value = value.replace(excl, '')
397
398 # Add varflags, assuming an exclusion list is set
399 if varflagsexcl:
400 varfdeps = []
401 for f in varflags:
402 if f not in varflagsexcl:
403 varfdeps.append('%s[%s]' % (key, f))
404 if varfdeps:
405 deps |= set(varfdeps)
406
407 deps |= set((vardeps or "").split())
408 deps -= set(varflags.get("vardepsexclude", "").split())
409 except Exception as e:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500410 bb.warn("Exception during build_dependencies for %s" % key)
411 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500412 return deps, value
413 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
414 #d.setVarFlag(key, "vardeps", deps)
415
416def generate_dependencies(d):
417
418 keys = set(key for key in d if not key.startswith("__"))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500419 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500420 varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True)
421
422 deps = {}
423 values = {}
424
425 tasklist = d.getVar('__BBTASKS', False) or []
426 for task in tasklist:
427 deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
428 newdeps = deps[task]
429 seen = set()
430 while newdeps:
431 nextdeps = newdeps
432 seen |= nextdeps
433 newdeps = set()
434 for dep in nextdeps:
435 if dep not in deps:
436 deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
437 newdeps |= deps[dep]
438 newdeps -= seen
439 #print "For %s: %s" % (task, str(deps[task]))
440 return tasklist, deps, values
441
442def inherits_class(klass, d):
443 val = d.getVar('__inherit_cache', False) or []
444 needle = os.path.join('classes', '%s.bbclass' % klass)
445 for v in val:
446 if v.endswith(needle):
447 return True
448 return False