blob: f6415a4674c297169befc83c3105a6afe76b9307 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Data' implementations
5
6Functions for interacting with the data structure used by the
7BitBake build tools.
8
9The expandKeys and update_data are the most expensive
10operations. At night the cookie monster came by and
11suggested 'give me cookies on setting the variables and
12things will work out'. Taking this suggestion into account
13applying the skills from the not yet passed 'Entwurf und
14Analyse von Algorithmen' lecture and the cookie
15monster seems to be right. We will track setVar more carefully
16to have faster update_data and expandKeys operations.
17
18This is a trade-off between speed and memory again but
19the speed is more critical here.
20"""
21
22# Copyright (C) 2003, 2004 Chris Larson
23# Copyright (C) 2005 Holger Hans Peter Freyther
24#
25# This program is free software; you can redistribute it and/or modify
26# it under the terms of the GNU General Public License version 2 as
27# published by the Free Software Foundation.
28#
29# This program is distributed in the hope that it will be useful,
30# but WITHOUT ANY WARRANTY; without even the implied warranty of
31# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32# GNU General Public License for more details.
33#
34# You should have received a copy of the GNU General Public License along
35# with this program; if not, write to the Free Software Foundation, Inc.,
36# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
37#
38# Based on functions from the base bb module, Copyright 2003 Holger Schurig
39
40import sys, os, re
41if sys.argv[0][-5:] == "pydoc":
42 path = os.path.dirname(os.path.dirname(sys.argv[1]))
43else:
44 path = os.path.dirname(os.path.dirname(sys.argv[0]))
45sys.path.insert(0, path)
46from itertools import groupby
47
48from bb import data_smart
49from bb import codeparser
50import bb
51
52logger = data_smart.logger
53_dict_type = data_smart.DataSmart
54
55def init():
56 """Return a new object representing the Bitbake data"""
57 return _dict_type()
58
59def init_db(parent = None):
60 """Return a new object representing the Bitbake data,
61 optionally based on an existing object"""
62 if parent is not None:
63 return parent.createCopy()
64 else:
65 return _dict_type()
66
67def createCopy(source):
68 """Link the source set to the destination
69 If one does not find the value in the destination set,
70 search will go on to the source set to get the value.
71 Value from source are copy-on-write. i.e. any try to
72 modify one of them will end up putting the modified value
73 in the destination set.
74 """
75 return source.createCopy()
76
77def initVar(var, d):
78 """Non-destructive var init for data structure"""
79 d.initVar(var)
80
81
82def setVar(var, value, d):
83 """Set a variable to a given value"""
84 d.setVar(var, value)
85
86
87def getVar(var, d, exp = False):
88 """Gets the value of a variable"""
89 return d.getVar(var, exp)
90
91
92def renameVar(key, newkey, d):
93 """Renames a variable from key to newkey"""
94 d.renameVar(key, newkey)
95
96def delVar(var, d):
97 """Removes a variable from the data set"""
98 d.delVar(var)
99
100def appendVar(var, value, d):
101 """Append additional value to a variable"""
102 d.appendVar(var, value)
103
104def setVarFlag(var, flag, flagvalue, d):
105 """Set a flag for a given variable to a given value"""
106 d.setVarFlag(var, flag, flagvalue)
107
108def getVarFlag(var, flag, d):
109 """Gets given flag from given var"""
110 return d.getVarFlag(var, flag)
111
112def delVarFlag(var, flag, d):
113 """Removes a given flag from the variable's flags"""
114 d.delVarFlag(var, flag)
115
116def setVarFlags(var, flags, d):
117 """Set the flags for a given variable
118
119 Note:
120 setVarFlags will not clear previous
121 flags. Think of this method as
122 addVarFlags
123 """
124 d.setVarFlags(var, flags)
125
126def getVarFlags(var, d):
127 """Gets a variable's flags"""
128 return d.getVarFlags(var)
129
130def delVarFlags(var, d):
131 """Removes a variable's flags"""
132 d.delVarFlags(var)
133
134def keys(d):
135 """Return a list of keys in d"""
136 return d.keys()
137
138
139__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
140__expand_python_regexp__ = re.compile(r"\${@.+?}")
141
142def expand(s, d, varname = None):
143 """Variable expansion using the data store"""
144 return d.expand(s, varname)
145
146def expandKeys(alterdata, readdata = None):
147 if readdata == None:
148 readdata = alterdata
149
150 todolist = {}
151 for key in alterdata:
152 if not '${' in key:
153 continue
154
155 ekey = expand(key, readdata)
156 if key == ekey:
157 continue
158 todolist[key] = ekey
159
160 # These two for loops are split for performance to maximise the
161 # usefulness of the expand cache
162 for key in sorted(todolist):
163 ekey = todolist[key]
164 newval = alterdata.getVar(ekey, False)
165 if newval is not None:
166 val = alterdata.getVar(key, False)
167 if val is not None:
168 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval))
169 alterdata.renameVar(key, ekey)
170
171def inheritFromOS(d, savedenv, permitted):
172 """Inherit variables from the initial environment."""
173 exportlist = bb.utils.preserved_envvars_exported()
174 for s in savedenv.keys():
175 if s in permitted:
176 try:
177 d.setVar(s, savedenv.getVar(s, True), op = 'from env')
178 if s in exportlist:
179 d.setVarFlag(s, "export", True, op = 'auto env export')
180 except TypeError:
181 pass
182
183def emit_var(var, o=sys.__stdout__, d = init(), all=False):
184 """Emit a variable to be sourced by a shell."""
185 if d.getVarFlag(var, "python"):
186 return False
187
188 export = d.getVarFlag(var, "export")
189 unexport = d.getVarFlag(var, "unexport")
190 func = d.getVarFlag(var, "func")
191 if not all and not export and not unexport and not func:
192 return False
193
194 try:
195 if all:
196 oval = d.getVar(var, False)
197 val = d.getVar(var, True)
198 except (KeyboardInterrupt, bb.build.FuncFailed):
199 raise
200 except Exception as exc:
201 o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
202 return False
203
204 if all:
205 d.varhistory.emit(var, oval, val, o, d)
206
207 if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
208 return False
209
210 varExpanded = d.expand(var)
211
212 if unexport:
213 o.write('unset %s\n' % varExpanded)
214 return False
215
216 if val is None:
217 return False
218
219 val = str(val)
220
221 if varExpanded.startswith("BASH_FUNC_"):
222 varExpanded = varExpanded[10:-2]
223 val = val[3:] # Strip off "() "
224 o.write("%s() %s\n" % (varExpanded, val))
225 o.write("export -f %s\n" % (varExpanded))
226 return True
227
228 if func:
229 # NOTE: should probably check for unbalanced {} within the var
230 o.write("%s() {\n%s\n}\n" % (varExpanded, val))
231 return 1
232
233 if export:
234 o.write('export ')
235
236 # if we're going to output this within doublequotes,
237 # to a shell, we need to escape the quotes in the var
238 alter = re.sub('"', '\\"', val)
239 alter = re.sub('\n', ' \\\n', alter)
240 alter = re.sub('\\$', '\\\\$', alter)
241 o.write('%s="%s"\n' % (varExpanded, alter))
242 return False
243
244def emit_env(o=sys.__stdout__, d = init(), all=False):
245 """Emits all items in the data store in a format such that it can be sourced by a shell."""
246
247 isfunc = lambda key: bool(d.getVarFlag(key, "func"))
248 keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
249 grouped = groupby(keys, isfunc)
250 for isfunc, keys in grouped:
251 for key in keys:
252 emit_var(key, o, d, all and not isfunc) and o.write('\n')
253
254def exported_keys(d):
255 return (key for key in d.keys() if not key.startswith('__') and
256 d.getVarFlag(key, 'export') and
257 not d.getVarFlag(key, 'unexport'))
258
259def exported_vars(d):
260 for key in exported_keys(d):
261 try:
262 value = d.getVar(key, True)
263 except Exception:
264 pass
265
266 if value is not None:
267 yield key, str(value)
268
269def emit_func(func, o=sys.__stdout__, d = init()):
270 """Emits all items in the data store in a format such that it can be sourced by a shell."""
271
272 keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func"))
273 for key in keys:
274 emit_var(key, o, d, False)
275
276 o.write('\n')
277 emit_var(func, o, d, False) and o.write('\n')
278 newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
279 newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
280 seen = set()
281 while newdeps:
282 deps = newdeps
283 seen |= deps
284 newdeps = set()
285 for dep in deps:
286 if d.getVarFlag(dep, "func") and not d.getVarFlag(dep, "python"):
287 emit_var(dep, o, d, False) and o.write('\n')
288 newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep, True))
289 newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
290 newdeps -= seen
291
292_functionfmt = """
293def {function}(d):
294{body}"""
295
296def emit_func_python(func, o=sys.__stdout__, d = init()):
297 """Emits all items in the data store in a format such that it can be sourced by a shell."""
298
299 def write_func(func, o, call = False):
300 body = d.getVar(func, True)
301 if not body.startswith("def"):
302 body = _functionfmt.format(function=func, body=body)
303
304 o.write(body.strip() + "\n\n")
305 if call:
306 o.write(func + "(d)" + "\n\n")
307
308 write_func(func, o, True)
309 pp = bb.codeparser.PythonParser(func, logger)
310 pp.parse_python(d.getVar(func, True))
311 newdeps = pp.execs
312 newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
313 seen = set()
314 while newdeps:
315 deps = newdeps
316 seen |= deps
317 newdeps = set()
318 for dep in deps:
319 if d.getVarFlag(dep, "func") and d.getVarFlag(dep, "python"):
320 write_func(dep, o)
321 pp = bb.codeparser.PythonParser(dep, logger)
322 pp.parse_python(d.getVar(dep, True))
323 newdeps |= pp.execs
324 newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
325 newdeps -= seen
326
327def update_data(d):
328 """Performs final steps upon the datastore, including application of overrides"""
329 d.finalize(parent = True)
330
331def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
332 deps = set()
333 try:
334 if key[-1] == ']':
335 vf = key[:-1].split('[')
336 value = d.getVarFlag(vf[0], vf[1], False)
337 parser = d.expandWithRefs(value, key)
338 deps |= parser.references
339 deps = deps | (keys & parser.execs)
340 return deps, value
341 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "vardepvalueexclude", "postfuncs", "prefuncs"]) or {}
342 vardeps = varflags.get("vardeps")
343 value = d.getVar(key, False)
344
345 def handle_contains(value, contains, d):
346 newvalue = ""
347 for k in sorted(contains):
348 l = (d.getVar(k, True) or "").split()
349 for word in sorted(contains[k]):
350 if word in l:
351 newvalue += "\n%s{%s} = Set" % (k, word)
352 else:
353 newvalue += "\n%s{%s} = Unset" % (k, word)
354 if not newvalue:
355 return value
356 if not value:
357 return newvalue
358 return value + newvalue
359
360 if "vardepvalue" in varflags:
361 value = varflags.get("vardepvalue")
362 elif varflags.get("func"):
363 if varflags.get("python"):
364 parsedvar = d.expandWithRefs(value, key)
365 parser = bb.codeparser.PythonParser(key, logger)
366 if parsedvar.value and "\t" in parsedvar.value:
367 logger.warn("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True)))
368 parser.parse_python(parsedvar.value)
369 deps = deps | parser.references
370 value = handle_contains(value, parser.contains, d)
371 else:
372 parsedvar = d.expandWithRefs(value, key)
373 parser = bb.codeparser.ShellParser(key, logger)
374 parser.parse_shell(parsedvar.value)
375 deps = deps | shelldeps
376 if vardeps is None:
377 parser.log.flush()
378 if "prefuncs" in varflags:
379 deps = deps | set(varflags["prefuncs"].split())
380 if "postfuncs" in varflags:
381 deps = deps | set(varflags["postfuncs"].split())
382 deps = deps | parsedvar.references
383 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
384 value = handle_contains(value, parsedvar.contains, d)
385 else:
386 parser = d.expandWithRefs(value, key)
387 deps |= parser.references
388 deps = deps | (keys & parser.execs)
389 value = handle_contains(value, parser.contains, d)
390
391 if "vardepvalueexclude" in varflags:
392 exclude = varflags.get("vardepvalueexclude")
393 for excl in exclude.split('|'):
394 if excl:
395 value = value.replace(excl, '')
396
397 # Add varflags, assuming an exclusion list is set
398 if varflagsexcl:
399 varfdeps = []
400 for f in varflags:
401 if f not in varflagsexcl:
402 varfdeps.append('%s[%s]' % (key, f))
403 if varfdeps:
404 deps |= set(varfdeps)
405
406 deps |= set((vardeps or "").split())
407 deps -= set(varflags.get("vardepsexclude", "").split())
408 except Exception as e:
409 raise bb.data_smart.ExpansionError(key, None, e)
410 return deps, value
411 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
412 #d.setVarFlag(key, "vardeps", deps)
413
414def generate_dependencies(d):
415
416 keys = set(key for key in d if not key.startswith("__"))
417 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export") and not d.getVarFlag(key, "unexport"))
418 varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True)
419
420 deps = {}
421 values = {}
422
423 tasklist = d.getVar('__BBTASKS', False) or []
424 for task in tasklist:
425 deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
426 newdeps = deps[task]
427 seen = set()
428 while newdeps:
429 nextdeps = newdeps
430 seen |= nextdeps
431 newdeps = set()
432 for dep in nextdeps:
433 if dep not in deps:
434 deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
435 newdeps |= deps[dep]
436 newdeps -= seen
437 #print "For %s: %s" % (task, str(deps[task]))
438 return tasklist, deps, values
439
440def inherits_class(klass, d):
441 val = d.getVar('__inherit_cache', False) or []
442 needle = os.path.join('classes', '%s.bbclass' % klass)
443 for v in val:
444 if v.endswith(needle):
445 return True
446 return False