blob: 134afaacc912a1901e75008aa3d2c7f695b576b5 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Data' implementations
5
6Functions for interacting with the data structure used by the
7BitBake build tools.
8
9The expandKeys and update_data are the most expensive
10operations. At night the cookie monster came by and
11suggested 'give me cookies on setting the variables and
12things will work out'. Taking this suggestion into account
13applying the skills from the not yet passed 'Entwurf und
14Analyse von Algorithmen' lecture and the cookie
15monster seems to be right. We will track setVar more carefully
16to have faster update_data and expandKeys operations.
17
18This is a trade-off between speed and memory again but
19the speed is more critical here.
20"""
21
22# Copyright (C) 2003, 2004 Chris Larson
23# Copyright (C) 2005 Holger Hans Peter Freyther
24#
25# This program is free software; you can redistribute it and/or modify
26# it under the terms of the GNU General Public License version 2 as
27# published by the Free Software Foundation.
28#
29# This program is distributed in the hope that it will be useful,
30# but WITHOUT ANY WARRANTY; without even the implied warranty of
31# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32# GNU General Public License for more details.
33#
34# You should have received a copy of the GNU General Public License along
35# with this program; if not, write to the Free Software Foundation, Inc.,
36# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
37#
38# Based on functions from the base bb module, Copyright 2003 Holger Schurig
39
40import sys, os, re
41if sys.argv[0][-5:] == "pydoc":
42 path = os.path.dirname(os.path.dirname(sys.argv[1]))
43else:
44 path = os.path.dirname(os.path.dirname(sys.argv[0]))
45sys.path.insert(0, path)
46from itertools import groupby
47
48from bb import data_smart
49from bb import codeparser
50import bb
51
52logger = data_smart.logger
53_dict_type = data_smart.DataSmart
54
55def init():
56 """Return a new object representing the Bitbake data"""
57 return _dict_type()
58
59def init_db(parent = None):
60 """Return a new object representing the Bitbake data,
61 optionally based on an existing object"""
62 if parent is not None:
63 return parent.createCopy()
64 else:
65 return _dict_type()
66
67def createCopy(source):
68 """Link the source set to the destination
69 If one does not find the value in the destination set,
70 search will go on to the source set to get the value.
71 Value from source are copy-on-write. i.e. any try to
72 modify one of them will end up putting the modified value
73 in the destination set.
74 """
75 return source.createCopy()
76
77def initVar(var, d):
78 """Non-destructive var init for data structure"""
79 d.initVar(var)
80
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081def keys(d):
82 """Return a list of keys in d"""
83 return d.keys()
84
85
86__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
87__expand_python_regexp__ = re.compile(r"\${@.+?}")
88
89def expand(s, d, varname = None):
90 """Variable expansion using the data store"""
91 return d.expand(s, varname)
92
93def expandKeys(alterdata, readdata = None):
94 if readdata == None:
95 readdata = alterdata
96
97 todolist = {}
98 for key in alterdata:
99 if not '${' in key:
100 continue
101
102 ekey = expand(key, readdata)
103 if key == ekey:
104 continue
105 todolist[key] = ekey
106
107 # These two for loops are split for performance to maximise the
108 # usefulness of the expand cache
109 for key in sorted(todolist):
110 ekey = todolist[key]
111 newval = alterdata.getVar(ekey, False)
112 if newval is not None:
113 val = alterdata.getVar(key, False)
114 if val is not None:
115 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval))
116 alterdata.renameVar(key, ekey)
117
118def inheritFromOS(d, savedenv, permitted):
119 """Inherit variables from the initial environment."""
120 exportlist = bb.utils.preserved_envvars_exported()
121 for s in savedenv.keys():
122 if s in permitted:
123 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500124 d.setVar(s, savedenv.getVar(s), op = 'from env')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500125 if s in exportlist:
126 d.setVarFlag(s, "export", True, op = 'auto env export')
127 except TypeError:
128 pass
129
130def emit_var(var, o=sys.__stdout__, d = init(), all=False):
131 """Emit a variable to be sourced by a shell."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600132 func = d.getVarFlag(var, "func", False)
133 if d.getVarFlag(var, 'python', False) and func:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500134 return False
135
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500136 export = d.getVarFlag(var, "export", False)
137 unexport = d.getVarFlag(var, "unexport", False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138 if not all and not export and not unexport and not func:
139 return False
140
141 try:
142 if all:
143 oval = d.getVar(var, False)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500144 val = d.getVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 except (KeyboardInterrupt, bb.build.FuncFailed):
146 raise
147 except Exception as exc:
148 o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
149 return False
150
151 if all:
152 d.varhistory.emit(var, oval, val, o, d)
153
154 if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
155 return False
156
157 varExpanded = d.expand(var)
158
159 if unexport:
160 o.write('unset %s\n' % varExpanded)
161 return False
162
163 if val is None:
164 return False
165
166 val = str(val)
167
168 if varExpanded.startswith("BASH_FUNC_"):
169 varExpanded = varExpanded[10:-2]
170 val = val[3:] # Strip off "() "
171 o.write("%s() %s\n" % (varExpanded, val))
172 o.write("export -f %s\n" % (varExpanded))
173 return True
174
175 if func:
176 # NOTE: should probably check for unbalanced {} within the var
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500177 val = val.rstrip('\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178 o.write("%s() {\n%s\n}\n" % (varExpanded, val))
179 return 1
180
181 if export:
182 o.write('export ')
183
184 # if we're going to output this within doublequotes,
185 # to a shell, we need to escape the quotes in the var
186 alter = re.sub('"', '\\"', val)
187 alter = re.sub('\n', ' \\\n', alter)
188 alter = re.sub('\\$', '\\\\$', alter)
189 o.write('%s="%s"\n' % (varExpanded, alter))
190 return False
191
192def emit_env(o=sys.__stdout__, d = init(), all=False):
193 """Emits all items in the data store in a format such that it can be sourced by a shell."""
194
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500195 isfunc = lambda key: bool(d.getVarFlag(key, "func", False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196 keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
197 grouped = groupby(keys, isfunc)
198 for isfunc, keys in grouped:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500199 for key in sorted(keys):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500200 emit_var(key, o, d, all and not isfunc) and o.write('\n')
201
202def exported_keys(d):
203 return (key for key in d.keys() if not key.startswith('__') and
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500204 d.getVarFlag(key, 'export', False) and
205 not d.getVarFlag(key, 'unexport', False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500206
207def exported_vars(d):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500208 k = list(exported_keys(d))
209 for key in k:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500211 value = d.getVar(key)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500212 except Exception as err:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500213 bb.warn("%s: Unable to export ${%s}: %s" % (d.getVar("FILE"), key, err))
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500214 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215
216 if value is not None:
217 yield key, str(value)
218
219def emit_func(func, o=sys.__stdout__, d = init()):
220 """Emits all items in the data store in a format such that it can be sourced by a shell."""
221
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500222 keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500223 for key in sorted(keys):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500224 emit_var(key, o, d, False)
225
226 o.write('\n')
227 emit_var(func, o, d, False) and o.write('\n')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
229 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500230 seen = set()
231 while newdeps:
232 deps = newdeps
233 seen |= deps
234 newdeps = set()
235 for dep in deps:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500236 if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500237 emit_var(dep, o, d, False) and o.write('\n')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500238 newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep))
239 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240 newdeps -= seen
241
242_functionfmt = """
243def {function}(d):
244{body}"""
245
246def emit_func_python(func, o=sys.__stdout__, d = init()):
247 """Emits all items in the data store in a format such that it can be sourced by a shell."""
248
249 def write_func(func, o, call = False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500250 body = d.getVar(func, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500251 if not body.startswith("def"):
252 body = _functionfmt.format(function=func, body=body)
253
254 o.write(body.strip() + "\n\n")
255 if call:
256 o.write(func + "(d)" + "\n\n")
257
258 write_func(func, o, True)
259 pp = bb.codeparser.PythonParser(func, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500260 pp.parse_python(d.getVar(func, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 newdeps = pp.execs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 seen = set()
264 while newdeps:
265 deps = newdeps
266 seen |= deps
267 newdeps = set()
268 for dep in deps:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500269 if d.getVarFlag(dep, "func", False) and d.getVarFlag(dep, "python", False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500270 write_func(dep, o)
271 pp = bb.codeparser.PythonParser(dep, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500272 pp.parse_python(d.getVar(dep, False))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273 newdeps |= pp.execs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500274 newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500275 newdeps -= seen
276
277def update_data(d):
278 """Performs final steps upon the datastore, including application of overrides"""
279 d.finalize(parent = True)
280
281def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
282 deps = set()
283 try:
284 if key[-1] == ']':
285 vf = key[:-1].split('[')
286 value = d.getVarFlag(vf[0], vf[1], False)
287 parser = d.expandWithRefs(value, key)
288 deps |= parser.references
289 deps = deps | (keys & parser.execs)
290 return deps, value
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600291 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500292 vardeps = varflags.get("vardeps")
293 value = d.getVar(key, False)
294
295 def handle_contains(value, contains, d):
296 newvalue = ""
297 for k in sorted(contains):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500298 l = (d.getVar(k) or "").split()
299 for item in sorted(contains[k]):
300 for word in item.split():
301 if not word in l:
302 newvalue += "\n%s{%s} = Unset" % (k, item)
303 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500304 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500305 newvalue += "\n%s{%s} = Set" % (k, item)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500306 if not newvalue:
307 return value
308 if not value:
309 return newvalue
310 return value + newvalue
311
312 if "vardepvalue" in varflags:
313 value = varflags.get("vardepvalue")
314 elif varflags.get("func"):
315 if varflags.get("python"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 parser = bb.codeparser.PythonParser(key, logger)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500317 if value and "\t" in value:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500318 logger.warning("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE")))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500319 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320 deps = deps | parser.references
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500321 deps = deps | (keys & parser.execs)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322 value = handle_contains(value, parser.contains, d)
323 else:
324 parsedvar = d.expandWithRefs(value, key)
325 parser = bb.codeparser.ShellParser(key, logger)
326 parser.parse_shell(parsedvar.value)
327 deps = deps | shelldeps
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500328 deps = deps | parsedvar.references
329 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
330 value = handle_contains(value, parsedvar.contains, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331 if vardeps is None:
332 parser.log.flush()
333 if "prefuncs" in varflags:
334 deps = deps | set(varflags["prefuncs"].split())
335 if "postfuncs" in varflags:
336 deps = deps | set(varflags["postfuncs"].split())
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600337 if "exports" in varflags:
338 deps = deps | set(varflags["exports"].split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500339 else:
340 parser = d.expandWithRefs(value, key)
341 deps |= parser.references
342 deps = deps | (keys & parser.execs)
343 value = handle_contains(value, parser.contains, d)
344
345 if "vardepvalueexclude" in varflags:
346 exclude = varflags.get("vardepvalueexclude")
347 for excl in exclude.split('|'):
348 if excl:
349 value = value.replace(excl, '')
350
351 # Add varflags, assuming an exclusion list is set
352 if varflagsexcl:
353 varfdeps = []
354 for f in varflags:
355 if f not in varflagsexcl:
356 varfdeps.append('%s[%s]' % (key, f))
357 if varfdeps:
358 deps |= set(varfdeps)
359
360 deps |= set((vardeps or "").split())
361 deps -= set(varflags.get("vardepsexclude", "").split())
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500362 except bb.parse.SkipRecipe:
363 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500364 except Exception as e:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500365 bb.warn("Exception during build_dependencies for %s" % key)
366 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500367 return deps, value
368 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
369 #d.setVarFlag(key, "vardeps", deps)
370
371def generate_dependencies(d):
372
373 keys = set(key for key in d if not key.startswith("__"))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500374 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500375 varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500376
377 deps = {}
378 values = {}
379
380 tasklist = d.getVar('__BBTASKS', False) or []
381 for task in tasklist:
382 deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
383 newdeps = deps[task]
384 seen = set()
385 while newdeps:
386 nextdeps = newdeps
387 seen |= nextdeps
388 newdeps = set()
389 for dep in nextdeps:
390 if dep not in deps:
391 deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
392 newdeps |= deps[dep]
393 newdeps -= seen
394 #print "For %s: %s" % (task, str(deps[task]))
395 return tasklist, deps, values
396
397def inherits_class(klass, d):
398 val = d.getVar('__inherit_cache', False) or []
399 needle = os.path.join('classes', '%s.bbclass' % klass)
400 for v in val:
401 if v.endswith(needle):
402 return True
403 return False