blob: 543372d15314888979553172202ab33d062cb3a8 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake Smart Dictionary Implementation
3
4Functions for interacting with the data structure used by the
5BitBake build tools.
6
7"""
8
9# Copyright (C) 2003, 2004 Chris Larson
10# Copyright (C) 2004, 2005 Seb Frankengul
11# Copyright (C) 2005, 2006 Holger Hans Peter Freyther
12# Copyright (C) 2005 Uli Luckas
13# Copyright (C) 2005 ROAD GmbH
14#
Brad Bishopc342db32019-05-15 21:57:59 -040015# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017# Based on functions from the base bb module, Copyright 2003 Holger Schurig
18
19import copy, re, sys, traceback
Andrew Geissler5199d832021-09-24 16:47:35 -050020from collections.abc import MutableMapping
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021import logging
22import hashlib
23import bb, bb.codeparser
24from bb import utils
25from bb.COW import COWDictBase
26
27logger = logging.getLogger("BitBake.Data")
28
Patrick Williams213cb262021-08-07 19:21:33 -050029__setvar_keyword__ = [":append", ":prepend", ":remove"]
30__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
31__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032__expand_python_regexp__ = re.compile(r"\${@.+?}")
Brad Bishop19323692019-04-05 15:28:33 -040033__whitespace_split__ = re.compile(r'(\s)')
34__override_regexp__ = re.compile(r'[a-z0-9]+')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035
36def infer_caller_details(loginfo, parent = False, varval = True):
37 """Save the caller the trouble of specifying everything."""
38 # Save effort.
39 if 'ignore' in loginfo and loginfo['ignore']:
40 return
41 # If nothing was provided, mark this as possibly unneeded.
42 if not loginfo:
43 loginfo['ignore'] = True
44 return
45 # Infer caller's likely values for variable (var) and value (value),
46 # to reduce clutter in the rest of the code.
47 above = None
48 def set_above():
49 try:
50 raise Exception
51 except Exception:
52 tb = sys.exc_info()[2]
53 if parent:
54 return tb.tb_frame.f_back.f_back.f_back
55 else:
56 return tb.tb_frame.f_back.f_back
57
58 if varval and ('variable' not in loginfo or 'detail' not in loginfo):
59 if not above:
60 above = set_above()
61 lcls = above.f_locals.items()
62 for k, v in lcls:
63 if k == 'value' and 'detail' not in loginfo:
64 loginfo['detail'] = v
65 if k == 'var' and 'variable' not in loginfo:
66 loginfo['variable'] = v
67 # Infer file/line/function from traceback
68 # Don't use traceback.extract_stack() since it fills the line contents which
69 # we don't need and that hits stat syscalls
70 if 'file' not in loginfo:
71 if not above:
72 above = set_above()
73 f = above.f_back
74 line = f.f_lineno
75 file = f.f_code.co_filename
76 func = f.f_code.co_name
77 loginfo['file'] = file
78 loginfo['line'] = line
79 if func not in loginfo:
80 loginfo['func'] = func
81
82class VariableParse:
83 def __init__(self, varname, d, val = None):
84 self.varname = varname
85 self.d = d
86 self.value = val
87
88 self.references = set()
89 self.execs = set()
90 self.contains = {}
91
92 def var_sub(self, match):
93 key = match.group()[2:-1]
94 if self.varname and key:
95 if self.varname == key:
96 raise Exception("variable %s references itself!" % self.varname)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080097 var = self.d.getVarFlag(key, "_content")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050098 self.references.add(key)
99 if var is not None:
100 return var
101 else:
102 return match.group()
103
104 def python_sub(self, match):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500105 if isinstance(match, str):
106 code = match
107 else:
108 code = match.group()[3:-1]
109
Brad Bishop19323692019-04-05 15:28:33 -0400110 if self.varname:
111 varname = 'Var <%s>' % self.varname
112 else:
113 varname = '<expansion>'
114 codeobj = compile(code.strip(), varname, "eval")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115
116 parser = bb.codeparser.PythonParser(self.varname, logger)
117 parser.parse_python(code)
118 if self.varname:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500119 vardeps = self.d.getVarFlag(self.varname, "vardeps")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120 if vardeps is None:
121 parser.log.flush()
122 else:
123 parser.log.flush()
124 self.references |= parser.references
125 self.execs |= parser.execs
126
127 for k in parser.contains:
128 if k not in self.contains:
129 self.contains[k] = parser.contains[k].copy()
130 else:
131 self.contains[k].update(parser.contains[k])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600132 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133 return str(value)
134
135
136class DataContext(dict):
137 def __init__(self, metadata, **kwargs):
138 self.metadata = metadata
139 dict.__init__(self, **kwargs)
140 self['d'] = metadata
141
142 def __missing__(self, key):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 value = self.metadata.getVar(key)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500144 if value is None or self.metadata.getVarFlag(key, 'func', False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 raise KeyError(key)
146 else:
147 return value
148
149class ExpansionError(Exception):
150 def __init__(self, varname, expression, exception):
151 self.expression = expression
152 self.variablename = varname
153 self.exception = exception
Andrew Geissler5199d832021-09-24 16:47:35 -0500154 self.varlist = [varname or expression or ""]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500155 if varname:
156 if expression:
157 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
158 else:
159 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception)
160 else:
161 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
162 Exception.__init__(self, self.msg)
163 self.args = (varname, expression, exception)
Andrew Geissler5199d832021-09-24 16:47:35 -0500164
165 def addVar(self, varname):
166 if varname:
167 self.varlist.append(varname)
168
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500169 def __str__(self):
Andrew Geissler5199d832021-09-24 16:47:35 -0500170 chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist)
171 return self.msg + chain
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500172
173class IncludeHistory(object):
174 def __init__(self, parent = None, filename = '[TOP LEVEL]'):
175 self.parent = parent
176 self.filename = filename
177 self.children = []
178 self.current = self
179
180 def copy(self):
181 new = IncludeHistory(self.parent, self.filename)
182 for c in self.children:
183 new.children.append(c)
184 return new
185
186 def include(self, filename):
187 newfile = IncludeHistory(self.current, filename)
188 self.current.children.append(newfile)
189 self.current = newfile
190 return self
191
192 def __enter__(self):
193 pass
194
195 def __exit__(self, a, b, c):
196 if self.current.parent:
197 self.current = self.current.parent
198 else:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500199 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500200 return False
201
202 def emit(self, o, level = 0):
203 """Emit an include history file, and its children."""
204 if level:
205 spaces = " " * (level - 1)
206 o.write("# %s%s" % (spaces, self.filename))
207 if len(self.children) > 0:
208 o.write(" includes:")
209 else:
210 o.write("#\n# INCLUDE HISTORY:\n#")
211 level = level + 1
212 for child in self.children:
213 o.write("\n")
214 child.emit(o, level)
215
216class VariableHistory(object):
217 def __init__(self, dataroot):
218 self.dataroot = dataroot
219 self.variables = COWDictBase.copy()
220
221 def copy(self):
222 new = VariableHistory(self.dataroot)
223 new.variables = self.variables.copy()
224 return new
225
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500226 def __getstate__(self):
227 vardict = {}
228 for k, v in self.variables.iteritems():
229 vardict[k] = v
230 return {'dataroot': self.dataroot,
231 'variables': vardict}
232
233 def __setstate__(self, state):
234 self.dataroot = state['dataroot']
235 self.variables = COWDictBase.copy()
236 for k, v in state['variables'].items():
237 self.variables[k] = v
238
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500239 def record(self, *kwonly, **loginfo):
240 if not self.dataroot._tracking:
241 return
242 if len(kwonly) > 0:
243 raise TypeError
244 infer_caller_details(loginfo, parent = True)
245 if 'ignore' in loginfo and loginfo['ignore']:
246 return
247 if 'op' not in loginfo or not loginfo['op']:
248 loginfo['op'] = 'set'
249 if 'detail' in loginfo:
250 loginfo['detail'] = str(loginfo['detail'])
251 if 'variable' not in loginfo or 'file' not in loginfo:
252 raise ValueError("record() missing variable or file.")
253 var = loginfo['variable']
254
255 if var not in self.variables:
256 self.variables[var] = []
257 if not isinstance(self.variables[var], list):
258 return
259 if 'nodups' in loginfo and loginfo in self.variables[var]:
260 return
261 self.variables[var].append(loginfo.copy())
262
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800263 def rename_variable_hist(self, oldvar, newvar):
264 if not self.dataroot._tracking:
265 return
266 if oldvar not in self.variables:
267 return
268 if newvar not in self.variables:
269 self.variables[newvar] = []
270 for i in self.variables[oldvar]:
271 self.variables[newvar].append(i.copy())
272
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273 def variable(self, var):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500274 varhistory = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500275 if var in self.variables:
276 varhistory.extend(self.variables[var])
277 return varhistory
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278
279 def emit(self, var, oval, val, o, d):
280 history = self.variable(var)
281
282 # Append override history
283 if var in d.overridedata:
284 for (r, override) in d.overridedata[var]:
285 for event in self.variable(r):
286 loginfo = event.copy()
Patrick Williams213cb262021-08-07 19:21:33 -0500287 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500288 continue
289 loginfo['variable'] = var
290 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
291 history.append(loginfo)
292
293 commentVal = re.sub('\n', '\n#', str(oval))
294 if history:
295 if len(history) == 1:
296 o.write("#\n# $%s\n" % var)
297 else:
298 o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
299 for event in history:
300 # o.write("# %s\n" % str(event))
301 if 'func' in event:
302 # If we have a function listed, this is internal
303 # code, not an operation in a config file, and the
304 # full path is distracting.
305 event['file'] = re.sub('.*/', '', event['file'])
306 display_func = ' [%s]' % event['func']
307 else:
308 display_func = ''
309 if 'flag' in event:
310 flag = '[%s] ' % (event['flag'])
311 else:
312 flag = ''
313 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail'])))
314 if len(history) > 1:
315 o.write("# pre-expansion value:\n")
316 o.write('# "%s"\n' % (commentVal))
317 else:
318 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var)
319 o.write('# "%s"\n' % (commentVal))
320
321 def get_variable_files(self, var):
322 """Get the files where operations are made on a variable"""
323 var_history = self.variable(var)
324 files = []
325 for event in var_history:
326 files.append(event['file'])
327 return files
328
329 def get_variable_lines(self, var, f):
330 """Get the line where a operation is made on a variable in file f"""
331 var_history = self.variable(var)
332 lines = []
333 for event in var_history:
334 if f== event['file']:
335 line = event['line']
336 lines.append(line)
337 return lines
338
Andrew Geissler82c905d2020-04-13 13:39:40 -0500339 def get_variable_items_files(self, var):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500340 """
341 Use variable history to map items added to a list variable and
342 the files in which they were added.
343 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500344 d = self.dataroot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500345 history = self.variable(var)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500346 finalitems = (d.getVar(var) or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347 filemap = {}
348 isset = False
349 for event in history:
350 if 'flag' in event:
351 continue
Patrick Williams213cb262021-08-07 19:21:33 -0500352 if event['op'] == ':remove':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500353 continue
354 if isset and event['op'] == 'set?':
355 continue
356 isset = True
357 items = d.expand(event['detail']).split()
358 for item in items:
359 # This is a little crude but is belt-and-braces to avoid us
360 # having to handle every possible operation type specifically
361 if item in finalitems and not item in filemap:
362 filemap[item] = event['file']
363 return filemap
364
365 def del_var_history(self, var, f=None, line=None):
366 """If file f and line are not given, the entire history of var is deleted"""
367 if var in self.variables:
368 if f and line:
369 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line]
370 else:
371 self.variables[var] = []
372
373class DataSmart(MutableMapping):
374 def __init__(self):
375 self.dict = {}
376
377 self.inchistory = IncludeHistory()
378 self.varhistory = VariableHistory(self)
379 self._tracking = False
380
381 self.expand_cache = {}
382
383 # cookie monster tribute
384 # Need to be careful about writes to overridedata as
385 # its only a shallow copy, could influence other data store
386 # copies!
387 self.overridedata = {}
388 self.overrides = None
389 self.overridevars = set(["OVERRIDES", "FILE"])
390 self.inoverride = False
391
392 def enableTracking(self):
393 self._tracking = True
394
395 def disableTracking(self):
396 self._tracking = False
397
398 def expandWithRefs(self, s, varname):
399
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600400 if not isinstance(s, str): # sanity check
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500401 return VariableParse(varname, self, s)
402
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500403 varparse = VariableParse(varname, self)
404
405 while s.find('${') != -1:
406 olds = s
407 try:
408 s = __expand_var_regexp__.sub(varparse.var_sub, s)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500409 try:
410 s = __expand_python_regexp__.sub(varparse.python_sub, s)
411 except SyntaxError as e:
412 # Likely unmatched brackets, just don't expand the expression
Andrew Geissler5199d832021-09-24 16:47:35 -0500413 if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500414 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415 if s == olds:
416 break
Andrew Geissler5199d832021-09-24 16:47:35 -0500417 except ExpansionError as e:
418 e.addVar(varname)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500419 raise
420 except bb.parse.SkipRecipe:
421 raise
Andrew Geissler5199d832021-09-24 16:47:35 -0500422 except bb.BBHandledException:
423 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500424 except Exception as exc:
Brad Bishop19323692019-04-05 15:28:33 -0400425 tb = sys.exc_info()[2]
426 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500427
428 varparse.value = s
429
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500430 return varparse
431
432 def expand(self, s, varname = None):
433 return self.expandWithRefs(s, varname).value
434
435 def finalize(self, parent = False):
436 return
437
438 def internal_finalize(self, parent = False):
439 """Performs final steps upon the datastore, including application of overrides"""
440 self.overrides = None
441
442 def need_overrides(self):
Patrick Williamsd7e96312015-09-22 08:09:05 -0500443 if self.overrides is not None:
444 return
445 if self.inoverride:
446 return
447 for count in range(5):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500448 self.inoverride = True
449 # Can end up here recursively so setup dummy values
450 self.overrides = []
451 self.overridesset = set()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500452 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500453 self.overridesset = set(self.overrides)
454 self.inoverride = False
455 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500456 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
Patrick Williamsd7e96312015-09-22 08:09:05 -0500457 if newoverrides == self.overrides:
458 break
459 self.overrides = newoverrides
460 self.overridesset = set(self.overrides)
461 else:
462 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463
464 def initVar(self, var):
465 self.expand_cache = {}
466 if not var in self.dict:
467 self.dict[var] = {}
468
469 def _findVar(self, var):
470 dest = self.dict
471 while dest:
472 if var in dest:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500473 return dest[var], self.overridedata.get(var, None)
474
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500475 if "_data" not in dest:
476 break
477 dest = dest["_data"]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500478 return None, self.overridedata.get(var, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500479
480 def _makeShadowCopy(self, var):
481 if var in self.dict:
482 return
483
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500484 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500485
486 if local_var:
487 self.dict[var] = copy.copy(local_var)
488 else:
489 self.initVar(var)
490
491
492 def setVar(self, var, value, **loginfo):
493 #print("var=" + str(var) + " val=" + str(value))
Patrick Williams213cb262021-08-07 19:21:33 -0500494
Andrew Geissler595f6302022-01-24 19:11:47 +0000495 if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var):
Patrick Williams213cb262021-08-07 19:21:33 -0500496 info = "%s" % var
497 if "filename" in loginfo:
498 info += " file: %s" % loginfo[filename]
499 if "lineno" in loginfo:
500 info += " line: %s" % loginfo[lineno]
501 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
502
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800503 self.expand_cache = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500504 parsing=False
505 if 'parsing' in loginfo:
506 parsing=True
507
508 if 'op' not in loginfo:
509 loginfo['op'] = "set"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800510
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500511 match = __setvar_regexp__.match(var)
512 if match and match.group("keyword") in __setvar_keyword__:
513 base = match.group('base')
514 keyword = match.group("keyword")
515 override = match.group('add')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500516 l = self.getVarFlag(base, keyword, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500517 l.append([value, override])
518 self.setVarFlag(base, keyword, l, ignore=True)
519 # And cause that to be recorded:
520 loginfo['detail'] = value
521 loginfo['variable'] = base
522 if override:
523 loginfo['op'] = '%s[%s]' % (keyword, override)
524 else:
525 loginfo['op'] = keyword
526 self.varhistory.record(**loginfo)
527 # todo make sure keyword is not __doc__ or __module__
528 # pay the cookie monster
529
530 # more cookies for the cookie monster
Patrick Williams213cb262021-08-07 19:21:33 -0500531 if ':' in var:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500532 self._setvar_update_overrides(base, **loginfo)
533
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500534 if base in self.overridevars:
Patrick Williamsd7e96312015-09-22 08:09:05 -0500535 self._setvar_update_overridevars(var, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500536 return
537
538 if not var in self.dict:
539 self._makeShadowCopy(var)
540
541 if not parsing:
Patrick Williams213cb262021-08-07 19:21:33 -0500542 if ":append" in self.dict[var]:
543 del self.dict[var][":append"]
544 if ":prepend" in self.dict[var]:
545 del self.dict[var][":prepend"]
546 if ":remove" in self.dict[var]:
547 del self.dict[var][":remove"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500548 if var in self.overridedata:
549 active = []
550 self.need_overrides()
551 for (r, o) in self.overridedata[var]:
552 if o in self.overridesset:
553 active.append(r)
Patrick Williams213cb262021-08-07 19:21:33 -0500554 elif ":" in o:
555 if set(o.split(":")).issubset(self.overridesset):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500556 active.append(r)
557 for a in active:
558 self.delVar(a)
559 del self.overridedata[var]
560
561 # more cookies for the cookie monster
Patrick Williams213cb262021-08-07 19:21:33 -0500562 if ':' in var:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500563 self._setvar_update_overrides(var, **loginfo)
564
565 # setting var
566 self.dict[var]["_content"] = value
567 self.varhistory.record(**loginfo)
568
569 if var in self.overridevars:
Patrick Williamsd7e96312015-09-22 08:09:05 -0500570 self._setvar_update_overridevars(var, value)
571
572 def _setvar_update_overridevars(self, var, value):
573 vardata = self.expandWithRefs(value, var)
574 new = vardata.references
575 new.update(vardata.contains.keys())
576 while not new.issubset(self.overridevars):
577 nextnew = set()
578 self.overridevars.update(new)
579 for i in new:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500580 vardata = self.expandWithRefs(self.getVar(i), i)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500581 nextnew.update(vardata.references)
582 nextnew.update(vardata.contains.keys())
583 new = nextnew
584 self.internal_finalize(True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500585
586 def _setvar_update_overrides(self, var, **loginfo):
587 # aka pay the cookie monster
Patrick Williams213cb262021-08-07 19:21:33 -0500588 override = var[var.rfind(':')+1:]
589 shortvar = var[:var.rfind(':')]
Brad Bishop19323692019-04-05 15:28:33 -0400590 while override and __override_regexp__.match(override):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500591 if shortvar not in self.overridedata:
592 self.overridedata[shortvar] = []
593 if [var, override] not in self.overridedata[shortvar]:
594 # Force CoW by recreating the list first
595 self.overridedata[shortvar] = list(self.overridedata[shortvar])
596 self.overridedata[shortvar].append([var, override])
597 override = None
Patrick Williams213cb262021-08-07 19:21:33 -0500598 if ":" in shortvar:
599 override = var[shortvar.rfind(':')+1:]
600 shortvar = var[:shortvar.rfind(':')]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500601 if len(shortvar) == 0:
602 override = None
603
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500604 def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500605 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
606
607 def renameVar(self, key, newkey, **loginfo):
608 """
609 Rename the variable key to newkey
610 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500611 if key == newkey:
612 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
613 return
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500614
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615 val = self.getVar(key, 0, parsing=True)
616 if val is not None:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800617 self.varhistory.rename_variable_hist(key, newkey)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500618 loginfo['variable'] = newkey
619 loginfo['op'] = 'rename from %s' % key
620 loginfo['detail'] = val
621 self.varhistory.record(**loginfo)
622 self.setVar(newkey, val, ignore=True, parsing=True)
623
624 for i in (__setvar_keyword__):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500625 src = self.getVarFlag(key, i, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500626 if src is None:
627 continue
628
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500629 dest = self.getVarFlag(newkey, i, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500630 dest.extend(src)
631 self.setVarFlag(newkey, i, dest, ignore=True)
632
633 if key in self.overridedata:
634 self.overridedata[newkey] = []
635 for (v, o) in self.overridedata[key]:
636 self.overridedata[newkey].append([v.replace(key, newkey), o])
637 self.renameVar(v, v.replace(key, newkey))
638
Patrick Williams213cb262021-08-07 19:21:33 -0500639 if ':' in newkey and val is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500640 self._setvar_update_overrides(newkey, **loginfo)
641
642 loginfo['variable'] = key
643 loginfo['op'] = 'rename (to)'
644 loginfo['detail'] = newkey
645 self.varhistory.record(**loginfo)
646 self.delVar(key, ignore=True)
647
648 def appendVar(self, var, value, **loginfo):
649 loginfo['op'] = 'append'
650 self.varhistory.record(**loginfo)
Patrick Williams213cb262021-08-07 19:21:33 -0500651 self.setVar(var + ":append", value, ignore=True, parsing=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500652
653 def prependVar(self, var, value, **loginfo):
654 loginfo['op'] = 'prepend'
655 self.varhistory.record(**loginfo)
Patrick Williams213cb262021-08-07 19:21:33 -0500656 self.setVar(var + ":prepend", value, ignore=True, parsing=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500657
658 def delVar(self, var, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800659 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500660
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500661 loginfo['detail'] = ""
662 loginfo['op'] = 'del'
663 self.varhistory.record(**loginfo)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500664 self.dict[var] = {}
665 if var in self.overridedata:
666 del self.overridedata[var]
Patrick Williams213cb262021-08-07 19:21:33 -0500667 if ':' in var:
668 override = var[var.rfind(':')+1:]
669 shortvar = var[:var.rfind(':')]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500670 while override and override.islower():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500671 try:
672 if shortvar in self.overridedata:
673 # Force CoW by recreating the list first
674 self.overridedata[shortvar] = list(self.overridedata[shortvar])
675 self.overridedata[shortvar].remove([var, override])
676 except ValueError as e:
677 pass
678 override = None
Patrick Williams213cb262021-08-07 19:21:33 -0500679 if ":" in shortvar:
680 override = var[shortvar.rfind(':')+1:]
681 shortvar = var[:shortvar.rfind(':')]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500682 if len(shortvar) == 0:
683 override = None
684
685 def setVarFlag(self, var, flag, value, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800686 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500687
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500688 if 'op' not in loginfo:
689 loginfo['op'] = "set"
690 loginfo['flag'] = flag
691 self.varhistory.record(**loginfo)
692 if not var in self.dict:
693 self._makeShadowCopy(var)
694 self.dict[var][flag] = value
695
Patrick Williams213cb262021-08-07 19:21:33 -0500696 if flag == "_defaultval" and ':' in var:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500697 self._setvar_update_overrides(var, **loginfo)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500698 if flag == "_defaultval" and var in self.overridevars:
699 self._setvar_update_overridevars(var, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500700
701 if flag == "unexport" or flag == "export":
702 if not "__exportlist" in self.dict:
703 self._makeShadowCopy("__exportlist")
704 if not "_content" in self.dict["__exportlist"]:
705 self.dict["__exportlist"]["_content"] = set()
706 self.dict["__exportlist"]["_content"].add(var)
707
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800708 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
709 if flag == "_content":
710 cachename = var
711 else:
712 if not flag:
713 bb.warn("Calling getVarFlag with flag unset is invalid")
714 return None
715 cachename = var + "[" + flag + "]"
716
717 if expand and cachename in self.expand_cache:
718 return self.expand_cache[cachename].value
719
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500720 local_var, overridedata = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500721 value = None
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800722 removes = set()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500723 if flag == "_content" and overridedata is not None and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500724 match = False
725 active = {}
726 self.need_overrides()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500727 for (r, o) in overridedata:
Patrick Williams213cb262021-08-07 19:21:33 -0500728 # FIXME What about double overrides both with "_" in the name?
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500729 if o in self.overridesset:
730 active[o] = r
Patrick Williams213cb262021-08-07 19:21:33 -0500731 elif ":" in o:
732 if set(o.split(":")).issubset(self.overridesset):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500733 active[o] = r
734
735 mod = True
736 while mod:
737 mod = False
738 for o in self.overrides:
739 for a in active.copy():
Patrick Williams213cb262021-08-07 19:21:33 -0500740 if a.endswith(":" + o):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741 t = active[a]
742 del active[a]
Patrick Williams213cb262021-08-07 19:21:33 -0500743 active[a.replace(":" + o, "")] = t
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500744 mod = True
745 elif a == o:
746 match = active[a]
747 del active[a]
748 if match:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800749 value, subparser = self.getVarFlag(match, "_content", False, retparser=True)
750 if hasattr(subparser, "removes"):
751 # We have to carry the removes from the overridden variable to apply at the
752 # end of processing
753 removes = subparser.removes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500754
755 if local_var is not None and value is None:
756 if flag in local_var:
757 value = copy.copy(local_var[flag])
758 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
759 value = copy.copy(local_var["_defaultval"])
760
761
Patrick Williams213cb262021-08-07 19:21:33 -0500762 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500763 self.need_overrides()
Patrick Williams213cb262021-08-07 19:21:33 -0500764 for (r, o) in local_var[":append"]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500765 match = True
766 if o:
Patrick Williams213cb262021-08-07 19:21:33 -0500767 for o2 in o.split(":"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500768 if not o2 in self.overrides:
769 match = False
770 if match:
Patrick Williams213cb262021-08-07 19:21:33 -0500771 if value is None:
772 value = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500773 value = value + r
774
Patrick Williams213cb262021-08-07 19:21:33 -0500775 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776 self.need_overrides()
Patrick Williams213cb262021-08-07 19:21:33 -0500777 for (r, o) in local_var[":prepend"]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500778
779 match = True
780 if o:
Patrick Williams213cb262021-08-07 19:21:33 -0500781 for o2 in o.split(":"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500782 if not o2 in self.overrides:
783 match = False
784 if match:
Patrick Williams213cb262021-08-07 19:21:33 -0500785 if value is None:
786 value = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500787 value = r + value
788
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800789 parser = None
790 if expand or retparser:
791 parser = self.expandWithRefs(value, cachename)
792 if expand:
793 value = parser.value
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500794
Patrick Williams213cb262021-08-07 19:21:33 -0500795 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500796 self.need_overrides()
Patrick Williams213cb262021-08-07 19:21:33 -0500797 for (r, o) in local_var[":remove"]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500798 match = True
799 if o:
Patrick Williams213cb262021-08-07 19:21:33 -0500800 for o2 in o.split(":"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500801 if not o2 in self.overrides:
802 match = False
803 if match:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800804 removes.add(r)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500805
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800806 if value and flag == "_content" and not parsing:
807 if removes and parser:
808 expanded_removes = {}
809 for r in removes:
810 expanded_removes[r] = self.expand(r).split()
811
812 parser.removes = set()
Andrew Geissler595f6302022-01-24 19:11:47 +0000813 val = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800814 for v in __whitespace_split__.split(parser.value):
815 skip = False
816 for r in removes:
817 if v in expanded_removes[r]:
818 parser.removes.add(r)
819 skip = True
820 if skip:
821 continue
Andrew Geissler595f6302022-01-24 19:11:47 +0000822 val.append(v)
823 parser.value = "".join(val)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800824 if expand:
825 value = parser.value
826
827 if parser:
828 self.expand_cache[cachename] = parser
829
830 if retparser:
831 return value, parser
832
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500833 return value
834
835 def delVarFlag(self, var, flag, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800836 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500837
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500838 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500839 if not local_var:
840 return
841 if not var in self.dict:
842 self._makeShadowCopy(var)
843
844 if var in self.dict and flag in self.dict[var]:
845 loginfo['detail'] = ""
846 loginfo['op'] = 'delFlag'
847 loginfo['flag'] = flag
848 self.varhistory.record(**loginfo)
849
850 del self.dict[var][flag]
851
852 def appendVarFlag(self, var, flag, value, **loginfo):
853 loginfo['op'] = 'append'
854 loginfo['flag'] = flag
855 self.varhistory.record(**loginfo)
856 newvalue = (self.getVarFlag(var, flag, False) or "") + value
857 self.setVarFlag(var, flag, newvalue, ignore=True)
858
859 def prependVarFlag(self, var, flag, value, **loginfo):
860 loginfo['op'] = 'prepend'
861 loginfo['flag'] = flag
862 self.varhistory.record(**loginfo)
863 newvalue = value + (self.getVarFlag(var, flag, False) or "")
864 self.setVarFlag(var, flag, newvalue, ignore=True)
865
866 def setVarFlags(self, var, flags, **loginfo):
867 self.expand_cache = {}
868 infer_caller_details(loginfo)
869 if not var in self.dict:
870 self._makeShadowCopy(var)
871
872 for i in flags:
873 if i == "_content":
874 continue
875 loginfo['flag'] = i
876 loginfo['detail'] = flags[i]
877 self.varhistory.record(**loginfo)
878 self.dict[var][i] = flags[i]
879
880 def getVarFlags(self, var, expand = False, internalflags=False):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500881 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500882 flags = {}
883
884 if local_var:
885 for i in local_var:
Patrick Williams213cb262021-08-07 19:21:33 -0500886 if i.startswith(("_", ":")) and not internalflags:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500887 continue
888 flags[i] = local_var[i]
889 if expand and i in expand:
890 flags[i] = self.expand(flags[i], var + "[" + i + "]")
891 if len(flags) == 0:
892 return None
893 return flags
894
895
896 def delVarFlags(self, var, **loginfo):
897 self.expand_cache = {}
898 if not var in self.dict:
899 self._makeShadowCopy(var)
900
901 if var in self.dict:
902 content = None
903
904 loginfo['op'] = 'delete flags'
905 self.varhistory.record(**loginfo)
906
907 # try to save the content
908 if "_content" in self.dict[var]:
909 content = self.dict[var]["_content"]
910 self.dict[var] = {}
911 self.dict[var]["_content"] = content
912 else:
913 del self.dict[var]
914
915 def createCopy(self):
916 """
917 Create a copy of self by setting _data to self
918 """
919 # we really want this to be a DataSmart...
920 data = DataSmart()
921 data.dict["_data"] = self.dict
922 data.varhistory = self.varhistory.copy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500923 data.varhistory.dataroot = data
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500924 data.inchistory = self.inchistory.copy()
925
926 data._tracking = self._tracking
927
928 data.overrides = None
929 data.overridevars = copy.copy(self.overridevars)
930 # Should really be a deepcopy but has heavy overhead.
931 # Instead, we're careful with writes.
932 data.overridedata = copy.copy(self.overridedata)
933
934 return data
935
936 def expandVarref(self, variable, parents=False):
937 """Find all references to variable in the data and expand it
938 in place, optionally descending to parent datastores."""
939
940 if parents:
941 keys = iter(self)
942 else:
943 keys = self.localkeys()
944
945 ref = '${%s}' % variable
946 value = self.getVar(variable, False)
947 for key in keys:
948 referrervalue = self.getVar(key, False)
949 if referrervalue and ref in referrervalue:
950 self.setVar(key, referrervalue.replace(ref, value))
951
952 def localkeys(self):
953 for key in self.dict:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500954 if key not in ['_data']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500955 yield key
956
957 def __iter__(self):
958 deleted = set()
959 overrides = set()
960 def keylist(d):
961 klist = set()
962 for key in d:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500963 if key in ["_data"]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500964 continue
965 if key in deleted:
966 continue
967 if key in overrides:
968 continue
969 if not d[key]:
970 deleted.add(key)
971 continue
972 klist.add(key)
973
974 if "_data" in d:
975 klist |= keylist(d["_data"])
976
977 return klist
978
979 self.need_overrides()
980 for var in self.overridedata:
981 for (r, o) in self.overridedata[var]:
982 if o in self.overridesset:
983 overrides.add(var)
Patrick Williams213cb262021-08-07 19:21:33 -0500984 elif ":" in o:
985 if set(o.split(":")).issubset(self.overridesset):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500986 overrides.add(var)
987
988 for k in keylist(self.dict):
989 yield k
990
991 for k in overrides:
992 yield k
993
994 def __len__(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600995 return len(frozenset(iter(self)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500996
997 def __getitem__(self, item):
998 value = self.getVar(item, False)
999 if value is None:
1000 raise KeyError(item)
1001 else:
1002 return value
1003
1004 def __setitem__(self, var, value):
1005 self.setVar(var, value)
1006
1007 def __delitem__(self, var):
1008 self.delVar(var)
1009
1010 def get_hash(self):
1011 data = {}
1012 d = self.createCopy()
1013 bb.data.expandKeys(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001014
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001015 config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001016 keys = set(key for key in iter(d) if not key.startswith("__"))
1017 for key in keys:
1018 if key in config_whitelist:
1019 continue
1020
1021 value = d.getVar(key, False) or ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001022 if type(value) is type(self):
1023 data.update({key:value.get_hash()})
1024 else:
1025 data.update({key:value})
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001026
Andrew Geisslerd1e89492021-02-12 15:35:20 -06001027 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001028 if not varflags:
1029 continue
1030 for f in varflags:
1031 if f == "_content":
1032 continue
1033 data.update({'%s[%s]' % (key, f):varflags[f]})
1034
1035 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
1036 bb_list = d.getVar(key, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001037 data.update({key:str(bb_list)})
1038
1039 if key == "__BBANONFUNCS":
1040 for i in bb_list:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001041 value = d.getVar(i, False) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001042 data.update({i:value})
1043
1044 data_str = str([(k, data[k]) for k in sorted(data.keys())])
Brad Bishop19323692019-04-05 15:28:33 -04001045 return hashlib.sha256(data_str.encode("utf-8")).hexdigest()