blob: 2328c334accba69f3c1d233d4cd7af0582c8e3ec [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake Smart Dictionary Implementation
3
4Functions for interacting with the data structure used by the
5BitBake build tools.
6
7"""
8
9# Copyright (C) 2003, 2004 Chris Larson
10# Copyright (C) 2004, 2005 Seb Frankengul
11# Copyright (C) 2005, 2006 Holger Hans Peter Freyther
12# Copyright (C) 2005 Uli Luckas
13# Copyright (C) 2005 ROAD GmbH
14#
Brad Bishopc342db32019-05-15 21:57:59 -040015# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016#
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017# Based on functions from the base bb module, Copyright 2003 Holger Schurig
18
19import copy, re, sys, traceback
20from collections import MutableMapping
21import logging
22import hashlib
23import bb, bb.codeparser
24from bb import utils
25from bb.COW import COWDictBase
26
27logger = logging.getLogger("BitBake.Data")
28
29__setvar_keyword__ = ["_append", "_prepend", "_remove"]
Brad Bishop19323692019-04-05 15:28:33 -040030__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
31__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032__expand_python_regexp__ = re.compile(r"\${@.+?}")
Brad Bishop19323692019-04-05 15:28:33 -040033__whitespace_split__ = re.compile(r'(\s)')
34__override_regexp__ = re.compile(r'[a-z0-9]+')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035
36def infer_caller_details(loginfo, parent = False, varval = True):
37 """Save the caller the trouble of specifying everything."""
38 # Save effort.
39 if 'ignore' in loginfo and loginfo['ignore']:
40 return
41 # If nothing was provided, mark this as possibly unneeded.
42 if not loginfo:
43 loginfo['ignore'] = True
44 return
45 # Infer caller's likely values for variable (var) and value (value),
46 # to reduce clutter in the rest of the code.
47 above = None
48 def set_above():
49 try:
50 raise Exception
51 except Exception:
52 tb = sys.exc_info()[2]
53 if parent:
54 return tb.tb_frame.f_back.f_back.f_back
55 else:
56 return tb.tb_frame.f_back.f_back
57
58 if varval and ('variable' not in loginfo or 'detail' not in loginfo):
59 if not above:
60 above = set_above()
61 lcls = above.f_locals.items()
62 for k, v in lcls:
63 if k == 'value' and 'detail' not in loginfo:
64 loginfo['detail'] = v
65 if k == 'var' and 'variable' not in loginfo:
66 loginfo['variable'] = v
67 # Infer file/line/function from traceback
68 # Don't use traceback.extract_stack() since it fills the line contents which
69 # we don't need and that hits stat syscalls
70 if 'file' not in loginfo:
71 if not above:
72 above = set_above()
73 f = above.f_back
74 line = f.f_lineno
75 file = f.f_code.co_filename
76 func = f.f_code.co_name
77 loginfo['file'] = file
78 loginfo['line'] = line
79 if func not in loginfo:
80 loginfo['func'] = func
81
82class VariableParse:
83 def __init__(self, varname, d, val = None):
84 self.varname = varname
85 self.d = d
86 self.value = val
87
88 self.references = set()
89 self.execs = set()
90 self.contains = {}
91
92 def var_sub(self, match):
93 key = match.group()[2:-1]
94 if self.varname and key:
95 if self.varname == key:
96 raise Exception("variable %s references itself!" % self.varname)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080097 var = self.d.getVarFlag(key, "_content")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050098 self.references.add(key)
99 if var is not None:
100 return var
101 else:
102 return match.group()
103
104 def python_sub(self, match):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500105 if isinstance(match, str):
106 code = match
107 else:
108 code = match.group()[3:-1]
109
Brad Bishop19323692019-04-05 15:28:33 -0400110 if self.varname:
111 varname = 'Var <%s>' % self.varname
112 else:
113 varname = '<expansion>'
114 codeobj = compile(code.strip(), varname, "eval")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115
116 parser = bb.codeparser.PythonParser(self.varname, logger)
117 parser.parse_python(code)
118 if self.varname:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500119 vardeps = self.d.getVarFlag(self.varname, "vardeps")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120 if vardeps is None:
121 parser.log.flush()
122 else:
123 parser.log.flush()
124 self.references |= parser.references
125 self.execs |= parser.execs
126
127 for k in parser.contains:
128 if k not in self.contains:
129 self.contains[k] = parser.contains[k].copy()
130 else:
131 self.contains[k].update(parser.contains[k])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600132 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133 return str(value)
134
135
136class DataContext(dict):
137 def __init__(self, metadata, **kwargs):
138 self.metadata = metadata
139 dict.__init__(self, **kwargs)
140 self['d'] = metadata
141
142 def __missing__(self, key):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 value = self.metadata.getVar(key)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500144 if value is None or self.metadata.getVarFlag(key, 'func', False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 raise KeyError(key)
146 else:
147 return value
148
149class ExpansionError(Exception):
150 def __init__(self, varname, expression, exception):
151 self.expression = expression
152 self.variablename = varname
153 self.exception = exception
154 if varname:
155 if expression:
156 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
157 else:
158 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception)
159 else:
160 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
161 Exception.__init__(self, self.msg)
162 self.args = (varname, expression, exception)
163 def __str__(self):
164 return self.msg
165
166class IncludeHistory(object):
167 def __init__(self, parent = None, filename = '[TOP LEVEL]'):
168 self.parent = parent
169 self.filename = filename
170 self.children = []
171 self.current = self
172
173 def copy(self):
174 new = IncludeHistory(self.parent, self.filename)
175 for c in self.children:
176 new.children.append(c)
177 return new
178
179 def include(self, filename):
180 newfile = IncludeHistory(self.current, filename)
181 self.current.children.append(newfile)
182 self.current = newfile
183 return self
184
185 def __enter__(self):
186 pass
187
188 def __exit__(self, a, b, c):
189 if self.current.parent:
190 self.current = self.current.parent
191 else:
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500192 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500193 return False
194
195 def emit(self, o, level = 0):
196 """Emit an include history file, and its children."""
197 if level:
198 spaces = " " * (level - 1)
199 o.write("# %s%s" % (spaces, self.filename))
200 if len(self.children) > 0:
201 o.write(" includes:")
202 else:
203 o.write("#\n# INCLUDE HISTORY:\n#")
204 level = level + 1
205 for child in self.children:
206 o.write("\n")
207 child.emit(o, level)
208
209class VariableHistory(object):
210 def __init__(self, dataroot):
211 self.dataroot = dataroot
212 self.variables = COWDictBase.copy()
213
214 def copy(self):
215 new = VariableHistory(self.dataroot)
216 new.variables = self.variables.copy()
217 return new
218
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500219 def __getstate__(self):
220 vardict = {}
221 for k, v in self.variables.iteritems():
222 vardict[k] = v
223 return {'dataroot': self.dataroot,
224 'variables': vardict}
225
226 def __setstate__(self, state):
227 self.dataroot = state['dataroot']
228 self.variables = COWDictBase.copy()
229 for k, v in state['variables'].items():
230 self.variables[k] = v
231
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500232 def record(self, *kwonly, **loginfo):
233 if not self.dataroot._tracking:
234 return
235 if len(kwonly) > 0:
236 raise TypeError
237 infer_caller_details(loginfo, parent = True)
238 if 'ignore' in loginfo and loginfo['ignore']:
239 return
240 if 'op' not in loginfo or not loginfo['op']:
241 loginfo['op'] = 'set'
242 if 'detail' in loginfo:
243 loginfo['detail'] = str(loginfo['detail'])
244 if 'variable' not in loginfo or 'file' not in loginfo:
245 raise ValueError("record() missing variable or file.")
246 var = loginfo['variable']
247
248 if var not in self.variables:
249 self.variables[var] = []
250 if not isinstance(self.variables[var], list):
251 return
252 if 'nodups' in loginfo and loginfo in self.variables[var]:
253 return
254 self.variables[var].append(loginfo.copy())
255
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800256 def rename_variable_hist(self, oldvar, newvar):
257 if not self.dataroot._tracking:
258 return
259 if oldvar not in self.variables:
260 return
261 if newvar not in self.variables:
262 self.variables[newvar] = []
263 for i in self.variables[oldvar]:
264 self.variables[newvar].append(i.copy())
265
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500266 def variable(self, var):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500267 varhistory = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500268 if var in self.variables:
269 varhistory.extend(self.variables[var])
270 return varhistory
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271
272 def emit(self, var, oval, val, o, d):
273 history = self.variable(var)
274
275 # Append override history
276 if var in d.overridedata:
277 for (r, override) in d.overridedata[var]:
278 for event in self.variable(r):
279 loginfo = event.copy()
280 if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
281 continue
282 loginfo['variable'] = var
283 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
284 history.append(loginfo)
285
286 commentVal = re.sub('\n', '\n#', str(oval))
287 if history:
288 if len(history) == 1:
289 o.write("#\n# $%s\n" % var)
290 else:
291 o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
292 for event in history:
293 # o.write("# %s\n" % str(event))
294 if 'func' in event:
295 # If we have a function listed, this is internal
296 # code, not an operation in a config file, and the
297 # full path is distracting.
298 event['file'] = re.sub('.*/', '', event['file'])
299 display_func = ' [%s]' % event['func']
300 else:
301 display_func = ''
302 if 'flag' in event:
303 flag = '[%s] ' % (event['flag'])
304 else:
305 flag = ''
306 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail'])))
307 if len(history) > 1:
308 o.write("# pre-expansion value:\n")
309 o.write('# "%s"\n' % (commentVal))
310 else:
311 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var)
312 o.write('# "%s"\n' % (commentVal))
313
314 def get_variable_files(self, var):
315 """Get the files where operations are made on a variable"""
316 var_history = self.variable(var)
317 files = []
318 for event in var_history:
319 files.append(event['file'])
320 return files
321
322 def get_variable_lines(self, var, f):
323 """Get the line where a operation is made on a variable in file f"""
324 var_history = self.variable(var)
325 lines = []
326 for event in var_history:
327 if f== event['file']:
328 line = event['line']
329 lines.append(line)
330 return lines
331
Andrew Geissler82c905d2020-04-13 13:39:40 -0500332 def get_variable_items_files(self, var):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500333 """
334 Use variable history to map items added to a list variable and
335 the files in which they were added.
336 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500337 d = self.dataroot
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500338 history = self.variable(var)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500339 finalitems = (d.getVar(var) or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500340 filemap = {}
341 isset = False
342 for event in history:
343 if 'flag' in event:
344 continue
345 if event['op'] == '_remove':
346 continue
347 if isset and event['op'] == 'set?':
348 continue
349 isset = True
350 items = d.expand(event['detail']).split()
351 for item in items:
352 # This is a little crude but is belt-and-braces to avoid us
353 # having to handle every possible operation type specifically
354 if item in finalitems and not item in filemap:
355 filemap[item] = event['file']
356 return filemap
357
358 def del_var_history(self, var, f=None, line=None):
359 """If file f and line are not given, the entire history of var is deleted"""
360 if var in self.variables:
361 if f and line:
362 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line]
363 else:
364 self.variables[var] = []
365
366class DataSmart(MutableMapping):
367 def __init__(self):
368 self.dict = {}
369
370 self.inchistory = IncludeHistory()
371 self.varhistory = VariableHistory(self)
372 self._tracking = False
373
374 self.expand_cache = {}
375
376 # cookie monster tribute
377 # Need to be careful about writes to overridedata as
378 # its only a shallow copy, could influence other data store
379 # copies!
380 self.overridedata = {}
381 self.overrides = None
382 self.overridevars = set(["OVERRIDES", "FILE"])
383 self.inoverride = False
384
385 def enableTracking(self):
386 self._tracking = True
387
388 def disableTracking(self):
389 self._tracking = False
390
391 def expandWithRefs(self, s, varname):
392
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600393 if not isinstance(s, str): # sanity check
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394 return VariableParse(varname, self, s)
395
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500396 varparse = VariableParse(varname, self)
397
398 while s.find('${') != -1:
399 olds = s
400 try:
401 s = __expand_var_regexp__.sub(varparse.var_sub, s)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500402 try:
403 s = __expand_python_regexp__.sub(varparse.python_sub, s)
404 except SyntaxError as e:
405 # Likely unmatched brackets, just don't expand the expression
406 if e.msg != "EOL while scanning string literal":
407 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500408 if s == olds:
409 break
410 except ExpansionError:
411 raise
412 except bb.parse.SkipRecipe:
413 raise
414 except Exception as exc:
Brad Bishop19323692019-04-05 15:28:33 -0400415 tb = sys.exc_info()[2]
416 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417
418 varparse.value = s
419
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500420 return varparse
421
422 def expand(self, s, varname = None):
423 return self.expandWithRefs(s, varname).value
424
425 def finalize(self, parent = False):
426 return
427
428 def internal_finalize(self, parent = False):
429 """Performs final steps upon the datastore, including application of overrides"""
430 self.overrides = None
431
432 def need_overrides(self):
Patrick Williamsd7e96312015-09-22 08:09:05 -0500433 if self.overrides is not None:
434 return
435 if self.inoverride:
436 return
437 for count in range(5):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438 self.inoverride = True
439 # Can end up here recursively so setup dummy values
440 self.overrides = []
441 self.overridesset = set()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500442 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500443 self.overridesset = set(self.overrides)
444 self.inoverride = False
445 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500446 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
Patrick Williamsd7e96312015-09-22 08:09:05 -0500447 if newoverrides == self.overrides:
448 break
449 self.overrides = newoverrides
450 self.overridesset = set(self.overrides)
451 else:
452 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500453
454 def initVar(self, var):
455 self.expand_cache = {}
456 if not var in self.dict:
457 self.dict[var] = {}
458
459 def _findVar(self, var):
460 dest = self.dict
461 while dest:
462 if var in dest:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500463 return dest[var], self.overridedata.get(var, None)
464
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500465 if "_data" not in dest:
466 break
467 dest = dest["_data"]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500468 return None, self.overridedata.get(var, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500469
470 def _makeShadowCopy(self, var):
471 if var in self.dict:
472 return
473
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500474 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500475
476 if local_var:
477 self.dict[var] = copy.copy(local_var)
478 else:
479 self.initVar(var)
480
481
482 def setVar(self, var, value, **loginfo):
483 #print("var=" + str(var) + " val=" + str(value))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800484 self.expand_cache = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500485 parsing=False
486 if 'parsing' in loginfo:
487 parsing=True
488
489 if 'op' not in loginfo:
490 loginfo['op'] = "set"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800491
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500492 match = __setvar_regexp__.match(var)
493 if match and match.group("keyword") in __setvar_keyword__:
494 base = match.group('base')
495 keyword = match.group("keyword")
496 override = match.group('add')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500497 l = self.getVarFlag(base, keyword, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500498 l.append([value, override])
499 self.setVarFlag(base, keyword, l, ignore=True)
500 # And cause that to be recorded:
501 loginfo['detail'] = value
502 loginfo['variable'] = base
503 if override:
504 loginfo['op'] = '%s[%s]' % (keyword, override)
505 else:
506 loginfo['op'] = keyword
507 self.varhistory.record(**loginfo)
508 # todo make sure keyword is not __doc__ or __module__
509 # pay the cookie monster
510
511 # more cookies for the cookie monster
512 if '_' in var:
513 self._setvar_update_overrides(base, **loginfo)
514
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500515 if base in self.overridevars:
Patrick Williamsd7e96312015-09-22 08:09:05 -0500516 self._setvar_update_overridevars(var, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500517 return
518
519 if not var in self.dict:
520 self._makeShadowCopy(var)
521
522 if not parsing:
523 if "_append" in self.dict[var]:
524 del self.dict[var]["_append"]
525 if "_prepend" in self.dict[var]:
526 del self.dict[var]["_prepend"]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500527 if "_remove" in self.dict[var]:
528 del self.dict[var]["_remove"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500529 if var in self.overridedata:
530 active = []
531 self.need_overrides()
532 for (r, o) in self.overridedata[var]:
533 if o in self.overridesset:
534 active.append(r)
535 elif "_" in o:
536 if set(o.split("_")).issubset(self.overridesset):
537 active.append(r)
538 for a in active:
539 self.delVar(a)
540 del self.overridedata[var]
541
542 # more cookies for the cookie monster
543 if '_' in var:
544 self._setvar_update_overrides(var, **loginfo)
545
546 # setting var
547 self.dict[var]["_content"] = value
548 self.varhistory.record(**loginfo)
549
550 if var in self.overridevars:
Patrick Williamsd7e96312015-09-22 08:09:05 -0500551 self._setvar_update_overridevars(var, value)
552
553 def _setvar_update_overridevars(self, var, value):
554 vardata = self.expandWithRefs(value, var)
555 new = vardata.references
556 new.update(vardata.contains.keys())
557 while not new.issubset(self.overridevars):
558 nextnew = set()
559 self.overridevars.update(new)
560 for i in new:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500561 vardata = self.expandWithRefs(self.getVar(i), i)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500562 nextnew.update(vardata.references)
563 nextnew.update(vardata.contains.keys())
564 new = nextnew
565 self.internal_finalize(True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500566
567 def _setvar_update_overrides(self, var, **loginfo):
568 # aka pay the cookie monster
569 override = var[var.rfind('_')+1:]
570 shortvar = var[:var.rfind('_')]
Brad Bishop19323692019-04-05 15:28:33 -0400571 while override and __override_regexp__.match(override):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500572 if shortvar not in self.overridedata:
573 self.overridedata[shortvar] = []
574 if [var, override] not in self.overridedata[shortvar]:
575 # Force CoW by recreating the list first
576 self.overridedata[shortvar] = list(self.overridedata[shortvar])
577 self.overridedata[shortvar].append([var, override])
578 override = None
579 if "_" in shortvar:
580 override = var[shortvar.rfind('_')+1:]
581 shortvar = var[:shortvar.rfind('_')]
582 if len(shortvar) == 0:
583 override = None
584
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500585 def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500586 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
587
588 def renameVar(self, key, newkey, **loginfo):
589 """
590 Rename the variable key to newkey
591 """
Andrew Geissler82c905d2020-04-13 13:39:40 -0500592 if key == newkey:
593 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
594 return
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500595
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500596 val = self.getVar(key, 0, parsing=True)
597 if val is not None:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800598 self.varhistory.rename_variable_hist(key, newkey)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500599 loginfo['variable'] = newkey
600 loginfo['op'] = 'rename from %s' % key
601 loginfo['detail'] = val
602 self.varhistory.record(**loginfo)
603 self.setVar(newkey, val, ignore=True, parsing=True)
604
605 for i in (__setvar_keyword__):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500606 src = self.getVarFlag(key, i, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607 if src is None:
608 continue
609
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500610 dest = self.getVarFlag(newkey, i, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611 dest.extend(src)
612 self.setVarFlag(newkey, i, dest, ignore=True)
613
614 if key in self.overridedata:
615 self.overridedata[newkey] = []
616 for (v, o) in self.overridedata[key]:
617 self.overridedata[newkey].append([v.replace(key, newkey), o])
618 self.renameVar(v, v.replace(key, newkey))
619
620 if '_' in newkey and val is None:
621 self._setvar_update_overrides(newkey, **loginfo)
622
623 loginfo['variable'] = key
624 loginfo['op'] = 'rename (to)'
625 loginfo['detail'] = newkey
626 self.varhistory.record(**loginfo)
627 self.delVar(key, ignore=True)
628
629 def appendVar(self, var, value, **loginfo):
630 loginfo['op'] = 'append'
631 self.varhistory.record(**loginfo)
632 self.setVar(var + "_append", value, ignore=True, parsing=True)
633
634 def prependVar(self, var, value, **loginfo):
635 loginfo['op'] = 'prepend'
636 self.varhistory.record(**loginfo)
637 self.setVar(var + "_prepend", value, ignore=True, parsing=True)
638
639 def delVar(self, var, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800640 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500641
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500642 loginfo['detail'] = ""
643 loginfo['op'] = 'del'
644 self.varhistory.record(**loginfo)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500645 self.dict[var] = {}
646 if var in self.overridedata:
647 del self.overridedata[var]
648 if '_' in var:
649 override = var[var.rfind('_')+1:]
650 shortvar = var[:var.rfind('_')]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500651 while override and override.islower():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500652 try:
653 if shortvar in self.overridedata:
654 # Force CoW by recreating the list first
655 self.overridedata[shortvar] = list(self.overridedata[shortvar])
656 self.overridedata[shortvar].remove([var, override])
657 except ValueError as e:
658 pass
659 override = None
660 if "_" in shortvar:
661 override = var[shortvar.rfind('_')+1:]
662 shortvar = var[:shortvar.rfind('_')]
663 if len(shortvar) == 0:
664 override = None
665
666 def setVarFlag(self, var, flag, value, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800667 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500668
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500669 if 'op' not in loginfo:
670 loginfo['op'] = "set"
671 loginfo['flag'] = flag
672 self.varhistory.record(**loginfo)
673 if not var in self.dict:
674 self._makeShadowCopy(var)
675 self.dict[var][flag] = value
676
677 if flag == "_defaultval" and '_' in var:
678 self._setvar_update_overrides(var, **loginfo)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500679 if flag == "_defaultval" and var in self.overridevars:
680 self._setvar_update_overridevars(var, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681
682 if flag == "unexport" or flag == "export":
683 if not "__exportlist" in self.dict:
684 self._makeShadowCopy("__exportlist")
685 if not "_content" in self.dict["__exportlist"]:
686 self.dict["__exportlist"]["_content"] = set()
687 self.dict["__exportlist"]["_content"].add(var)
688
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800689 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
690 if flag == "_content":
691 cachename = var
692 else:
693 if not flag:
694 bb.warn("Calling getVarFlag with flag unset is invalid")
695 return None
696 cachename = var + "[" + flag + "]"
697
698 if expand and cachename in self.expand_cache:
699 return self.expand_cache[cachename].value
700
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500701 local_var, overridedata = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500702 value = None
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800703 removes = set()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500704 if flag == "_content" and overridedata is not None and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500705 match = False
706 active = {}
707 self.need_overrides()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500708 for (r, o) in overridedata:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500709 # What about double overrides both with "_" in the name?
710 if o in self.overridesset:
711 active[o] = r
712 elif "_" in o:
713 if set(o.split("_")).issubset(self.overridesset):
714 active[o] = r
715
716 mod = True
717 while mod:
718 mod = False
719 for o in self.overrides:
720 for a in active.copy():
721 if a.endswith("_" + o):
722 t = active[a]
723 del active[a]
724 active[a.replace("_" + o, "")] = t
725 mod = True
726 elif a == o:
727 match = active[a]
728 del active[a]
729 if match:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800730 value, subparser = self.getVarFlag(match, "_content", False, retparser=True)
731 if hasattr(subparser, "removes"):
732 # We have to carry the removes from the overridden variable to apply at the
733 # end of processing
734 removes = subparser.removes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500735
736 if local_var is not None and value is None:
737 if flag in local_var:
738 value = copy.copy(local_var[flag])
739 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
740 value = copy.copy(local_var["_defaultval"])
741
742
743 if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
744 if not value:
745 value = ""
746 self.need_overrides()
747 for (r, o) in local_var["_append"]:
748 match = True
749 if o:
750 for o2 in o.split("_"):
751 if not o2 in self.overrides:
752 match = False
753 if match:
754 value = value + r
755
756 if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
757 if not value:
758 value = ""
759 self.need_overrides()
760 for (r, o) in local_var["_prepend"]:
761
762 match = True
763 if o:
764 for o2 in o.split("_"):
765 if not o2 in self.overrides:
766 match = False
767 if match:
768 value = r + value
769
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800770 parser = None
771 if expand or retparser:
772 parser = self.expandWithRefs(value, cachename)
773 if expand:
774 value = parser.value
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500775
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800776 if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500777 self.need_overrides()
778 for (r, o) in local_var["_remove"]:
779 match = True
780 if o:
781 for o2 in o.split("_"):
782 if not o2 in self.overrides:
783 match = False
784 if match:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800785 removes.add(r)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500786
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800787 if value and flag == "_content" and not parsing:
788 if removes and parser:
789 expanded_removes = {}
790 for r in removes:
791 expanded_removes[r] = self.expand(r).split()
792
793 parser.removes = set()
794 val = ""
795 for v in __whitespace_split__.split(parser.value):
796 skip = False
797 for r in removes:
798 if v in expanded_removes[r]:
799 parser.removes.add(r)
800 skip = True
801 if skip:
802 continue
803 val = val + v
804 parser.value = val
805 if expand:
806 value = parser.value
807
808 if parser:
809 self.expand_cache[cachename] = parser
810
811 if retparser:
812 return value, parser
813
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500814 return value
815
816 def delVarFlag(self, var, flag, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800817 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500818
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500819 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500820 if not local_var:
821 return
822 if not var in self.dict:
823 self._makeShadowCopy(var)
824
825 if var in self.dict and flag in self.dict[var]:
826 loginfo['detail'] = ""
827 loginfo['op'] = 'delFlag'
828 loginfo['flag'] = flag
829 self.varhistory.record(**loginfo)
830
831 del self.dict[var][flag]
832
833 def appendVarFlag(self, var, flag, value, **loginfo):
834 loginfo['op'] = 'append'
835 loginfo['flag'] = flag
836 self.varhistory.record(**loginfo)
837 newvalue = (self.getVarFlag(var, flag, False) or "") + value
838 self.setVarFlag(var, flag, newvalue, ignore=True)
839
840 def prependVarFlag(self, var, flag, value, **loginfo):
841 loginfo['op'] = 'prepend'
842 loginfo['flag'] = flag
843 self.varhistory.record(**loginfo)
844 newvalue = value + (self.getVarFlag(var, flag, False) or "")
845 self.setVarFlag(var, flag, newvalue, ignore=True)
846
847 def setVarFlags(self, var, flags, **loginfo):
848 self.expand_cache = {}
849 infer_caller_details(loginfo)
850 if not var in self.dict:
851 self._makeShadowCopy(var)
852
853 for i in flags:
854 if i == "_content":
855 continue
856 loginfo['flag'] = i
857 loginfo['detail'] = flags[i]
858 self.varhistory.record(**loginfo)
859 self.dict[var][i] = flags[i]
860
861 def getVarFlags(self, var, expand = False, internalflags=False):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500862 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500863 flags = {}
864
865 if local_var:
866 for i in local_var:
867 if i.startswith("_") and not internalflags:
868 continue
869 flags[i] = local_var[i]
870 if expand and i in expand:
871 flags[i] = self.expand(flags[i], var + "[" + i + "]")
872 if len(flags) == 0:
873 return None
874 return flags
875
876
877 def delVarFlags(self, var, **loginfo):
878 self.expand_cache = {}
879 if not var in self.dict:
880 self._makeShadowCopy(var)
881
882 if var in self.dict:
883 content = None
884
885 loginfo['op'] = 'delete flags'
886 self.varhistory.record(**loginfo)
887
888 # try to save the content
889 if "_content" in self.dict[var]:
890 content = self.dict[var]["_content"]
891 self.dict[var] = {}
892 self.dict[var]["_content"] = content
893 else:
894 del self.dict[var]
895
896 def createCopy(self):
897 """
898 Create a copy of self by setting _data to self
899 """
900 # we really want this to be a DataSmart...
901 data = DataSmart()
902 data.dict["_data"] = self.dict
903 data.varhistory = self.varhistory.copy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500904 data.varhistory.dataroot = data
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500905 data.inchistory = self.inchistory.copy()
906
907 data._tracking = self._tracking
908
909 data.overrides = None
910 data.overridevars = copy.copy(self.overridevars)
911 # Should really be a deepcopy but has heavy overhead.
912 # Instead, we're careful with writes.
913 data.overridedata = copy.copy(self.overridedata)
914
915 return data
916
917 def expandVarref(self, variable, parents=False):
918 """Find all references to variable in the data and expand it
919 in place, optionally descending to parent datastores."""
920
921 if parents:
922 keys = iter(self)
923 else:
924 keys = self.localkeys()
925
926 ref = '${%s}' % variable
927 value = self.getVar(variable, False)
928 for key in keys:
929 referrervalue = self.getVar(key, False)
930 if referrervalue and ref in referrervalue:
931 self.setVar(key, referrervalue.replace(ref, value))
932
933 def localkeys(self):
934 for key in self.dict:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500935 if key not in ['_data']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500936 yield key
937
938 def __iter__(self):
939 deleted = set()
940 overrides = set()
941 def keylist(d):
942 klist = set()
943 for key in d:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500944 if key in ["_data"]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500945 continue
946 if key in deleted:
947 continue
948 if key in overrides:
949 continue
950 if not d[key]:
951 deleted.add(key)
952 continue
953 klist.add(key)
954
955 if "_data" in d:
956 klist |= keylist(d["_data"])
957
958 return klist
959
960 self.need_overrides()
961 for var in self.overridedata:
962 for (r, o) in self.overridedata[var]:
963 if o in self.overridesset:
964 overrides.add(var)
965 elif "_" in o:
966 if set(o.split("_")).issubset(self.overridesset):
967 overrides.add(var)
968
969 for k in keylist(self.dict):
970 yield k
971
972 for k in overrides:
973 yield k
974
975 def __len__(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600976 return len(frozenset(iter(self)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500977
978 def __getitem__(self, item):
979 value = self.getVar(item, False)
980 if value is None:
981 raise KeyError(item)
982 else:
983 return value
984
985 def __setitem__(self, var, value):
986 self.setVar(var, value)
987
988 def __delitem__(self, var):
989 self.delVar(var)
990
991 def get_hash(self):
992 data = {}
993 d = self.createCopy()
994 bb.data.expandKeys(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500995
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500996 config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500997 keys = set(key for key in iter(d) if not key.startswith("__"))
998 for key in keys:
999 if key in config_whitelist:
1000 continue
1001
1002 value = d.getVar(key, False) or ""
Andrew Geissler82c905d2020-04-13 13:39:40 -05001003 if type(value) is type(self):
1004 data.update({key:value.get_hash()})
1005 else:
1006 data.update({key:value})
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001007
Andrew Geisslerd1e89492021-02-12 15:35:20 -06001008 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"])
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001009 if not varflags:
1010 continue
1011 for f in varflags:
1012 if f == "_content":
1013 continue
1014 data.update({'%s[%s]' % (key, f):varflags[f]})
1015
1016 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
1017 bb_list = d.getVar(key, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001018 data.update({key:str(bb_list)})
1019
1020 if key == "__BBANONFUNCS":
1021 for i in bb_list:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001022 value = d.getVar(i, False) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001023 data.update({i:value})
1024
1025 data_str = str([(k, data[k]) for k in sorted(data.keys())])
Brad Bishop19323692019-04-05 15:28:33 -04001026 return hashlib.sha256(data_str.encode("utf-8")).hexdigest()