blob: 7b09af5cf1bd5eb95a10261691eaf8b7b1d5d3f0 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake Smart Dictionary Implementation
5
6Functions for interacting with the data structure used by the
7BitBake build tools.
8
9"""
10
11# Copyright (C) 2003, 2004 Chris Larson
12# Copyright (C) 2004, 2005 Seb Frankengul
13# Copyright (C) 2005, 2006 Holger Hans Peter Freyther
14# Copyright (C) 2005 Uli Luckas
15# Copyright (C) 2005 ROAD GmbH
16#
17# This program is free software; you can redistribute it and/or modify
18# it under the terms of the GNU General Public License version 2 as
19# published by the Free Software Foundation.
20#
21# This program is distributed in the hope that it will be useful,
22# but WITHOUT ANY WARRANTY; without even the implied warranty of
23# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24# GNU General Public License for more details.
25#
26# You should have received a copy of the GNU General Public License along
27# with this program; if not, write to the Free Software Foundation, Inc.,
28# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
29# Based on functions from the base bb module, Copyright 2003 Holger Schurig
30
31import copy, re, sys, traceback
32from collections import MutableMapping
33import logging
34import hashlib
35import bb, bb.codeparser
36from bb import utils
37from bb.COW import COWDictBase
38
39logger = logging.getLogger("BitBake.Data")
40
41__setvar_keyword__ = ["_append", "_prepend", "_remove"]
Brad Bishopd7bf8c12018-02-25 22:55:05 -050042__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050043__expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050044__expand_python_regexp__ = re.compile(r"\${@.+?}")
45
46def infer_caller_details(loginfo, parent = False, varval = True):
47 """Save the caller the trouble of specifying everything."""
48 # Save effort.
49 if 'ignore' in loginfo and loginfo['ignore']:
50 return
51 # If nothing was provided, mark this as possibly unneeded.
52 if not loginfo:
53 loginfo['ignore'] = True
54 return
55 # Infer caller's likely values for variable (var) and value (value),
56 # to reduce clutter in the rest of the code.
57 above = None
58 def set_above():
59 try:
60 raise Exception
61 except Exception:
62 tb = sys.exc_info()[2]
63 if parent:
64 return tb.tb_frame.f_back.f_back.f_back
65 else:
66 return tb.tb_frame.f_back.f_back
67
68 if varval and ('variable' not in loginfo or 'detail' not in loginfo):
69 if not above:
70 above = set_above()
71 lcls = above.f_locals.items()
72 for k, v in lcls:
73 if k == 'value' and 'detail' not in loginfo:
74 loginfo['detail'] = v
75 if k == 'var' and 'variable' not in loginfo:
76 loginfo['variable'] = v
77 # Infer file/line/function from traceback
78 # Don't use traceback.extract_stack() since it fills the line contents which
79 # we don't need and that hits stat syscalls
80 if 'file' not in loginfo:
81 if not above:
82 above = set_above()
83 f = above.f_back
84 line = f.f_lineno
85 file = f.f_code.co_filename
86 func = f.f_code.co_name
87 loginfo['file'] = file
88 loginfo['line'] = line
89 if func not in loginfo:
90 loginfo['func'] = func
91
92class VariableParse:
93 def __init__(self, varname, d, val = None):
94 self.varname = varname
95 self.d = d
96 self.value = val
97
98 self.references = set()
99 self.execs = set()
100 self.contains = {}
101
102 def var_sub(self, match):
103 key = match.group()[2:-1]
104 if self.varname and key:
105 if self.varname == key:
106 raise Exception("variable %s references itself!" % self.varname)
107 if key in self.d.expand_cache:
108 varparse = self.d.expand_cache[key]
109 var = varparse.value
110 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500111 var = self.d.getVarFlag(key, "_content")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500112 self.references.add(key)
113 if var is not None:
114 return var
115 else:
116 return match.group()
117
118 def python_sub(self, match):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500119 if isinstance(match, str):
120 code = match
121 else:
122 code = match.group()[3:-1]
123
124 if "_remote_data" in self.d:
125 connector = self.d["_remote_data"]
126 return connector.expandPythonRef(self.varname, code, self.d)
127
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 codeobj = compile(code.strip(), self.varname or "<expansion>", "eval")
129
130 parser = bb.codeparser.PythonParser(self.varname, logger)
131 parser.parse_python(code)
132 if self.varname:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500133 vardeps = self.d.getVarFlag(self.varname, "vardeps")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500134 if vardeps is None:
135 parser.log.flush()
136 else:
137 parser.log.flush()
138 self.references |= parser.references
139 self.execs |= parser.execs
140
141 for k in parser.contains:
142 if k not in self.contains:
143 self.contains[k] = parser.contains[k].copy()
144 else:
145 self.contains[k].update(parser.contains[k])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600146 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500147 return str(value)
148
149
150class DataContext(dict):
151 def __init__(self, metadata, **kwargs):
152 self.metadata = metadata
153 dict.__init__(self, **kwargs)
154 self['d'] = metadata
155
156 def __missing__(self, key):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500157 value = self.metadata.getVar(key)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500158 if value is None or self.metadata.getVarFlag(key, 'func', False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500159 raise KeyError(key)
160 else:
161 return value
162
163class ExpansionError(Exception):
164 def __init__(self, varname, expression, exception):
165 self.expression = expression
166 self.variablename = varname
167 self.exception = exception
168 if varname:
169 if expression:
170 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
171 else:
172 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception)
173 else:
174 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
175 Exception.__init__(self, self.msg)
176 self.args = (varname, expression, exception)
177 def __str__(self):
178 return self.msg
179
180class IncludeHistory(object):
181 def __init__(self, parent = None, filename = '[TOP LEVEL]'):
182 self.parent = parent
183 self.filename = filename
184 self.children = []
185 self.current = self
186
187 def copy(self):
188 new = IncludeHistory(self.parent, self.filename)
189 for c in self.children:
190 new.children.append(c)
191 return new
192
193 def include(self, filename):
194 newfile = IncludeHistory(self.current, filename)
195 self.current.children.append(newfile)
196 self.current = newfile
197 return self
198
199 def __enter__(self):
200 pass
201
202 def __exit__(self, a, b, c):
203 if self.current.parent:
204 self.current = self.current.parent
205 else:
206 bb.warn("Include log: Tried to finish '%s' at top level." % filename)
207 return False
208
209 def emit(self, o, level = 0):
210 """Emit an include history file, and its children."""
211 if level:
212 spaces = " " * (level - 1)
213 o.write("# %s%s" % (spaces, self.filename))
214 if len(self.children) > 0:
215 o.write(" includes:")
216 else:
217 o.write("#\n# INCLUDE HISTORY:\n#")
218 level = level + 1
219 for child in self.children:
220 o.write("\n")
221 child.emit(o, level)
222
223class VariableHistory(object):
224 def __init__(self, dataroot):
225 self.dataroot = dataroot
226 self.variables = COWDictBase.copy()
227
228 def copy(self):
229 new = VariableHistory(self.dataroot)
230 new.variables = self.variables.copy()
231 return new
232
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 def __getstate__(self):
234 vardict = {}
235 for k, v in self.variables.iteritems():
236 vardict[k] = v
237 return {'dataroot': self.dataroot,
238 'variables': vardict}
239
240 def __setstate__(self, state):
241 self.dataroot = state['dataroot']
242 self.variables = COWDictBase.copy()
243 for k, v in state['variables'].items():
244 self.variables[k] = v
245
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500246 def record(self, *kwonly, **loginfo):
247 if not self.dataroot._tracking:
248 return
249 if len(kwonly) > 0:
250 raise TypeError
251 infer_caller_details(loginfo, parent = True)
252 if 'ignore' in loginfo and loginfo['ignore']:
253 return
254 if 'op' not in loginfo or not loginfo['op']:
255 loginfo['op'] = 'set'
256 if 'detail' in loginfo:
257 loginfo['detail'] = str(loginfo['detail'])
258 if 'variable' not in loginfo or 'file' not in loginfo:
259 raise ValueError("record() missing variable or file.")
260 var = loginfo['variable']
261
262 if var not in self.variables:
263 self.variables[var] = []
264 if not isinstance(self.variables[var], list):
265 return
266 if 'nodups' in loginfo and loginfo in self.variables[var]:
267 return
268 self.variables[var].append(loginfo.copy())
269
270 def variable(self, var):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 remote_connector = self.dataroot.getVar('_remote_data', False)
272 if remote_connector:
273 varhistory = remote_connector.getVarHistory(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500275 varhistory = []
276
277 if var in self.variables:
278 varhistory.extend(self.variables[var])
279 return varhistory
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500280
281 def emit(self, var, oval, val, o, d):
282 history = self.variable(var)
283
284 # Append override history
285 if var in d.overridedata:
286 for (r, override) in d.overridedata[var]:
287 for event in self.variable(r):
288 loginfo = event.copy()
289 if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
290 continue
291 loginfo['variable'] = var
292 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
293 history.append(loginfo)
294
295 commentVal = re.sub('\n', '\n#', str(oval))
296 if history:
297 if len(history) == 1:
298 o.write("#\n# $%s\n" % var)
299 else:
300 o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
301 for event in history:
302 # o.write("# %s\n" % str(event))
303 if 'func' in event:
304 # If we have a function listed, this is internal
305 # code, not an operation in a config file, and the
306 # full path is distracting.
307 event['file'] = re.sub('.*/', '', event['file'])
308 display_func = ' [%s]' % event['func']
309 else:
310 display_func = ''
311 if 'flag' in event:
312 flag = '[%s] ' % (event['flag'])
313 else:
314 flag = ''
315 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail'])))
316 if len(history) > 1:
317 o.write("# pre-expansion value:\n")
318 o.write('# "%s"\n' % (commentVal))
319 else:
320 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var)
321 o.write('# "%s"\n' % (commentVal))
322
323 def get_variable_files(self, var):
324 """Get the files where operations are made on a variable"""
325 var_history = self.variable(var)
326 files = []
327 for event in var_history:
328 files.append(event['file'])
329 return files
330
331 def get_variable_lines(self, var, f):
332 """Get the line where a operation is made on a variable in file f"""
333 var_history = self.variable(var)
334 lines = []
335 for event in var_history:
336 if f== event['file']:
337 line = event['line']
338 lines.append(line)
339 return lines
340
341 def get_variable_items_files(self, var, d):
342 """
343 Use variable history to map items added to a list variable and
344 the files in which they were added.
345 """
346 history = self.variable(var)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500347 finalitems = (d.getVar(var) or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348 filemap = {}
349 isset = False
350 for event in history:
351 if 'flag' in event:
352 continue
353 if event['op'] == '_remove':
354 continue
355 if isset and event['op'] == 'set?':
356 continue
357 isset = True
358 items = d.expand(event['detail']).split()
359 for item in items:
360 # This is a little crude but is belt-and-braces to avoid us
361 # having to handle every possible operation type specifically
362 if item in finalitems and not item in filemap:
363 filemap[item] = event['file']
364 return filemap
365
366 def del_var_history(self, var, f=None, line=None):
367 """If file f and line are not given, the entire history of var is deleted"""
368 if var in self.variables:
369 if f and line:
370 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line]
371 else:
372 self.variables[var] = []
373
374class DataSmart(MutableMapping):
375 def __init__(self):
376 self.dict = {}
377
378 self.inchistory = IncludeHistory()
379 self.varhistory = VariableHistory(self)
380 self._tracking = False
381
382 self.expand_cache = {}
383
384 # cookie monster tribute
385 # Need to be careful about writes to overridedata as
386 # its only a shallow copy, could influence other data store
387 # copies!
388 self.overridedata = {}
389 self.overrides = None
390 self.overridevars = set(["OVERRIDES", "FILE"])
391 self.inoverride = False
392
393 def enableTracking(self):
394 self._tracking = True
395
396 def disableTracking(self):
397 self._tracking = False
398
399 def expandWithRefs(self, s, varname):
400
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600401 if not isinstance(s, str): # sanity check
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 return VariableParse(varname, self, s)
403
404 if varname and varname in self.expand_cache:
405 return self.expand_cache[varname]
406
407 varparse = VariableParse(varname, self)
408
409 while s.find('${') != -1:
410 olds = s
411 try:
412 s = __expand_var_regexp__.sub(varparse.var_sub, s)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500413 try:
414 s = __expand_python_regexp__.sub(varparse.python_sub, s)
415 except SyntaxError as e:
416 # Likely unmatched brackets, just don't expand the expression
417 if e.msg != "EOL while scanning string literal":
418 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500419 if s == olds:
420 break
421 except ExpansionError:
422 raise
423 except bb.parse.SkipRecipe:
424 raise
425 except Exception as exc:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600426 raise ExpansionError(varname, s, exc) from exc
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500427
428 varparse.value = s
429
430 if varname:
431 self.expand_cache[varname] = varparse
432
433 return varparse
434
435 def expand(self, s, varname = None):
436 return self.expandWithRefs(s, varname).value
437
438 def finalize(self, parent = False):
439 return
440
441 def internal_finalize(self, parent = False):
442 """Performs final steps upon the datastore, including application of overrides"""
443 self.overrides = None
444
445 def need_overrides(self):
Patrick Williamsd7e96312015-09-22 08:09:05 -0500446 if self.overrides is not None:
447 return
448 if self.inoverride:
449 return
450 for count in range(5):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500451 self.inoverride = True
452 # Can end up here recursively so setup dummy values
453 self.overrides = []
454 self.overridesset = set()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500455 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500456 self.overridesset = set(self.overrides)
457 self.inoverride = False
458 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500459 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
Patrick Williamsd7e96312015-09-22 08:09:05 -0500460 if newoverrides == self.overrides:
461 break
462 self.overrides = newoverrides
463 self.overridesset = set(self.overrides)
464 else:
465 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500466
467 def initVar(self, var):
468 self.expand_cache = {}
469 if not var in self.dict:
470 self.dict[var] = {}
471
472 def _findVar(self, var):
473 dest = self.dict
474 while dest:
475 if var in dest:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500476 return dest[var], self.overridedata.get(var, None)
477
478 if "_remote_data" in dest:
479 connector = dest["_remote_data"]["_content"]
480 return connector.getVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500481
482 if "_data" not in dest:
483 break
484 dest = dest["_data"]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500485 return None, self.overridedata.get(var, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500486
487 def _makeShadowCopy(self, var):
488 if var in self.dict:
489 return
490
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500491 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500492
493 if local_var:
494 self.dict[var] = copy.copy(local_var)
495 else:
496 self.initVar(var)
497
498
499 def setVar(self, var, value, **loginfo):
500 #print("var=" + str(var) + " val=" + str(value))
501 parsing=False
502 if 'parsing' in loginfo:
503 parsing=True
504
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500505 if '_remote_data' in self.dict:
506 connector = self.dict["_remote_data"]["_content"]
507 res = connector.setVar(var, value)
508 if not res:
509 return
510
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500511 if 'op' not in loginfo:
512 loginfo['op'] = "set"
513 self.expand_cache = {}
514 match = __setvar_regexp__.match(var)
515 if match and match.group("keyword") in __setvar_keyword__:
516 base = match.group('base')
517 keyword = match.group("keyword")
518 override = match.group('add')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500519 l = self.getVarFlag(base, keyword, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500520 l.append([value, override])
521 self.setVarFlag(base, keyword, l, ignore=True)
522 # And cause that to be recorded:
523 loginfo['detail'] = value
524 loginfo['variable'] = base
525 if override:
526 loginfo['op'] = '%s[%s]' % (keyword, override)
527 else:
528 loginfo['op'] = keyword
529 self.varhistory.record(**loginfo)
530 # todo make sure keyword is not __doc__ or __module__
531 # pay the cookie monster
532
533 # more cookies for the cookie monster
534 if '_' in var:
535 self._setvar_update_overrides(base, **loginfo)
536
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500537 if base in self.overridevars:
Patrick Williamsd7e96312015-09-22 08:09:05 -0500538 self._setvar_update_overridevars(var, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500539 return
540
541 if not var in self.dict:
542 self._makeShadowCopy(var)
543
544 if not parsing:
545 if "_append" in self.dict[var]:
546 del self.dict[var]["_append"]
547 if "_prepend" in self.dict[var]:
548 del self.dict[var]["_prepend"]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500549 if "_remove" in self.dict[var]:
550 del self.dict[var]["_remove"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500551 if var in self.overridedata:
552 active = []
553 self.need_overrides()
554 for (r, o) in self.overridedata[var]:
555 if o in self.overridesset:
556 active.append(r)
557 elif "_" in o:
558 if set(o.split("_")).issubset(self.overridesset):
559 active.append(r)
560 for a in active:
561 self.delVar(a)
562 del self.overridedata[var]
563
564 # more cookies for the cookie monster
565 if '_' in var:
566 self._setvar_update_overrides(var, **loginfo)
567
568 # setting var
569 self.dict[var]["_content"] = value
570 self.varhistory.record(**loginfo)
571
572 if var in self.overridevars:
Patrick Williamsd7e96312015-09-22 08:09:05 -0500573 self._setvar_update_overridevars(var, value)
574
575 def _setvar_update_overridevars(self, var, value):
576 vardata = self.expandWithRefs(value, var)
577 new = vardata.references
578 new.update(vardata.contains.keys())
579 while not new.issubset(self.overridevars):
580 nextnew = set()
581 self.overridevars.update(new)
582 for i in new:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500583 vardata = self.expandWithRefs(self.getVar(i), i)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500584 nextnew.update(vardata.references)
585 nextnew.update(vardata.contains.keys())
586 new = nextnew
587 self.internal_finalize(True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500588
589 def _setvar_update_overrides(self, var, **loginfo):
590 # aka pay the cookie monster
591 override = var[var.rfind('_')+1:]
592 shortvar = var[:var.rfind('_')]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500593 while override and override.islower():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500594 if shortvar not in self.overridedata:
595 self.overridedata[shortvar] = []
596 if [var, override] not in self.overridedata[shortvar]:
597 # Force CoW by recreating the list first
598 self.overridedata[shortvar] = list(self.overridedata[shortvar])
599 self.overridedata[shortvar].append([var, override])
600 override = None
601 if "_" in shortvar:
602 override = var[shortvar.rfind('_')+1:]
603 shortvar = var[:shortvar.rfind('_')]
604 if len(shortvar) == 0:
605 override = None
606
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500607 def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500608 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
609
610 def renameVar(self, key, newkey, **loginfo):
611 """
612 Rename the variable key to newkey
613 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500614 if '_remote_data' in self.dict:
615 connector = self.dict["_remote_data"]["_content"]
616 res = connector.renameVar(key, newkey)
617 if not res:
618 return
619
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500620 val = self.getVar(key, 0, parsing=True)
621 if val is not None:
622 loginfo['variable'] = newkey
623 loginfo['op'] = 'rename from %s' % key
624 loginfo['detail'] = val
625 self.varhistory.record(**loginfo)
626 self.setVar(newkey, val, ignore=True, parsing=True)
627
628 for i in (__setvar_keyword__):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500629 src = self.getVarFlag(key, i, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500630 if src is None:
631 continue
632
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500633 dest = self.getVarFlag(newkey, i, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500634 dest.extend(src)
635 self.setVarFlag(newkey, i, dest, ignore=True)
636
637 if key in self.overridedata:
638 self.overridedata[newkey] = []
639 for (v, o) in self.overridedata[key]:
640 self.overridedata[newkey].append([v.replace(key, newkey), o])
641 self.renameVar(v, v.replace(key, newkey))
642
643 if '_' in newkey and val is None:
644 self._setvar_update_overrides(newkey, **loginfo)
645
646 loginfo['variable'] = key
647 loginfo['op'] = 'rename (to)'
648 loginfo['detail'] = newkey
649 self.varhistory.record(**loginfo)
650 self.delVar(key, ignore=True)
651
652 def appendVar(self, var, value, **loginfo):
653 loginfo['op'] = 'append'
654 self.varhistory.record(**loginfo)
655 self.setVar(var + "_append", value, ignore=True, parsing=True)
656
657 def prependVar(self, var, value, **loginfo):
658 loginfo['op'] = 'prepend'
659 self.varhistory.record(**loginfo)
660 self.setVar(var + "_prepend", value, ignore=True, parsing=True)
661
662 def delVar(self, var, **loginfo):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500663 if '_remote_data' in self.dict:
664 connector = self.dict["_remote_data"]["_content"]
665 res = connector.delVar(var)
666 if not res:
667 return
668
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500669 loginfo['detail'] = ""
670 loginfo['op'] = 'del'
671 self.varhistory.record(**loginfo)
672 self.expand_cache = {}
673 self.dict[var] = {}
674 if var in self.overridedata:
675 del self.overridedata[var]
676 if '_' in var:
677 override = var[var.rfind('_')+1:]
678 shortvar = var[:var.rfind('_')]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500679 while override and override.islower():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500680 try:
681 if shortvar in self.overridedata:
682 # Force CoW by recreating the list first
683 self.overridedata[shortvar] = list(self.overridedata[shortvar])
684 self.overridedata[shortvar].remove([var, override])
685 except ValueError as e:
686 pass
687 override = None
688 if "_" in shortvar:
689 override = var[shortvar.rfind('_')+1:]
690 shortvar = var[:shortvar.rfind('_')]
691 if len(shortvar) == 0:
692 override = None
693
694 def setVarFlag(self, var, flag, value, **loginfo):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500695 if '_remote_data' in self.dict:
696 connector = self.dict["_remote_data"]["_content"]
697 res = connector.setVarFlag(var, flag, value)
698 if not res:
699 return
700
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500701 self.expand_cache = {}
702 if 'op' not in loginfo:
703 loginfo['op'] = "set"
704 loginfo['flag'] = flag
705 self.varhistory.record(**loginfo)
706 if not var in self.dict:
707 self._makeShadowCopy(var)
708 self.dict[var][flag] = value
709
710 if flag == "_defaultval" and '_' in var:
711 self._setvar_update_overrides(var, **loginfo)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500712 if flag == "_defaultval" and var in self.overridevars:
713 self._setvar_update_overridevars(var, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500714
715 if flag == "unexport" or flag == "export":
716 if not "__exportlist" in self.dict:
717 self._makeShadowCopy("__exportlist")
718 if not "_content" in self.dict["__exportlist"]:
719 self.dict["__exportlist"]["_content"] = set()
720 self.dict["__exportlist"]["_content"].add(var)
721
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500722 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False):
723 local_var, overridedata = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500724 value = None
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500725 if flag == "_content" and overridedata is not None and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500726 match = False
727 active = {}
728 self.need_overrides()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500729 for (r, o) in overridedata:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500730 # What about double overrides both with "_" in the name?
731 if o in self.overridesset:
732 active[o] = r
733 elif "_" in o:
734 if set(o.split("_")).issubset(self.overridesset):
735 active[o] = r
736
737 mod = True
738 while mod:
739 mod = False
740 for o in self.overrides:
741 for a in active.copy():
742 if a.endswith("_" + o):
743 t = active[a]
744 del active[a]
745 active[a.replace("_" + o, "")] = t
746 mod = True
747 elif a == o:
748 match = active[a]
749 del active[a]
750 if match:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500751 value = self.getVar(match, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500752
753 if local_var is not None and value is None:
754 if flag in local_var:
755 value = copy.copy(local_var[flag])
756 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
757 value = copy.copy(local_var["_defaultval"])
758
759
760 if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
761 if not value:
762 value = ""
763 self.need_overrides()
764 for (r, o) in local_var["_append"]:
765 match = True
766 if o:
767 for o2 in o.split("_"):
768 if not o2 in self.overrides:
769 match = False
770 if match:
771 value = value + r
772
773 if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
774 if not value:
775 value = ""
776 self.need_overrides()
777 for (r, o) in local_var["_prepend"]:
778
779 match = True
780 if o:
781 for o2 in o.split("_"):
782 if not o2 in self.overrides:
783 match = False
784 if match:
785 value = r + value
786
787 if expand and value:
788 # Only getvar (flag == _content) hits the expand cache
789 cachename = None
790 if flag == "_content":
791 cachename = var
792 else:
793 cachename = var + "[" + flag + "]"
794 value = self.expand(value, cachename)
795
796 if value and flag == "_content" and local_var is not None and "_remove" in local_var:
797 removes = []
798 self.need_overrides()
799 for (r, o) in local_var["_remove"]:
800 match = True
801 if o:
802 for o2 in o.split("_"):
803 if not o2 in self.overrides:
804 match = False
805 if match:
806 removes.extend(self.expand(r).split())
807
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500808 if removes:
809 filtered = filter(lambda v: v not in removes,
810 value.split())
811 value = " ".join(filtered)
812 if expand and var in self.expand_cache:
813 # We need to ensure the expand cache has the correct value
814 # flag == "_content" here
815 self.expand_cache[var].value = value
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500816 return value
817
818 def delVarFlag(self, var, flag, **loginfo):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500819 if '_remote_data' in self.dict:
820 connector = self.dict["_remote_data"]["_content"]
821 res = connector.delVarFlag(var, flag)
822 if not res:
823 return
824
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500825 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500826 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500827 if not local_var:
828 return
829 if not var in self.dict:
830 self._makeShadowCopy(var)
831
832 if var in self.dict and flag in self.dict[var]:
833 loginfo['detail'] = ""
834 loginfo['op'] = 'delFlag'
835 loginfo['flag'] = flag
836 self.varhistory.record(**loginfo)
837
838 del self.dict[var][flag]
839
840 def appendVarFlag(self, var, flag, value, **loginfo):
841 loginfo['op'] = 'append'
842 loginfo['flag'] = flag
843 self.varhistory.record(**loginfo)
844 newvalue = (self.getVarFlag(var, flag, False) or "") + value
845 self.setVarFlag(var, flag, newvalue, ignore=True)
846
847 def prependVarFlag(self, var, flag, value, **loginfo):
848 loginfo['op'] = 'prepend'
849 loginfo['flag'] = flag
850 self.varhistory.record(**loginfo)
851 newvalue = value + (self.getVarFlag(var, flag, False) or "")
852 self.setVarFlag(var, flag, newvalue, ignore=True)
853
854 def setVarFlags(self, var, flags, **loginfo):
855 self.expand_cache = {}
856 infer_caller_details(loginfo)
857 if not var in self.dict:
858 self._makeShadowCopy(var)
859
860 for i in flags:
861 if i == "_content":
862 continue
863 loginfo['flag'] = i
864 loginfo['detail'] = flags[i]
865 self.varhistory.record(**loginfo)
866 self.dict[var][i] = flags[i]
867
868 def getVarFlags(self, var, expand = False, internalflags=False):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500869 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500870 flags = {}
871
872 if local_var:
873 for i in local_var:
874 if i.startswith("_") and not internalflags:
875 continue
876 flags[i] = local_var[i]
877 if expand and i in expand:
878 flags[i] = self.expand(flags[i], var + "[" + i + "]")
879 if len(flags) == 0:
880 return None
881 return flags
882
883
884 def delVarFlags(self, var, **loginfo):
885 self.expand_cache = {}
886 if not var in self.dict:
887 self._makeShadowCopy(var)
888
889 if var in self.dict:
890 content = None
891
892 loginfo['op'] = 'delete flags'
893 self.varhistory.record(**loginfo)
894
895 # try to save the content
896 if "_content" in self.dict[var]:
897 content = self.dict[var]["_content"]
898 self.dict[var] = {}
899 self.dict[var]["_content"] = content
900 else:
901 del self.dict[var]
902
903 def createCopy(self):
904 """
905 Create a copy of self by setting _data to self
906 """
907 # we really want this to be a DataSmart...
908 data = DataSmart()
909 data.dict["_data"] = self.dict
910 data.varhistory = self.varhistory.copy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500911 data.varhistory.dataroot = data
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500912 data.inchistory = self.inchistory.copy()
913
914 data._tracking = self._tracking
915
916 data.overrides = None
917 data.overridevars = copy.copy(self.overridevars)
918 # Should really be a deepcopy but has heavy overhead.
919 # Instead, we're careful with writes.
920 data.overridedata = copy.copy(self.overridedata)
921
922 return data
923
924 def expandVarref(self, variable, parents=False):
925 """Find all references to variable in the data and expand it
926 in place, optionally descending to parent datastores."""
927
928 if parents:
929 keys = iter(self)
930 else:
931 keys = self.localkeys()
932
933 ref = '${%s}' % variable
934 value = self.getVar(variable, False)
935 for key in keys:
936 referrervalue = self.getVar(key, False)
937 if referrervalue and ref in referrervalue:
938 self.setVar(key, referrervalue.replace(ref, value))
939
940 def localkeys(self):
941 for key in self.dict:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500942 if key not in ['_data', '_remote_data']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500943 yield key
944
945 def __iter__(self):
946 deleted = set()
947 overrides = set()
948 def keylist(d):
949 klist = set()
950 for key in d:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500951 if key in ["_data", "_remote_data"]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500952 continue
953 if key in deleted:
954 continue
955 if key in overrides:
956 continue
957 if not d[key]:
958 deleted.add(key)
959 continue
960 klist.add(key)
961
962 if "_data" in d:
963 klist |= keylist(d["_data"])
964
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500965 if "_remote_data" in d:
966 connector = d["_remote_data"]["_content"]
967 for key in connector.getKeys():
968 if key in deleted:
969 continue
970 klist.add(key)
971
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500972 return klist
973
974 self.need_overrides()
975 for var in self.overridedata:
976 for (r, o) in self.overridedata[var]:
977 if o in self.overridesset:
978 overrides.add(var)
979 elif "_" in o:
980 if set(o.split("_")).issubset(self.overridesset):
981 overrides.add(var)
982
983 for k in keylist(self.dict):
984 yield k
985
986 for k in overrides:
987 yield k
988
989 def __len__(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600990 return len(frozenset(iter(self)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500991
992 def __getitem__(self, item):
993 value = self.getVar(item, False)
994 if value is None:
995 raise KeyError(item)
996 else:
997 return value
998
999 def __setitem__(self, var, value):
1000 self.setVar(var, value)
1001
1002 def __delitem__(self, var):
1003 self.delVar(var)
1004
1005 def get_hash(self):
1006 data = {}
1007 d = self.createCopy()
1008 bb.data.expandKeys(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001009
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001010 config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001011 keys = set(key for key in iter(d) if not key.startswith("__"))
1012 for key in keys:
1013 if key in config_whitelist:
1014 continue
1015
1016 value = d.getVar(key, False) or ""
1017 data.update({key:value})
1018
1019 varflags = d.getVarFlags(key, internalflags = True)
1020 if not varflags:
1021 continue
1022 for f in varflags:
1023 if f == "_content":
1024 continue
1025 data.update({'%s[%s]' % (key, f):varflags[f]})
1026
1027 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
1028 bb_list = d.getVar(key, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001029 data.update({key:str(bb_list)})
1030
1031 if key == "__BBANONFUNCS":
1032 for i in bb_list:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001033 value = d.getVar(i, False) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001034 data.update({i:value})
1035
1036 data_str = str([(k, data[k]) for k in sorted(data.keys())])
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001037 return hashlib.md5(data_str.encode("utf-8")).hexdigest()