blob: 6b94fc4b42b4a9819f236ffcfad7bf0ce019bb16 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake Smart Dictionary Implementation
5
6Functions for interacting with the data structure used by the
7BitBake build tools.
8
9"""
10
11# Copyright (C) 2003, 2004 Chris Larson
12# Copyright (C) 2004, 2005 Seb Frankengul
13# Copyright (C) 2005, 2006 Holger Hans Peter Freyther
14# Copyright (C) 2005 Uli Luckas
15# Copyright (C) 2005 ROAD GmbH
16#
17# This program is free software; you can redistribute it and/or modify
18# it under the terms of the GNU General Public License version 2 as
19# published by the Free Software Foundation.
20#
21# This program is distributed in the hope that it will be useful,
22# but WITHOUT ANY WARRANTY; without even the implied warranty of
23# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24# GNU General Public License for more details.
25#
26# You should have received a copy of the GNU General Public License along
27# with this program; if not, write to the Free Software Foundation, Inc.,
28# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
29# Based on functions from the base bb module, Copyright 2003 Holger Schurig
30
31import copy, re, sys, traceback
32from collections import MutableMapping
33import logging
34import hashlib
35import bb, bb.codeparser
36from bb import utils
37from bb.COW import COWDictBase
38
39logger = logging.getLogger("BitBake.Data")
40
41__setvar_keyword__ = ["_append", "_prepend", "_remove"]
Brad Bishopd7bf8c12018-02-25 22:55:05 -050042__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050043__expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050044__expand_python_regexp__ = re.compile(r"\${@.+?}")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080045__whitespace_split__ = re.compile('(\s)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046
47def infer_caller_details(loginfo, parent = False, varval = True):
48 """Save the caller the trouble of specifying everything."""
49 # Save effort.
50 if 'ignore' in loginfo and loginfo['ignore']:
51 return
52 # If nothing was provided, mark this as possibly unneeded.
53 if not loginfo:
54 loginfo['ignore'] = True
55 return
56 # Infer caller's likely values for variable (var) and value (value),
57 # to reduce clutter in the rest of the code.
58 above = None
59 def set_above():
60 try:
61 raise Exception
62 except Exception:
63 tb = sys.exc_info()[2]
64 if parent:
65 return tb.tb_frame.f_back.f_back.f_back
66 else:
67 return tb.tb_frame.f_back.f_back
68
69 if varval and ('variable' not in loginfo or 'detail' not in loginfo):
70 if not above:
71 above = set_above()
72 lcls = above.f_locals.items()
73 for k, v in lcls:
74 if k == 'value' and 'detail' not in loginfo:
75 loginfo['detail'] = v
76 if k == 'var' and 'variable' not in loginfo:
77 loginfo['variable'] = v
78 # Infer file/line/function from traceback
79 # Don't use traceback.extract_stack() since it fills the line contents which
80 # we don't need and that hits stat syscalls
81 if 'file' not in loginfo:
82 if not above:
83 above = set_above()
84 f = above.f_back
85 line = f.f_lineno
86 file = f.f_code.co_filename
87 func = f.f_code.co_name
88 loginfo['file'] = file
89 loginfo['line'] = line
90 if func not in loginfo:
91 loginfo['func'] = func
92
93class VariableParse:
94 def __init__(self, varname, d, val = None):
95 self.varname = varname
96 self.d = d
97 self.value = val
98
99 self.references = set()
100 self.execs = set()
101 self.contains = {}
102
103 def var_sub(self, match):
104 key = match.group()[2:-1]
105 if self.varname and key:
106 if self.varname == key:
107 raise Exception("variable %s references itself!" % self.varname)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800108 var = self.d.getVarFlag(key, "_content")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500109 self.references.add(key)
110 if var is not None:
111 return var
112 else:
113 return match.group()
114
115 def python_sub(self, match):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500116 if isinstance(match, str):
117 code = match
118 else:
119 code = match.group()[3:-1]
120
121 if "_remote_data" in self.d:
122 connector = self.d["_remote_data"]
123 return connector.expandPythonRef(self.varname, code, self.d)
124
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500125 codeobj = compile(code.strip(), self.varname or "<expansion>", "eval")
126
127 parser = bb.codeparser.PythonParser(self.varname, logger)
128 parser.parse_python(code)
129 if self.varname:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500130 vardeps = self.d.getVarFlag(self.varname, "vardeps")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500131 if vardeps is None:
132 parser.log.flush()
133 else:
134 parser.log.flush()
135 self.references |= parser.references
136 self.execs |= parser.execs
137
138 for k in parser.contains:
139 if k not in self.contains:
140 self.contains[k] = parser.contains[k].copy()
141 else:
142 self.contains[k].update(parser.contains[k])
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600143 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 return str(value)
145
146
147class DataContext(dict):
148 def __init__(self, metadata, **kwargs):
149 self.metadata = metadata
150 dict.__init__(self, **kwargs)
151 self['d'] = metadata
152
153 def __missing__(self, key):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500154 value = self.metadata.getVar(key)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500155 if value is None or self.metadata.getVarFlag(key, 'func', False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500156 raise KeyError(key)
157 else:
158 return value
159
160class ExpansionError(Exception):
161 def __init__(self, varname, expression, exception):
162 self.expression = expression
163 self.variablename = varname
164 self.exception = exception
165 if varname:
166 if expression:
167 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
168 else:
169 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception)
170 else:
171 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
172 Exception.__init__(self, self.msg)
173 self.args = (varname, expression, exception)
174 def __str__(self):
175 return self.msg
176
177class IncludeHistory(object):
178 def __init__(self, parent = None, filename = '[TOP LEVEL]'):
179 self.parent = parent
180 self.filename = filename
181 self.children = []
182 self.current = self
183
184 def copy(self):
185 new = IncludeHistory(self.parent, self.filename)
186 for c in self.children:
187 new.children.append(c)
188 return new
189
190 def include(self, filename):
191 newfile = IncludeHistory(self.current, filename)
192 self.current.children.append(newfile)
193 self.current = newfile
194 return self
195
196 def __enter__(self):
197 pass
198
199 def __exit__(self, a, b, c):
200 if self.current.parent:
201 self.current = self.current.parent
202 else:
203 bb.warn("Include log: Tried to finish '%s' at top level." % filename)
204 return False
205
206 def emit(self, o, level = 0):
207 """Emit an include history file, and its children."""
208 if level:
209 spaces = " " * (level - 1)
210 o.write("# %s%s" % (spaces, self.filename))
211 if len(self.children) > 0:
212 o.write(" includes:")
213 else:
214 o.write("#\n# INCLUDE HISTORY:\n#")
215 level = level + 1
216 for child in self.children:
217 o.write("\n")
218 child.emit(o, level)
219
220class VariableHistory(object):
221 def __init__(self, dataroot):
222 self.dataroot = dataroot
223 self.variables = COWDictBase.copy()
224
225 def copy(self):
226 new = VariableHistory(self.dataroot)
227 new.variables = self.variables.copy()
228 return new
229
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500230 def __getstate__(self):
231 vardict = {}
232 for k, v in self.variables.iteritems():
233 vardict[k] = v
234 return {'dataroot': self.dataroot,
235 'variables': vardict}
236
237 def __setstate__(self, state):
238 self.dataroot = state['dataroot']
239 self.variables = COWDictBase.copy()
240 for k, v in state['variables'].items():
241 self.variables[k] = v
242
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500243 def record(self, *kwonly, **loginfo):
244 if not self.dataroot._tracking:
245 return
246 if len(kwonly) > 0:
247 raise TypeError
248 infer_caller_details(loginfo, parent = True)
249 if 'ignore' in loginfo and loginfo['ignore']:
250 return
251 if 'op' not in loginfo or not loginfo['op']:
252 loginfo['op'] = 'set'
253 if 'detail' in loginfo:
254 loginfo['detail'] = str(loginfo['detail'])
255 if 'variable' not in loginfo or 'file' not in loginfo:
256 raise ValueError("record() missing variable or file.")
257 var = loginfo['variable']
258
259 if var not in self.variables:
260 self.variables[var] = []
261 if not isinstance(self.variables[var], list):
262 return
263 if 'nodups' in loginfo and loginfo in self.variables[var]:
264 return
265 self.variables[var].append(loginfo.copy())
266
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800267 def rename_variable_hist(self, oldvar, newvar):
268 if not self.dataroot._tracking:
269 return
270 if oldvar not in self.variables:
271 return
272 if newvar not in self.variables:
273 self.variables[newvar] = []
274 for i in self.variables[oldvar]:
275 self.variables[newvar].append(i.copy())
276
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500277 def variable(self, var):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500278 remote_connector = self.dataroot.getVar('_remote_data', False)
279 if remote_connector:
280 varhistory = remote_connector.getVarHistory(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500281 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500282 varhistory = []
283
284 if var in self.variables:
285 varhistory.extend(self.variables[var])
286 return varhistory
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500287
288 def emit(self, var, oval, val, o, d):
289 history = self.variable(var)
290
291 # Append override history
292 if var in d.overridedata:
293 for (r, override) in d.overridedata[var]:
294 for event in self.variable(r):
295 loginfo = event.copy()
296 if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
297 continue
298 loginfo['variable'] = var
299 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
300 history.append(loginfo)
301
302 commentVal = re.sub('\n', '\n#', str(oval))
303 if history:
304 if len(history) == 1:
305 o.write("#\n# $%s\n" % var)
306 else:
307 o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
308 for event in history:
309 # o.write("# %s\n" % str(event))
310 if 'func' in event:
311 # If we have a function listed, this is internal
312 # code, not an operation in a config file, and the
313 # full path is distracting.
314 event['file'] = re.sub('.*/', '', event['file'])
315 display_func = ' [%s]' % event['func']
316 else:
317 display_func = ''
318 if 'flag' in event:
319 flag = '[%s] ' % (event['flag'])
320 else:
321 flag = ''
322 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail'])))
323 if len(history) > 1:
324 o.write("# pre-expansion value:\n")
325 o.write('# "%s"\n' % (commentVal))
326 else:
327 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var)
328 o.write('# "%s"\n' % (commentVal))
329
330 def get_variable_files(self, var):
331 """Get the files where operations are made on a variable"""
332 var_history = self.variable(var)
333 files = []
334 for event in var_history:
335 files.append(event['file'])
336 return files
337
338 def get_variable_lines(self, var, f):
339 """Get the line where a operation is made on a variable in file f"""
340 var_history = self.variable(var)
341 lines = []
342 for event in var_history:
343 if f== event['file']:
344 line = event['line']
345 lines.append(line)
346 return lines
347
348 def get_variable_items_files(self, var, d):
349 """
350 Use variable history to map items added to a list variable and
351 the files in which they were added.
352 """
353 history = self.variable(var)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500354 finalitems = (d.getVar(var) or '').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 filemap = {}
356 isset = False
357 for event in history:
358 if 'flag' in event:
359 continue
360 if event['op'] == '_remove':
361 continue
362 if isset and event['op'] == 'set?':
363 continue
364 isset = True
365 items = d.expand(event['detail']).split()
366 for item in items:
367 # This is a little crude but is belt-and-braces to avoid us
368 # having to handle every possible operation type specifically
369 if item in finalitems and not item in filemap:
370 filemap[item] = event['file']
371 return filemap
372
373 def del_var_history(self, var, f=None, line=None):
374 """If file f and line are not given, the entire history of var is deleted"""
375 if var in self.variables:
376 if f and line:
377 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line]
378 else:
379 self.variables[var] = []
380
381class DataSmart(MutableMapping):
382 def __init__(self):
383 self.dict = {}
384
385 self.inchistory = IncludeHistory()
386 self.varhistory = VariableHistory(self)
387 self._tracking = False
388
389 self.expand_cache = {}
390
391 # cookie monster tribute
392 # Need to be careful about writes to overridedata as
393 # its only a shallow copy, could influence other data store
394 # copies!
395 self.overridedata = {}
396 self.overrides = None
397 self.overridevars = set(["OVERRIDES", "FILE"])
398 self.inoverride = False
399
400 def enableTracking(self):
401 self._tracking = True
402
403 def disableTracking(self):
404 self._tracking = False
405
406 def expandWithRefs(self, s, varname):
407
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600408 if not isinstance(s, str): # sanity check
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500409 return VariableParse(varname, self, s)
410
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500411 varparse = VariableParse(varname, self)
412
413 while s.find('${') != -1:
414 olds = s
415 try:
416 s = __expand_var_regexp__.sub(varparse.var_sub, s)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500417 try:
418 s = __expand_python_regexp__.sub(varparse.python_sub, s)
419 except SyntaxError as e:
420 # Likely unmatched brackets, just don't expand the expression
421 if e.msg != "EOL while scanning string literal":
422 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500423 if s == olds:
424 break
425 except ExpansionError:
426 raise
427 except bb.parse.SkipRecipe:
428 raise
429 except Exception as exc:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600430 raise ExpansionError(varname, s, exc) from exc
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500431
432 varparse.value = s
433
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500434 return varparse
435
436 def expand(self, s, varname = None):
437 return self.expandWithRefs(s, varname).value
438
439 def finalize(self, parent = False):
440 return
441
442 def internal_finalize(self, parent = False):
443 """Performs final steps upon the datastore, including application of overrides"""
444 self.overrides = None
445
446 def need_overrides(self):
Patrick Williamsd7e96312015-09-22 08:09:05 -0500447 if self.overrides is not None:
448 return
449 if self.inoverride:
450 return
451 for count in range(5):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500452 self.inoverride = True
453 # Can end up here recursively so setup dummy values
454 self.overrides = []
455 self.overridesset = set()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500456 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500457 self.overridesset = set(self.overrides)
458 self.inoverride = False
459 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500460 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
Patrick Williamsd7e96312015-09-22 08:09:05 -0500461 if newoverrides == self.overrides:
462 break
463 self.overrides = newoverrides
464 self.overridesset = set(self.overrides)
465 else:
466 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500467
468 def initVar(self, var):
469 self.expand_cache = {}
470 if not var in self.dict:
471 self.dict[var] = {}
472
473 def _findVar(self, var):
474 dest = self.dict
475 while dest:
476 if var in dest:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500477 return dest[var], self.overridedata.get(var, None)
478
479 if "_remote_data" in dest:
480 connector = dest["_remote_data"]["_content"]
481 return connector.getVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500482
483 if "_data" not in dest:
484 break
485 dest = dest["_data"]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500486 return None, self.overridedata.get(var, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500487
488 def _makeShadowCopy(self, var):
489 if var in self.dict:
490 return
491
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500492 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500493
494 if local_var:
495 self.dict[var] = copy.copy(local_var)
496 else:
497 self.initVar(var)
498
499
500 def setVar(self, var, value, **loginfo):
501 #print("var=" + str(var) + " val=" + str(value))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800502 self.expand_cache = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500503 parsing=False
504 if 'parsing' in loginfo:
505 parsing=True
506
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500507 if '_remote_data' in self.dict:
508 connector = self.dict["_remote_data"]["_content"]
509 res = connector.setVar(var, value)
510 if not res:
511 return
512
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500513 if 'op' not in loginfo:
514 loginfo['op'] = "set"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800515
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500516 match = __setvar_regexp__.match(var)
517 if match and match.group("keyword") in __setvar_keyword__:
518 base = match.group('base')
519 keyword = match.group("keyword")
520 override = match.group('add')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500521 l = self.getVarFlag(base, keyword, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500522 l.append([value, override])
523 self.setVarFlag(base, keyword, l, ignore=True)
524 # And cause that to be recorded:
525 loginfo['detail'] = value
526 loginfo['variable'] = base
527 if override:
528 loginfo['op'] = '%s[%s]' % (keyword, override)
529 else:
530 loginfo['op'] = keyword
531 self.varhistory.record(**loginfo)
532 # todo make sure keyword is not __doc__ or __module__
533 # pay the cookie monster
534
535 # more cookies for the cookie monster
536 if '_' in var:
537 self._setvar_update_overrides(base, **loginfo)
538
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500539 if base in self.overridevars:
Patrick Williamsd7e96312015-09-22 08:09:05 -0500540 self._setvar_update_overridevars(var, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500541 return
542
543 if not var in self.dict:
544 self._makeShadowCopy(var)
545
546 if not parsing:
547 if "_append" in self.dict[var]:
548 del self.dict[var]["_append"]
549 if "_prepend" in self.dict[var]:
550 del self.dict[var]["_prepend"]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500551 if "_remove" in self.dict[var]:
552 del self.dict[var]["_remove"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500553 if var in self.overridedata:
554 active = []
555 self.need_overrides()
556 for (r, o) in self.overridedata[var]:
557 if o in self.overridesset:
558 active.append(r)
559 elif "_" in o:
560 if set(o.split("_")).issubset(self.overridesset):
561 active.append(r)
562 for a in active:
563 self.delVar(a)
564 del self.overridedata[var]
565
566 # more cookies for the cookie monster
567 if '_' in var:
568 self._setvar_update_overrides(var, **loginfo)
569
570 # setting var
571 self.dict[var]["_content"] = value
572 self.varhistory.record(**loginfo)
573
574 if var in self.overridevars:
Patrick Williamsd7e96312015-09-22 08:09:05 -0500575 self._setvar_update_overridevars(var, value)
576
577 def _setvar_update_overridevars(self, var, value):
578 vardata = self.expandWithRefs(value, var)
579 new = vardata.references
580 new.update(vardata.contains.keys())
581 while not new.issubset(self.overridevars):
582 nextnew = set()
583 self.overridevars.update(new)
584 for i in new:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500585 vardata = self.expandWithRefs(self.getVar(i), i)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500586 nextnew.update(vardata.references)
587 nextnew.update(vardata.contains.keys())
588 new = nextnew
589 self.internal_finalize(True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500590
591 def _setvar_update_overrides(self, var, **loginfo):
592 # aka pay the cookie monster
593 override = var[var.rfind('_')+1:]
594 shortvar = var[:var.rfind('_')]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500595 while override and override.islower():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500596 if shortvar not in self.overridedata:
597 self.overridedata[shortvar] = []
598 if [var, override] not in self.overridedata[shortvar]:
599 # Force CoW by recreating the list first
600 self.overridedata[shortvar] = list(self.overridedata[shortvar])
601 self.overridedata[shortvar].append([var, override])
602 override = None
603 if "_" in shortvar:
604 override = var[shortvar.rfind('_')+1:]
605 shortvar = var[:shortvar.rfind('_')]
606 if len(shortvar) == 0:
607 override = None
608
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500609 def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500610 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
611
612 def renameVar(self, key, newkey, **loginfo):
613 """
614 Rename the variable key to newkey
615 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500616 if '_remote_data' in self.dict:
617 connector = self.dict["_remote_data"]["_content"]
618 res = connector.renameVar(key, newkey)
619 if not res:
620 return
621
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500622 val = self.getVar(key, 0, parsing=True)
623 if val is not None:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800624 self.varhistory.rename_variable_hist(key, newkey)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500625 loginfo['variable'] = newkey
626 loginfo['op'] = 'rename from %s' % key
627 loginfo['detail'] = val
628 self.varhistory.record(**loginfo)
629 self.setVar(newkey, val, ignore=True, parsing=True)
630
631 for i in (__setvar_keyword__):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500632 src = self.getVarFlag(key, i, False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500633 if src is None:
634 continue
635
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500636 dest = self.getVarFlag(newkey, i, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500637 dest.extend(src)
638 self.setVarFlag(newkey, i, dest, ignore=True)
639
640 if key in self.overridedata:
641 self.overridedata[newkey] = []
642 for (v, o) in self.overridedata[key]:
643 self.overridedata[newkey].append([v.replace(key, newkey), o])
644 self.renameVar(v, v.replace(key, newkey))
645
646 if '_' in newkey and val is None:
647 self._setvar_update_overrides(newkey, **loginfo)
648
649 loginfo['variable'] = key
650 loginfo['op'] = 'rename (to)'
651 loginfo['detail'] = newkey
652 self.varhistory.record(**loginfo)
653 self.delVar(key, ignore=True)
654
655 def appendVar(self, var, value, **loginfo):
656 loginfo['op'] = 'append'
657 self.varhistory.record(**loginfo)
658 self.setVar(var + "_append", value, ignore=True, parsing=True)
659
660 def prependVar(self, var, value, **loginfo):
661 loginfo['op'] = 'prepend'
662 self.varhistory.record(**loginfo)
663 self.setVar(var + "_prepend", value, ignore=True, parsing=True)
664
665 def delVar(self, var, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800666 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500667 if '_remote_data' in self.dict:
668 connector = self.dict["_remote_data"]["_content"]
669 res = connector.delVar(var)
670 if not res:
671 return
672
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500673 loginfo['detail'] = ""
674 loginfo['op'] = 'del'
675 self.varhistory.record(**loginfo)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500676 self.dict[var] = {}
677 if var in self.overridedata:
678 del self.overridedata[var]
679 if '_' in var:
680 override = var[var.rfind('_')+1:]
681 shortvar = var[:var.rfind('_')]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500682 while override and override.islower():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500683 try:
684 if shortvar in self.overridedata:
685 # Force CoW by recreating the list first
686 self.overridedata[shortvar] = list(self.overridedata[shortvar])
687 self.overridedata[shortvar].remove([var, override])
688 except ValueError as e:
689 pass
690 override = None
691 if "_" in shortvar:
692 override = var[shortvar.rfind('_')+1:]
693 shortvar = var[:shortvar.rfind('_')]
694 if len(shortvar) == 0:
695 override = None
696
697 def setVarFlag(self, var, flag, value, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800698 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500699 if '_remote_data' in self.dict:
700 connector = self.dict["_remote_data"]["_content"]
701 res = connector.setVarFlag(var, flag, value)
702 if not res:
703 return
704
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500705 if 'op' not in loginfo:
706 loginfo['op'] = "set"
707 loginfo['flag'] = flag
708 self.varhistory.record(**loginfo)
709 if not var in self.dict:
710 self._makeShadowCopy(var)
711 self.dict[var][flag] = value
712
713 if flag == "_defaultval" and '_' in var:
714 self._setvar_update_overrides(var, **loginfo)
Patrick Williamsd7e96312015-09-22 08:09:05 -0500715 if flag == "_defaultval" and var in self.overridevars:
716 self._setvar_update_overridevars(var, value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500717
718 if flag == "unexport" or flag == "export":
719 if not "__exportlist" in self.dict:
720 self._makeShadowCopy("__exportlist")
721 if not "_content" in self.dict["__exportlist"]:
722 self.dict["__exportlist"]["_content"] = set()
723 self.dict["__exportlist"]["_content"].add(var)
724
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800725 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
726 if flag == "_content":
727 cachename = var
728 else:
729 if not flag:
730 bb.warn("Calling getVarFlag with flag unset is invalid")
731 return None
732 cachename = var + "[" + flag + "]"
733
734 if expand and cachename in self.expand_cache:
735 return self.expand_cache[cachename].value
736
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500737 local_var, overridedata = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500738 value = None
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800739 removes = set()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500740 if flag == "_content" and overridedata is not None and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741 match = False
742 active = {}
743 self.need_overrides()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500744 for (r, o) in overridedata:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500745 # What about double overrides both with "_" in the name?
746 if o in self.overridesset:
747 active[o] = r
748 elif "_" in o:
749 if set(o.split("_")).issubset(self.overridesset):
750 active[o] = r
751
752 mod = True
753 while mod:
754 mod = False
755 for o in self.overrides:
756 for a in active.copy():
757 if a.endswith("_" + o):
758 t = active[a]
759 del active[a]
760 active[a.replace("_" + o, "")] = t
761 mod = True
762 elif a == o:
763 match = active[a]
764 del active[a]
765 if match:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800766 value, subparser = self.getVarFlag(match, "_content", False, retparser=True)
767 if hasattr(subparser, "removes"):
768 # We have to carry the removes from the overridden variable to apply at the
769 # end of processing
770 removes = subparser.removes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500771
772 if local_var is not None and value is None:
773 if flag in local_var:
774 value = copy.copy(local_var[flag])
775 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
776 value = copy.copy(local_var["_defaultval"])
777
778
779 if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
780 if not value:
781 value = ""
782 self.need_overrides()
783 for (r, o) in local_var["_append"]:
784 match = True
785 if o:
786 for o2 in o.split("_"):
787 if not o2 in self.overrides:
788 match = False
789 if match:
790 value = value + r
791
792 if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
793 if not value:
794 value = ""
795 self.need_overrides()
796 for (r, o) in local_var["_prepend"]:
797
798 match = True
799 if o:
800 for o2 in o.split("_"):
801 if not o2 in self.overrides:
802 match = False
803 if match:
804 value = r + value
805
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800806 parser = None
807 if expand or retparser:
808 parser = self.expandWithRefs(value, cachename)
809 if expand:
810 value = parser.value
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500811
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800812 if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500813 self.need_overrides()
814 for (r, o) in local_var["_remove"]:
815 match = True
816 if o:
817 for o2 in o.split("_"):
818 if not o2 in self.overrides:
819 match = False
820 if match:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800821 removes.add(r)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500822
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800823 if value and flag == "_content" and not parsing:
824 if removes and parser:
825 expanded_removes = {}
826 for r in removes:
827 expanded_removes[r] = self.expand(r).split()
828
829 parser.removes = set()
830 val = ""
831 for v in __whitespace_split__.split(parser.value):
832 skip = False
833 for r in removes:
834 if v in expanded_removes[r]:
835 parser.removes.add(r)
836 skip = True
837 if skip:
838 continue
839 val = val + v
840 parser.value = val
841 if expand:
842 value = parser.value
843
844 if parser:
845 self.expand_cache[cachename] = parser
846
847 if retparser:
848 return value, parser
849
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500850 return value
851
852 def delVarFlag(self, var, flag, **loginfo):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800853 self.expand_cache = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500854 if '_remote_data' in self.dict:
855 connector = self.dict["_remote_data"]["_content"]
856 res = connector.delVarFlag(var, flag)
857 if not res:
858 return
859
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500860 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500861 if not local_var:
862 return
863 if not var in self.dict:
864 self._makeShadowCopy(var)
865
866 if var in self.dict and flag in self.dict[var]:
867 loginfo['detail'] = ""
868 loginfo['op'] = 'delFlag'
869 loginfo['flag'] = flag
870 self.varhistory.record(**loginfo)
871
872 del self.dict[var][flag]
873
874 def appendVarFlag(self, var, flag, value, **loginfo):
875 loginfo['op'] = 'append'
876 loginfo['flag'] = flag
877 self.varhistory.record(**loginfo)
878 newvalue = (self.getVarFlag(var, flag, False) or "") + value
879 self.setVarFlag(var, flag, newvalue, ignore=True)
880
881 def prependVarFlag(self, var, flag, value, **loginfo):
882 loginfo['op'] = 'prepend'
883 loginfo['flag'] = flag
884 self.varhistory.record(**loginfo)
885 newvalue = value + (self.getVarFlag(var, flag, False) or "")
886 self.setVarFlag(var, flag, newvalue, ignore=True)
887
888 def setVarFlags(self, var, flags, **loginfo):
889 self.expand_cache = {}
890 infer_caller_details(loginfo)
891 if not var in self.dict:
892 self._makeShadowCopy(var)
893
894 for i in flags:
895 if i == "_content":
896 continue
897 loginfo['flag'] = i
898 loginfo['detail'] = flags[i]
899 self.varhistory.record(**loginfo)
900 self.dict[var][i] = flags[i]
901
902 def getVarFlags(self, var, expand = False, internalflags=False):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500903 local_var, _ = self._findVar(var)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500904 flags = {}
905
906 if local_var:
907 for i in local_var:
908 if i.startswith("_") and not internalflags:
909 continue
910 flags[i] = local_var[i]
911 if expand and i in expand:
912 flags[i] = self.expand(flags[i], var + "[" + i + "]")
913 if len(flags) == 0:
914 return None
915 return flags
916
917
918 def delVarFlags(self, var, **loginfo):
919 self.expand_cache = {}
920 if not var in self.dict:
921 self._makeShadowCopy(var)
922
923 if var in self.dict:
924 content = None
925
926 loginfo['op'] = 'delete flags'
927 self.varhistory.record(**loginfo)
928
929 # try to save the content
930 if "_content" in self.dict[var]:
931 content = self.dict[var]["_content"]
932 self.dict[var] = {}
933 self.dict[var]["_content"] = content
934 else:
935 del self.dict[var]
936
937 def createCopy(self):
938 """
939 Create a copy of self by setting _data to self
940 """
941 # we really want this to be a DataSmart...
942 data = DataSmart()
943 data.dict["_data"] = self.dict
944 data.varhistory = self.varhistory.copy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500945 data.varhistory.dataroot = data
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500946 data.inchistory = self.inchistory.copy()
947
948 data._tracking = self._tracking
949
950 data.overrides = None
951 data.overridevars = copy.copy(self.overridevars)
952 # Should really be a deepcopy but has heavy overhead.
953 # Instead, we're careful with writes.
954 data.overridedata = copy.copy(self.overridedata)
955
956 return data
957
958 def expandVarref(self, variable, parents=False):
959 """Find all references to variable in the data and expand it
960 in place, optionally descending to parent datastores."""
961
962 if parents:
963 keys = iter(self)
964 else:
965 keys = self.localkeys()
966
967 ref = '${%s}' % variable
968 value = self.getVar(variable, False)
969 for key in keys:
970 referrervalue = self.getVar(key, False)
971 if referrervalue and ref in referrervalue:
972 self.setVar(key, referrervalue.replace(ref, value))
973
974 def localkeys(self):
975 for key in self.dict:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500976 if key not in ['_data', '_remote_data']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500977 yield key
978
979 def __iter__(self):
980 deleted = set()
981 overrides = set()
982 def keylist(d):
983 klist = set()
984 for key in d:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500985 if key in ["_data", "_remote_data"]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500986 continue
987 if key in deleted:
988 continue
989 if key in overrides:
990 continue
991 if not d[key]:
992 deleted.add(key)
993 continue
994 klist.add(key)
995
996 if "_data" in d:
997 klist |= keylist(d["_data"])
998
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500999 if "_remote_data" in d:
1000 connector = d["_remote_data"]["_content"]
1001 for key in connector.getKeys():
1002 if key in deleted:
1003 continue
1004 klist.add(key)
1005
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001006 return klist
1007
1008 self.need_overrides()
1009 for var in self.overridedata:
1010 for (r, o) in self.overridedata[var]:
1011 if o in self.overridesset:
1012 overrides.add(var)
1013 elif "_" in o:
1014 if set(o.split("_")).issubset(self.overridesset):
1015 overrides.add(var)
1016
1017 for k in keylist(self.dict):
1018 yield k
1019
1020 for k in overrides:
1021 yield k
1022
1023 def __len__(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001024 return len(frozenset(iter(self)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001025
1026 def __getitem__(self, item):
1027 value = self.getVar(item, False)
1028 if value is None:
1029 raise KeyError(item)
1030 else:
1031 return value
1032
1033 def __setitem__(self, var, value):
1034 self.setVar(var, value)
1035
1036 def __delitem__(self, var):
1037 self.delVar(var)
1038
1039 def get_hash(self):
1040 data = {}
1041 d = self.createCopy()
1042 bb.data.expandKeys(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001043
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001044 config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001045 keys = set(key for key in iter(d) if not key.startswith("__"))
1046 for key in keys:
1047 if key in config_whitelist:
1048 continue
1049
1050 value = d.getVar(key, False) or ""
1051 data.update({key:value})
1052
1053 varflags = d.getVarFlags(key, internalflags = True)
1054 if not varflags:
1055 continue
1056 for f in varflags:
1057 if f == "_content":
1058 continue
1059 data.update({'%s[%s]' % (key, f):varflags[f]})
1060
1061 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
1062 bb_list = d.getVar(key, False) or []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001063 data.update({key:str(bb_list)})
1064
1065 if key == "__BBANONFUNCS":
1066 for i in bb_list:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001067 value = d.getVar(i, False) or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001068 data.update({i:value})
1069
1070 data_str = str([(k, data[k]) for k in sorted(data.keys())])
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001071 return hashlib.md5(data_str.encode("utf-8")).hexdigest()