blob: 0352e45231c15fe0087107737c5f1af7a988e9c3 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001import hashlib
2import logging
3import os
4import re
5import tempfile
6import bb.data
7
8logger = logging.getLogger('BitBake.SigGen')
9
10try:
11 import cPickle as pickle
12except ImportError:
13 import pickle
14 logger.info('Importing cPickle failed. Falling back to a very slow implementation.')
15
16def init(d):
17 siggens = [obj for obj in globals().itervalues()
18 if type(obj) is type and issubclass(obj, SignatureGenerator)]
19
20 desired = d.getVar("BB_SIGNATURE_HANDLER", True) or "noop"
21 for sg in siggens:
22 if desired == sg.name:
23 return sg(d)
24 break
25 else:
26 logger.error("Invalid signature generator '%s', using default 'noop'\n"
27 "Available generators: %s", desired,
28 ', '.join(obj.name for obj in siggens))
29 return SignatureGenerator(d)
30
31class SignatureGenerator(object):
32 """
33 """
34 name = "noop"
35
36 def __init__(self, data):
37 self.taskhash = {}
38 self.runtaskdeps = {}
39 self.file_checksum_values = {}
40
41 def finalise(self, fn, d, varient):
42 return
43
44 def get_taskhash(self, fn, task, deps, dataCache):
45 return "0"
46
47 def set_taskdata(self, hashes, deps, checksum):
48 return
49
50 def stampfile(self, stampbase, file_name, taskname, extrainfo):
51 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
52
53 def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
54 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
55
56 def dump_sigtask(self, fn, task, stampbase, runtime):
57 return
58
59 def invalidate_task(self, task, d, fn):
60 bb.build.del_stamp(task, d, fn)
61
62 def dump_sigs(self, dataCache, options):
63 return
64
65 def get_taskdata(self):
66 return (self.runtaskdeps, self.taskhash, self.file_checksum_values)
67
68 def set_taskdata(self, data):
69 self.runtaskdeps, self.taskhash, self.file_checksum_values = data
70
71
72class SignatureGeneratorBasic(SignatureGenerator):
73 """
74 """
75 name = "basic"
76
77 def __init__(self, data):
78 self.basehash = {}
79 self.taskhash = {}
80 self.taskdeps = {}
81 self.runtaskdeps = {}
82 self.file_checksum_values = {}
Patrick Williamsf1e5d692016-03-30 15:21:19 -050083 self.taints = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084 self.gendeps = {}
85 self.lookupcache = {}
86 self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
87 self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split())
88 self.taskwhitelist = None
89 self.init_rundepcheck(data)
90
91 def init_rundepcheck(self, data):
92 self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None
93 if self.taskwhitelist:
94 self.twl = re.compile(self.taskwhitelist)
95 else:
96 self.twl = None
97
98 def _build_data(self, fn, d):
99
100 tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
101
102 taskdeps = {}
103 basehash = {}
104
105 for task in tasklist:
106 data = lookupcache[task]
107
108 if data is None:
109 bb.error("Task %s from %s seems to be empty?!" % (task, fn))
110 data = ''
111
112 gendeps[task] -= self.basewhitelist
113 newdeps = gendeps[task]
114 seen = set()
115 while newdeps:
116 nextdeps = newdeps
117 seen |= nextdeps
118 newdeps = set()
119 for dep in nextdeps:
120 if dep in self.basewhitelist:
121 continue
122 gendeps[dep] -= self.basewhitelist
123 newdeps |= gendeps[dep]
124 newdeps -= seen
125
126 alldeps = sorted(seen)
127 for dep in alldeps:
128 data = data + dep
129 var = lookupcache[dep]
130 if var is not None:
131 data = data + str(var)
132 self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest()
133 taskdeps[task] = alldeps
134
135 self.taskdeps[fn] = taskdeps
136 self.gendeps[fn] = gendeps
137 self.lookupcache[fn] = lookupcache
138
139 return taskdeps
140
141 def finalise(self, fn, d, variant):
142
143 if variant:
144 fn = "virtual:" + variant + ":" + fn
145
146 try:
147 taskdeps = self._build_data(fn, d)
148 except:
149 bb.note("Error during finalise of %s" % fn)
150 raise
151
152 #Slow but can be useful for debugging mismatched basehashes
153 #for task in self.taskdeps[fn]:
154 # self.dump_sigtask(fn, task, d.getVar("STAMP", True), False)
155
156 for task in taskdeps:
157 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
158
159 def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
160 # Return True if we should keep the dependency, False to drop it
161 # We only manipulate the dependencies for packages not in the whitelist
162 if self.twl and not self.twl.search(recipename):
163 # then process the actual dependencies
164 if self.twl.search(depname):
165 return False
166 return True
167
168 def read_taint(self, fn, task, stampbase):
169 taint = None
170 try:
171 with open(stampbase + '.' + task + '.taint', 'r') as taintf:
172 taint = taintf.read()
173 except IOError:
174 pass
175 return taint
176
177 def get_taskhash(self, fn, task, deps, dataCache):
178 k = fn + "." + task
179 data = dataCache.basetaskhash[k]
180 self.runtaskdeps[k] = []
181 self.file_checksum_values[k] = {}
182 recipename = dataCache.pkg_fn[fn]
183 for dep in sorted(deps, key=clean_basepath):
184 depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')]
185 if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
186 continue
187 if dep not in self.taskhash:
188 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep)
189 data = data + self.taskhash[dep]
190 self.runtaskdeps[k].append(dep)
191
192 if task in dataCache.file_checksums[fn]:
193 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
194 for (f,cs) in checksums:
195 self.file_checksum_values[k][f] = cs
196 if cs:
197 data = data + cs
198
199 taskdep = dataCache.task_deps[fn]
200 if 'nostamp' in taskdep and task in taskdep['nostamp']:
201 # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
202 import uuid
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500203 taint = str(uuid.uuid4())
204 data = data + taint
205 self.taints[k] = "nostamp:" + taint
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500206
207 taint = self.read_taint(fn, task, dataCache.stamp[fn])
208 if taint:
209 data = data + taint
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500210 self.taints[k] = taint
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500211 logger.warn("%s is tainted from a forced run" % k)
212
213 h = hashlib.md5(data).hexdigest()
214 self.taskhash[k] = h
215 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
216 return h
217
218 def dump_sigtask(self, fn, task, stampbase, runtime):
219 k = fn + "." + task
220 if runtime == "customfile":
221 sigfile = stampbase
222 elif runtime and k in self.taskhash:
223 sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k]
224 else:
225 sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k]
226
227 bb.utils.mkdirhier(os.path.dirname(sigfile))
228
229 data = {}
230 data['basewhitelist'] = self.basewhitelist
231 data['taskwhitelist'] = self.taskwhitelist
232 data['taskdeps'] = self.taskdeps[fn][task]
233 data['basehash'] = self.basehash[k]
234 data['gendeps'] = {}
235 data['varvals'] = {}
236 data['varvals'][task] = self.lookupcache[fn][task]
237 for dep in self.taskdeps[fn][task]:
238 if dep in self.basewhitelist:
239 continue
240 data['gendeps'][dep] = self.gendeps[fn][dep]
241 data['varvals'][dep] = self.lookupcache[fn][dep]
242
243 if runtime and k in self.taskhash:
244 data['runtaskdeps'] = self.runtaskdeps[k]
245 data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k].items()]
246 data['runtaskhashes'] = {}
247 for dep in data['runtaskdeps']:
248 data['runtaskhashes'][dep] = self.taskhash[dep]
249
250 taint = self.read_taint(fn, task, stampbase)
251 if taint:
252 data['taint'] = taint
253
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500254 if runtime and k in self.taints:
255 if 'nostamp:' in self.taints[k]:
256 data['taint'] = self.taints[k]
257
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500258 fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
259 try:
260 with os.fdopen(fd, "wb") as stream:
261 p = pickle.dump(data, stream, -1)
262 stream.flush()
263 os.chmod(tmpfile, 0664)
264 os.rename(tmpfile, sigfile)
265 except (OSError, IOError) as err:
266 try:
267 os.unlink(tmpfile)
268 except OSError:
269 pass
270 raise err
271
272 def dump_sigs(self, dataCache, options):
273 for fn in self.taskdeps:
274 for task in self.taskdeps[fn]:
275 k = fn + "." + task
276 if k not in self.taskhash:
277 continue
278 if dataCache.basetaskhash[k] != self.basehash[k]:
279 bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k)
280 bb.error("The mismatched hashes were %s and %s" % (dataCache.basetaskhash[k], self.basehash[k]))
281 self.dump_sigtask(fn, task, dataCache.stamp[fn], True)
282
283class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
284 name = "basichash"
285
286 def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
287 if taskname != "do_setscene" and taskname.endswith("_setscene"):
288 k = fn + "." + taskname[:-9]
289 else:
290 k = fn + "." + taskname
291 if clean:
292 h = "*"
293 elif k in self.taskhash:
294 h = self.taskhash[k]
295 else:
296 # If k is not in basehash, then error
297 h = self.basehash[k]
298 return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
299
300 def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
301 return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
302
303 def invalidate_task(self, task, d, fn):
304 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
305 bb.build.write_taint(task, d, fn)
306
307def dump_this_task(outfile, d):
308 import bb.parse
309 fn = d.getVar("BB_FILENAME", True)
310 task = "do_" + d.getVar("BB_CURRENTTASK", True)
311 bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile")
312
313def clean_basepath(a):
314 b = a.rsplit("/", 2)[1] + a.rsplit("/", 2)[2]
315 if a.startswith("virtual:"):
316 b = b + ":" + a.rsplit(":", 1)[0]
317 return b
318
319def clean_basepaths(a):
320 b = {}
321 for x in a:
322 b[clean_basepath(x)] = a[x]
323 return b
324
325def clean_basepaths_list(a):
326 b = []
327 for x in a:
328 b.append(clean_basepath(x))
329 return b
330
331def compare_sigfiles(a, b, recursecb = None):
332 output = []
333
334 p1 = pickle.Unpickler(open(a, "rb"))
335 a_data = p1.load()
336 p2 = pickle.Unpickler(open(b, "rb"))
337 b_data = p2.load()
338
339 def dict_diff(a, b, whitelist=set()):
340 sa = set(a.keys())
341 sb = set(b.keys())
342 common = sa & sb
343 changed = set()
344 for i in common:
345 if a[i] != b[i] and i not in whitelist:
346 changed.add(i)
347 added = sb - sa
348 removed = sa - sb
349 return changed, added, removed
350
351 def file_checksums_diff(a, b):
352 from collections import Counter
353 # Handle old siginfo format
354 if isinstance(a, dict):
355 a = [(os.path.basename(f), cs) for f, cs in a.items()]
356 if isinstance(b, dict):
357 b = [(os.path.basename(f), cs) for f, cs in b.items()]
358 # Compare lists, ensuring we can handle duplicate filenames if they exist
359 removedcount = Counter(a)
360 removedcount.subtract(b)
361 addedcount = Counter(b)
362 addedcount.subtract(a)
363 added = []
364 for x in b:
365 if addedcount[x] > 0:
366 addedcount[x] -= 1
367 added.append(x)
368 removed = []
369 changed = []
370 for x in a:
371 if removedcount[x] > 0:
372 removedcount[x] -= 1
373 for y in added:
374 if y[0] == x[0]:
375 changed.append((x[0], x[1], y[1]))
376 added.remove(y)
377 break
378 else:
379 removed.append(x)
380 added = [x[0] for x in added]
381 removed = [x[0] for x in removed]
382 return changed, added, removed
383
384 if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
385 output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist']))
386 if a_data['basewhitelist'] and b_data['basewhitelist']:
387 output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
388
389 if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
390 output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
391 if a_data['taskwhitelist'] and b_data['taskwhitelist']:
392 output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
393
394 if a_data['taskdeps'] != b_data['taskdeps']:
395 output.append("Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
396
397 if a_data['basehash'] != b_data['basehash']:
398 output.append("basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash']))
399
400 changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
401 if changed:
402 for dep in changed:
403 output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
404 if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
405 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
406 if added:
407 for dep in added:
408 output.append("Dependency on variable %s was added" % (dep))
409 if removed:
410 for dep in removed:
411 output.append("Dependency on Variable %s was removed" % (dep))
412
413
414 changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
415 if changed:
416 for dep in changed:
417 output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
418
419 changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
420 if changed:
421 for f, old, new in changed:
422 output.append("Checksum for file %s changed from %s to %s" % (f, old, new))
423 if added:
424 for f in added:
425 output.append("Dependency on checksum of file %s was added" % (f))
426 if removed:
427 for f in removed:
428 output.append("Dependency on checksum of file %s was removed" % (f))
429
430
431 if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']):
432 changed = ["Number of task dependencies changed"]
433 else:
434 changed = []
435 for idx, task in enumerate(a_data['runtaskdeps']):
436 a = a_data['runtaskdeps'][idx]
437 b = b_data['runtaskdeps'][idx]
438 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b]:
439 changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b]))
440
441 if changed:
442 output.append("runtaskdeps changed from %s to %s" % (clean_basepaths_list(a_data['runtaskdeps']), clean_basepaths_list(b_data['runtaskdeps'])))
443 output.append("\n".join(changed))
444
445
446 if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
447 a = a_data['runtaskhashes']
448 b = b_data['runtaskhashes']
449 changed, added, removed = dict_diff(a, b)
450 if added:
451 for dep in added:
452 bdep_found = False
453 if removed:
454 for bdep in removed:
455 if b[dep] == a[bdep]:
456 #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
457 bdep_found = True
458 if not bdep_found:
459 output.append("Dependency on task %s was added with hash %s" % (clean_basepath(dep), b[dep]))
460 if removed:
461 for dep in removed:
462 adep_found = False
463 if added:
464 for adep in added:
465 if b[adep] == a[dep]:
466 #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
467 adep_found = True
468 if not adep_found:
469 output.append("Dependency on task %s was removed with hash %s" % (clean_basepath(dep), a[dep]))
470 if changed:
471 for dep in changed:
472 output.append("Hash for dependent task %s changed from %s to %s" % (clean_basepath(dep), a[dep], b[dep]))
473 if callable(recursecb):
474 # If a dependent hash changed, might as well print the line above and then defer to the changes in
475 # that hash since in all likelyhood, they're the same changes this task also saw.
476 recout = recursecb(dep, a[dep], b[dep])
477 if recout:
478 output = [output[-1]] + recout
479
480 a_taint = a_data.get('taint', None)
481 b_taint = b_data.get('taint', None)
482 if a_taint != b_taint:
483 output.append("Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint))
484
485 return output
486
487
488def dump_sigfile(a):
489 output = []
490
491 p1 = pickle.Unpickler(open(a, "rb"))
492 a_data = p1.load()
493
494 output.append("basewhitelist: %s" % (a_data['basewhitelist']))
495
496 output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
497
498 output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
499
500 output.append("basehash: %s" % (a_data['basehash']))
501
502 for dep in a_data['gendeps']:
503 output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
504
505 for dep in a_data['varvals']:
506 output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
507
508 if 'runtaskdeps' in a_data:
509 output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
510
511 if 'file_checksum_values' in a_data:
512 output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
513
514 if 'runtaskhashes' in a_data:
515 for dep in a_data['runtaskhashes']:
516 output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
517
518 if 'taint' in a_data:
519 output.append("Tainted (by forced/invalidated task): %s" % a_data['taint'])
520
521 data = a_data['basehash']
522 for dep in a_data['runtaskdeps']:
523 data = data + a_data['runtaskhashes'][dep]
524
525 for c in a_data['file_checksum_values']:
526 data = data + c[1]
527
528 if 'taint' in a_data:
529 data = data + a_data['taint']
530
531 h = hashlib.md5(data).hexdigest()
532 output.append("Computed Hash is %s" % h)
533
534 return output