blob: e047c217e5dff53e7354c87ed03892f3d4f16250 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
Patrick Williamsc124f4f2015-09-15 14:41:29 -05005import hashlib
6import logging
7import os
8import re
9import tempfile
Patrick Williamsc0f7c042017-02-23 20:41:17 -060010import pickle
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011import bb.data
Brad Bishop6e60e8b2018-02-01 10:27:11 -050012import difflib
13import simplediff
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050014from bb.checksum import FileChecksumCache
Brad Bishop08902b02019-08-20 09:16:51 -040015from bb import runqueue
Brad Bishopa34c0302019-09-23 22:34:48 -040016import hashserv
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017
18logger = logging.getLogger('BitBake.SigGen')
19
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020def init(d):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060021 siggens = [obj for obj in globals().values()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022 if type(obj) is type and issubclass(obj, SignatureGenerator)]
23
Brad Bishop6e60e8b2018-02-01 10:27:11 -050024 desired = d.getVar("BB_SIGNATURE_HANDLER") or "noop"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025 for sg in siggens:
26 if desired == sg.name:
27 return sg(d)
28 break
29 else:
30 logger.error("Invalid signature generator '%s', using default 'noop'\n"
31 "Available generators: %s", desired,
32 ', '.join(obj.name for obj in siggens))
33 return SignatureGenerator(d)
34
35class SignatureGenerator(object):
36 """
37 """
38 name = "noop"
39
40 def __init__(self, data):
Brad Bishop37a0e4d2017-12-04 01:01:44 -050041 self.basehash = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042 self.taskhash = {}
43 self.runtaskdeps = {}
44 self.file_checksum_values = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050045 self.taints = {}
Brad Bishop08902b02019-08-20 09:16:51 -040046 self.unitaskhashes = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047
48 def finalise(self, fn, d, varient):
49 return
50
Brad Bishop08902b02019-08-20 09:16:51 -040051 def get_unihash(self, tid):
52 return self.taskhash[tid]
Brad Bishop19323692019-04-05 15:28:33 -040053
Brad Bishop08902b02019-08-20 09:16:51 -040054 def get_taskhash(self, tid, deps, dataCache):
55 self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest()
56 return self.taskhash[tid]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050057
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050058 def writeout_file_checksum_cache(self):
59 """Write/update the file checksum cache onto disk"""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060 return
61
62 def stampfile(self, stampbase, file_name, taskname, extrainfo):
63 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
64
65 def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
66 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
67
68 def dump_sigtask(self, fn, task, stampbase, runtime):
69 return
70
71 def invalidate_task(self, task, d, fn):
72 bb.build.del_stamp(task, d, fn)
73
74 def dump_sigs(self, dataCache, options):
75 return
76
77 def get_taskdata(self):
Brad Bishop08902b02019-08-20 09:16:51 -040078 return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079
80 def set_taskdata(self, data):
Brad Bishop08902b02019-08-20 09:16:51 -040081 self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes = data
Patrick Williamsc124f4f2015-09-15 14:41:29 -050082
Brad Bishopd7bf8c12018-02-25 22:55:05 -050083 def reset(self, data):
84 self.__init__(data)
85
Brad Bishop08902b02019-08-20 09:16:51 -040086 def get_taskhashes(self):
87 return self.taskhash, self.unitaskhashes
88
89 def set_taskhashes(self, hashes):
90 self.taskhash, self.unitaskhashes = hashes
91
92 def save_unitaskhashes(self):
93 return
94
Brad Bishopa34c0302019-09-23 22:34:48 -040095 def set_setscene_tasks(self, setscene_tasks):
96 return
Brad Bishopd7bf8c12018-02-25 22:55:05 -050097
Patrick Williamsc124f4f2015-09-15 14:41:29 -050098class SignatureGeneratorBasic(SignatureGenerator):
99 """
100 """
101 name = "basic"
102
103 def __init__(self, data):
104 self.basehash = {}
105 self.taskhash = {}
106 self.taskdeps = {}
107 self.runtaskdeps = {}
108 self.file_checksum_values = {}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500109 self.taints = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500110 self.gendeps = {}
111 self.lookupcache = {}
Brad Bishopa34c0302019-09-23 22:34:48 -0400112 self.setscenetasks = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500113 self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500114 self.taskwhitelist = None
115 self.init_rundepcheck(data)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500116 checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500117 if checksum_cache_file:
118 self.checksum_cache = FileChecksumCache()
119 self.checksum_cache.init_cache(data, checksum_cache_file)
120 else:
121 self.checksum_cache = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122
Brad Bishop08902b02019-08-20 09:16:51 -0400123 self.unihash_cache = bb.cache.SimpleCache("1")
124 self.unitaskhashes = self.unihash_cache.init_cache(data, "bb_unihashes.dat", {})
125
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 def init_rundepcheck(self, data):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500127 self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 if self.taskwhitelist:
129 self.twl = re.compile(self.taskwhitelist)
130 else:
131 self.twl = None
132
133 def _build_data(self, fn, d):
134
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500135 ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500136 tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
137
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800138 taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500139
140 for task in tasklist:
Brad Bishop08902b02019-08-20 09:16:51 -0400141 tid = fn + ":" + task
142 if not ignore_mismatch and tid in self.basehash and self.basehash[tid] != basehash[tid]:
143 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (tid, self.basehash[tid], basehash[tid]))
Brad Bishopc342db32019-05-15 21:57:59 -0400144 bb.error("The following commands may help:")
145 cmd = "$ bitbake %s -c%s" % (d.getVar('PN'), task)
146 # Make sure sigdata is dumped before run printdiff
147 bb.error("%s -Snone" % cmd)
148 bb.error("Then:")
149 bb.error("%s -Sprintdiff\n" % cmd)
Brad Bishop08902b02019-08-20 09:16:51 -0400150 self.basehash[tid] = basehash[tid]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500151
152 self.taskdeps[fn] = taskdeps
153 self.gendeps[fn] = gendeps
154 self.lookupcache[fn] = lookupcache
155
156 return taskdeps
157
Brad Bishopa34c0302019-09-23 22:34:48 -0400158 def set_setscene_tasks(self, setscene_tasks):
159 self.setscenetasks = setscene_tasks
160
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 def finalise(self, fn, d, variant):
162
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600163 mc = d.getVar("__BBMULTICONFIG", False) or ""
164 if variant or mc:
165 fn = bb.cache.realfn2virtual(fn, variant, mc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500166
167 try:
168 taskdeps = self._build_data(fn, d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500169 except bb.parse.SkipRecipe:
170 raise
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500171 except:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500172 bb.warn("Error during finalise of %s" % fn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173 raise
174
175 #Slow but can be useful for debugging mismatched basehashes
176 #for task in self.taskdeps[fn]:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500177 # self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178
179 for task in taskdeps:
Brad Bishop08902b02019-08-20 09:16:51 -0400180 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500181
182 def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
183 # Return True if we should keep the dependency, False to drop it
184 # We only manipulate the dependencies for packages not in the whitelist
185 if self.twl and not self.twl.search(recipename):
186 # then process the actual dependencies
187 if self.twl.search(depname):
188 return False
189 return True
190
191 def read_taint(self, fn, task, stampbase):
192 taint = None
193 try:
194 with open(stampbase + '.' + task + '.taint', 'r') as taintf:
195 taint = taintf.read()
196 except IOError:
197 pass
198 return taint
199
Brad Bishop08902b02019-08-20 09:16:51 -0400200 def get_taskhash(self, tid, deps, dataCache):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800201
Brad Bishop08902b02019-08-20 09:16:51 -0400202 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800203
Brad Bishop08902b02019-08-20 09:16:51 -0400204 data = dataCache.basetaskhash[tid]
205 self.basehash[tid] = data
206 self.runtaskdeps[tid] = []
207 self.file_checksum_values[tid] = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500208 recipename = dataCache.pkg_fn[fn]
209 for dep in sorted(deps, key=clean_basepath):
Brad Bishop08902b02019-08-20 09:16:51 -0400210 (depmc, _, deptaskname, depfn) = bb.runqueue.split_tid_mcfn(dep)
211 if mc != depmc:
Andrew Geissler99467da2019-02-25 18:54:23 -0600212 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400213 depname = dataCache.pkg_fn[depfn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500214 if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
215 continue
216 if dep not in self.taskhash:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800217 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
Brad Bishop19323692019-04-05 15:28:33 -0400218 data = data + self.get_unihash(dep)
Brad Bishop08902b02019-08-20 09:16:51 -0400219 self.runtaskdeps[tid].append(dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500220
221 if task in dataCache.file_checksums[fn]:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500222 if self.checksum_cache:
223 checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename)
224 else:
225 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226 for (f,cs) in checksums:
Brad Bishop08902b02019-08-20 09:16:51 -0400227 self.file_checksum_values[tid].append((f,cs))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228 if cs:
229 data = data + cs
230
231 taskdep = dataCache.task_deps[fn]
232 if 'nostamp' in taskdep and task in taskdep['nostamp']:
233 # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
234 import uuid
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500235 taint = str(uuid.uuid4())
236 data = data + taint
Brad Bishop08902b02019-08-20 09:16:51 -0400237 self.taints[tid] = "nostamp:" + taint
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238
239 taint = self.read_taint(fn, task, dataCache.stamp[fn])
240 if taint:
241 data = data + taint
Brad Bishop08902b02019-08-20 09:16:51 -0400242 self.taints[tid] = taint
243 logger.warning("%s is tainted from a forced run" % tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244
Brad Bishop19323692019-04-05 15:28:33 -0400245 h = hashlib.sha256(data.encode("utf-8")).hexdigest()
Brad Bishop08902b02019-08-20 09:16:51 -0400246 self.taskhash[tid] = h
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500247 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
248 return h
249
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500250 def writeout_file_checksum_cache(self):
251 """Write/update the file checksum cache onto disk"""
252 if self.checksum_cache:
253 self.checksum_cache.save_extras()
254 self.checksum_cache.save_merge()
255 else:
256 bb.fetch2.fetcher_parse_save()
257 bb.fetch2.fetcher_parse_done()
258
Brad Bishop08902b02019-08-20 09:16:51 -0400259 def save_unitaskhashes(self):
260 self.unihash_cache.save(self.unitaskhashes)
261
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500262 def dump_sigtask(self, fn, task, stampbase, runtime):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500263
Brad Bishop08902b02019-08-20 09:16:51 -0400264 tid = fn + ":" + task
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500265 referencestamp = stampbase
266 if isinstance(runtime, str) and runtime.startswith("customfile"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500267 sigfile = stampbase
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500268 referencestamp = runtime[11:]
Brad Bishop08902b02019-08-20 09:16:51 -0400269 elif runtime and tid in self.taskhash:
270 sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[tid]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271 else:
Brad Bishop08902b02019-08-20 09:16:51 -0400272 sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[tid]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500273
274 bb.utils.mkdirhier(os.path.dirname(sigfile))
275
276 data = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500277 data['task'] = task
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278 data['basewhitelist'] = self.basewhitelist
279 data['taskwhitelist'] = self.taskwhitelist
280 data['taskdeps'] = self.taskdeps[fn][task]
Brad Bishop08902b02019-08-20 09:16:51 -0400281 data['basehash'] = self.basehash[tid]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500282 data['gendeps'] = {}
283 data['varvals'] = {}
284 data['varvals'][task] = self.lookupcache[fn][task]
285 for dep in self.taskdeps[fn][task]:
286 if dep in self.basewhitelist:
287 continue
288 data['gendeps'][dep] = self.gendeps[fn][dep]
289 data['varvals'][dep] = self.lookupcache[fn][dep]
290
Brad Bishop08902b02019-08-20 09:16:51 -0400291 if runtime and tid in self.taskhash:
292 data['runtaskdeps'] = self.runtaskdeps[tid]
293 data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[tid]]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500294 data['runtaskhashes'] = {}
295 for dep in data['runtaskdeps']:
Brad Bishop19323692019-04-05 15:28:33 -0400296 data['runtaskhashes'][dep] = self.get_unihash(dep)
Brad Bishop08902b02019-08-20 09:16:51 -0400297 data['taskhash'] = self.taskhash[tid]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500298
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500299 taint = self.read_taint(fn, task, referencestamp)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500300 if taint:
301 data['taint'] = taint
302
Brad Bishop08902b02019-08-20 09:16:51 -0400303 if runtime and tid in self.taints:
304 if 'nostamp:' in self.taints[tid]:
305 data['taint'] = self.taints[tid]
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500306
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500307 computed_basehash = calc_basehash(data)
Brad Bishop08902b02019-08-20 09:16:51 -0400308 if computed_basehash != self.basehash[tid]:
309 bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[tid], tid))
310 if runtime and tid in self.taskhash:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500311 computed_taskhash = calc_taskhash(data)
Brad Bishop08902b02019-08-20 09:16:51 -0400312 if computed_taskhash != self.taskhash[tid]:
313 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid))
314 sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash)
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500315
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
317 try:
318 with os.fdopen(fd, "wb") as stream:
319 p = pickle.dump(data, stream, -1)
320 stream.flush()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600321 os.chmod(tmpfile, 0o664)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322 os.rename(tmpfile, sigfile)
323 except (OSError, IOError) as err:
324 try:
325 os.unlink(tmpfile)
326 except OSError:
327 pass
328 raise err
329
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500330 def dump_sigfn(self, fn, dataCaches, options):
331 if fn in self.taskdeps:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500332 for task in self.taskdeps[fn]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600333 tid = fn + ":" + task
Brad Bishop08902b02019-08-20 09:16:51 -0400334 mc = bb.runqueue.mc_from_tid(tid)
335 if tid not in self.taskhash:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500336 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400337 if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]:
338 bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % tid)
339 bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid]))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600340 self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500341
342class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
343 name = "basichash"
344
Brad Bishop08902b02019-08-20 09:16:51 -0400345 def get_stampfile_hash(self, tid):
346 if tid in self.taskhash:
347 return self.taskhash[tid]
Brad Bishop19323692019-04-05 15:28:33 -0400348
349 # If task is not in basehash, then error
Brad Bishop08902b02019-08-20 09:16:51 -0400350 return self.basehash[tid]
Brad Bishop19323692019-04-05 15:28:33 -0400351
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352 def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
353 if taskname != "do_setscene" and taskname.endswith("_setscene"):
Brad Bishop08902b02019-08-20 09:16:51 -0400354 tid = fn + ":" + taskname[:-9]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 else:
Brad Bishop08902b02019-08-20 09:16:51 -0400356 tid = fn + ":" + taskname
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500357 if clean:
358 h = "*"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500359 else:
Brad Bishop08902b02019-08-20 09:16:51 -0400360 h = self.get_stampfile_hash(tid)
Brad Bishop19323692019-04-05 15:28:33 -0400361
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500362 return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
363
364 def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
365 return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800366
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500367 def invalidate_task(self, task, d, fn):
368 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
369 bb.build.write_taint(task, d, fn)
370
Brad Bishop08902b02019-08-20 09:16:51 -0400371class SignatureGeneratorUniHashMixIn(object):
372 def get_taskdata(self):
373 return (self.server, self.method) + super().get_taskdata()
374
375 def set_taskdata(self, data):
376 self.server, self.method = data[:2]
377 super().set_taskdata(data[2:])
378
Brad Bishopa34c0302019-09-23 22:34:48 -0400379 def client(self):
380 if getattr(self, '_client', None) is None:
381 self._client = hashserv.create_client(self.server)
382 return self._client
383
Brad Bishop08902b02019-08-20 09:16:51 -0400384 def __get_task_unihash_key(self, tid):
385 # TODO: The key only *needs* to be the taskhash, the tid is just
386 # convenient
387 return '%s:%s' % (tid, self.taskhash[tid])
388
389 def get_stampfile_hash(self, tid):
390 if tid in self.taskhash:
391 # If a unique hash is reported, use it as the stampfile hash. This
392 # ensures that if a task won't be re-run if the taskhash changes,
393 # but it would result in the same output hash
394 unihash = self.unitaskhashes.get(self.__get_task_unihash_key(tid), None)
395 if unihash is not None:
396 return unihash
397
398 return super().get_stampfile_hash(tid)
399
400 def set_unihash(self, tid, unihash):
401 self.unitaskhashes[self.__get_task_unihash_key(tid)] = unihash
402
403 def get_unihash(self, tid):
Brad Bishop08902b02019-08-20 09:16:51 -0400404 taskhash = self.taskhash[tid]
405
Brad Bishopa34c0302019-09-23 22:34:48 -0400406 # If its not a setscene task we can return
407 if self.setscenetasks and tid not in self.setscenetasks:
408 return taskhash
409
Brad Bishop08902b02019-08-20 09:16:51 -0400410 key = self.__get_task_unihash_key(tid)
411
412 # TODO: This cache can grow unbounded. It probably only needs to keep
413 # for each task
414 unihash = self.unitaskhashes.get(key, None)
415 if unihash is not None:
416 return unihash
417
418 # In the absence of being able to discover a unique hash from the
419 # server, make it be equivalent to the taskhash. The unique "hash" only
420 # really needs to be a unique string (not even necessarily a hash), but
421 # making it match the taskhash has a few advantages:
422 #
423 # 1) All of the sstate code that assumes hashes can be the same
424 # 2) It provides maximal compatibility with builders that don't use
425 # an equivalency server
426 # 3) The value is easy for multiple independent builders to derive the
427 # same unique hash from the same input. This means that if the
428 # independent builders find the same taskhash, but it isn't reported
429 # to the server, there is a better chance that they will agree on
430 # the unique hash.
431 unihash = taskhash
432
433 try:
Brad Bishopa34c0302019-09-23 22:34:48 -0400434 data = self.client().get_unihash(self.method, self.taskhash[tid])
435 if data:
436 unihash = data
Brad Bishop08902b02019-08-20 09:16:51 -0400437 # A unique hash equal to the taskhash is not very interesting,
438 # so it is reported it at debug level 2. If they differ, that
439 # is much more interesting, so it is reported at debug level 1
440 bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
441 else:
442 bb.debug(2, 'No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
Brad Bishopa34c0302019-09-23 22:34:48 -0400443 except hashserv.HashConnectionError as e:
444 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
Brad Bishop08902b02019-08-20 09:16:51 -0400445
446 self.unitaskhashes[key] = unihash
447 return unihash
448
449 def report_unihash(self, path, task, d):
Brad Bishop08902b02019-08-20 09:16:51 -0400450 import importlib
451
452 taskhash = d.getVar('BB_TASKHASH')
453 unihash = d.getVar('BB_UNIHASH')
454 report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
455 tempdir = d.getVar('T')
456 fn = d.getVar('BB_FILENAME')
457 key = fn + ':do_' + task + ':' + taskhash
458
459 # Sanity checks
460 cache_unihash = self.unitaskhashes.get(key, None)
461 if cache_unihash is None:
462 bb.fatal('%s not in unihash cache. Please report this error' % key)
463
464 if cache_unihash != unihash:
465 bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash))
466
467 sigfile = None
468 sigfile_name = "depsig.do_%s.%d" % (task, os.getpid())
469 sigfile_link = "depsig.do_%s" % task
470
471 try:
472 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
473
474 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
475
476 if "." in self.method:
477 (module, method) = self.method.rsplit('.', 1)
478 locs['method'] = getattr(importlib.import_module(module), method)
479 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
480 else:
481 outhash = bb.utils.better_eval(self.method + '(path, sigfile, task, d)', locs)
482
483 try:
Brad Bishopa34c0302019-09-23 22:34:48 -0400484 extra_data = {}
485
486 owner = d.getVar('SSTATE_HASHEQUIV_OWNER')
487 if owner:
488 extra_data['owner'] = owner
Brad Bishop08902b02019-08-20 09:16:51 -0400489
490 if report_taskdata:
491 sigfile.seek(0)
492
Brad Bishopa34c0302019-09-23 22:34:48 -0400493 extra_data['PN'] = d.getVar('PN')
494 extra_data['PV'] = d.getVar('PV')
495 extra_data['PR'] = d.getVar('PR')
496 extra_data['task'] = task
497 extra_data['outhash_siginfo'] = sigfile.read().decode('utf-8')
Brad Bishop08902b02019-08-20 09:16:51 -0400498
Brad Bishopa34c0302019-09-23 22:34:48 -0400499 data = self.client().report_unihash(taskhash, self.method, outhash, unihash, extra_data)
500 new_unihash = data['unihash']
Brad Bishop08902b02019-08-20 09:16:51 -0400501
502 if new_unihash != unihash:
503 bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
504 bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
505 else:
506 bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
Brad Bishopa34c0302019-09-23 22:34:48 -0400507 except hashserv.HashConnectionError as e:
508 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
Brad Bishop08902b02019-08-20 09:16:51 -0400509 finally:
510 if sigfile:
511 sigfile.close()
512
513 sigfile_link_path = os.path.join(tempdir, sigfile_link)
514 bb.utils.remove(sigfile_link_path)
515
516 try:
517 os.symlink(sigfile_name, sigfile_link_path)
518 except OSError:
519 pass
520
521
522#
523# Dummy class used for bitbake-selftest
524#
525class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureGeneratorBasicHash):
526 name = "TestEquivHash"
527 def init_rundepcheck(self, data):
528 super().init_rundepcheck(data)
Brad Bishopa34c0302019-09-23 22:34:48 -0400529 self.server = data.getVar('BB_HASHSERVE')
Brad Bishop08902b02019-08-20 09:16:51 -0400530 self.method = "sstate_output_hash"
531
532
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500533def dump_this_task(outfile, d):
534 import bb.parse
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500535 fn = d.getVar("BB_FILENAME")
536 task = "do_" + d.getVar("BB_CURRENTTASK")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500537 referencestamp = bb.build.stamp_internal(task, d, None, True)
538 bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500539
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500540def init_colors(enable_color):
541 """Initialise colour dict for passing to compare_sigfiles()"""
542 # First set up the colours
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800543 colors = {'color_title': '\033[1m',
544 'color_default': '\033[0m',
545 'color_add': '\033[0;32m',
546 'color_remove': '\033[0;31m',
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500547 }
548 # Leave all keys present but clear the values
549 if not enable_color:
550 for k in colors.keys():
551 colors[k] = ''
552 return colors
553
554def worddiff_str(oldstr, newstr, colors=None):
555 if not colors:
556 colors = init_colors(False)
557 diff = simplediff.diff(oldstr.split(' '), newstr.split(' '))
558 ret = []
559 for change, value in diff:
560 value = ' '.join(value)
561 if change == '=':
562 ret.append(value)
563 elif change == '+':
564 item = '{color_add}{{+{value}+}}{color_default}'.format(value=value, **colors)
565 ret.append(item)
566 elif change == '-':
567 item = '{color_remove}[-{value}-]{color_default}'.format(value=value, **colors)
568 ret.append(item)
569 whitespace_note = ''
570 if oldstr != newstr and ' '.join(oldstr.split()) == ' '.join(newstr.split()):
571 whitespace_note = ' (whitespace changed)'
572 return '"%s"%s' % (' '.join(ret), whitespace_note)
573
574def list_inline_diff(oldlist, newlist, colors=None):
575 if not colors:
576 colors = init_colors(False)
577 diff = simplediff.diff(oldlist, newlist)
578 ret = []
579 for change, value in diff:
580 value = ' '.join(value)
581 if change == '=':
582 ret.append("'%s'" % value)
583 elif change == '+':
584 item = '{color_add}+{value}{color_default}'.format(value=value, **colors)
585 ret.append(item)
586 elif change == '-':
587 item = '{color_remove}-{value}{color_default}'.format(value=value, **colors)
588 ret.append(item)
589 return '[%s]' % (', '.join(ret))
590
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500591def clean_basepath(a):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500592 mc = None
Brad Bishop15ae2502019-06-18 21:44:24 -0400593 if a.startswith("mc:"):
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500594 _, mc, a = a.split(":", 2)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500595 b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500596 if a.startswith("virtual:"):
597 b = b + ":" + a.rsplit(":", 1)[0]
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500598 if mc:
Brad Bishop15ae2502019-06-18 21:44:24 -0400599 b = b + ":mc:" + mc
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500600 return b
601
602def clean_basepaths(a):
603 b = {}
604 for x in a:
605 b[clean_basepath(x)] = a[x]
606 return b
607
608def clean_basepaths_list(a):
609 b = []
610 for x in a:
611 b.append(clean_basepath(x))
612 return b
613
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500614def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615 output = []
616
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500617 colors = init_colors(color)
618 def color_format(formatstr, **values):
619 """
620 Return colour formatted string.
621 NOTE: call with the format string, not an already formatted string
622 containing values (otherwise you could have trouble with { and }
623 characters)
624 """
625 if not formatstr.endswith('{color_default}'):
626 formatstr += '{color_default}'
627 # In newer python 3 versions you can pass both of these directly,
628 # but we only require 3.4 at the moment
629 formatparams = {}
630 formatparams.update(colors)
631 formatparams.update(values)
632 return formatstr.format(**formatparams)
633
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600634 with open(a, 'rb') as f:
635 p1 = pickle.Unpickler(f)
636 a_data = p1.load()
637 with open(b, 'rb') as f:
638 p2 = pickle.Unpickler(f)
639 b_data = p2.load()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500640
641 def dict_diff(a, b, whitelist=set()):
642 sa = set(a.keys())
643 sb = set(b.keys())
644 common = sa & sb
645 changed = set()
646 for i in common:
647 if a[i] != b[i] and i not in whitelist:
648 changed.add(i)
649 added = sb - sa
650 removed = sa - sb
651 return changed, added, removed
652
653 def file_checksums_diff(a, b):
654 from collections import Counter
655 # Handle old siginfo format
656 if isinstance(a, dict):
657 a = [(os.path.basename(f), cs) for f, cs in a.items()]
658 if isinstance(b, dict):
659 b = [(os.path.basename(f), cs) for f, cs in b.items()]
660 # Compare lists, ensuring we can handle duplicate filenames if they exist
661 removedcount = Counter(a)
662 removedcount.subtract(b)
663 addedcount = Counter(b)
664 addedcount.subtract(a)
665 added = []
666 for x in b:
667 if addedcount[x] > 0:
668 addedcount[x] -= 1
669 added.append(x)
670 removed = []
671 changed = []
672 for x in a:
673 if removedcount[x] > 0:
674 removedcount[x] -= 1
675 for y in added:
676 if y[0] == x[0]:
677 changed.append((x[0], x[1], y[1]))
678 added.remove(y)
679 break
680 else:
681 removed.append(x)
682 added = [x[0] for x in added]
683 removed = [x[0] for x in removed]
684 return changed, added, removed
685
686 if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500687 output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist']))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500688 if a_data['basewhitelist'] and b_data['basewhitelist']:
689 output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
690
691 if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500692 output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist']))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500693 if a_data['taskwhitelist'] and b_data['taskwhitelist']:
694 output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
695
696 if a_data['taskdeps'] != b_data['taskdeps']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500697 output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500698
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500699 if a_data['basehash'] != b_data['basehash'] and not collapsed:
700 output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500701
702 changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
703 if changed:
704 for dep in changed:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500705 output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500706 if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
707 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
708 if added:
709 for dep in added:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500710 output.append(color_format("{color_title}Dependency on variable %s was added") % (dep))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500711 if removed:
712 for dep in removed:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500713 output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500714
715
716 changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
717 if changed:
718 for dep in changed:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500719 oldval = a_data['varvals'][dep]
720 newval = b_data['varvals'][dep]
721 if newval and oldval and ('\n' in oldval or '\n' in newval):
722 diff = difflib.unified_diff(oldval.splitlines(), newval.splitlines(), lineterm='')
723 # Cut off the first two lines, since we aren't interested in
724 # the old/new filename (they are blank anyway in this case)
725 difflines = list(diff)[2:]
726 if color:
727 # Add colour to diff output
728 for i, line in enumerate(difflines):
729 if line.startswith('+'):
730 line = color_format('{color_add}{line}', line=line)
731 difflines[i] = line
732 elif line.startswith('-'):
733 line = color_format('{color_remove}{line}', line=line)
734 difflines[i] = line
735 output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff='\n'.join(difflines)))
736 elif newval and oldval and (' ' in oldval or ' ' in newval):
737 output.append(color_format("{color_title}Variable {var} value changed:{color_default}\n{diff}", var=dep, diff=worddiff_str(oldval, newval, colors)))
738 else:
739 output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500740
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600741 if not 'file_checksum_values' in a_data:
742 a_data['file_checksum_values'] = {}
743 if not 'file_checksum_values' in b_data:
744 b_data['file_checksum_values'] = {}
745
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500746 changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
747 if changed:
748 for f, old, new in changed:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500749 output.append(color_format("{color_title}Checksum for file %s changed{color_default} from %s to %s") % (f, old, new))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500750 if added:
751 for f in added:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500752 output.append(color_format("{color_title}Dependency on checksum of file %s was added") % (f))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500753 if removed:
754 for f in removed:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500755 output.append(color_format("{color_title}Dependency on checksum of file %s was removed") % (f))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500756
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600757 if not 'runtaskdeps' in a_data:
758 a_data['runtaskdeps'] = {}
759 if not 'runtaskdeps' in b_data:
760 b_data['runtaskdeps'] = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500761
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500762 if not collapsed:
763 if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']):
764 changed = ["Number of task dependencies changed"]
765 else:
766 changed = []
767 for idx, task in enumerate(a_data['runtaskdeps']):
768 a = a_data['runtaskdeps'][idx]
769 b = b_data['runtaskdeps'][idx]
770 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
771 changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500772
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500773 if changed:
774 clean_a = clean_basepaths_list(a_data['runtaskdeps'])
775 clean_b = clean_basepaths_list(b_data['runtaskdeps'])
776 if clean_a != clean_b:
777 output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
778 else:
779 output.append(color_format("{color_title}runtaskdeps changed:"))
780 output.append("\n".join(changed))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500781
782
783 if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
784 a = a_data['runtaskhashes']
785 b = b_data['runtaskhashes']
786 changed, added, removed = dict_diff(a, b)
787 if added:
788 for dep in added:
789 bdep_found = False
790 if removed:
791 for bdep in removed:
792 if b[dep] == a[bdep]:
793 #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
794 bdep_found = True
795 if not bdep_found:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500796 output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500797 if removed:
798 for dep in removed:
799 adep_found = False
800 if added:
801 for adep in added:
802 if b[adep] == a[dep]:
803 #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
804 adep_found = True
805 if not adep_found:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500806 output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500807 if changed:
808 for dep in changed:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500809 if not collapsed:
810 output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500811 if callable(recursecb):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500812 recout = recursecb(dep, a[dep], b[dep])
813 if recout:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500814 if collapsed:
815 output.extend(recout)
816 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800817 # If a dependent hash changed, might as well print the line above and then defer to the changes in
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500818 # that hash since in all likelyhood, they're the same changes this task also saw.
819 output = [output[-1]] + recout
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500820
821 a_taint = a_data.get('taint', None)
822 b_taint = b_data.get('taint', None)
823 if a_taint != b_taint:
Brad Bishop96ff1982019-08-19 13:50:42 -0400824 if a_taint and a_taint.startswith('nostamp:'):
Brad Bishopc342db32019-05-15 21:57:59 -0400825 a_taint = a_taint.replace('nostamp:', 'nostamp(uuid4):')
Brad Bishop96ff1982019-08-19 13:50:42 -0400826 if b_taint and b_taint.startswith('nostamp:'):
Brad Bishopc342db32019-05-15 21:57:59 -0400827 b_taint = b_taint.replace('nostamp:', 'nostamp(uuid4):')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500828 output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500829
830 return output
831
832
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500833def calc_basehash(sigdata):
834 task = sigdata['task']
835 basedata = sigdata['varvals'][task]
836
837 if basedata is None:
838 basedata = ''
839
840 alldeps = sigdata['taskdeps']
841 for dep in alldeps:
842 basedata = basedata + dep
843 val = sigdata['varvals'][dep]
844 if val is not None:
845 basedata = basedata + str(val)
846
Brad Bishop19323692019-04-05 15:28:33 -0400847 return hashlib.sha256(basedata.encode("utf-8")).hexdigest()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500848
849def calc_taskhash(sigdata):
850 data = sigdata['basehash']
851
852 for dep in sigdata['runtaskdeps']:
853 data = data + sigdata['runtaskhashes'][dep]
854
855 for c in sigdata['file_checksum_values']:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500856 if c[1]:
857 data = data + c[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500858
859 if 'taint' in sigdata:
860 if 'nostamp:' in sigdata['taint']:
861 data = data + sigdata['taint'][8:]
862 else:
863 data = data + sigdata['taint']
864
Brad Bishop19323692019-04-05 15:28:33 -0400865 return hashlib.sha256(data.encode("utf-8")).hexdigest()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500866
867
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500868def dump_sigfile(a):
869 output = []
870
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600871 with open(a, 'rb') as f:
872 p1 = pickle.Unpickler(f)
873 a_data = p1.load()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500874
875 output.append("basewhitelist: %s" % (a_data['basewhitelist']))
876
877 output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
878
879 output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
880
881 output.append("basehash: %s" % (a_data['basehash']))
882
883 for dep in a_data['gendeps']:
884 output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
885
886 for dep in a_data['varvals']:
887 output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
888
889 if 'runtaskdeps' in a_data:
890 output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
891
892 if 'file_checksum_values' in a_data:
893 output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
894
895 if 'runtaskhashes' in a_data:
896 for dep in a_data['runtaskhashes']:
897 output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
898
899 if 'taint' in a_data:
Brad Bishopc342db32019-05-15 21:57:59 -0400900 if a_data['taint'].startswith('nostamp:'):
901 msg = a_data['taint'].replace('nostamp:', 'nostamp(uuid4):')
902 else:
903 msg = a_data['taint']
904 output.append("Tainted (by forced/invalidated task): %s" % msg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500905
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500906 if 'task' in a_data:
907 computed_basehash = calc_basehash(a_data)
908 output.append("Computed base hash is %s and from file %s" % (computed_basehash, a_data['basehash']))
909 else:
910 output.append("Unable to compute base hash")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500911
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500912 computed_taskhash = calc_taskhash(a_data)
913 output.append("Computed task hash is %s" % computed_taskhash)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500914
915 return output