blob: ae7ef14453fda7166825a76b439c2e46cfaa22f3 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
Patrick Williams92b42cb2022-09-03 06:53:57 -05002# Copyright OpenEmbedded Contributors
3#
Brad Bishopc342db32019-05-15 21:57:59 -04004# SPDX-License-Identifier: GPL-2.0-only
5#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006import bb.siggen
Andrew Geisslerd25ed322020-06-27 00:28:28 -05007import bb.runqueue
Brad Bishop316dfdd2018-06-25 12:45:53 -04008import oe
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009
Andrew Geisslerd25ed322020-06-27 00:28:28 -050010def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011 # Return True if we should keep the dependency, False to drop it
12 def isNative(x):
13 return x.endswith("-native")
14 def isCross(x):
15 return "-cross-" in x
16 def isNativeSDK(x):
17 return x.startswith("nativesdk-")
Andrew Geisslerd25ed322020-06-27 00:28:28 -050018 def isKernel(mc, fn):
19 inherits = " ".join(dataCaches[mc].inherits[fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020 return inherits.find("/module-base.bbclass") != -1 or inherits.find("/linux-kernel-base.bbclass") != -1
Andrew Geisslerd25ed322020-06-27 00:28:28 -050021 def isPackageGroup(mc, fn):
22 inherits = " ".join(dataCaches[mc].inherits[fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023 return "/packagegroup.bbclass" in inherits
Andrew Geisslerd25ed322020-06-27 00:28:28 -050024 def isAllArch(mc, fn):
25 inherits = " ".join(dataCaches[mc].inherits[fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026 return "/allarch.bbclass" in inherits
Andrew Geisslerd25ed322020-06-27 00:28:28 -050027 def isImage(mc, fn):
28 return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn])
Patrick Williamsdb4c27e2022-08-05 08:10:29 -050029 def isSPDXTask(task):
30 return task in ("do_create_spdx", "do_create_runtime_spdx")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050031
Andrew Geisslerd25ed322020-06-27 00:28:28 -050032 depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep)
33 mc, _ = bb.runqueue.split_mc(fn)
34
Patrick Williams7784c422022-11-17 07:29:11 -060035 # We can skip the rm_work task signature to avoid running the task
36 # when we remove some tasks from the dependencie chain
37 # i.e INHERIT:remove = "create-spdx" will trigger the do_rm_work
38 if task == "do_rm_work":
39 return False
40
Patrick Williamsdb4c27e2022-08-05 08:10:29 -050041 # Keep all dependencies between SPDX tasks in the signature. SPDX documents
42 # are linked together by hashes, which means if a dependent document changes,
43 # all downstream documents must be re-written (even if they are "safe"
44 # dependencies).
45 if isSPDXTask(task) and isSPDXTask(deptaskname):
46 return True
47
Andrew Geisslerd25ed322020-06-27 00:28:28 -050048 # (Almost) always include our own inter-task dependencies (unless it comes
49 # from a mcdepends). The exception is the special
50 # do_kernel_configme->do_unpack_and_patch dependency from archiver.bbclass.
51 if recipename == depname and depmc == mc:
52 if task == "do_kernel_configme" and deptaskname == "do_unpack_and_patch":
Brad Bishop6e60e8b2018-02-01 10:27:11 -050053 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054 return True
55
Patrick Williamsc124f4f2015-09-15 14:41:29 -050056 # Exclude well defined recipe->dependency
57 if "%s->%s" % (recipename, depname) in siggen.saferecipedeps:
58 return False
59
Brad Bishop316dfdd2018-06-25 12:45:53 -040060 # Check for special wildcard
61 if "*->%s" % depname in siggen.saferecipedeps and recipename != depname:
62 return False
63
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064 # Don't change native/cross/nativesdk recipe dependencies any further
65 if isNative(recipename) or isCross(recipename) or isNativeSDK(recipename):
66 return True
67
68 # Only target packages beyond here
69
70 # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes
Andrew Geisslerd25ed322020-06-27 00:28:28 -050071 if isPackageGroup(mc, fn) and isAllArch(mc, fn) and not isNative(depname):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080072 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050073
74 # Exclude well defined machine specific configurations which don't change ABI
Andrew Geisslerd25ed322020-06-27 00:28:28 -050075 if depname in siggen.abisaferecipes and not isImage(mc, fn):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 return False
77
78 # Kernel modules are well namespaced. We don't want to depend on the kernel's checksum
Patrick Williams213cb262021-08-07 19:21:33 -050079 # if we're just doing an RRECOMMENDS:xxx = "kernel-module-*", not least because the checksum
Patrick Williamsc124f4f2015-09-15 14:41:29 -050080 # is machine specific.
81 # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes)
82 # and we reccomend a kernel-module, we exclude the dependency.
Andrew Geisslerd25ed322020-06-27 00:28:28 -050083 if dataCaches and isKernel(depmc, depmcfn) and not isKernel(mc, fn):
84 for pkg in dataCaches[mc].runrecs[fn]:
85 if " ".join(dataCaches[mc].runrecs[fn][pkg]).find("kernel-module-") != -1:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050086 return False
87
88 # Default to keep dependencies
89 return True
90
91def sstate_lockedsigs(d):
92 sigs = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -050093 types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050094 for t in types:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050095 siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t
Brad Bishop6e60e8b2018-02-01 10:27:11 -050096 lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050097 for ls in lockedsigs:
98 pn, task, h = ls.split(":", 2)
99 if pn not in sigs:
100 sigs[pn] = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500101 sigs[pn][task] = [h, siggen_lockedsigs_var]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 return sigs
103
Brad Bishop00e122a2019-10-05 11:10:57 -0400104class SignatureGeneratorOEBasicHashMixIn(object):
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500105 supports_multiconfig_datacaches = True
106
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500107 def init_rundepcheck(self, data):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500108 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split()
109 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500110 self.lockedsigs = sstate_lockedsigs(data)
111 self.lockedhashes = {}
112 self.lockedpnmap = {}
113 self.lockedhashfn = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500114 self.machine = data.getVar("MACHINE")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 self.mismatch_msgs = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500116 self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500117 "").split()
118 self.unlockedrecipes = { k: "" for k in self.unlockedrecipes }
Andrew Geissler82c905d2020-04-13 13:39:40 -0500119 self._internal = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120 pass
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500121
122 def tasks_resolved(self, virtmap, virtpnmap, dataCache):
123 # Translate virtual/xxx entries to PN values
124 newabisafe = []
125 for a in self.abisaferecipes:
126 if a in virtpnmap:
127 newabisafe.append(virtpnmap[a])
128 else:
129 newabisafe.append(a)
130 self.abisaferecipes = newabisafe
131 newsafedeps = []
132 for a in self.saferecipedeps:
133 a1, a2 = a.split("->")
134 if a1 in virtpnmap:
135 a1 = virtpnmap[a1]
136 if a2 in virtpnmap:
137 a2 = virtpnmap[a2]
138 newsafedeps.append(a1 + "->" + a2)
139 self.saferecipedeps = newsafedeps
140
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500141 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None):
142 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143
144 def get_taskdata(self):
Brad Bishop00e122a2019-10-05 11:10:57 -0400145 return (self.lockedpnmap, self.lockedhashfn, self.lockedhashes) + super().get_taskdata()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500146
147 def set_taskdata(self, data):
Brad Bishop00e122a2019-10-05 11:10:57 -0400148 self.lockedpnmap, self.lockedhashfn, self.lockedhashes = data[:3]
149 super().set_taskdata(data[3:])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500150
151 def dump_sigs(self, dataCache, options):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600152 sigfile = os.getcwd() + "/locked-sigs.inc"
153 bb.plain("Writing locked sigs to %s" % sigfile)
154 self.dump_lockedsigs(sigfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500155 return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
156
Andrew Geissler82c905d2020-04-13 13:39:40 -0500157
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500158 def get_taskhash(self, tid, deps, dataCaches):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500159 if tid in self.lockedhashes:
160 if self.lockedhashes[tid]:
161 return self.lockedhashes[tid]
162 else:
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500163 return super().get_taskhash(tid, deps, dataCaches)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500164
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500165 h = super().get_taskhash(tid, deps, dataCaches)
Brad Bishop08902b02019-08-20 09:16:51 -0400166
167 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500168
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500169 recipename = dataCaches[mc].pkg_fn[fn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 self.lockedpnmap[fn] = recipename
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500171 self.lockedhashfn[fn] = dataCaches[mc].hashfn[fn]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500172
173 unlocked = False
174 if recipename in self.unlockedrecipes:
175 unlocked = True
176 else:
177 def recipename_from_dep(dep):
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500178 (depmc, _, _, depfn) = bb.runqueue.split_tid_mcfn(dep)
179 return dataCaches[depmc].pkg_fn[depfn]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500180
181 # If any unlocked recipe is in the direct dependencies then the
182 # current recipe should be unlocked as well.
Brad Bishop08902b02019-08-20 09:16:51 -0400183 depnames = [ recipename_from_dep(x) for x in deps if mc == bb.runqueue.mc_from_tid(x)]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500184 if any(x in y for y in depnames for x in self.unlockedrecipes):
185 self.unlockedrecipes[recipename] = ''
186 unlocked = True
187
188 if not unlocked and recipename in self.lockedsigs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500189 if task in self.lockedsigs[recipename]:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500190 h_locked = self.lockedsigs[recipename][task][0]
191 var = self.lockedsigs[recipename][task][1]
Brad Bishop08902b02019-08-20 09:16:51 -0400192 self.lockedhashes[tid] = h_locked
Andrew Geissler82c905d2020-04-13 13:39:40 -0500193 self._internal = True
194 unihash = self.get_unihash(tid)
195 self._internal = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196 #bb.warn("Using %s %s %s" % (recipename, task, h))
197
Brad Bishop00e122a2019-10-05 11:10:57 -0400198 if h != h_locked and h_locked != unihash:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500199 self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s'
200 % (recipename, task, h, h_locked, var))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500201
202 return h_locked
Andrew Geissler82c905d2020-04-13 13:39:40 -0500203
204 self.lockedhashes[tid] = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500205 #bb.warn("%s %s %s" % (recipename, task, h))
206 return h
207
Andrew Geissler82c905d2020-04-13 13:39:40 -0500208 def get_stampfile_hash(self, tid):
209 if tid in self.lockedhashes and self.lockedhashes[tid]:
210 return self.lockedhashes[tid]
211 return super().get_stampfile_hash(tid)
212
Brad Bishop00e122a2019-10-05 11:10:57 -0400213 def get_unihash(self, tid):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500214 if tid in self.lockedhashes and self.lockedhashes[tid] and not self._internal:
Brad Bishop00e122a2019-10-05 11:10:57 -0400215 return self.lockedhashes[tid]
216 return super().get_unihash(tid)
217
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218 def dump_sigtask(self, fn, task, stampbase, runtime):
Brad Bishop08902b02019-08-20 09:16:51 -0400219 tid = fn + ":" + task
Andrew Geissler82c905d2020-04-13 13:39:40 -0500220 if tid in self.lockedhashes and self.lockedhashes[tid]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221 return
222 super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
223
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600224 def dump_lockedsigs(self, sigfile, taskfilter=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225 types = {}
Brad Bishop08902b02019-08-20 09:16:51 -0400226 for tid in self.runtaskdeps:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500227 if taskfilter:
Brad Bishop08902b02019-08-20 09:16:51 -0400228 if not tid in taskfilter:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500229 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400230 fn = bb.runqueue.fn_from_tid(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231 t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
232 t = 't-' + t.replace('_', '-')
233 if t not in types:
234 types[t] = []
Brad Bishop08902b02019-08-20 09:16:51 -0400235 types[t].append(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500236
237 with open(sigfile, "w") as f:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500238 l = sorted(types)
239 for t in l:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
241 types[t].sort()
Brad Bishop08902b02019-08-20 09:16:51 -0400242 sortedtid = sorted(types[t], key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)])
243 for tid in sortedtid:
244 (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
245 if tid not in self.taskhash:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500246 continue
Brad Bishop00e122a2019-10-05 11:10:57 -0400247 f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.get_unihash(tid) + " \\\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248 f.write(' "\n')
Patrick Williams213cb262021-08-07 19:21:33 -0500249 f.write('SIGGEN_LOCKEDSIGS_TYPES:%s = "%s"' % (self.machine, " ".join(l)))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500250
Andrew Geissler09036742021-06-25 14:25:14 -0500251 def dump_siglist(self, sigfile, path_prefix_strip=None):
252 def strip_fn(fn):
253 nonlocal path_prefix_strip
254 if not path_prefix_strip:
255 return fn
256
257 fn_exp = fn.split(":")
258 if fn_exp[-1].startswith(path_prefix_strip):
259 fn_exp[-1] = fn_exp[-1][len(path_prefix_strip):]
260
261 return ":".join(fn_exp)
262
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500263 with open(sigfile, "w") as f:
264 tasks = []
265 for taskitem in self.taskhash:
Brad Bishop08902b02019-08-20 09:16:51 -0400266 (fn, task) = taskitem.rsplit(":", 1)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500267 pn = self.lockedpnmap[fn]
Andrew Geissler09036742021-06-25 14:25:14 -0500268 tasks.append((pn, task, strip_fn(fn), self.taskhash[taskitem]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500269 for (pn, task, fn, taskhash) in sorted(tasks):
Brad Bishop08902b02019-08-20 09:16:51 -0400270 f.write('%s:%s %s %s\n' % (pn, task, fn, taskhash))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271
Brad Bishop08902b02019-08-20 09:16:51 -0400272 def checkhashes(self, sq_data, missed, found, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500273 warn_msgs = []
274 error_msgs = []
275 sstate_missing_msgs = []
276
Brad Bishop08902b02019-08-20 09:16:51 -0400277 for tid in sq_data['hash']:
278 if tid not in found:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500279 for pn in self.lockedsigs:
Brad Bishop08902b02019-08-20 09:16:51 -0400280 taskname = bb.runqueue.taskname_from_tid(tid)
281 if sq_data['hash'][tid] in iter(self.lockedsigs[pn].values()):
282 if taskname == 'do_shared_workdir':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500283 continue
284 sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
Brad Bishop08902b02019-08-20 09:16:51 -0400285 % (pn, taskname, sq_data['hash'][tid]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500286
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500287 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500288 if checklevel == 'warn':
289 warn_msgs += self.mismatch_msgs
290 elif checklevel == 'error':
291 error_msgs += self.mismatch_msgs
292
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500293 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500294 if checklevel == 'warn':
295 warn_msgs += sstate_missing_msgs
296 elif checklevel == 'error':
297 error_msgs += sstate_missing_msgs
298
299 if warn_msgs:
300 bb.warn("\n".join(warn_msgs))
301 if error_msgs:
302 bb.fatal("\n".join(error_msgs))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500303
Brad Bishop00e122a2019-10-05 11:10:57 -0400304class SignatureGeneratorOEBasicHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorBasicHash):
305 name = "OEBasicHash"
306
307class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorUniHashMixIn, bb.siggen.SignatureGeneratorBasicHash):
Brad Bishop19323692019-04-05 15:28:33 -0400308 name = "OEEquivHash"
309
310 def init_rundepcheck(self, data):
311 super().init_rundepcheck(data)
Brad Bishopa34c0302019-09-23 22:34:48 -0400312 self.server = data.getVar('BB_HASHSERVE')
Brad Bishop08902b02019-08-20 09:16:51 -0400313 if not self.server:
Brad Bishopa34c0302019-09-23 22:34:48 -0400314 bb.fatal("OEEquivHash requires BB_HASHSERVE to be set")
Brad Bishop19323692019-04-05 15:28:33 -0400315 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD')
Brad Bishop08902b02019-08-20 09:16:51 -0400316 if not self.method:
317 bb.fatal("OEEquivHash requires SSTATE_HASHEQUIV_METHOD to be set")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318
319# Insert these classes into siggen's namespace so it can see and select them
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash
Brad Bishop19323692019-04-05 15:28:33 -0400321bb.siggen.SignatureGeneratorOEEquivHash = SignatureGeneratorOEEquivHash
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322
323
324def find_siginfo(pn, taskname, taskhashlist, d):
325 """ Find signature data files for comparison purposes """
326
327 import fnmatch
328 import glob
329
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500330 if not taskname:
331 # We have to derive pn and taskname
332 key = pn
Brad Bishop08902b02019-08-20 09:16:51 -0400333 splitit = key.split('.bb:')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334 taskname = splitit[1]
335 pn = os.path.basename(splitit[0]).split('_')[0]
336 if key.startswith('virtual:native:'):
337 pn = pn + '-native'
338
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500339 hashfiles = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500340 filedates = {}
341
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500342 def get_hashval(siginfo):
343 if siginfo.endswith('.siginfo'):
344 return siginfo.rpartition(':')[2].partition('_')[0]
345 else:
346 return siginfo.rpartition('.')[2]
347
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348 # First search in stamps dir
349 localdata = d.createCopy()
350 localdata.setVar('MULTIMACH_TARGET_SYS', '*')
351 localdata.setVar('PN', pn)
352 localdata.setVar('PV', '*')
353 localdata.setVar('PR', '*')
354 localdata.setVar('EXTENDPE', '')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500355 stamp = localdata.getVar('STAMP')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500356 if pn.startswith("gcc-source"):
357 # gcc-source shared workdir is a special case :(
358 stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}")
359
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360 filespec = '%s.%s.sigdata.*' % (stamp, taskname)
361 foundall = False
362 import glob
363 for fullpath in glob.glob(filespec):
364 match = False
365 if taskhashlist:
366 for taskhash in taskhashlist:
367 if fullpath.endswith('.%s' % taskhash):
368 hashfiles[taskhash] = fullpath
369 if len(hashfiles) == len(taskhashlist):
370 foundall = True
371 break
372 else:
373 try:
374 filedates[fullpath] = os.stat(fullpath).st_mtime
375 except OSError:
376 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500377 hashval = get_hashval(fullpath)
378 hashfiles[hashval] = fullpath
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500379
380 if not taskhashlist or (len(filedates) < 2 and not foundall):
381 # That didn't work, look in sstate-cache
Brad Bishop19323692019-04-05 15:28:33 -0400382 hashes = taskhashlist or ['?' * 64]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500383 localdata = bb.data.createCopy(d)
384 for hashval in hashes:
385 localdata.setVar('PACKAGE_ARCH', '*')
386 localdata.setVar('TARGET_VENDOR', '*')
387 localdata.setVar('TARGET_OS', '*')
388 localdata.setVar('PN', pn)
389 localdata.setVar('PV', '*')
390 localdata.setVar('PR', '*')
391 localdata.setVar('BB_TASKHASH', hashval)
Patrick Williams03907ee2022-05-01 06:28:52 -0500392 localdata.setVar('SSTATE_CURRTASK', taskname[3:])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500393 swspec = localdata.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394 if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec:
395 localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}')
396 elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn:
397 localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williams03907ee2022-05-01 06:28:52 -0500398 filespec = '%s.siginfo' % localdata.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500399
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500400 matchedfiles = glob.glob(filespec)
401 for fullpath in matchedfiles:
402 actual_hashval = get_hashval(fullpath)
403 if actual_hashval in hashfiles:
404 continue
405 hashfiles[hashval] = fullpath
406 if not taskhashlist:
407 try:
408 filedates[fullpath] = os.stat(fullpath).st_mtime
409 except:
410 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500411
412 if taskhashlist:
413 return hashfiles
414 else:
415 return filedates
416
417bb.siggen.find_siginfo = find_siginfo
418
419
420def sstate_get_manifest_filename(task, d):
421 """
422 Return the sstate manifest file path for a particular task.
423 Also returns the datastore that can be used to query related variables.
424 """
425 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500426 extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500427 if extrainf:
428 d2.setVar("SSTATE_MANMACH", extrainf)
429 return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400430
431def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache):
432 d2 = d
433 variant = ''
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800434 curr_variant = ''
435 if d.getVar("BBEXTENDCURR") == "multilib":
436 curr_variant = d.getVar("BBEXTENDVARIANT")
437 if "virtclass-multilib" not in d.getVar("OVERRIDES"):
438 curr_variant = "invalid"
Brad Bishop316dfdd2018-06-25 12:45:53 -0400439 if taskdata2.startswith("virtual:multilib"):
440 variant = taskdata2.split(":")[2]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800441 if curr_variant != variant:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400442 if variant not in multilibcache:
443 multilibcache[variant] = oe.utils.get_multilib_datastore(variant, d)
444 d2 = multilibcache[variant]
445
446 if taskdata.endswith("-native"):
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600447 pkgarchs = ["${BUILD_ARCH}", "${BUILD_ARCH}_${ORIGNATIVELSBSTRING}"]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400448 elif taskdata.startswith("nativesdk-"):
449 pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"]
450 elif "-cross-canadian" in taskdata:
451 pkgarchs = ["${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}"]
452 elif "-cross-" in taskdata:
Andrew Geissler9aee5002022-03-30 16:27:02 +0000453 pkgarchs = ["${BUILD_ARCH}"]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400454 elif "-crosssdk" in taskdata:
455 pkgarchs = ["${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"]
456 else:
457 pkgarchs = ['${MACHINE_ARCH}']
458 pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split()))
459 pkgarchs.append('allarch')
460 pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}')
461
Andrew Geissler517393d2023-01-13 08:55:19 -0600462 searched_manifests = []
463
Brad Bishop316dfdd2018-06-25 12:45:53 -0400464 for pkgarch in pkgarchs:
465 manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, taskdata, taskname))
466 if os.path.exists(manifest):
467 return manifest, d2
Andrew Geissler517393d2023-01-13 08:55:19 -0600468 searched_manifests.append(manifest)
469 bb.fatal("The sstate manifest for task '%s:%s' (multilib variant '%s') could not be found.\nThe pkgarchs considered were: %s.\nBut none of these manifests exists:\n %s"
470 % (taskdata, taskname, variant, d2.expand(", ".join(pkgarchs)),"\n ".join(searched_manifests)))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400471 return None, d2
472
Brad Bishop19323692019-04-05 15:28:33 -0400473def OEOuthashBasic(path, sigfile, task, d):
474 """
475 Basic output hash function
476
477 Calculates the output hash of a task by hashing all output file metadata,
478 and file contents.
479 """
480 import hashlib
481 import stat
482 import pwd
483 import grp
Patrick Williams93c203f2021-10-06 16:15:23 -0500484 import re
485 import fnmatch
Brad Bishop19323692019-04-05 15:28:33 -0400486
487 def update_hash(s):
488 s = s.encode('utf-8')
489 h.update(s)
490 if sigfile:
491 sigfile.write(s)
492
493 h = hashlib.sha256()
494 prev_dir = os.getcwd()
Patrick Williams93c203f2021-10-06 16:15:23 -0500495 corebase = d.getVar("COREBASE")
496 tmpdir = d.getVar("TMPDIR")
Brad Bishop19323692019-04-05 15:28:33 -0400497 include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
Andrew Geisslerf0343792020-11-18 10:42:21 -0600498 if "package_write_" in task or task == "package_qa":
499 include_owners = False
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600500 include_timestamps = False
Andrew Geissler5199d832021-09-24 16:47:35 -0500501 include_root = True
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600502 if task == "package":
Andrew Geisslereff27472021-10-29 15:35:00 -0500503 include_timestamps = True
Andrew Geissler5199d832021-09-24 16:47:35 -0500504 include_root = False
Andrew Geissler595f6302022-01-24 19:11:47 +0000505 hash_version = d.getVar('HASHEQUIV_HASH_VERSION')
506 extra_sigdata = d.getVar("HASHEQUIV_EXTRA_SIGDATA")
Brad Bishop19323692019-04-05 15:28:33 -0400507
Patrick Williams93c203f2021-10-06 16:15:23 -0500508 filemaps = {}
509 for m in (d.getVar('SSTATE_HASHEQUIV_FILEMAP') or '').split():
510 entry = m.split(":")
511 if len(entry) != 3 or entry[0] != task:
512 continue
513 filemaps.setdefault(entry[1], [])
514 filemaps[entry[1]].append(entry[2])
515
Brad Bishop19323692019-04-05 15:28:33 -0400516 try:
517 os.chdir(path)
Patrick Williams93c203f2021-10-06 16:15:23 -0500518 basepath = os.path.normpath(path)
Brad Bishop19323692019-04-05 15:28:33 -0400519
520 update_hash("OEOuthashBasic\n")
Andrew Geissler595f6302022-01-24 19:11:47 +0000521 if hash_version:
522 update_hash(hash_version + "\n")
523
524 if extra_sigdata:
525 update_hash(extra_sigdata + "\n")
Brad Bishop19323692019-04-05 15:28:33 -0400526
527 # It is only currently useful to get equivalent hashes for things that
528 # can be restored from sstate. Since the sstate object is named using
529 # SSTATE_PKGSPEC and the task name, those should be included in the
530 # output hash calculation.
531 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC'))
532 update_hash("task=%s\n" % task)
533
534 for root, dirs, files in os.walk('.', topdown=True):
535 # Sort directories to ensure consistent ordering when recursing
536 dirs.sort()
537 files.sort()
538
539 def process(path):
540 s = os.lstat(path)
541
542 if stat.S_ISDIR(s.st_mode):
543 update_hash('d')
544 elif stat.S_ISCHR(s.st_mode):
545 update_hash('c')
546 elif stat.S_ISBLK(s.st_mode):
547 update_hash('b')
548 elif stat.S_ISSOCK(s.st_mode):
549 update_hash('s')
550 elif stat.S_ISLNK(s.st_mode):
551 update_hash('l')
552 elif stat.S_ISFIFO(s.st_mode):
553 update_hash('p')
554 else:
555 update_hash('-')
556
557 def add_perm(mask, on, off='-'):
558 if mask & s.st_mode:
559 update_hash(on)
560 else:
561 update_hash(off)
562
563 add_perm(stat.S_IRUSR, 'r')
564 add_perm(stat.S_IWUSR, 'w')
565 if stat.S_ISUID & s.st_mode:
566 add_perm(stat.S_IXUSR, 's', 'S')
567 else:
568 add_perm(stat.S_IXUSR, 'x')
569
Brad Bishop19323692019-04-05 15:28:33 -0400570 if include_owners:
Andrew Geisslereff27472021-10-29 15:35:00 -0500571 # Group/other permissions are only relevant in pseudo context
572 add_perm(stat.S_IRGRP, 'r')
573 add_perm(stat.S_IWGRP, 'w')
574 if stat.S_ISGID & s.st_mode:
575 add_perm(stat.S_IXGRP, 's', 'S')
576 else:
577 add_perm(stat.S_IXGRP, 'x')
578
579 add_perm(stat.S_IROTH, 'r')
580 add_perm(stat.S_IWOTH, 'w')
581 if stat.S_ISVTX & s.st_mode:
582 update_hash('t')
583 else:
584 add_perm(stat.S_IXOTH, 'x')
585
Andrew Geissler82c905d2020-04-13 13:39:40 -0500586 try:
587 update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
588 update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600589 except KeyError as e:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500590 bb.warn("KeyError in %s" % path)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600591 msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match "
592 "any user/group on target. This may be due to host contamination." % (e, path, s.st_uid, s.st_gid))
593 raise Exception(msg).with_traceback(e.__traceback__)
Brad Bishop19323692019-04-05 15:28:33 -0400594
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600595 if include_timestamps:
596 update_hash(" %10d" % s.st_mtime)
597
Brad Bishop19323692019-04-05 15:28:33 -0400598 update_hash(" ")
599 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
600 update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
601 else:
602 update_hash(" " * 9)
603
Patrick Williams93c203f2021-10-06 16:15:23 -0500604 filterfile = False
605 for entry in filemaps:
606 if fnmatch.fnmatch(path, entry):
607 filterfile = True
608
Brad Bishop19323692019-04-05 15:28:33 -0400609 update_hash(" ")
Patrick Williams93c203f2021-10-06 16:15:23 -0500610 if stat.S_ISREG(s.st_mode) and not filterfile:
Brad Bishop19323692019-04-05 15:28:33 -0400611 update_hash("%10d" % s.st_size)
612 else:
613 update_hash(" " * 10)
614
615 update_hash(" ")
616 fh = hashlib.sha256()
617 if stat.S_ISREG(s.st_mode):
618 # Hash file contents
Patrick Williams93c203f2021-10-06 16:15:23 -0500619 if filterfile:
620 # Need to ignore paths in crossscripts and postinst-useradd files.
621 with open(path, 'rb') as d:
622 chunk = d.read()
623 chunk = chunk.replace(bytes(basepath, encoding='utf8'), b'')
624 for entry in filemaps:
625 if not fnmatch.fnmatch(path, entry):
626 continue
627 for r in filemaps[entry]:
628 if r.startswith("regex-"):
629 chunk = re.sub(bytes(r[6:], encoding='utf8'), b'', chunk)
630 else:
631 chunk = chunk.replace(bytes(r, encoding='utf8'), b'')
Brad Bishop19323692019-04-05 15:28:33 -0400632 fh.update(chunk)
Patrick Williams93c203f2021-10-06 16:15:23 -0500633 else:
634 with open(path, 'rb') as d:
635 for chunk in iter(lambda: d.read(4096), b""):
636 fh.update(chunk)
Brad Bishop19323692019-04-05 15:28:33 -0400637 update_hash(fh.hexdigest())
638 else:
639 update_hash(" " * len(fh.hexdigest()))
640
641 update_hash(" %s" % path)
642
643 if stat.S_ISLNK(s.st_mode):
644 update_hash(" -> %s" % os.readlink(path))
645
646 update_hash("\n")
647
648 # Process this directory and all its child files
Andrew Geissler5199d832021-09-24 16:47:35 -0500649 if include_root or root != ".":
650 process(root)
Brad Bishop19323692019-04-05 15:28:33 -0400651 for f in files:
652 if f == 'fixmepath':
653 continue
654 process(os.path.join(root, f))
Andrew Geissler6aa7eec2023-03-03 12:41:14 -0600655
656 for dir in dirs:
657 if os.path.islink(os.path.join(root, dir)):
658 process(os.path.join(root, dir))
Brad Bishop19323692019-04-05 15:28:33 -0400659 finally:
660 os.chdir(prev_dir)
661
662 return h.hexdigest()
663
Brad Bishop316dfdd2018-06-25 12:45:53 -0400664