blob: abcd96231e72f87a8fb8f75e8ce7c341f7bf9074 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004import bb.siggen
Andrew Geisslerd25ed322020-06-27 00:28:28 -05005import bb.runqueue
Brad Bishop316dfdd2018-06-25 12:45:53 -04006import oe
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007
Andrew Geisslerd25ed322020-06-27 00:28:28 -05008def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05009 # Return True if we should keep the dependency, False to drop it
10 def isNative(x):
11 return x.endswith("-native")
12 def isCross(x):
13 return "-cross-" in x
14 def isNativeSDK(x):
15 return x.startswith("nativesdk-")
Andrew Geisslerd25ed322020-06-27 00:28:28 -050016 def isKernel(mc, fn):
17 inherits = " ".join(dataCaches[mc].inherits[fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018 return inherits.find("/module-base.bbclass") != -1 or inherits.find("/linux-kernel-base.bbclass") != -1
Andrew Geisslerd25ed322020-06-27 00:28:28 -050019 def isPackageGroup(mc, fn):
20 inherits = " ".join(dataCaches[mc].inherits[fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050021 return "/packagegroup.bbclass" in inherits
Andrew Geisslerd25ed322020-06-27 00:28:28 -050022 def isAllArch(mc, fn):
23 inherits = " ".join(dataCaches[mc].inherits[fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024 return "/allarch.bbclass" in inherits
Andrew Geisslerd25ed322020-06-27 00:28:28 -050025 def isImage(mc, fn):
26 return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn])
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027
Andrew Geisslerd25ed322020-06-27 00:28:28 -050028 depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep)
29 mc, _ = bb.runqueue.split_mc(fn)
30
31 # (Almost) always include our own inter-task dependencies (unless it comes
32 # from a mcdepends). The exception is the special
33 # do_kernel_configme->do_unpack_and_patch dependency from archiver.bbclass.
34 if recipename == depname and depmc == mc:
35 if task == "do_kernel_configme" and deptaskname == "do_unpack_and_patch":
Brad Bishop6e60e8b2018-02-01 10:27:11 -050036 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 return True
38
Patrick Williamsc124f4f2015-09-15 14:41:29 -050039 # Exclude well defined recipe->dependency
40 if "%s->%s" % (recipename, depname) in siggen.saferecipedeps:
41 return False
42
Brad Bishop316dfdd2018-06-25 12:45:53 -040043 # Check for special wildcard
44 if "*->%s" % depname in siggen.saferecipedeps and recipename != depname:
45 return False
46
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047 # Don't change native/cross/nativesdk recipe dependencies any further
48 if isNative(recipename) or isCross(recipename) or isNativeSDK(recipename):
49 return True
50
51 # Only target packages beyond here
52
53 # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes
Andrew Geisslerd25ed322020-06-27 00:28:28 -050054 if isPackageGroup(mc, fn) and isAllArch(mc, fn) and not isNative(depname):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080055 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050056
57 # Exclude well defined machine specific configurations which don't change ABI
Andrew Geisslerd25ed322020-06-27 00:28:28 -050058 if depname in siggen.abisaferecipes and not isImage(mc, fn):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059 return False
60
61 # Kernel modules are well namespaced. We don't want to depend on the kernel's checksum
Patrick Williams213cb262021-08-07 19:21:33 -050062 # if we're just doing an RRECOMMENDS:xxx = "kernel-module-*", not least because the checksum
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063 # is machine specific.
64 # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes)
65 # and we reccomend a kernel-module, we exclude the dependency.
Andrew Geisslerd25ed322020-06-27 00:28:28 -050066 if dataCaches and isKernel(depmc, depmcfn) and not isKernel(mc, fn):
67 for pkg in dataCaches[mc].runrecs[fn]:
68 if " ".join(dataCaches[mc].runrecs[fn][pkg]).find("kernel-module-") != -1:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069 return False
70
71 # Default to keep dependencies
72 return True
73
74def sstate_lockedsigs(d):
75 sigs = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -050076 types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050077 for t in types:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050078 siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t
Brad Bishop6e60e8b2018-02-01 10:27:11 -050079 lockedsigs = (d.getVar(siggen_lockedsigs_var) or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050080 for ls in lockedsigs:
81 pn, task, h = ls.split(":", 2)
82 if pn not in sigs:
83 sigs[pn] = {}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050084 sigs[pn][task] = [h, siggen_lockedsigs_var]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085 return sigs
86
87class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
88 name = "OEBasic"
89 def init_rundepcheck(self, data):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050090 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split()
91 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050092 pass
Andrew Geisslerd25ed322020-06-27 00:28:28 -050093 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None):
94 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095
Brad Bishop00e122a2019-10-05 11:10:57 -040096class SignatureGeneratorOEBasicHashMixIn(object):
Andrew Geisslerd25ed322020-06-27 00:28:28 -050097 supports_multiconfig_datacaches = True
98
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099 def init_rundepcheck(self, data):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500100 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split()
101 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 self.lockedsigs = sstate_lockedsigs(data)
103 self.lockedhashes = {}
104 self.lockedpnmap = {}
105 self.lockedhashfn = {}
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500106 self.machine = data.getVar("MACHINE")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500107 self.mismatch_msgs = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500108 self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES") or
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500109 "").split()
110 self.unlockedrecipes = { k: "" for k in self.unlockedrecipes }
Andrew Geissler82c905d2020-04-13 13:39:40 -0500111 self._internal = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500112 pass
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500113
114 def tasks_resolved(self, virtmap, virtpnmap, dataCache):
115 # Translate virtual/xxx entries to PN values
116 newabisafe = []
117 for a in self.abisaferecipes:
118 if a in virtpnmap:
119 newabisafe.append(virtpnmap[a])
120 else:
121 newabisafe.append(a)
122 self.abisaferecipes = newabisafe
123 newsafedeps = []
124 for a in self.saferecipedeps:
125 a1, a2 = a.split("->")
126 if a1 in virtpnmap:
127 a1 = virtpnmap[a1]
128 if a2 in virtpnmap:
129 a2 = virtpnmap[a2]
130 newsafedeps.append(a1 + "->" + a2)
131 self.saferecipedeps = newsafedeps
132
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500133 def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None):
134 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135
136 def get_taskdata(self):
Brad Bishop00e122a2019-10-05 11:10:57 -0400137 return (self.lockedpnmap, self.lockedhashfn, self.lockedhashes) + super().get_taskdata()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138
139 def set_taskdata(self, data):
Brad Bishop00e122a2019-10-05 11:10:57 -0400140 self.lockedpnmap, self.lockedhashfn, self.lockedhashes = data[:3]
141 super().set_taskdata(data[3:])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500142
143 def dump_sigs(self, dataCache, options):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600144 sigfile = os.getcwd() + "/locked-sigs.inc"
145 bb.plain("Writing locked sigs to %s" % sigfile)
146 self.dump_lockedsigs(sigfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500147 return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
148
Andrew Geissler82c905d2020-04-13 13:39:40 -0500149
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500150 def get_taskhash(self, tid, deps, dataCaches):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500151 if tid in self.lockedhashes:
152 if self.lockedhashes[tid]:
153 return self.lockedhashes[tid]
154 else:
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500155 return super().get_taskhash(tid, deps, dataCaches)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500156
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500157 h = super().get_taskhash(tid, deps, dataCaches)
Brad Bishop08902b02019-08-20 09:16:51 -0400158
159 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500161 recipename = dataCaches[mc].pkg_fn[fn]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500162 self.lockedpnmap[fn] = recipename
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500163 self.lockedhashfn[fn] = dataCaches[mc].hashfn[fn]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500164
165 unlocked = False
166 if recipename in self.unlockedrecipes:
167 unlocked = True
168 else:
169 def recipename_from_dep(dep):
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500170 (depmc, _, _, depfn) = bb.runqueue.split_tid_mcfn(dep)
171 return dataCaches[depmc].pkg_fn[depfn]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500172
173 # If any unlocked recipe is in the direct dependencies then the
174 # current recipe should be unlocked as well.
Brad Bishop08902b02019-08-20 09:16:51 -0400175 depnames = [ recipename_from_dep(x) for x in deps if mc == bb.runqueue.mc_from_tid(x)]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500176 if any(x in y for y in depnames for x in self.unlockedrecipes):
177 self.unlockedrecipes[recipename] = ''
178 unlocked = True
179
180 if not unlocked and recipename in self.lockedsigs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500181 if task in self.lockedsigs[recipename]:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500182 h_locked = self.lockedsigs[recipename][task][0]
183 var = self.lockedsigs[recipename][task][1]
Brad Bishop08902b02019-08-20 09:16:51 -0400184 self.lockedhashes[tid] = h_locked
Andrew Geissler82c905d2020-04-13 13:39:40 -0500185 self._internal = True
186 unihash = self.get_unihash(tid)
187 self._internal = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500188 #bb.warn("Using %s %s %s" % (recipename, task, h))
189
Brad Bishop00e122a2019-10-05 11:10:57 -0400190 if h != h_locked and h_locked != unihash:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500191 self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s'
192 % (recipename, task, h, h_locked, var))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500193
194 return h_locked
Andrew Geissler82c905d2020-04-13 13:39:40 -0500195
196 self.lockedhashes[tid] = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 #bb.warn("%s %s %s" % (recipename, task, h))
198 return h
199
Andrew Geissler82c905d2020-04-13 13:39:40 -0500200 def get_stampfile_hash(self, tid):
201 if tid in self.lockedhashes and self.lockedhashes[tid]:
202 return self.lockedhashes[tid]
203 return super().get_stampfile_hash(tid)
204
Brad Bishop00e122a2019-10-05 11:10:57 -0400205 def get_unihash(self, tid):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500206 if tid in self.lockedhashes and self.lockedhashes[tid] and not self._internal:
Brad Bishop00e122a2019-10-05 11:10:57 -0400207 return self.lockedhashes[tid]
208 return super().get_unihash(tid)
209
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210 def dump_sigtask(self, fn, task, stampbase, runtime):
Brad Bishop08902b02019-08-20 09:16:51 -0400211 tid = fn + ":" + task
Andrew Geissler82c905d2020-04-13 13:39:40 -0500212 if tid in self.lockedhashes and self.lockedhashes[tid]:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500213 return
214 super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
215
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600216 def dump_lockedsigs(self, sigfile, taskfilter=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500217 types = {}
Brad Bishop08902b02019-08-20 09:16:51 -0400218 for tid in self.runtaskdeps:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500219 if taskfilter:
Brad Bishop08902b02019-08-20 09:16:51 -0400220 if not tid in taskfilter:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400222 fn = bb.runqueue.fn_from_tid(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223 t = self.lockedhashfn[fn].split(" ")[1].split(":")[5]
224 t = 't-' + t.replace('_', '-')
225 if t not in types:
226 types[t] = []
Brad Bishop08902b02019-08-20 09:16:51 -0400227 types[t].append(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500228
229 with open(sigfile, "w") as f:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500230 l = sorted(types)
231 for t in l:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500232 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % t)
233 types[t].sort()
Brad Bishop08902b02019-08-20 09:16:51 -0400234 sortedtid = sorted(types[t], key=lambda tid: self.lockedpnmap[bb.runqueue.fn_from_tid(tid)])
235 for tid in sortedtid:
236 (_, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
237 if tid not in self.taskhash:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238 continue
Brad Bishop00e122a2019-10-05 11:10:57 -0400239 f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.get_unihash(tid) + " \\\n")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240 f.write(' "\n')
Patrick Williams213cb262021-08-07 19:21:33 -0500241 f.write('SIGGEN_LOCKEDSIGS_TYPES:%s = "%s"' % (self.machine, " ".join(l)))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500242
Andrew Geissler09036742021-06-25 14:25:14 -0500243 def dump_siglist(self, sigfile, path_prefix_strip=None):
244 def strip_fn(fn):
245 nonlocal path_prefix_strip
246 if not path_prefix_strip:
247 return fn
248
249 fn_exp = fn.split(":")
250 if fn_exp[-1].startswith(path_prefix_strip):
251 fn_exp[-1] = fn_exp[-1][len(path_prefix_strip):]
252
253 return ":".join(fn_exp)
254
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500255 with open(sigfile, "w") as f:
256 tasks = []
257 for taskitem in self.taskhash:
Brad Bishop08902b02019-08-20 09:16:51 -0400258 (fn, task) = taskitem.rsplit(":", 1)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500259 pn = self.lockedpnmap[fn]
Andrew Geissler09036742021-06-25 14:25:14 -0500260 tasks.append((pn, task, strip_fn(fn), self.taskhash[taskitem]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500261 for (pn, task, fn, taskhash) in sorted(tasks):
Brad Bishop08902b02019-08-20 09:16:51 -0400262 f.write('%s:%s %s %s\n' % (pn, task, fn, taskhash))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263
Brad Bishop08902b02019-08-20 09:16:51 -0400264 def checkhashes(self, sq_data, missed, found, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500265 warn_msgs = []
266 error_msgs = []
267 sstate_missing_msgs = []
268
Brad Bishop08902b02019-08-20 09:16:51 -0400269 for tid in sq_data['hash']:
270 if tid not in found:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271 for pn in self.lockedsigs:
Brad Bishop08902b02019-08-20 09:16:51 -0400272 taskname = bb.runqueue.taskname_from_tid(tid)
273 if sq_data['hash'][tid] in iter(self.lockedsigs[pn].values()):
274 if taskname == 'do_shared_workdir':
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500275 continue
276 sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
Brad Bishop08902b02019-08-20 09:16:51 -0400277 % (pn, taskname, sq_data['hash'][tid]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500279 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500280 if checklevel == 'warn':
281 warn_msgs += self.mismatch_msgs
282 elif checklevel == 'error':
283 error_msgs += self.mismatch_msgs
284
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500285 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500286 if checklevel == 'warn':
287 warn_msgs += sstate_missing_msgs
288 elif checklevel == 'error':
289 error_msgs += sstate_missing_msgs
290
291 if warn_msgs:
292 bb.warn("\n".join(warn_msgs))
293 if error_msgs:
294 bb.fatal("\n".join(error_msgs))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295
Brad Bishop00e122a2019-10-05 11:10:57 -0400296class SignatureGeneratorOEBasicHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorBasicHash):
297 name = "OEBasicHash"
298
299class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHashMixIn, bb.siggen.SignatureGeneratorUniHashMixIn, bb.siggen.SignatureGeneratorBasicHash):
Brad Bishop19323692019-04-05 15:28:33 -0400300 name = "OEEquivHash"
301
302 def init_rundepcheck(self, data):
303 super().init_rundepcheck(data)
Brad Bishopa34c0302019-09-23 22:34:48 -0400304 self.server = data.getVar('BB_HASHSERVE')
Brad Bishop08902b02019-08-20 09:16:51 -0400305 if not self.server:
Brad Bishopa34c0302019-09-23 22:34:48 -0400306 bb.fatal("OEEquivHash requires BB_HASHSERVE to be set")
Brad Bishop19323692019-04-05 15:28:33 -0400307 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD')
Brad Bishop08902b02019-08-20 09:16:51 -0400308 if not self.method:
309 bb.fatal("OEEquivHash requires SSTATE_HASHEQUIV_METHOD to be set")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500310
311# Insert these classes into siggen's namespace so it can see and select them
312bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic
313bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash
Brad Bishop19323692019-04-05 15:28:33 -0400314bb.siggen.SignatureGeneratorOEEquivHash = SignatureGeneratorOEEquivHash
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315
316
317def find_siginfo(pn, taskname, taskhashlist, d):
318 """ Find signature data files for comparison purposes """
319
320 import fnmatch
321 import glob
322
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323 if not taskname:
324 # We have to derive pn and taskname
325 key = pn
Brad Bishop08902b02019-08-20 09:16:51 -0400326 splitit = key.split('.bb:')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327 taskname = splitit[1]
328 pn = os.path.basename(splitit[0]).split('_')[0]
329 if key.startswith('virtual:native:'):
330 pn = pn + '-native'
331
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500332 hashfiles = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500333 filedates = {}
334
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500335 def get_hashval(siginfo):
336 if siginfo.endswith('.siginfo'):
337 return siginfo.rpartition(':')[2].partition('_')[0]
338 else:
339 return siginfo.rpartition('.')[2]
340
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500341 # First search in stamps dir
342 localdata = d.createCopy()
343 localdata.setVar('MULTIMACH_TARGET_SYS', '*')
344 localdata.setVar('PN', pn)
345 localdata.setVar('PV', '*')
346 localdata.setVar('PR', '*')
347 localdata.setVar('EXTENDPE', '')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500348 stamp = localdata.getVar('STAMP')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500349 if pn.startswith("gcc-source"):
350 # gcc-source shared workdir is a special case :(
351 stamp = localdata.expand("${STAMPS_DIR}/work-shared/gcc-${PV}-${PR}")
352
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500353 filespec = '%s.%s.sigdata.*' % (stamp, taskname)
354 foundall = False
355 import glob
356 for fullpath in glob.glob(filespec):
357 match = False
358 if taskhashlist:
359 for taskhash in taskhashlist:
360 if fullpath.endswith('.%s' % taskhash):
361 hashfiles[taskhash] = fullpath
362 if len(hashfiles) == len(taskhashlist):
363 foundall = True
364 break
365 else:
366 try:
367 filedates[fullpath] = os.stat(fullpath).st_mtime
368 except OSError:
369 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500370 hashval = get_hashval(fullpath)
371 hashfiles[hashval] = fullpath
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500372
373 if not taskhashlist or (len(filedates) < 2 and not foundall):
374 # That didn't work, look in sstate-cache
Brad Bishop19323692019-04-05 15:28:33 -0400375 hashes = taskhashlist or ['?' * 64]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500376 localdata = bb.data.createCopy(d)
377 for hashval in hashes:
378 localdata.setVar('PACKAGE_ARCH', '*')
379 localdata.setVar('TARGET_VENDOR', '*')
380 localdata.setVar('TARGET_OS', '*')
381 localdata.setVar('PN', pn)
382 localdata.setVar('PV', '*')
383 localdata.setVar('PR', '*')
384 localdata.setVar('BB_TASKHASH', hashval)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500385 swspec = localdata.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500386 if taskname in ['do_fetch', 'do_unpack', 'do_patch', 'do_populate_lic', 'do_preconfigure'] and swspec:
387 localdata.setVar('SSTATE_PKGSPEC', '${SSTATE_SWSPEC}')
388 elif pn.endswith('-native') or "-cross-" in pn or "-crosssdk-" in pn:
389 localdata.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
390 sstatename = taskname[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500391 filespec = '%s_%s.*.siginfo' % (localdata.getVar('SSTATE_PKG'), sstatename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500392
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500393 matchedfiles = glob.glob(filespec)
394 for fullpath in matchedfiles:
395 actual_hashval = get_hashval(fullpath)
396 if actual_hashval in hashfiles:
397 continue
398 hashfiles[hashval] = fullpath
399 if not taskhashlist:
400 try:
401 filedates[fullpath] = os.stat(fullpath).st_mtime
402 except:
403 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404
405 if taskhashlist:
406 return hashfiles
407 else:
408 return filedates
409
410bb.siggen.find_siginfo = find_siginfo
411
412
413def sstate_get_manifest_filename(task, d):
414 """
415 Return the sstate manifest file path for a particular task.
416 Also returns the datastore that can be used to query related variables.
417 """
418 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500419 extrainf = d.getVarFlag("do_" + task, 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500420 if extrainf:
421 d2.setVar("SSTATE_MANMACH", extrainf)
422 return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400423
424def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache):
425 d2 = d
426 variant = ''
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800427 curr_variant = ''
428 if d.getVar("BBEXTENDCURR") == "multilib":
429 curr_variant = d.getVar("BBEXTENDVARIANT")
430 if "virtclass-multilib" not in d.getVar("OVERRIDES"):
431 curr_variant = "invalid"
Brad Bishop316dfdd2018-06-25 12:45:53 -0400432 if taskdata2.startswith("virtual:multilib"):
433 variant = taskdata2.split(":")[2]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800434 if curr_variant != variant:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400435 if variant not in multilibcache:
436 multilibcache[variant] = oe.utils.get_multilib_datastore(variant, d)
437 d2 = multilibcache[variant]
438
439 if taskdata.endswith("-native"):
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600440 pkgarchs = ["${BUILD_ARCH}", "${BUILD_ARCH}_${ORIGNATIVELSBSTRING}"]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400441 elif taskdata.startswith("nativesdk-"):
442 pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"]
443 elif "-cross-canadian" in taskdata:
444 pkgarchs = ["${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}"]
445 elif "-cross-" in taskdata:
446 pkgarchs = ["${BUILD_ARCH}_${TARGET_ARCH}"]
447 elif "-crosssdk" in taskdata:
448 pkgarchs = ["${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"]
449 else:
450 pkgarchs = ['${MACHINE_ARCH}']
451 pkgarchs = pkgarchs + list(reversed(d2.getVar("PACKAGE_EXTRA_ARCHS").split()))
452 pkgarchs.append('allarch')
453 pkgarchs.append('${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX}')
454
455 for pkgarch in pkgarchs:
456 manifest = d2.expand("${SSTATE_MANIFESTS}/manifest-%s-%s.%s" % (pkgarch, taskdata, taskname))
457 if os.path.exists(manifest):
458 return manifest, d2
William A. Kennington IIIac69b482021-06-02 12:28:27 -0700459 bb.fatal("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant))
Brad Bishop316dfdd2018-06-25 12:45:53 -0400460 return None, d2
461
Brad Bishop19323692019-04-05 15:28:33 -0400462def OEOuthashBasic(path, sigfile, task, d):
463 """
464 Basic output hash function
465
466 Calculates the output hash of a task by hashing all output file metadata,
467 and file contents.
468 """
469 import hashlib
470 import stat
471 import pwd
472 import grp
Patrick Williams93c203f2021-10-06 16:15:23 -0500473 import re
474 import fnmatch
Brad Bishop19323692019-04-05 15:28:33 -0400475
476 def update_hash(s):
477 s = s.encode('utf-8')
478 h.update(s)
479 if sigfile:
480 sigfile.write(s)
481
482 h = hashlib.sha256()
483 prev_dir = os.getcwd()
Patrick Williams93c203f2021-10-06 16:15:23 -0500484 corebase = d.getVar("COREBASE")
485 tmpdir = d.getVar("TMPDIR")
Brad Bishop19323692019-04-05 15:28:33 -0400486 include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
Andrew Geisslerf0343792020-11-18 10:42:21 -0600487 if "package_write_" in task or task == "package_qa":
488 include_owners = False
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600489 include_timestamps = False
Andrew Geissler5199d832021-09-24 16:47:35 -0500490 include_root = True
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600491 if task == "package":
Andrew Geisslereff27472021-10-29 15:35:00 -0500492 include_timestamps = True
Andrew Geissler5199d832021-09-24 16:47:35 -0500493 include_root = False
Andrew Geissler595f6302022-01-24 19:11:47 +0000494 hash_version = d.getVar('HASHEQUIV_HASH_VERSION')
495 extra_sigdata = d.getVar("HASHEQUIV_EXTRA_SIGDATA")
Brad Bishop19323692019-04-05 15:28:33 -0400496
Patrick Williams93c203f2021-10-06 16:15:23 -0500497 filemaps = {}
498 for m in (d.getVar('SSTATE_HASHEQUIV_FILEMAP') or '').split():
499 entry = m.split(":")
500 if len(entry) != 3 or entry[0] != task:
501 continue
502 filemaps.setdefault(entry[1], [])
503 filemaps[entry[1]].append(entry[2])
504
Brad Bishop19323692019-04-05 15:28:33 -0400505 try:
506 os.chdir(path)
Patrick Williams93c203f2021-10-06 16:15:23 -0500507 basepath = os.path.normpath(path)
Brad Bishop19323692019-04-05 15:28:33 -0400508
509 update_hash("OEOuthashBasic\n")
Andrew Geissler595f6302022-01-24 19:11:47 +0000510 if hash_version:
511 update_hash(hash_version + "\n")
512
513 if extra_sigdata:
514 update_hash(extra_sigdata + "\n")
Brad Bishop19323692019-04-05 15:28:33 -0400515
516 # It is only currently useful to get equivalent hashes for things that
517 # can be restored from sstate. Since the sstate object is named using
518 # SSTATE_PKGSPEC and the task name, those should be included in the
519 # output hash calculation.
520 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC'))
521 update_hash("task=%s\n" % task)
522
523 for root, dirs, files in os.walk('.', topdown=True):
524 # Sort directories to ensure consistent ordering when recursing
525 dirs.sort()
526 files.sort()
527
528 def process(path):
529 s = os.lstat(path)
530
531 if stat.S_ISDIR(s.st_mode):
532 update_hash('d')
533 elif stat.S_ISCHR(s.st_mode):
534 update_hash('c')
535 elif stat.S_ISBLK(s.st_mode):
536 update_hash('b')
537 elif stat.S_ISSOCK(s.st_mode):
538 update_hash('s')
539 elif stat.S_ISLNK(s.st_mode):
540 update_hash('l')
541 elif stat.S_ISFIFO(s.st_mode):
542 update_hash('p')
543 else:
544 update_hash('-')
545
546 def add_perm(mask, on, off='-'):
547 if mask & s.st_mode:
548 update_hash(on)
549 else:
550 update_hash(off)
551
552 add_perm(stat.S_IRUSR, 'r')
553 add_perm(stat.S_IWUSR, 'w')
554 if stat.S_ISUID & s.st_mode:
555 add_perm(stat.S_IXUSR, 's', 'S')
556 else:
557 add_perm(stat.S_IXUSR, 'x')
558
Brad Bishop19323692019-04-05 15:28:33 -0400559 if include_owners:
Andrew Geisslereff27472021-10-29 15:35:00 -0500560 # Group/other permissions are only relevant in pseudo context
561 add_perm(stat.S_IRGRP, 'r')
562 add_perm(stat.S_IWGRP, 'w')
563 if stat.S_ISGID & s.st_mode:
564 add_perm(stat.S_IXGRP, 's', 'S')
565 else:
566 add_perm(stat.S_IXGRP, 'x')
567
568 add_perm(stat.S_IROTH, 'r')
569 add_perm(stat.S_IWOTH, 'w')
570 if stat.S_ISVTX & s.st_mode:
571 update_hash('t')
572 else:
573 add_perm(stat.S_IXOTH, 'x')
574
Andrew Geissler82c905d2020-04-13 13:39:40 -0500575 try:
576 update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
577 update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600578 except KeyError as e:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500579 bb.warn("KeyError in %s" % path)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600580 msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match "
581 "any user/group on target. This may be due to host contamination." % (e, path, s.st_uid, s.st_gid))
582 raise Exception(msg).with_traceback(e.__traceback__)
Brad Bishop19323692019-04-05 15:28:33 -0400583
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600584 if include_timestamps:
585 update_hash(" %10d" % s.st_mtime)
586
Brad Bishop19323692019-04-05 15:28:33 -0400587 update_hash(" ")
588 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
589 update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
590 else:
591 update_hash(" " * 9)
592
Patrick Williams93c203f2021-10-06 16:15:23 -0500593 filterfile = False
594 for entry in filemaps:
595 if fnmatch.fnmatch(path, entry):
596 filterfile = True
597
Brad Bishop19323692019-04-05 15:28:33 -0400598 update_hash(" ")
Patrick Williams93c203f2021-10-06 16:15:23 -0500599 if stat.S_ISREG(s.st_mode) and not filterfile:
Brad Bishop19323692019-04-05 15:28:33 -0400600 update_hash("%10d" % s.st_size)
601 else:
602 update_hash(" " * 10)
603
604 update_hash(" ")
605 fh = hashlib.sha256()
606 if stat.S_ISREG(s.st_mode):
607 # Hash file contents
Patrick Williams93c203f2021-10-06 16:15:23 -0500608 if filterfile:
609 # Need to ignore paths in crossscripts and postinst-useradd files.
610 with open(path, 'rb') as d:
611 chunk = d.read()
612 chunk = chunk.replace(bytes(basepath, encoding='utf8'), b'')
613 for entry in filemaps:
614 if not fnmatch.fnmatch(path, entry):
615 continue
616 for r in filemaps[entry]:
617 if r.startswith("regex-"):
618 chunk = re.sub(bytes(r[6:], encoding='utf8'), b'', chunk)
619 else:
620 chunk = chunk.replace(bytes(r, encoding='utf8'), b'')
Brad Bishop19323692019-04-05 15:28:33 -0400621 fh.update(chunk)
Patrick Williams93c203f2021-10-06 16:15:23 -0500622 else:
623 with open(path, 'rb') as d:
624 for chunk in iter(lambda: d.read(4096), b""):
625 fh.update(chunk)
Brad Bishop19323692019-04-05 15:28:33 -0400626 update_hash(fh.hexdigest())
627 else:
628 update_hash(" " * len(fh.hexdigest()))
629
630 update_hash(" %s" % path)
631
632 if stat.S_ISLNK(s.st_mode):
633 update_hash(" -> %s" % os.readlink(path))
634
635 update_hash("\n")
636
637 # Process this directory and all its child files
Andrew Geissler5199d832021-09-24 16:47:35 -0500638 if include_root or root != ".":
639 process(root)
Brad Bishop19323692019-04-05 15:28:33 -0400640 for f in files:
641 if f == 'fixmepath':
642 continue
643 process(os.path.join(root, f))
644 finally:
645 os.chdir(prev_dir)
646
647 return h.hexdigest()
648
Brad Bishop316dfdd2018-06-25 12:45:53 -0400649