blob: 96655ff24fff0281133227def69a4fcb39f829f1 [file] [log] [blame]
Patrick Williams92b42cb2022-09-03 06:53:57 -05001#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
Andrew Geisslerfc113ea2023-03-31 09:59:46 -05007SSTATE_VERSION = "11"
Patrick Williams92b42cb2022-09-03 06:53:57 -05008
9SSTATE_ZSTD_CLEVEL ??= "8"
10
11SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
12SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
13
14def generate_sstatefn(spec, hash, taskname, siginfo, d):
15 if taskname is None:
16 return ""
17 extension = ".tar.zst"
18 # 8 chars reserved for siginfo
19 limit = 254 - 8
20 if siginfo:
21 limit = 254
22 extension = ".tar.zst.siginfo"
23 if not hash:
24 hash = "INVALID"
25 fn = spec + hash + "_" + taskname + extension
26 # If the filename is too long, attempt to reduce it
27 if len(fn) > limit:
28 components = spec.split(":")
29 # Fields 0,5,6 are mandatory, 1 is most useful, 2,3,4 are just for information
30 # 7 is for the separators
31 avail = (limit - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3
32 components[2] = components[2][:avail]
33 components[3] = components[3][:avail]
34 components[4] = components[4][:avail]
35 spec = ":".join(components)
36 fn = spec + hash + "_" + taskname + extension
37 if len(fn) > limit:
38 bb.fatal("Unable to reduce sstate name to less than 255 chararacters")
39 return hash[:2] + "/" + hash[2:4] + "/" + fn
40
41SSTATE_PKGARCH = "${PACKAGE_ARCH}"
42SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
43SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
44SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d.getVar('SSTATE_CURRTASK'), False, d)}"
45SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
46SSTATE_EXTRAPATH = ""
47SSTATE_EXTRAPATHWILDCARD = ""
48SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tar.zst*"
49
50# explicitly make PV to depend on evaluated value of PV variable
51PV[vardepvalue] = "${PV}"
52
53# We don't want the sstate to depend on things like the distro string
54# of the system, we let the sstate paths take care of this.
55SSTATE_EXTRAPATH[vardepvalue] = ""
56SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
57
Patrick Williams92b42cb2022-09-03 06:53:57 -050058# Avoid docbook/sgml catalog warnings for now
59SSTATE_ALLOW_OVERLAP_FILES += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
60# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
61SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/"
62SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/"
63# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used
64SSTATE_ALLOW_OVERLAP_FILES += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/"
65# Archive the sources for many architectures in one deploy folder
66SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_SRC}"
67# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files
68SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_IMAGE}/ovmf"
69SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_IMAGE}/grub-efi"
70SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_IMAGE}/systemd-boot"
71SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_IMAGE}/microcode"
72
73SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
74SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
75SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
76SSTATE_HASHEQUIV_FILEMAP ?= " \
77 populate_sysroot:*/postinst-useradd-*:${TMPDIR} \
78 populate_sysroot:*/postinst-useradd-*:${COREBASE} \
79 populate_sysroot:*/postinst-useradd-*:regex-\s(PATH|PSEUDO_IGNORE_PATHS|HOME|LOGNAME|OMP_NUM_THREADS|USER)=.*\s \
80 populate_sysroot:*/crossscripts/*:${TMPDIR} \
81 populate_sysroot:*/crossscripts/*:${COREBASE} \
82 "
83
84BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
85
Andrew Geissler220dafd2023-10-04 10:18:08 -050086SSTATE_ARCHS_TUNEPKG ??= "${TUNE_PKGARCH}"
Patrick Williams92b42cb2022-09-03 06:53:57 -050087SSTATE_ARCHS = " \
88 ${BUILD_ARCH} \
89 ${BUILD_ARCH}_${ORIGNATIVELSBSTRING} \
90 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
91 ${SDK_ARCH}_${SDK_OS} \
Andrew Geissler220dafd2023-10-04 10:18:08 -050092 ${SDK_ARCH}_${SDK_ARCH}-${SDKPKGSUFFIX} \
Patrick Williams92b42cb2022-09-03 06:53:57 -050093 allarch \
Andrew Geissler220dafd2023-10-04 10:18:08 -050094 ${SSTATE_ARCHS_TUNEPKG} \
Patrick Williams92b42cb2022-09-03 06:53:57 -050095 ${PACKAGE_EXTRA_ARCHS} \
96 ${MACHINE_ARCH}"
97SSTATE_ARCHS[vardepsexclude] = "ORIGNATIVELSBSTRING"
98
99SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
100
101SSTATECREATEFUNCS += "sstate_hardcode_path"
102SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
103SSTATEPOSTCREATEFUNCS = ""
104SSTATEPREINSTFUNCS = ""
105SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
106SSTATEPOSTINSTFUNCS = ""
107EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
108
109# Check whether sstate exists for tasks that support sstate and are in the
110# locked signatures file.
111SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
112
113# Check whether the task's computed hash matches the task's hash in the
114# locked signatures file.
115SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
116
117# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
118# not sign)
119SSTATE_SIG_KEY ?= ""
120SSTATE_SIG_PASSPHRASE ?= ""
121# Whether to verify the GnUPG signatures when extracting sstate archives
122SSTATE_VERIFY_SIG ?= "0"
123# List of signatures to consider valid.
124SSTATE_VALID_SIGS ??= ""
125SSTATE_VALID_SIGS[vardepvalue] = ""
126
127SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
128SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
129 the output hash for a task, which in turn is used to determine equivalency. \
130 "
131
132SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
133SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
134 hash equivalency server, such as PN, PV, taskname, etc. This information \
135 is very useful for developers looking at task data, but may leak sensitive \
136 data if the equivalence server is public. \
137 "
138
139python () {
140 if bb.data.inherits_class('native', d):
141 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
142 elif bb.data.inherits_class('crosssdk', d):
143 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
144 elif bb.data.inherits_class('cross', d):
145 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}"))
146 elif bb.data.inherits_class('nativesdk', d):
147 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
148 elif bb.data.inherits_class('cross-canadian', d):
149 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
150 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
151 d.setVar('SSTATE_PKGARCH', "allarch")
152 else:
153 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
154
155 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
156 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
157 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
158 d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/")
159
160 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
161 d.setVar('SSTATETASKS', " ".join(unique_tasks))
162 for task in unique_tasks:
163 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
164 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
165 d.setVarFlag(task, 'network', '1')
166 d.setVarFlag(task + "_setscene", 'network', '1')
167}
168
169def sstate_init(task, d):
170 ss = {}
171 ss['task'] = task
172 ss['dirs'] = []
173 ss['plaindirs'] = []
174 ss['lockfiles'] = []
175 ss['lockfiles-shared'] = []
176 return ss
177
178def sstate_state_fromvars(d, task = None):
179 if task is None:
180 task = d.getVar('BB_CURRENTTASK')
181 if not task:
182 bb.fatal("sstate code running without task context?!")
183 task = task.replace("_setscene", "")
184
185 if task.startswith("do_"):
186 task = task[3:]
187 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
188 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
189 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
190 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
191 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
192 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
193 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
194 if not task or len(inputs) != len(outputs):
195 bb.fatal("sstate variables not setup correctly?!")
196
197 if task == "populate_lic":
198 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
199 d.setVar("SSTATE_EXTRAPATH", "")
200 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
201
202 ss = sstate_init(task, d)
203 for i in range(len(inputs)):
204 sstate_add(ss, inputs[i], outputs[i], d)
205 ss['lockfiles'] = lockfiles
206 ss['lockfiles-shared'] = lockfilesshared
207 ss['plaindirs'] = plaindirs
208 ss['interceptfuncs'] = interceptfuncs
209 ss['fixmedir'] = fixmedir
210 return ss
211
212def sstate_add(ss, source, dest, d):
213 if not source.endswith("/"):
214 source = source + "/"
215 if not dest.endswith("/"):
216 dest = dest + "/"
217 source = os.path.normpath(source)
218 dest = os.path.normpath(dest)
219 srcbase = os.path.basename(source)
220 ss['dirs'].append([srcbase, source, dest])
221 return ss
222
223def sstate_install(ss, d):
224 import oe.path
225 import oe.sstatesig
226 import subprocess
227
228 sharedfiles = []
229 shareddirs = []
230 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
231
232 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
233
234 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
235
236 if os.access(manifest, os.R_OK):
237 bb.fatal("Package already staged (%s)?!" % manifest)
238
239 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
240
241 locks = []
242 for lock in ss['lockfiles-shared']:
243 locks.append(bb.utils.lockfile(lock, True))
244 for lock in ss['lockfiles']:
245 locks.append(bb.utils.lockfile(lock))
246
247 for state in ss['dirs']:
248 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
249 for walkroot, dirs, files in os.walk(state[1]):
250 for file in files:
251 srcpath = os.path.join(walkroot, file)
252 dstpath = srcpath.replace(state[1], state[2])
253 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
254 sharedfiles.append(dstpath)
255 for dir in dirs:
256 srcdir = os.path.join(walkroot, dir)
257 dstdir = srcdir.replace(state[1], state[2])
258 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
259 if os.path.islink(srcdir):
260 sharedfiles.append(dstdir)
261 continue
262 if not dstdir.endswith("/"):
263 dstdir = dstdir + "/"
264 shareddirs.append(dstdir)
265
266 # Check the file list for conflicts against files which already exist
267 overlap_allowed = (d.getVar("SSTATE_ALLOW_OVERLAP_FILES") or "").split()
268 match = []
269 for f in sharedfiles:
Andrew Geissler220dafd2023-10-04 10:18:08 -0500270 if os.path.exists(f):
Patrick Williams92b42cb2022-09-03 06:53:57 -0500271 f = os.path.normpath(f)
272 realmatch = True
273 for w in overlap_allowed:
274 w = os.path.normpath(w)
275 if f.startswith(w):
276 realmatch = False
277 break
278 if realmatch:
279 match.append(f)
Andrew Geissler220dafd2023-10-04 10:18:08 -0500280 sstate_search_cmd = "grep -rlF '%s' %s --exclude=index-* | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williams92b42cb2022-09-03 06:53:57 -0500281 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
282 if search_output:
283 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
284 else:
285 match.append(" (not matched to any task)")
286 if match:
Andrew Geissler220dafd2023-10-04 10:18:08 -0500287 bb.fatal("Recipe %s is trying to install files into a shared " \
288 "area when those files already exist. The files and the manifests listing " \
289 "them are:\n %s\n"
290 "Please adjust the recipes so only one recipe provides a given file. " % \
Patrick Williams92b42cb2022-09-03 06:53:57 -0500291 (d.getVar('PN'), "\n ".join(match)))
Patrick Williams92b42cb2022-09-03 06:53:57 -0500292
293 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
294 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
295 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
296
297 # Write out the manifest
298 f = open(manifest, "w")
299 for file in sharedfiles:
300 f.write(file + "\n")
301
302 # We want to ensure that directories appear at the end of the manifest
303 # so that when we test to see if they should be deleted any contents
304 # added by the task will have been removed first.
305 dirs = sorted(shareddirs, key=len)
306 # Must remove children first, which will have a longer path than the parent
307 for di in reversed(dirs):
308 f.write(di + "\n")
309 f.close()
310
311 # Append to the list of manifests for this PACKAGE_ARCH
312
313 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
314 l = bb.utils.lockfile(i + ".lock")
315 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
316 manifests = []
317 if os.path.exists(i):
318 with open(i, "r") as f:
319 manifests = f.readlines()
320 # We append new entries, we don't remove older entries which may have the same
321 # manifest name but different versions from stamp/workdir. See below.
322 if filedata not in manifests:
323 with open(i, "a+") as f:
324 f.write(filedata)
325 bb.utils.unlockfile(l)
326
327 # Run the actual file install
328 for state in ss['dirs']:
329 if os.path.exists(state[1]):
330 oe.path.copyhardlinktree(state[1], state[2])
331
332 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
333 # All hooks should run in the SSTATE_INSTDIR
334 bb.build.exec_func(postinst, d, (sstateinst,))
335
336 for lock in locks:
337 bb.utils.unlockfile(lock)
338
Patrick Williams169d7bc2024-01-05 11:33:25 -0600339sstate_install[vardepsexclude] += "SSTATE_ALLOW_OVERLAP_FILES SSTATE_MANMACH SSTATE_MANFILEPREFIX"
Patrick Williams92b42cb2022-09-03 06:53:57 -0500340sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
341
342def sstate_installpkg(ss, d):
343 from oe.gpg_sign import get_signer
344
345 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
346 d.setVar("SSTATE_CURRTASK", ss['task'])
347 sstatefetch = d.getVar('SSTATE_PKGNAME')
348 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williams864cc432023-02-09 14:54:44 -0600349 verify_sig = bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False)
Patrick Williams92b42cb2022-09-03 06:53:57 -0500350
Patrick Williams864cc432023-02-09 14:54:44 -0600351 if not os.path.exists(sstatepkg) or (verify_sig and not os.path.exists(sstatepkg + '.sig')):
Patrick Williams92b42cb2022-09-03 06:53:57 -0500352 pstaging_fetch(sstatefetch, d)
353
354 if not os.path.isfile(sstatepkg):
355 bb.note("Sstate package %s does not exist" % sstatepkg)
356 return False
357
358 sstate_clean(ss, d)
359
360 d.setVar('SSTATE_INSTDIR', sstateinst)
361
Patrick Williams864cc432023-02-09 14:54:44 -0600362 if verify_sig:
Patrick Williams92b42cb2022-09-03 06:53:57 -0500363 if not os.path.isfile(sstatepkg + '.sig'):
364 bb.warn("No signature file for sstate package %s, skipping acceleration..." % sstatepkg)
365 return False
366 signer = get_signer(d, 'local')
367 if not signer.verify(sstatepkg + '.sig', d.getVar("SSTATE_VALID_SIGS")):
368 bb.warn("Cannot verify signature on sstate package %s, skipping acceleration..." % sstatepkg)
369 return False
370
371 # Empty sstateinst directory, ensure its clean
372 if os.path.exists(sstateinst):
373 oe.path.remove(sstateinst)
374 bb.utils.mkdirhier(sstateinst)
375
376 sstateinst = d.getVar("SSTATE_INSTDIR")
377 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
378
379 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
380 # All hooks should run in the SSTATE_INSTDIR
381 bb.build.exec_func(f, d, (sstateinst,))
382
383 return sstate_installpkgdir(ss, d)
384
385def sstate_installpkgdir(ss, d):
386 import oe.path
387 import subprocess
388
389 sstateinst = d.getVar("SSTATE_INSTDIR")
390 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
391
392 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
393 # All hooks should run in the SSTATE_INSTDIR
394 bb.build.exec_func(f, d, (sstateinst,))
395
396 def prepdir(dir):
397 # remove dir if it exists, ensure any parent directories do exist
398 if os.path.exists(dir):
399 oe.path.remove(dir)
400 bb.utils.mkdirhier(dir)
401 oe.path.remove(dir)
402
403 for state in ss['dirs']:
404 prepdir(state[1])
405 bb.utils.rename(sstateinst + state[0], state[1])
406 sstate_install(ss, d)
407
408 for plain in ss['plaindirs']:
409 workdir = d.getVar('WORKDIR')
410 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
411 src = sstateinst + "/" + plain.replace(workdir, '')
412 if sharedworkdir in plain:
413 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
414 dest = plain
415 bb.utils.mkdirhier(src)
416 prepdir(dest)
417 bb.utils.rename(src, dest)
418
419 return True
420
421python sstate_hardcode_path_unpack () {
422 # Fixup hardcoded paths
423 #
424 # Note: The logic below must match the reverse logic in
425 # sstate_hardcode_path(d)
426 import subprocess
427
428 sstateinst = d.getVar('SSTATE_INSTDIR')
429 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
430 fixmefn = sstateinst + "fixmepath"
431 if os.path.isfile(fixmefn):
432 staging_target = d.getVar('RECIPE_SYSROOT')
433 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
434
435 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
436 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
437 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
438 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
439 else:
440 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
441
442 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
443 for fixmevar in extra_staging_fixmes.split():
444 fixme_path = d.getVar(fixmevar)
445 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
446
447 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
448 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
449
450 # Defer do_populate_sysroot relocation command
451 if sstatefixmedir:
452 bb.utils.mkdirhier(sstatefixmedir)
453 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
454 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
455 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
456 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
457 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
458 f.write(sstate_hardcode_cmd)
459 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
460 return
461
462 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
463 subprocess.check_call(sstate_hardcode_cmd, shell=True)
464
465 # Need to remove this or we'd copy it into the target directory and may
466 # conflict with another writer
467 os.remove(fixmefn)
468}
469
470def sstate_clean_cachefile(ss, d):
471 import oe.path
472
473 if d.getVarFlag('do_%s' % ss['task'], 'task'):
474 d.setVar("SSTATE_PATH_CURRTASK", ss['task'])
475 sstatepkgfile = d.getVar('SSTATE_PATHSPEC')
476 bb.note("Removing %s" % sstatepkgfile)
477 oe.path.remove(sstatepkgfile)
478
479def sstate_clean_cachefiles(d):
480 for task in (d.getVar('SSTATETASKS') or "").split():
481 ld = d.createCopy()
482 ss = sstate_state_fromvars(ld, task)
483 sstate_clean_cachefile(ss, ld)
484
485def sstate_clean_manifest(manifest, d, canrace=False, prefix=None):
486 import oe.path
487
488 mfile = open(manifest)
489 entries = mfile.readlines()
490 mfile.close()
491
492 for entry in entries:
493 entry = entry.strip()
494 if prefix and not entry.startswith("/"):
495 entry = prefix + "/" + entry
496 bb.debug(2, "Removing manifest: %s" % entry)
497 # We can race against another package populating directories as we're removing them
498 # so we ignore errors here.
499 try:
500 if entry.endswith("/"):
501 if os.path.islink(entry[:-1]):
502 os.remove(entry[:-1])
503 elif os.path.exists(entry) and len(os.listdir(entry)) == 0 and not canrace:
504 # Removing directories whilst builds are in progress exposes a race. Only
505 # do it in contexts where it is safe to do so.
506 os.rmdir(entry[:-1])
507 else:
508 os.remove(entry)
509 except OSError:
510 pass
511
512 postrm = manifest + ".postrm"
513 if os.path.exists(manifest + ".postrm"):
514 import subprocess
515 os.chmod(postrm, 0o755)
516 subprocess.check_call(postrm, shell=True)
517 oe.path.remove(postrm)
518
519 oe.path.remove(manifest)
520
521def sstate_clean(ss, d):
522 import oe.path
523 import glob
524
525 d2 = d.createCopy()
526 stamp_clean = d.getVar("STAMPCLEAN")
527 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
528 if extrainf:
529 d2.setVar("SSTATE_MANMACH", extrainf)
530 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
531 else:
532 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
533
534 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
535
536 if os.path.exists(manifest):
537 locks = []
538 for lock in ss['lockfiles-shared']:
539 locks.append(bb.utils.lockfile(lock))
540 for lock in ss['lockfiles']:
541 locks.append(bb.utils.lockfile(lock))
542
543 sstate_clean_manifest(manifest, d, canrace=True)
544
545 for lock in locks:
546 bb.utils.unlockfile(lock)
547
548 # Remove the current and previous stamps, but keep the sigdata.
549 #
550 # The glob() matches do_task* which may match multiple tasks, for
551 # example: do_package and do_package_write_ipk, so we need to
552 # exactly match *.do_task.* and *.do_task_setscene.*
553 rm_stamp = '.do_%s.' % ss['task']
554 rm_setscene = '.do_%s_setscene.' % ss['task']
555 # For BB_SIGNATURE_HANDLER = "noop"
556 rm_nohash = ".do_%s" % ss['task']
557 for stfile in glob.glob(wildcard_stfile):
558 # Keep the sigdata
559 if ".sigdata." in stfile or ".sigbasedata." in stfile:
560 continue
561 # Preserve taint files in the stamps directory
562 if stfile.endswith('.taint'):
563 continue
564 if rm_stamp in stfile or rm_setscene in stfile or \
565 stfile.endswith(rm_nohash):
566 oe.path.remove(stfile)
567
568sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
569
570CLEANFUNCS += "sstate_cleanall"
571
572python sstate_cleanall() {
573 bb.note("Removing shared state for package %s" % d.getVar('PN'))
574
575 manifest_dir = d.getVar('SSTATE_MANIFESTS')
576 if not os.path.exists(manifest_dir):
577 return
578
579 tasks = d.getVar('SSTATETASKS').split()
580 for name in tasks:
581 ld = d.createCopy()
582 shared_state = sstate_state_fromvars(ld, name)
583 sstate_clean(shared_state, ld)
584}
585
586python sstate_hardcode_path () {
587 import subprocess, platform
588
589 # Need to remove hardcoded paths and fix these when we install the
590 # staging packages.
591 #
592 # Note: the logic in this function needs to match the reverse logic
593 # in sstate_installpkg(ss, d)
594
595 staging_target = d.getVar('RECIPE_SYSROOT')
596 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
597 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
598
599 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host
600 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
601 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
602 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
603 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
604 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
605 else:
606 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
607 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
608
609 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
610 for fixmevar in extra_staging_fixmes.split():
611 fixme_path = d.getVar(fixmevar)
612 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
613 sstate_grep_cmd += " -e '%s'" % (fixme_path)
614
615 fixmefn = sstate_builddir + "fixmepath"
616
617 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
618 sstate_filelist_cmd = "tee %s" % (fixmefn)
619
620 # fixmepath file needs relative paths, drop sstate_builddir prefix
621 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
622
623 xargs_no_empty_run_cmd = '--no-run-if-empty'
624 if platform.system() == 'Darwin':
625 xargs_no_empty_run_cmd = ''
626
627 # Limit the fixpaths and sed operations based on the initial grep search
628 # This has the side effect of making sure the vfs cache is hot
629 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
630
631 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
632 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
633
634 # If the fixmefn is empty, remove it..
635 if os.stat(fixmefn).st_size == 0:
636 os.remove(fixmefn)
637 else:
638 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
639 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
640}
641
642def sstate_package(ss, d):
643 import oe.path
644 import time
645
646 tmpdir = d.getVar('TMPDIR')
647
648 fixtime = False
649 if ss['task'] == "package":
650 fixtime = True
651
652 def fixtimestamp(root, path):
653 f = os.path.join(root, path)
654 if os.lstat(f).st_mtime > sde:
655 os.utime(f, (sde, sde), follow_symlinks=False)
656
657 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
658 sde = int(d.getVar("SOURCE_DATE_EPOCH") or time.time())
659 d.setVar("SSTATE_CURRTASK", ss['task'])
660 bb.utils.remove(sstatebuild, recurse=True)
661 bb.utils.mkdirhier(sstatebuild)
662 for state in ss['dirs']:
663 if not os.path.exists(state[1]):
664 continue
665 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
666 # Find and error for absolute symlinks. We could attempt to relocate but its not
667 # clear where the symlink is relative to in this context. We could add that markup
668 # to sstate tasks but there aren't many of these so better just avoid them entirely.
669 for walkroot, dirs, files in os.walk(state[1]):
670 for file in files + dirs:
671 if fixtime:
672 fixtimestamp(walkroot, file)
673 srcpath = os.path.join(walkroot, file)
674 if not os.path.islink(srcpath):
675 continue
676 link = os.readlink(srcpath)
677 if not os.path.isabs(link):
678 continue
679 if not link.startswith(tmpdir):
680 continue
681 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
682 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
683 bb.utils.rename(state[1], sstatebuild + state[0])
684
685 workdir = d.getVar('WORKDIR')
686 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
687 for plain in ss['plaindirs']:
688 pdir = plain.replace(workdir, sstatebuild)
689 if sharedworkdir in plain:
690 pdir = plain.replace(sharedworkdir, sstatebuild)
691 bb.utils.mkdirhier(plain)
692 bb.utils.mkdirhier(pdir)
693 bb.utils.rename(plain, pdir)
694 if fixtime:
695 fixtimestamp(pdir, "")
696 for walkroot, dirs, files in os.walk(pdir):
697 for file in files + dirs:
698 fixtimestamp(walkroot, file)
699
700 d.setVar('SSTATE_BUILDDIR', sstatebuild)
701 d.setVar('SSTATE_INSTDIR', sstatebuild)
702
703 if d.getVar('SSTATE_SKIP_CREATION') == '1':
704 return
705
Patrick Williams73bd93f2024-02-20 08:07:48 -0600706 sstate_create_package = ['sstate_report_unihash', 'sstate_create_and_sign_package']
Patrick Williams92b42cb2022-09-03 06:53:57 -0500707
708 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
709 sstate_create_package + \
710 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
711 # All hooks should run in SSTATE_BUILDDIR.
712 bb.build.exec_func(f, d, (sstatebuild,))
713
714 # SSTATE_PKG may have been changed by sstate_report_unihash
715 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
716 if not os.path.exists(siginfo):
717 bb.siggen.dump_this_task(siginfo, d)
718 else:
719 try:
720 os.utime(siginfo, None)
721 except PermissionError:
722 pass
723 except OSError as e:
724 # Handle read-only file systems gracefully
725 import errno
726 if e.errno != errno.EROFS:
727 raise e
728
729 return
730
731sstate_package[vardepsexclude] += "SSTATE_SIG_KEY"
732
733def pstaging_fetch(sstatefetch, d):
734 import bb.fetch2
735
736 # Only try and fetch if the user has configured a mirror
737 mirrors = d.getVar('SSTATE_MIRRORS')
738 if not mirrors:
739 return
740
741 # Copy the data object and override DL_DIR and SRC_URI
742 localdata = bb.data.createCopy(d)
743
744 dldir = localdata.expand("${SSTATE_DIR}")
745 bb.utils.mkdirhier(dldir)
746
747 localdata.delVar('MIRRORS')
748 localdata.setVar('FILESPATH', dldir)
749 localdata.setVar('DL_DIR', dldir)
750 localdata.setVar('PREMIRRORS', mirrors)
Patrick Williams92b42cb2022-09-03 06:53:57 -0500751
752 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
753 # we'll want to allow network access for the current set of fetches.
754 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
755 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
756 localdata.delVar('BB_NO_NETWORK')
757
758 # Try a fetch from the sstate mirror, if it fails just return and
759 # we will build the package
760 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
761 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
762 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
763 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
764
765 for srcuri in uris:
Andrew Geissler87f5cff2022-09-30 13:13:31 -0500766 localdata.delVar('SRC_URI')
Patrick Williams92b42cb2022-09-03 06:53:57 -0500767 localdata.setVar('SRC_URI', srcuri)
768 try:
769 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
770 fetcher.checkstatus()
771 fetcher.download()
772
773 except bb.fetch2.BBFetchException:
774 pass
775
Patrick Williams92b42cb2022-09-03 06:53:57 -0500776def sstate_setscene(d):
777 shared_state = sstate_state_fromvars(d)
778 accelerate = sstate_installpkg(shared_state, d)
779 if not accelerate:
780 msg = "No sstate archive obtainable, will run full task instead."
781 bb.warn(msg)
782 raise bb.BBHandledException(msg)
783
784python sstate_task_prefunc () {
785 shared_state = sstate_state_fromvars(d)
786 sstate_clean(shared_state, d)
787}
788sstate_task_prefunc[dirs] = "${WORKDIR}"
789
790python sstate_task_postfunc () {
791 shared_state = sstate_state_fromvars(d)
792
793 for intercept in shared_state['interceptfuncs']:
794 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
795
796 omask = os.umask(0o002)
797 if omask != 0o002:
798 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
799 sstate_package(shared_state, d)
800 os.umask(omask)
801
802 sstateinst = d.getVar("SSTATE_INSTDIR")
803 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
804
805 sstate_installpkgdir(shared_state, d)
806
807 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
808}
809sstate_task_postfunc[dirs] = "${WORKDIR}"
810
Patrick Williams73bd93f2024-02-20 08:07:48 -0600811# Create a sstate package
812# If enabled, sign the package.
813# Package and signature are created in a sub-directory
814# and renamed in place once created.
815python sstate_create_and_sign_package () {
816 from pathlib import Path
817
818 # Best effort touch
819 def touch(file):
820 try:
821 file.touch()
822 except:
823 pass
824
825 def update_file(src, dst, force=False):
826 if dst.is_symlink() and not dst.exists():
827 force=True
828 try:
829 # This relies on that src is a temporary file that can be renamed
830 # or left as is.
831 if force:
832 src.rename(dst)
833 else:
834 os.link(src, dst)
835 return True
836 except:
837 pass
838
839 if dst.exists():
840 touch(dst)
841
842 return False
843
844 sign_pkg = (
845 bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG")) and
846 bool(d.getVar("SSTATE_SIG_KEY"))
847 )
848
849 sstate_pkg = Path(d.getVar("SSTATE_PKG"))
850 sstate_pkg_sig = Path(str(sstate_pkg) + ".sig")
851 if sign_pkg:
852 if sstate_pkg.exists() and sstate_pkg_sig.exists():
853 touch(sstate_pkg)
854 touch(sstate_pkg_sig)
855 return
856 else:
857 if sstate_pkg.exists():
858 touch(sstate_pkg)
859 return
860
861 # Create the required sstate directory if it is not present.
862 if not sstate_pkg.parent.is_dir():
863 with bb.utils.umask(0o002):
864 bb.utils.mkdirhier(str(sstate_pkg.parent))
865
866 if sign_pkg:
867 from tempfile import TemporaryDirectory
868 with TemporaryDirectory(dir=sstate_pkg.parent) as tmp_dir:
869 tmp_pkg = Path(tmp_dir) / sstate_pkg.name
870 d.setVar("TMP_SSTATE_PKG", str(tmp_pkg))
871 bb.build.exec_func('sstate_archive_package', d)
872
873 from oe.gpg_sign import get_signer
874 signer = get_signer(d, 'local')
875 signer.detach_sign(str(tmp_pkg), d.getVar('SSTATE_SIG_KEY'), None,
876 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
877
878 tmp_pkg_sig = Path(tmp_dir) / sstate_pkg_sig.name
879 if not update_file(tmp_pkg_sig, sstate_pkg_sig):
880 # If the created signature file could not be copied into place,
881 # then we should not use the sstate package either.
882 return
883
884 # If the .sig file was updated, then the sstate package must also
885 # be updated.
886 update_file(tmp_pkg, sstate_pkg, force=True)
887 else:
888 from tempfile import NamedTemporaryFile
889 with NamedTemporaryFile(prefix=sstate_pkg.name, dir=sstate_pkg.parent) as tmp_pkg_fd:
890 tmp_pkg = tmp_pkg_fd.name
891 d.setVar("TMP_SSTATE_PKG", str(tmp_pkg))
892 bb.build.exec_func('sstate_archive_package',d)
893 update_file(tmp_pkg, sstate_pkg)
894 # update_file() may have renamed tmp_pkg, which must exist when the
895 # NamedTemporaryFile() context handler ends.
896 touch(Path(tmp_pkg))
897
Patrick Williams169d7bc2024-01-05 11:33:25 -0600898}
Patrick Williams92b42cb2022-09-03 06:53:57 -0500899
Patrick Williams92b42cb2022-09-03 06:53:57 -0500900# Shell function to generate a sstate package from a directory
901# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
Patrick Williams73bd93f2024-02-20 08:07:48 -0600902# The calling function handles moving the sstate package into the final
903# destination.
904sstate_archive_package () {
Patrick Williams92b42cb2022-09-03 06:53:57 -0500905 OPT="-cS"
906 ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
907 # Use pzstd if available
908 if [ -x "$(command -v pzstd)" ]; then
909 ZSTD="pzstd -${SSTATE_ZSTD_CLEVEL} -p ${ZSTD_THREADS}"
910 fi
911
912 # Need to handle empty directories
913 if [ "$(ls -A)" ]; then
914 set +e
Patrick Williams73bd93f2024-02-20 08:07:48 -0600915 tar -I "$ZSTD" $OPT -f ${TMP_SSTATE_PKG} *
Patrick Williams92b42cb2022-09-03 06:53:57 -0500916 ret=$?
917 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
918 exit 1
919 fi
920 set -e
921 else
Patrick Williams73bd93f2024-02-20 08:07:48 -0600922 tar -I "$ZSTD" $OPT --file=${TMP_SSTATE_PKG} --files-from=/dev/null
Patrick Williams92b42cb2022-09-03 06:53:57 -0500923 fi
Patrick Williams73bd93f2024-02-20 08:07:48 -0600924 chmod 0664 ${TMP_SSTATE_PKG}
Patrick Williams92b42cb2022-09-03 06:53:57 -0500925}
926
Patrick Williams92b42cb2022-09-03 06:53:57 -0500927
928python sstate_report_unihash() {
929 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
930
931 if report_unihash:
932 ss = sstate_state_fromvars(d)
933 report_unihash(os.getcwd(), ss['task'], d)
934}
935
936#
937# Shell function to decompress and prepare a package for installation
938# Will be run from within SSTATE_INSTDIR.
939#
940sstate_unpack_package () {
941 ZSTD="zstd -T${ZSTD_THREADS}"
942 # Use pzstd if available
943 if [ -x "$(command -v pzstd)" ]; then
944 ZSTD="pzstd -p ${ZSTD_THREADS}"
945 fi
946
947 tar -I "$ZSTD" -xvpf ${SSTATE_PKG}
948 # update .siginfo atime on local/NFS mirror if it is a symbolic link
949 [ ! -h ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch -a ${SSTATE_PKG}.siginfo 2>/dev/null || true
950 # update each symbolic link instead of any referenced file
951 touch --no-dereference ${SSTATE_PKG} 2>/dev/null || true
952 [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig 2>/dev/null || true
953 [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo 2>/dev/null || true
954}
955
956BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
957
958def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
Andrew Geissler517393d2023-01-13 08:55:19 -0600959 import itertools
960
Patrick Williams92b42cb2022-09-03 06:53:57 -0500961 found = set()
962 missed = set()
963
964 def gethash(task):
965 return sq_data['unihash'][task]
966
967 def getpathcomponents(task, d):
968 # Magic data from BB_HASHFILENAME
969 splithashfn = sq_data['hashfn'][task].split(" ")
970 spec = splithashfn[1]
971 if splithashfn[0] == "True":
972 extrapath = d.getVar("NATIVELSBSTRING") + "/"
973 else:
974 extrapath = ""
975
976 tname = bb.runqueue.taskname_from_tid(task)[3:]
977
978 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
979 spec = splithashfn[2]
980 extrapath = ""
981
982 return spec, extrapath, tname
983
984 def getsstatefile(tid, siginfo, d):
985 spec, extrapath, tname = getpathcomponents(tid, d)
986 return extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d)
987
988 for tid in sq_data['hash']:
989
990 sstatefile = d.expand("${SSTATE_DIR}/" + getsstatefile(tid, siginfo, d))
991
992 if os.path.exists(sstatefile):
Patrick Williamsac13d5f2023-11-24 18:59:46 -0600993 oe.utils.touch(sstatefile)
Patrick Williams92b42cb2022-09-03 06:53:57 -0500994 found.add(tid)
995 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
996 else:
997 missed.add(tid)
998 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
999
1000 foundLocal = len(found)
1001 mirrors = d.getVar("SSTATE_MIRRORS")
1002 if mirrors:
1003 # Copy the data object and override DL_DIR and SRC_URI
1004 localdata = bb.data.createCopy(d)
1005
1006 dldir = localdata.expand("${SSTATE_DIR}")
1007 localdata.delVar('MIRRORS')
1008 localdata.setVar('FILESPATH', dldir)
1009 localdata.setVar('DL_DIR', dldir)
1010 localdata.setVar('PREMIRRORS', mirrors)
1011
1012 bb.debug(2, "SState using premirror of: %s" % mirrors)
1013
1014 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
1015 # we'll want to allow network access for the current set of fetches.
1016 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
1017 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
1018 localdata.delVar('BB_NO_NETWORK')
1019
1020 from bb.fetch2 import FetchConnectionCache
1021 def checkstatus_init():
1022 while not connection_cache_pool.full():
1023 connection_cache_pool.put(FetchConnectionCache())
1024
1025 def checkstatus_end():
1026 while not connection_cache_pool.empty():
1027 connection_cache = connection_cache_pool.get()
1028 connection_cache.close_connections()
1029
1030 def checkstatus(arg):
1031 (tid, sstatefile) = arg
1032
1033 connection_cache = connection_cache_pool.get()
1034 localdata2 = bb.data.createCopy(localdata)
1035 srcuri = "file://" + sstatefile
1036 localdata2.setVar('SRC_URI', srcuri)
1037 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
1038
1039 import traceback
1040
1041 try:
1042 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
1043 connection_cache=connection_cache)
1044 fetcher.checkstatus()
1045 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
1046 found.add(tid)
1047 missed.remove(tid)
1048 except bb.fetch2.FetchError as e:
1049 bb.debug(2, "SState: Unsuccessful fetch test for %s (%s)\n%s" % (srcuri, repr(e), traceback.format_exc()))
1050 except Exception as e:
1051 bb.error("SState: cannot test %s: %s\n%s" % (srcuri, repr(e), traceback.format_exc()))
1052
1053 connection_cache_pool.put(connection_cache)
1054
1055 if progress:
Andrew Geissler517393d2023-01-13 08:55:19 -06001056 bb.event.fire(bb.event.ProcessProgress(msg, next(cnt_tasks_done)), d)
Andrew Geissler6aa7eec2023-03-03 12:41:14 -06001057 bb.event.check_for_interrupts(d)
Patrick Williams92b42cb2022-09-03 06:53:57 -05001058
1059 tasklist = []
1060 for tid in missed:
1061 sstatefile = d.expand(getsstatefile(tid, siginfo, d))
1062 tasklist.append((tid, sstatefile))
1063
1064 if tasklist:
1065 nproc = min(int(d.getVar("BB_NUMBER_THREADS")), len(tasklist))
1066
Andrew Geissler517393d2023-01-13 08:55:19 -06001067 ## thread-safe counter
1068 cnt_tasks_done = itertools.count(start = 1)
Patrick Williams92b42cb2022-09-03 06:53:57 -05001069 progress = len(tasklist) >= 100
1070 if progress:
1071 msg = "Checking sstate mirror object availability"
1072 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
1073
1074 # Have to setup the fetcher environment here rather than in each thread as it would race
1075 fetcherenv = bb.fetch2.get_fetcher_environment(d)
1076 with bb.utils.environment(**fetcherenv):
1077 bb.event.enable_threadlock()
1078 import concurrent.futures
1079 from queue import Queue
1080 connection_cache_pool = Queue(nproc)
1081 checkstatus_init()
1082 with concurrent.futures.ThreadPoolExecutor(max_workers=nproc) as executor:
1083 executor.map(checkstatus, tasklist.copy())
1084 checkstatus_end()
1085 bb.event.disable_threadlock()
1086
1087 if progress:
1088 bb.event.fire(bb.event.ProcessFinished(msg), d)
1089
1090 inheritlist = d.getVar("INHERIT")
1091 if "toaster" in inheritlist:
1092 evdata = {'missed': [], 'found': []};
1093 for tid in missed:
1094 sstatefile = d.expand(getsstatefile(tid, False, d))
1095 evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
1096 for tid in found:
1097 sstatefile = d.expand(getsstatefile(tid, False, d))
1098 evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
1099 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
1100
1101 if summary:
1102 # Print some summary statistics about the current task completion and how much sstate
1103 # reuse there was. Avoid divide by zero errors.
1104 total = len(sq_data['hash'])
1105 complete = 0
1106 if currentcount:
1107 complete = (len(found) + currentcount) / (total + currentcount) * 100
1108 match = 0
1109 if total:
1110 match = len(found) / total * 100
1111 bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" %
1112 (total, foundLocal, len(found)-foundLocal, len(missed), currentcount, match, complete))
1113
1114 if hasattr(bb.parse.siggen, "checkhashes"):
1115 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
1116
1117 return found
1118setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT"
1119
1120BB_SETSCENE_DEPVALID = "setscene_depvalid"
1121
1122def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
1123 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
1124 # task is included in taskdependees too
1125 # Return - False - We need this dependency
1126 # - True - We can skip this dependency
1127 import re
1128
1129 def logit(msg, log):
1130 if log is not None:
1131 log.append(msg)
1132 else:
1133 bb.debug(2, msg)
1134
1135 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
1136
Patrick Williams7784c422022-11-17 07:29:11 -06001137 directtasks = ["do_populate_lic", "do_deploy_source_date_epoch", "do_shared_workdir", "do_stash_locale", "do_gcc_stash_builddir", "do_create_spdx", "do_deploy_archives"]
Patrick Williams92b42cb2022-09-03 06:53:57 -05001138
1139 def isNativeCross(x):
1140 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
1141
1142 # We only need to trigger deploy_source_date_epoch through direct dependencies
1143 if taskdependees[task][1] in directtasks:
1144 return True
1145
1146 # We only need to trigger packagedata through direct dependencies
1147 # but need to preserve packagedata on packagedata links
1148 if taskdependees[task][1] == "do_packagedata":
1149 for dep in taskdependees:
1150 if taskdependees[dep][1] == "do_packagedata":
1151 return False
1152 return True
1153
1154 for dep in taskdependees:
1155 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
1156 if task == dep:
1157 continue
1158 if dep in notneeded:
1159 continue
1160 # do_package_write_* and do_package doesn't need do_package
1161 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
1162 continue
1163 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
1164 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
1165 return False
1166 # do_package/packagedata/package_qa/deploy don't need do_populate_sysroot
1167 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa', 'do_deploy']:
1168 continue
1169 # Native/Cross packages don't exist and are noexec anyway
1170 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
1171 continue
1172
Patrick Williams92b42cb2022-09-03 06:53:57 -05001173 # Consider sysroot depending on sysroot tasks
1174 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
1175 # Allow excluding certain recursive dependencies. If a recipe needs it should add a
1176 # specific dependency itself, rather than relying on one of its dependees to pull
1177 # them in.
1178 # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
1179 not_needed = False
1180 excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT')
1181 if excludedeps is None:
1182 # Cache the regular expressions for speed
1183 excludedeps = []
1184 for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split():
1185 excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))
1186 d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps)
1187 for excl in excludedeps:
1188 if excl[0].match(taskdependees[dep][0]):
1189 if excl[1].match(taskdependees[task][0]):
1190 not_needed = True
1191 break
1192 if not_needed:
1193 continue
1194 # For meta-extsdk-toolchain we want all sysroot dependencies
1195 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
1196 return False
1197 # Native/Cross populate_sysroot need their dependencies
1198 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
1199 return False
1200 # Target populate_sysroot depended on by cross tools need to be installed
1201 if isNativeCross(taskdependees[dep][0]):
1202 return False
1203 # Native/cross tools depended upon by target sysroot are not needed
1204 # Add an exception for shadow-native as required by useradd.bbclass
1205 if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
1206 continue
1207 # Target populate_sysroot need their dependencies
1208 return False
1209
1210 if taskdependees[dep][1] in directtasks:
1211 continue
1212
1213 # Safe fallthrough default
1214 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
1215 return False
1216 return True
1217
1218addhandler sstate_eventhandler
1219sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
1220python sstate_eventhandler() {
1221 d = e.data
1222 writtensstate = d.getVar('SSTATE_CURRTASK')
1223 if not writtensstate:
1224 taskname = d.getVar("BB_RUNTASK")[3:]
1225 spec = d.getVar('SSTATE_PKGSPEC')
1226 swspec = d.getVar('SSTATE_SWSPEC')
1227 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1228 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1229 d.setVar("SSTATE_EXTRAPATH", "")
1230 d.setVar("SSTATE_CURRTASK", taskname)
1231 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
1232 if not os.path.exists(siginfo):
1233 bb.siggen.dump_this_task(siginfo, d)
1234 else:
Patrick Williamsac13d5f2023-11-24 18:59:46 -06001235 oe.utils.touch(siginfo)
Patrick Williams92b42cb2022-09-03 06:53:57 -05001236}
1237
1238SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1"
1239
1240#
1241# Event handler which removes manifests and stamps file for recipes which are no
1242# longer 'reachable' in a build where they once were. 'Reachable' refers to
1243# whether a recipe is parsed so recipes in a layer which was removed would no
1244# longer be reachable. Switching between systemd and sysvinit where recipes
1245# became skipped would be another example.
1246#
1247# Also optionally removes the workdir of those tasks/recipes
1248#
1249addhandler sstate_eventhandler_reachablestamps
1250sstate_eventhandler_reachablestamps[eventmask] = "bb.event.ReachableStamps"
1251python sstate_eventhandler_reachablestamps() {
1252 import glob
1253 d = e.data
1254 stamps = e.stamps.values()
1255 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
1256 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1257 preservestamps = []
1258 if os.path.exists(preservestampfile):
1259 with open(preservestampfile, 'r') as f:
1260 preservestamps = f.readlines()
1261 seen = []
1262
1263 # The machine index contains all the stamps this machine has ever seen in this build directory.
1264 # We should only remove things which this machine once accessed but no longer does.
1265 machineindex = set()
1266 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1267 mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
1268 if os.path.exists(mi):
1269 with open(mi, "r") as f:
1270 machineindex = set(line.strip() for line in f.readlines())
1271
1272 for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
1273 toremove = []
1274 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1275 if not os.path.exists(i):
1276 continue
1277 manseen = set()
1278 ignore = []
1279 with open(i, "r") as f:
1280 lines = f.readlines()
1281 for l in reversed(lines):
1282 try:
1283 (stamp, manifest, workdir) = l.split()
1284 # The index may have multiple entries for the same manifest as the code above only appends
1285 # new entries and there may be an entry with matching manifest but differing version in stamp/workdir.
1286 # The last entry in the list is the valid one, any earlier entries with matching manifests
1287 # should be ignored.
1288 if manifest in manseen:
1289 ignore.append(l)
1290 continue
1291 manseen.add(manifest)
1292 if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
1293 toremove.append(l)
1294 if stamp not in seen:
1295 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
1296 seen.append(stamp)
1297 except ValueError:
1298 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
1299
1300 if toremove:
1301 msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a)
1302 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
1303
1304 removed = 0
1305 for r in toremove:
1306 (stamp, manifest, workdir) = r.split()
1307 for m in glob.glob(manifest + ".*"):
1308 if m.endswith(".postrm"):
1309 continue
1310 sstate_clean_manifest(m, d)
1311 bb.utils.remove(stamp + "*")
1312 if removeworkdir:
1313 bb.utils.remove(workdir, recurse = True)
1314 lines.remove(r)
1315 removed = removed + 1
1316 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
Andrew Geissler6aa7eec2023-03-03 12:41:14 -06001317 bb.event.check_for_interrupts(d)
Patrick Williams92b42cb2022-09-03 06:53:57 -05001318
1319 bb.event.fire(bb.event.ProcessFinished(msg), d)
1320
1321 with open(i, "w") as f:
1322 for l in lines:
1323 if l in ignore:
1324 continue
1325 f.write(l)
1326 machineindex |= set(stamps)
1327 with open(mi, "w") as f:
1328 for l in machineindex:
1329 f.write(l + "\n")
1330
1331 if preservestamps:
1332 os.remove(preservestampfile)
1333}
1334
1335
1336#
1337# Bitbake can generate an event showing which setscene tasks are 'stale',
1338# i.e. which ones will be rerun. These are ones where a stamp file is present but
1339# it is stable (e.g. taskhash doesn't match). With that list we can go through
1340# the manifests for matching tasks and "uninstall" those manifests now. We do
1341# this now rather than mid build since the distribution of files between sstate
1342# objects may have changed, new tasks may run first and if those new tasks overlap
1343# with the stale tasks, we'd see overlapping files messages and failures. Thankfully
1344# removing these files is fast.
1345#
1346addhandler sstate_eventhandler_stalesstate
1347sstate_eventhandler_stalesstate[eventmask] = "bb.event.StaleSetSceneTasks"
1348python sstate_eventhandler_stalesstate() {
1349 d = e.data
1350 tasks = e.tasks
1351
1352 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1353
1354 for a in list(set(d.getVar("SSTATE_ARCHS").split())):
1355 toremove = []
1356 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1357 if not os.path.exists(i):
1358 continue
1359 with open(i, "r") as f:
1360 lines = f.readlines()
1361 for l in lines:
1362 try:
1363 (stamp, manifest, workdir) = l.split()
1364 for tid in tasks:
1365 for s in tasks[tid]:
1366 if s.startswith(stamp):
1367 taskname = bb.runqueue.taskname_from_tid(tid)[3:]
1368 manname = manifest + "." + taskname
1369 if os.path.exists(manname):
1370 bb.debug(2, "Sstate for %s is stale, removing related manifest %s" % (tid, manname))
1371 toremove.append((manname, tid, tasks[tid]))
1372 break
1373 except ValueError:
1374 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
1375
1376 if toremove:
1377 msg = "Removing %d stale sstate objects for arch %s" % (len(toremove), a)
1378 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
1379
1380 removed = 0
1381 for (manname, tid, stamps) in toremove:
1382 sstate_clean_manifest(manname, d)
1383 for stamp in stamps:
1384 bb.utils.remove(stamp)
1385 removed = removed + 1
1386 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
Andrew Geissler6aa7eec2023-03-03 12:41:14 -06001387 bb.event.check_for_interrupts(d)
Patrick Williams92b42cb2022-09-03 06:53:57 -05001388
1389 bb.event.fire(bb.event.ProcessFinished(msg), d)
1390}