blob: 92a73114bb59e84a9fbdf74371b69da9c805754d [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001SSTATE_VERSION = "3"
2
3SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
4SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
5
Andrew Geissler82c905d2020-04-13 13:39:40 -05006def generate_sstatefn(spec, hash, taskname, siginfo, d):
7 if taskname is None:
8 return ""
9 extension = ".tgz"
10 # 8 chars reserved for siginfo
11 limit = 254 - 8
12 if siginfo:
13 limit = 254
14 extension = ".tgz.siginfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015 if not hash:
16 hash = "INVALID"
Andrew Geissler82c905d2020-04-13 13:39:40 -050017 fn = spec + hash + "_" + taskname + extension
18 # If the filename is too long, attempt to reduce it
19 if len(fn) > limit:
20 components = spec.split(":")
21 # Fields 0,5,6 are mandatory, 1 is most useful, 2,3,4 are just for information
22 # 7 is for the separators
23 avail = (254 - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3
24 components[2] = components[2][:avail]
25 components[3] = components[3][:avail]
26 components[4] = components[4][:avail]
27 spec = ":".join(components)
28 fn = spec + hash + "_" + taskname + extension
29 if len(fn) > limit:
30 bb.fatal("Unable to reduce sstate name to less than 255 chararacters")
31 return hash[:2] + "/" + hash[2:4] + "/" + fn
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032
33SSTATE_PKGARCH = "${PACKAGE_ARCH}"
34SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050035SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
Andrew Geissler82c905d2020-04-13 13:39:40 -050036SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d.getVar('SSTATE_CURRTASK'), False, d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
38SSTATE_EXTRAPATH = ""
39SSTATE_EXTRAPATHWILDCARD = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -050040SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041
Patrick Williamsc0f7c042017-02-23 20:41:17 -060042# explicitly make PV to depend on evaluated value of PV variable
43PV[vardepvalue] = "${PV}"
44
Patrick Williamsc124f4f2015-09-15 14:41:29 -050045# We don't want the sstate to depend on things like the distro string
46# of the system, we let the sstate paths take care of this.
47SSTATE_EXTRAPATH[vardepvalue] = ""
Brad Bishop19323692019-04-05 15:28:33 -040048SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049
50# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080051SSTATE_DUPWHITELIST = "${DEPLOY_DIR}/licenses/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052# Avoid docbook/sgml catalog warnings for now
53SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
Brad Bishop316dfdd2018-06-25 12:45:53 -040054# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
55SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/"
56SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080057# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used
58SSTATE_DUPWHITELIST += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059# Archive the sources for many architectures in one deploy folder
60SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080061# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files
62SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/ovmf"
63SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/grub-efi"
64SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/systemd-boot"
65SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/microcode"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066
Brad Bishop6e60e8b2018-02-01 10:27:11 -050067SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
68SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
69SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050071BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
73SSTATE_ARCHS = " \
74 ${BUILD_ARCH} \
Andrew Geissler6ce62a22020-11-30 19:58:47 -060075 ${BUILD_ARCH}_${ORIGNATIVELSBSTRING} \
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
77 ${BUILD_ARCH}_${TARGET_ARCH} \
78 ${SDK_ARCH}_${SDK_OS} \
79 ${SDK_ARCH}_${PACKAGE_ARCH} \
80 allarch \
81 ${PACKAGE_ARCH} \
Brad Bishop316dfdd2018-06-25 12:45:53 -040082 ${PACKAGE_EXTRA_ARCHS} \
83 ${MACHINE_ARCH}"
Andrew Geissler6ce62a22020-11-30 19:58:47 -060084SSTATE_ARCHS[vardepsexclude] = "ORIGNATIVELSBSTRING"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
86SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
87
88SSTATECREATEFUNCS = "sstate_hardcode_path"
Brad Bishop19323692019-04-05 15:28:33 -040089SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090SSTATEPOSTCREATEFUNCS = ""
91SSTATEPREINSTFUNCS = ""
92SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
93SSTATEPOSTINSTFUNCS = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050094EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050096# Check whether sstate exists for tasks that support sstate and are in the
97# locked signatures file.
98SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
99
100# Check whether the task's computed hash matches the task's hash in the
101# locked signatures file.
102SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500103
104# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
105# not sign)
106SSTATE_SIG_KEY ?= ""
107SSTATE_SIG_PASSPHRASE ?= ""
108# Whether to verify the GnUPG signatures when extracting sstate archives
109SSTATE_VERIFY_SIG ?= "0"
110
Brad Bishop19323692019-04-05 15:28:33 -0400111SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
112SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
113 the output hash for a task, which in turn is used to determine equivalency. \
114 "
115
Brad Bishop19323692019-04-05 15:28:33 -0400116SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
117SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
118 hash equivalency server, such as PN, PV, taskname, etc. This information \
119 is very useful for developers looking at task data, but may leak sensitive \
120 data if the equivalence server is public. \
121 "
122
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123python () {
124 if bb.data.inherits_class('native', d):
125 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
126 elif bb.data.inherits_class('crosssdk', d):
127 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
128 elif bb.data.inherits_class('cross', d):
129 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TARGET_ARCH}"))
130 elif bb.data.inherits_class('nativesdk', d):
131 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
132 elif bb.data.inherits_class('cross-canadian', d):
133 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500134 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135 d.setVar('SSTATE_PKGARCH', "allarch")
136 else:
137 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
138
139 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
140 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500141 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
Brad Bishop19323692019-04-05 15:28:33 -0400142 d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500144 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 d.setVar('SSTATETASKS', " ".join(unique_tasks))
146 for task in unique_tasks:
147 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
148 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
149}
150
151def sstate_init(task, d):
152 ss = {}
153 ss['task'] = task
154 ss['dirs'] = []
155 ss['plaindirs'] = []
156 ss['lockfiles'] = []
157 ss['lockfiles-shared'] = []
158 return ss
159
160def sstate_state_fromvars(d, task = None):
161 if task is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 task = d.getVar('BB_CURRENTTASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500163 if not task:
164 bb.fatal("sstate code running without task context?!")
165 task = task.replace("_setscene", "")
166
167 if task.startswith("do_"):
168 task = task[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500169 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
170 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
171 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
172 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
173 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
174 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
175 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176 if not task or len(inputs) != len(outputs):
177 bb.fatal("sstate variables not setup correctly?!")
178
179 if task == "populate_lic":
180 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
181 d.setVar("SSTATE_EXTRAPATH", "")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500182 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183
184 ss = sstate_init(task, d)
185 for i in range(len(inputs)):
186 sstate_add(ss, inputs[i], outputs[i], d)
187 ss['lockfiles'] = lockfiles
188 ss['lockfiles-shared'] = lockfilesshared
189 ss['plaindirs'] = plaindirs
190 ss['interceptfuncs'] = interceptfuncs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500191 ss['fixmedir'] = fixmedir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192 return ss
193
194def sstate_add(ss, source, dest, d):
195 if not source.endswith("/"):
196 source = source + "/"
197 if not dest.endswith("/"):
198 dest = dest + "/"
199 source = os.path.normpath(source)
200 dest = os.path.normpath(dest)
201 srcbase = os.path.basename(source)
202 ss['dirs'].append([srcbase, source, dest])
203 return ss
204
205def sstate_install(ss, d):
206 import oe.path
207 import oe.sstatesig
208 import subprocess
209
210 sharedfiles = []
211 shareddirs = []
212 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
213
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500214 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
215
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
217
218 if os.access(manifest, os.R_OK):
219 bb.fatal("Package already staged (%s)?!" % manifest)
220
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600221 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
222
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223 locks = []
224 for lock in ss['lockfiles-shared']:
225 locks.append(bb.utils.lockfile(lock, True))
226 for lock in ss['lockfiles']:
227 locks.append(bb.utils.lockfile(lock))
228
229 for state in ss['dirs']:
230 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
231 for walkroot, dirs, files in os.walk(state[1]):
232 for file in files:
233 srcpath = os.path.join(walkroot, file)
234 dstpath = srcpath.replace(state[1], state[2])
235 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
236 sharedfiles.append(dstpath)
237 for dir in dirs:
238 srcdir = os.path.join(walkroot, dir)
239 dstdir = srcdir.replace(state[1], state[2])
240 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500241 if os.path.islink(srcdir):
242 sharedfiles.append(dstdir)
243 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 if not dstdir.endswith("/"):
245 dstdir = dstdir + "/"
246 shareddirs.append(dstdir)
247
248 # Check the file list for conflicts against files which already exist
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500249 whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250 match = []
251 for f in sharedfiles:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500252 if os.path.exists(f) and not os.path.islink(f):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500253 f = os.path.normpath(f)
254 realmatch = True
255 for w in whitelist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 w = os.path.normpath(w)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 if f.startswith(w):
258 realmatch = False
259 break
260 if realmatch:
261 match.append(f)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500264 if search_output:
265 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
266 else:
267 match.append(" (not matched to any task)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 if match:
269 bb.error("The recipe %s is trying to install files into a shared " \
270 "area when those files already exist. Those files and their manifest " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 "location are:\n %s\nPlease verify which recipe should provide the " \
272 "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \
273 "break things - if not now, possibly in the future (we've seen builds fail " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 "several months later). If the system knew how to recover from this " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500275 "automatically it would, however there are several different scenarios " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500276 "which can result in this and we don't know which one this is. It may be " \
277 "you have switched providers of something like virtual/kernel (e.g. from " \
278 "linux-yocto to linux-yocto-dev), in that case you need to execute the " \
279 "clean task for both recipes and it will resolve this error. It may be " \
280 "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500281 "those recipes should again resolve this error, however switching " \
282 "DISTRO_FEATURES on an existing build directory is not supported - you " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500283 "should really clean out tmp and rebuild (reusing sstate should be safe). " \
284 "It could be the overlapping files detected are harmless in which case " \
285 "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \
286 "also be your build is including two different conflicting versions of " \
287 "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \
288 "be to resolve the conflict. If in doubt, please ask on the mailing list, " \
289 "sharing the error and filelist above." % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500290 (d.getVar('PN'), "\n ".join(match)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500291 bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.")
292
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500293 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
294 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
295 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
296
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297 # Write out the manifest
298 f = open(manifest, "w")
299 for file in sharedfiles:
300 f.write(file + "\n")
301
302 # We want to ensure that directories appear at the end of the manifest
303 # so that when we test to see if they should be deleted any contents
304 # added by the task will have been removed first.
305 dirs = sorted(shareddirs, key=len)
306 # Must remove children first, which will have a longer path than the parent
307 for di in reversed(dirs):
308 f.write(di + "\n")
309 f.close()
310
311 # Append to the list of manifests for this PACKAGE_ARCH
312
313 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
314 l = bb.utils.lockfile(i + ".lock")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500315 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 manifests = []
317 if os.path.exists(i):
318 with open(i, "r") as f:
319 manifests = f.readlines()
William A. Kennington IIIac69b482021-06-02 12:28:27 -0700320 # We append new entries, we don't remove older entries which may have the same
321 # manifest name but different versions from stamp/workdir. See below.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322 if filedata not in manifests:
323 with open(i, "a+") as f:
324 f.write(filedata)
325 bb.utils.unlockfile(l)
326
327 # Run the actual file install
328 for state in ss['dirs']:
329 if os.path.exists(state[1]):
330 oe.path.copyhardlinktree(state[1], state[2])
331
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500332 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500333 # All hooks should run in the SSTATE_INSTDIR
334 bb.build.exec_func(postinst, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335
336 for lock in locks:
337 bb.utils.unlockfile(lock)
338
339sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX"
340sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
341
342def sstate_installpkg(ss, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500343 from oe.gpg_sign import get_signer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500345 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
Andrew Geissler82c905d2020-04-13 13:39:40 -0500346 d.setVar("SSTATE_CURRTASK", ss['task'])
347 sstatefetch = d.getVar('SSTATE_PKGNAME')
348 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500349
350 if not os.path.exists(sstatepkg):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800351 pstaging_fetch(sstatefetch, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
353 if not os.path.isfile(sstatepkg):
Brad Bishop08902b02019-08-20 09:16:51 -0400354 bb.note("Sstate package %s does not exist" % sstatepkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 return False
356
357 sstate_clean(ss, d)
358
359 d.setVar('SSTATE_INSTDIR', sstateinst)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500361 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500362 if not os.path.isfile(sstatepkg + '.sig'):
363 bb.warn("No signature file for sstate package %s, skipping acceleration..." % sstatepkg)
364 return False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500365 signer = get_signer(d, 'local')
366 if not signer.verify(sstatepkg + '.sig'):
Brad Bishop08902b02019-08-20 09:16:51 -0400367 bb.warn("Cannot verify signature on sstate package %s, skipping acceleration..." % sstatepkg)
368 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500370 # Empty sstateinst directory, ensure its clean
371 if os.path.exists(sstateinst):
372 oe.path.remove(sstateinst)
373 bb.utils.mkdirhier(sstateinst)
374
375 sstateinst = d.getVar("SSTATE_INSTDIR")
376 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
377
378 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500379 # All hooks should run in the SSTATE_INSTDIR
380 bb.build.exec_func(f, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500382 return sstate_installpkgdir(ss, d)
383
384def sstate_installpkgdir(ss, d):
385 import oe.path
386 import subprocess
387
388 sstateinst = d.getVar("SSTATE_INSTDIR")
389 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
390
391 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
392 # All hooks should run in the SSTATE_INSTDIR
393 bb.build.exec_func(f, d, (sstateinst,))
394
395 def prepdir(dir):
396 # remove dir if it exists, ensure any parent directories do exist
397 if os.path.exists(dir):
398 oe.path.remove(dir)
399 bb.utils.mkdirhier(dir)
400 oe.path.remove(dir)
401
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 for state in ss['dirs']:
403 prepdir(state[1])
Andrew Geisslerc926e172021-05-07 16:11:35 -0500404 bb.utils.rename(sstateinst + state[0], state[1])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500405 sstate_install(ss, d)
406
407 for plain in ss['plaindirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500408 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500409 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500410 src = sstateinst + "/" + plain.replace(workdir, '')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500411 if sharedworkdir in plain:
412 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413 dest = plain
414 bb.utils.mkdirhier(src)
415 prepdir(dest)
Andrew Geisslerc926e172021-05-07 16:11:35 -0500416 bb.utils.rename(src, dest)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417
418 return True
419
420python sstate_hardcode_path_unpack () {
421 # Fixup hardcoded paths
422 #
423 # Note: The logic below must match the reverse logic in
424 # sstate_hardcode_path(d)
425 import subprocess
426
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500427 sstateinst = d.getVar('SSTATE_INSTDIR')
428 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
429 fixmefn = sstateinst + "fixmepath"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500430 if os.path.isfile(fixmefn):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500431 staging_target = d.getVar('RECIPE_SYSROOT')
432 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500433
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500434 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500436 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
437 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
438 else:
439 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500441 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500442 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500443 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
445
446 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
447 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
448
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500449 # Defer do_populate_sysroot relocation command
450 if sstatefixmedir:
451 bb.utils.mkdirhier(sstatefixmedir)
452 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
453 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
454 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
455 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
456 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
457 f.write(sstate_hardcode_cmd)
458 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
459 return
460
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500461 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500462 subprocess.check_call(sstate_hardcode_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800464 # Need to remove this or we'd copy it into the target directory and may
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500465 # conflict with another writer
466 os.remove(fixmefn)
467}
468
469def sstate_clean_cachefile(ss, d):
470 import oe.path
471
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300472 if d.getVarFlag('do_%s' % ss['task'], 'task'):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500473 d.setVar("SSTATE_PATH_CURRTASK", ss['task'])
474 sstatepkgfile = d.getVar('SSTATE_PATHSPEC')
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300475 bb.note("Removing %s" % sstatepkgfile)
476 oe.path.remove(sstatepkgfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500477
478def sstate_clean_cachefiles(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500479 for task in (d.getVar('SSTATETASKS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480 ld = d.createCopy()
481 ss = sstate_state_fromvars(ld, task)
482 sstate_clean_cachefile(ss, ld)
483
Andrew Geissler5f350902021-07-23 13:09:54 -0400484def sstate_clean_manifest(manifest, d, canrace=False, prefix=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500485 import oe.path
486
487 mfile = open(manifest)
488 entries = mfile.readlines()
489 mfile.close()
490
491 for entry in entries:
492 entry = entry.strip()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500493 if prefix and not entry.startswith("/"):
494 entry = prefix + "/" + entry
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 bb.debug(2, "Removing manifest: %s" % entry)
496 # We can race against another package populating directories as we're removing them
497 # so we ignore errors here.
498 try:
499 if entry.endswith("/"):
500 if os.path.islink(entry[:-1]):
501 os.remove(entry[:-1])
Andrew Geissler5f350902021-07-23 13:09:54 -0400502 elif os.path.exists(entry) and len(os.listdir(entry)) == 0 and not canrace:
503 # Removing directories whilst builds are in progress exposes a race. Only
504 # do it in contexts where it is safe to do so.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505 os.rmdir(entry[:-1])
506 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500507 os.remove(entry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500508 except OSError:
509 pass
510
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600511 postrm = manifest + ".postrm"
512 if os.path.exists(manifest + ".postrm"):
513 import subprocess
514 os.chmod(postrm, 0o755)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500515 subprocess.check_call(postrm, shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600516 oe.path.remove(postrm)
517
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500518 oe.path.remove(manifest)
519
520def sstate_clean(ss, d):
521 import oe.path
522 import glob
523
524 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500525 stamp_clean = d.getVar("STAMPCLEAN")
526 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500527 if extrainf:
528 d2.setVar("SSTATE_MANMACH", extrainf)
529 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
530 else:
531 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
532
533 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
534
535 if os.path.exists(manifest):
536 locks = []
537 for lock in ss['lockfiles-shared']:
538 locks.append(bb.utils.lockfile(lock))
539 for lock in ss['lockfiles']:
540 locks.append(bb.utils.lockfile(lock))
541
Andrew Geissler5f350902021-07-23 13:09:54 -0400542 sstate_clean_manifest(manifest, d, canrace=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500543
544 for lock in locks:
545 bb.utils.unlockfile(lock)
546
547 # Remove the current and previous stamps, but keep the sigdata.
548 #
549 # The glob() matches do_task* which may match multiple tasks, for
550 # example: do_package and do_package_write_ipk, so we need to
551 # exactly match *.do_task.* and *.do_task_setscene.*
552 rm_stamp = '.do_%s.' % ss['task']
553 rm_setscene = '.do_%s_setscene.' % ss['task']
554 # For BB_SIGNATURE_HANDLER = "noop"
555 rm_nohash = ".do_%s" % ss['task']
556 for stfile in glob.glob(wildcard_stfile):
557 # Keep the sigdata
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500558 if ".sigdata." in stfile or ".sigbasedata." in stfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500559 continue
560 # Preserve taint files in the stamps directory
561 if stfile.endswith('.taint'):
562 continue
563 if rm_stamp in stfile or rm_setscene in stfile or \
564 stfile.endswith(rm_nohash):
565 oe.path.remove(stfile)
566
567sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
568
569CLEANFUNCS += "sstate_cleanall"
570
571python sstate_cleanall() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500572 bb.note("Removing shared state for package %s" % d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500573
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500574 manifest_dir = d.getVar('SSTATE_MANIFESTS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575 if not os.path.exists(manifest_dir):
576 return
577
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500578 tasks = d.getVar('SSTATETASKS').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579 for name in tasks:
580 ld = d.createCopy()
581 shared_state = sstate_state_fromvars(ld, name)
582 sstate_clean(shared_state, ld)
583}
584
585python sstate_hardcode_path () {
586 import subprocess, platform
587
588 # Need to remove hardcoded paths and fix these when we install the
589 # staging packages.
590 #
591 # Note: the logic in this function needs to match the reverse logic
592 # in sstate_installpkg(ss, d)
593
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500594 staging_target = d.getVar('RECIPE_SYSROOT')
595 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
596 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500597
Brad Bishop316dfdd2018-06-25 12:45:53 -0400598 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500599 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500600 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500601 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
602 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400603 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500604 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400605 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
606 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500608 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500609 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500610 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500612 sstate_grep_cmd += " -e '%s'" % (fixme_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500613
614 fixmefn = sstate_builddir + "fixmepath"
615
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500616 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500617 sstate_filelist_cmd = "tee %s" % (fixmefn)
618
619 # fixmepath file needs relative paths, drop sstate_builddir prefix
620 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
621
622 xargs_no_empty_run_cmd = '--no-run-if-empty'
623 if platform.system() == 'Darwin':
624 xargs_no_empty_run_cmd = ''
625
626 # Limit the fixpaths and sed operations based on the initial grep search
627 # This has the side effect of making sure the vfs cache is hot
628 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
629
630 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500631 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500632
633 # If the fixmefn is empty, remove it..
634 if os.stat(fixmefn).st_size == 0:
635 os.remove(fixmefn)
636 else:
637 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500638 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500639}
640
641def sstate_package(ss, d):
642 import oe.path
Andrew Geissler5199d832021-09-24 16:47:35 -0500643 import time
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500644
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500645 tmpdir = d.getVar('TMPDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500646
Andrew Geissler5199d832021-09-24 16:47:35 -0500647 fixtime = False
648 if ss['task'] == "package":
649 fixtime = True
650
651 def fixtimestamp(root, path):
652 f = os.path.join(root, path)
653 if os.lstat(f).st_mtime > sde:
654 os.utime(f, (sde, sde), follow_symlinks=False)
655
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500656 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
Andrew Geissler5199d832021-09-24 16:47:35 -0500657 sde = int(d.getVar("SOURCE_DATE_EPOCH") or time.time())
Andrew Geissler82c905d2020-04-13 13:39:40 -0500658 d.setVar("SSTATE_CURRTASK", ss['task'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500659 bb.utils.remove(sstatebuild, recurse=True)
660 bb.utils.mkdirhier(sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500661 for state in ss['dirs']:
662 if not os.path.exists(state[1]):
663 continue
664 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500665 # Find and error for absolute symlinks. We could attempt to relocate but its not
666 # clear where the symlink is relative to in this context. We could add that markup
667 # to sstate tasks but there aren't many of these so better just avoid them entirely.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500668 for walkroot, dirs, files in os.walk(state[1]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500669 for file in files + dirs:
Andrew Geissler5199d832021-09-24 16:47:35 -0500670 if fixtime:
671 fixtimestamp(walkroot, file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672 srcpath = os.path.join(walkroot, file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500673 if not os.path.islink(srcpath):
674 continue
675 link = os.readlink(srcpath)
676 if not os.path.isabs(link):
677 continue
678 if not link.startswith(tmpdir):
679 continue
680 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
Andrew Geisslerc926e172021-05-07 16:11:35 -0500682 bb.utils.rename(state[1], sstatebuild + state[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500683
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500684 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500685 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500686 for plain in ss['plaindirs']:
687 pdir = plain.replace(workdir, sstatebuild)
Brad Bishop977dc1a2019-02-06 16:01:43 -0500688 if sharedworkdir in plain:
689 pdir = plain.replace(sharedworkdir, sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500690 bb.utils.mkdirhier(plain)
691 bb.utils.mkdirhier(pdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -0500692 bb.utils.rename(plain, pdir)
Andrew Geissler5199d832021-09-24 16:47:35 -0500693 if fixtime:
694 fixtimestamp(pdir, "")
695 for walkroot, dirs, files in os.walk(pdir):
696 for file in files + dirs:
697 fixtimestamp(walkroot, file)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500698
699 d.setVar('SSTATE_BUILDDIR', sstatebuild)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500700 d.setVar('SSTATE_INSTDIR', sstatebuild)
701
702 if d.getVar('SSTATE_SKIP_CREATION') == '1':
703 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500704
Brad Bishop08902b02019-08-20 09:16:51 -0400705 sstate_create_package = ['sstate_report_unihash', 'sstate_create_package']
706 if d.getVar('SSTATE_SIG_KEY'):
707 sstate_create_package.append('sstate_sign_package')
708
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500709 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
Brad Bishop08902b02019-08-20 09:16:51 -0400710 sstate_create_package + \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500711 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500712 # All hooks should run in SSTATE_BUILDDIR.
713 bb.build.exec_func(f, d, (sstatebuild,))
714
Andrew Geissler82c905d2020-04-13 13:39:40 -0500715 # SSTATE_PKG may have been changed by sstate_report_unihash
716 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
717 if not os.path.exists(siginfo):
718 bb.siggen.dump_this_task(siginfo, d)
719 else:
Andrew Geisslerc182c622020-05-15 14:13:32 -0500720 try:
721 os.utime(siginfo, None)
722 except PermissionError:
723 pass
Andrew Geissler5f350902021-07-23 13:09:54 -0400724 except OSError as e:
725 # Handle read-only file systems gracefully
Patrick Williams0ca19cc2021-08-16 14:03:13 -0500726 import errno
Andrew Geissler5f350902021-07-23 13:09:54 -0400727 if e.errno != errno.EROFS:
728 raise e
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500729
730 return
731
William A. Kennington IIIac69b482021-06-02 12:28:27 -0700732sstate_package[vardepsexclude] += "SSTATE_SIG_KEY"
733
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800734def pstaging_fetch(sstatefetch, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500735 import bb.fetch2
736
737 # Only try and fetch if the user has configured a mirror
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500738 mirrors = d.getVar('SSTATE_MIRRORS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500739 if not mirrors:
740 return
741
742 # Copy the data object and override DL_DIR and SRC_URI
743 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500744
745 dldir = localdata.expand("${SSTATE_DIR}")
746 bb.utils.mkdirhier(dldir)
747
748 localdata.delVar('MIRRORS')
749 localdata.setVar('FILESPATH', dldir)
750 localdata.setVar('DL_DIR', dldir)
751 localdata.setVar('PREMIRRORS', mirrors)
Andrew Geissler5199d832021-09-24 16:47:35 -0500752 localdata.setVar('SRCPV', d.getVar('SRCPV'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500753
754 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
755 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400756 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
757 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500758 localdata.delVar('BB_NO_NETWORK')
759
760 # Try a fetch from the sstate mirror, if it fails just return and
761 # we will build the package
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600762 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
763 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500764 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600765 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500766
767 for srcuri in uris:
768 localdata.setVar('SRC_URI', srcuri)
769 try:
770 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500771 fetcher.checkstatus()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500772 fetcher.download()
773
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500774 except bb.fetch2.BBFetchException:
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500775 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500776
Andrew Geissler5199d832021-09-24 16:47:35 -0500777pstaging_fetch[vardepsexclude] += "SRCPV"
778
779
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500780def sstate_setscene(d):
781 shared_state = sstate_state_fromvars(d)
782 accelerate = sstate_installpkg(shared_state, d)
783 if not accelerate:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600784 bb.fatal("No suitable staging package found")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500785
786python sstate_task_prefunc () {
787 shared_state = sstate_state_fromvars(d)
788 sstate_clean(shared_state, d)
789}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500790sstate_task_prefunc[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500791
792python sstate_task_postfunc () {
793 shared_state = sstate_state_fromvars(d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500794
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500795 for intercept in shared_state['interceptfuncs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500796 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
797
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600798 omask = os.umask(0o002)
799 if omask != 0o002:
800 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500801 sstate_package(shared_state, d)
802 os.umask(omask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500803
804 sstateinst = d.getVar("SSTATE_INSTDIR")
805 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
806
807 sstate_installpkgdir(shared_state, d)
808
809 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500810}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500811sstate_task_postfunc[dirs] = "${WORKDIR}"
812
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500813
814#
815# Shell function to generate a sstate package from a directory
816# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
817#
818sstate_create_package () {
Andrew Geissler82c905d2020-04-13 13:39:40 -0500819 # Exit early if it already exists
Brad Bishop08902b02019-08-20 09:16:51 -0400820 if [ -e ${SSTATE_PKG} ]; then
Andrew Geisslerc182c622020-05-15 14:13:32 -0500821 [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
Brad Bishop08902b02019-08-20 09:16:51 -0400822 return
823 fi
824
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500825 mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
Andrew Geissler82c905d2020-04-13 13:39:40 -0500826 TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
827
828 # Use pigz if available
829 OPT="-czS"
830 if [ -x "$(command -v pigz)" ]; then
831 OPT="-I pigz -cS"
832 fi
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800833
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500834 # Need to handle empty directories
835 if [ "$(ls -A)" ]; then
836 set +e
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800837 tar $OPT -f $TFILE *
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500838 ret=$?
839 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
840 exit 1
841 fi
842 set -e
843 else
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800844 tar $OPT --file=$TFILE --files-from=/dev/null
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500845 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500846 chmod 0664 $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400847 # Skip if it was already created by some other process
848 if [ ! -e ${SSTATE_PKG} ]; then
Andrew Geissler82c905d2020-04-13 13:39:40 -0500849 # Move into place using ln to attempt an atomic op.
850 # Abort if it already exists
851 ln $TFILE ${SSTATE_PKG} && rm $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400852 else
853 rm $TFILE
854 fi
Andrew Geisslerc182c622020-05-15 14:13:32 -0500855 [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500856}
857
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500858python sstate_sign_package () {
859 from oe.gpg_sign import get_signer
860
Brad Bishop08902b02019-08-20 09:16:51 -0400861
862 signer = get_signer(d, 'local')
863 sstate_pkg = d.getVar('SSTATE_PKG')
864 if os.path.exists(sstate_pkg + '.sig'):
865 os.unlink(sstate_pkg + '.sig')
866 signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
867 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500868}
869
Brad Bishop19323692019-04-05 15:28:33 -0400870python sstate_report_unihash() {
871 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
872
873 if report_unihash:
874 ss = sstate_state_fromvars(d)
875 report_unihash(os.getcwd(), ss['task'], d)
876}
877
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500878#
879# Shell function to decompress and prepare a package for installation
880# Will be run from within SSTATE_INSTDIR.
881#
882sstate_unpack_package () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500883 tar -xvzf ${SSTATE_PKG}
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500884 # update .siginfo atime on local/NFS mirror
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500885 [ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500886 # Use "! -w ||" to return true for read only files
887 [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500888 [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
889 [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500890}
891
892BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
893
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500894def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
Brad Bishop08902b02019-08-20 09:16:51 -0400895 found = set()
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600896 foundLocal = set()
897 foundNet = set()
Brad Bishop08902b02019-08-20 09:16:51 -0400898 missed = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500899
Brad Bishop19323692019-04-05 15:28:33 -0400900 def gethash(task):
Brad Bishop08902b02019-08-20 09:16:51 -0400901 return sq_data['unihash'][task]
Brad Bishop19323692019-04-05 15:28:33 -0400902
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500903 def getpathcomponents(task, d):
904 # Magic data from BB_HASHFILENAME
Brad Bishop08902b02019-08-20 09:16:51 -0400905 splithashfn = sq_data['hashfn'][task].split(" ")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500906 spec = splithashfn[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500907 if splithashfn[0] == "True":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500908 extrapath = d.getVar("NATIVELSBSTRING") + "/"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500909 else:
910 extrapath = ""
Brad Bishop08902b02019-08-20 09:16:51 -0400911
912 tname = bb.runqueue.taskname_from_tid(task)[3:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500913
914 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
915 spec = splithashfn[2]
916 extrapath = ""
917
918 return spec, extrapath, tname
919
920
Brad Bishop08902b02019-08-20 09:16:51 -0400921 for tid in sq_data['hash']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500922
Brad Bishop08902b02019-08-20 09:16:51 -0400923 spec, extrapath, tname = getpathcomponents(tid, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500924
Andrew Geissler82c905d2020-04-13 13:39:40 -0500925 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500926
927 if os.path.exists(sstatefile):
928 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
Brad Bishop08902b02019-08-20 09:16:51 -0400929 found.add(tid)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600930 foundLocal.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500931 continue
932 else:
Brad Bishop08902b02019-08-20 09:16:51 -0400933 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500934 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
935
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500936 mirrors = d.getVar("SSTATE_MIRRORS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500937 if mirrors:
938 # Copy the data object and override DL_DIR and SRC_URI
939 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500940
941 dldir = localdata.expand("${SSTATE_DIR}")
942 localdata.delVar('MIRRORS')
943 localdata.setVar('FILESPATH', dldir)
944 localdata.setVar('DL_DIR', dldir)
945 localdata.setVar('PREMIRRORS', mirrors)
946
947 bb.debug(2, "SState using premirror of: %s" % mirrors)
948
949 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
950 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400951 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
952 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500953 localdata.delVar('BB_NO_NETWORK')
954
955 from bb.fetch2 import FetchConnectionCache
956 def checkstatus_init(thread_worker):
957 thread_worker.connection_cache = FetchConnectionCache()
958
959 def checkstatus_end(thread_worker):
960 thread_worker.connection_cache.close_connections()
961
962 def checkstatus(thread_worker, arg):
Brad Bishop08902b02019-08-20 09:16:51 -0400963 (tid, sstatefile) = arg
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500964
965 localdata2 = bb.data.createCopy(localdata)
966 srcuri = "file://" + sstatefile
967 localdata.setVar('SRC_URI', srcuri)
968 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
969
970 try:
971 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
972 connection_cache=thread_worker.connection_cache)
973 fetcher.checkstatus()
974 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
Brad Bishop08902b02019-08-20 09:16:51 -0400975 found.add(tid)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600976 foundNet.add(tid)
Brad Bishop08902b02019-08-20 09:16:51 -0400977 if tid in missed:
978 missed.remove(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500979 except:
Brad Bishop08902b02019-08-20 09:16:51 -0400980 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500981 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600982 pass
Andrew Geissler82c905d2020-04-13 13:39:40 -0500983 if len(tasklist) >= min_tasks:
984 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500985
986 tasklist = []
Andrew Geissler82c905d2020-04-13 13:39:40 -0500987 min_tasks = 100
Brad Bishop08902b02019-08-20 09:16:51 -0400988 for tid in sq_data['hash']:
989 if tid in found:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500990 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400991 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500992 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
Brad Bishop08902b02019-08-20 09:16:51 -0400993 tasklist.append((tid, sstatefile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500994
995 if tasklist:
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500996 nproc = min(int(d.getVar("BB_NUMBER_THREADS")), len(tasklist))
997
Andrew Geissler82c905d2020-04-13 13:39:40 -0500998 if len(tasklist) >= min_tasks:
999 msg = "Checking sstate mirror object availability"
1000 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001001
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001002 bb.event.enable_threadlock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001003 pool = oe.utils.ThreadedPool(nproc, len(tasklist),
1004 worker_init=checkstatus_init, worker_end=checkstatus_end)
1005 for t in tasklist:
1006 pool.add_task(checkstatus, t)
1007 pool.start()
1008 pool.wait_completion()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001009 bb.event.disable_threadlock()
1010
Andrew Geissler82c905d2020-04-13 13:39:40 -05001011 if len(tasklist) >= min_tasks:
1012 bb.event.fire(bb.event.ProcessFinished(msg), d)
Brad Bishop96ff1982019-08-19 13:50:42 -04001013
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001014 inheritlist = d.getVar("INHERIT")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001015 if "toaster" in inheritlist:
1016 evdata = {'missed': [], 'found': []};
Brad Bishop08902b02019-08-20 09:16:51 -04001017 for tid in missed:
1018 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001019 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
Brad Bishop08902b02019-08-20 09:16:51 -04001020 evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
1021 for tid in found:
1022 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001023 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
Brad Bishop08902b02019-08-20 09:16:51 -04001024 evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001025 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
1026
Brad Bishop1d80a2e2019-11-15 16:35:03 -05001027 if summary:
1028 # Print some summary statistics about the current task completion and how much sstate
1029 # reuse there was. Avoid divide by zero errors.
1030 total = len(sq_data['hash'])
1031 complete = 0
1032 if currentcount:
1033 complete = (len(found) + currentcount) / (total + currentcount) * 100
1034 match = 0
1035 if total:
1036 match = len(found) / total * 100
Andrew Geissler9b4d8b02021-02-19 12:26:16 -06001037 bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001038
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001039 if hasattr(bb.parse.siggen, "checkhashes"):
Brad Bishop08902b02019-08-20 09:16:51 -04001040 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001041
Brad Bishop08902b02019-08-20 09:16:51 -04001042 return found
Patrick Williams213cb262021-08-07 19:21:33 -05001043setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001044
1045BB_SETSCENE_DEPVALID = "setscene_depvalid"
1046
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001047def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001048 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
1049 # task is included in taskdependees too
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001050 # Return - False - We need this dependency
1051 # - True - We can skip this dependency
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001052 import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001053
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001054 def logit(msg, log):
1055 if log is not None:
1056 log.append(msg)
1057 else:
1058 bb.debug(2, msg)
1059
1060 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001061
Andrew Geissler5199d832021-09-24 16:47:35 -05001062 directtasks = ["do_populate_lic", "do_deploy_source_date_epoch", "do_shared_workdir", "do_stash_locale", "do_gcc_stash_builddir"]
1063
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001064 def isNativeCross(x):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001065 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001066
Andrew Geissler5199d832021-09-24 16:47:35 -05001067 # We only need to trigger deploy_source_date_epoch through direct dependencies
1068 if taskdependees[task][1] in directtasks:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001069 return True
1070
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001071 # We only need to trigger packagedata through direct dependencies
1072 # but need to preserve packagedata on packagedata links
1073 if taskdependees[task][1] == "do_packagedata":
1074 for dep in taskdependees:
1075 if taskdependees[dep][1] == "do_packagedata":
1076 return False
1077 return True
1078
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001079 for dep in taskdependees:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001080 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001081 if task == dep:
1082 continue
1083 if dep in notneeded:
1084 continue
1085 # do_package_write_* and do_package doesn't need do_package
1086 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
1087 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001088 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
1089 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
1090 return False
Andrew Geissler5199d832021-09-24 16:47:35 -05001091 # do_package/packagedata/package_qa/deploy don't need do_populate_sysroot
1092 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa', 'do_deploy']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001093 continue
1094 # Native/Cross packages don't exist and are noexec anyway
1095 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
1096 continue
1097
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001098 # This is due to the [depends] in useradd.bbclass complicating matters
1099 # The logic *is* reversed here due to the way hard setscene dependencies are injected
1100 if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
1101 continue
1102
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001103 # Consider sysroot depending on sysroot tasks
1104 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001105 # Allow excluding certain recursive dependencies. If a recipe needs it should add a
1106 # specific dependency itself, rather than relying on one of its dependees to pull
1107 # them in.
1108 # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
1109 not_needed = False
1110 excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT')
1111 if excludedeps is None:
1112 # Cache the regular expressions for speed
1113 excludedeps = []
1114 for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split():
1115 excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))
1116 d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps)
1117 for excl in excludedeps:
1118 if excl[0].match(taskdependees[dep][0]):
1119 if excl[1].match(taskdependees[task][0]):
1120 not_needed = True
1121 break
1122 if not_needed:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001123 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001124 # For meta-extsdk-toolchain we want all sysroot dependencies
1125 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
1126 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001127 # Native/Cross populate_sysroot need their dependencies
1128 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
1129 return False
1130 # Target populate_sysroot depended on by cross tools need to be installed
1131 if isNativeCross(taskdependees[dep][0]):
1132 return False
1133 # Native/cross tools depended upon by target sysroot are not needed
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001134 # Add an exception for shadow-native as required by useradd.bbclass
1135 if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001136 continue
1137 # Target populate_sysroot need their dependencies
1138 return False
1139
Andrew Geissler5199d832021-09-24 16:47:35 -05001140 if taskdependees[dep][1] in directtasks:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001141 continue
1142
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001143 # Safe fallthrough default
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001144 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001145 return False
1146 return True
1147
1148addhandler sstate_eventhandler
1149sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
1150python sstate_eventhandler() {
1151 d = e.data
Andrew Geissler82c905d2020-04-13 13:39:40 -05001152 writtensstate = d.getVar('SSTATE_CURRTASK')
1153 if not writtensstate:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001154 taskname = d.getVar("BB_RUNTASK")[3:]
1155 spec = d.getVar('SSTATE_PKGSPEC')
1156 swspec = d.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001157 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1158 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1159 d.setVar("SSTATE_EXTRAPATH", "")
Andrew Geissler82c905d2020-04-13 13:39:40 -05001160 d.setVar("SSTATE_CURRTASK", taskname)
1161 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
1162 if not os.path.exists(siginfo):
1163 bb.siggen.dump_this_task(siginfo, d)
1164 else:
Andrew Geisslerc182c622020-05-15 14:13:32 -05001165 try:
1166 os.utime(siginfo, None)
1167 except PermissionError:
1168 pass
Andrew Geissler5f350902021-07-23 13:09:54 -04001169 except OSError as e:
1170 # Handle read-only file systems gracefully
Patrick Williams0ca19cc2021-08-16 14:03:13 -05001171 import errno
Andrew Geissler5f350902021-07-23 13:09:54 -04001172 if e.errno != errno.EROFS:
1173 raise e
Andrew Geisslerc182c622020-05-15 14:13:32 -05001174
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001175}
1176
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001177SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001178
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001179#
1180# Event handler which removes manifests and stamps file for recipes which are no
1181# longer 'reachable' in a build where they once were. 'Reachable' refers to
1182# whether a recipe is parsed so recipes in a layer which was removed would no
1183# longer be reachable. Switching between systemd and sysvinit where recipes
1184# became skipped would be another example.
1185#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001186# Also optionally removes the workdir of those tasks/recipes
1187#
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001188addhandler sstate_eventhandler_reachablestamps
1189sstate_eventhandler_reachablestamps[eventmask] = "bb.event.ReachableStamps"
1190python sstate_eventhandler_reachablestamps() {
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001191 import glob
1192 d = e.data
1193 stamps = e.stamps.values()
1194 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001195 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1196 preservestamps = []
1197 if os.path.exists(preservestampfile):
1198 with open(preservestampfile, 'r') as f:
1199 preservestamps = f.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001200 seen = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001201
1202 # The machine index contains all the stamps this machine has ever seen in this build directory.
1203 # We should only remove things which this machine once accessed but no longer does.
1204 machineindex = set()
1205 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1206 mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
1207 if os.path.exists(mi):
1208 with open(mi, "r") as f:
1209 machineindex = set(line.strip() for line in f.readlines())
1210
Brad Bishop316dfdd2018-06-25 12:45:53 -04001211 for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001212 toremove = []
1213 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1214 if not os.path.exists(i):
1215 continue
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001216 manseen = set()
1217 ignore = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001218 with open(i, "r") as f:
1219 lines = f.readlines()
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001220 for l in reversed(lines):
Brad Bishop19323692019-04-05 15:28:33 -04001221 try:
1222 (stamp, manifest, workdir) = l.split()
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001223 # The index may have multiple entries for the same manifest as the code above only appends
1224 # new entries and there may be an entry with matching manifest but differing version in stamp/workdir.
1225 # The last entry in the list is the valid one, any earlier entries with matching manifests
1226 # should be ignored.
1227 if manifest in manseen:
1228 ignore.append(l)
1229 continue
1230 manseen.add(manifest)
Brad Bishop19323692019-04-05 15:28:33 -04001231 if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
1232 toremove.append(l)
1233 if stamp not in seen:
1234 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
1235 seen.append(stamp)
1236 except ValueError:
1237 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001238
1239 if toremove:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001240 msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a)
1241 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001242
Brad Bishop316dfdd2018-06-25 12:45:53 -04001243 removed = 0
1244 for r in toremove:
1245 (stamp, manifest, workdir) = r.split()
1246 for m in glob.glob(manifest + ".*"):
1247 if m.endswith(".postrm"):
1248 continue
1249 sstate_clean_manifest(m, d)
1250 bb.utils.remove(stamp + "*")
1251 if removeworkdir:
1252 bb.utils.remove(workdir, recurse = True)
1253 lines.remove(r)
1254 removed = removed + 1
1255 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1256
1257 bb.event.fire(bb.event.ProcessFinished(msg), d)
1258
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001259 with open(i, "w") as f:
1260 for l in lines:
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001261 if l in ignore:
1262 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001263 f.write(l)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001264 machineindex |= set(stamps)
1265 with open(mi, "w") as f:
1266 for l in machineindex:
1267 f.write(l + "\n")
1268
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001269 if preservestamps:
1270 os.remove(preservestampfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001271}
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001272
1273
1274#
1275# Bitbake can generate an event showing which setscene tasks are 'stale',
1276# i.e. which ones will be rerun. These are ones where a stamp file is present but
1277# it is stable (e.g. taskhash doesn't match). With that list we can go through
1278# the manifests for matching tasks and "uninstall" those manifests now. We do
1279# this now rather than mid build since the distribution of files between sstate
1280# objects may have changed, new tasks may run first and if those new tasks overlap
1281# with the stale tasks, we'd see overlapping files messages and failures. Thankfully
1282# removing these files is fast.
1283#
1284addhandler sstate_eventhandler_stalesstate
1285sstate_eventhandler_stalesstate[eventmask] = "bb.event.StaleSetSceneTasks"
1286python sstate_eventhandler_stalesstate() {
1287 d = e.data
1288 tasks = e.tasks
1289
1290 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1291
1292 for a in list(set(d.getVar("SSTATE_ARCHS").split())):
1293 toremove = []
1294 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1295 if not os.path.exists(i):
1296 continue
1297 with open(i, "r") as f:
1298 lines = f.readlines()
1299 for l in lines:
1300 try:
1301 (stamp, manifest, workdir) = l.split()
1302 for tid in tasks:
1303 for s in tasks[tid]:
1304 if s.startswith(stamp):
1305 taskname = bb.runqueue.taskname_from_tid(tid)[3:]
1306 manname = manifest + "." + taskname
1307 if os.path.exists(manname):
1308 bb.debug(2, "Sstate for %s is stale, removing related manifest %s" % (tid, manname))
1309 toremove.append((manname, tid, tasks[tid]))
1310 break
1311 except ValueError:
1312 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
1313
1314 if toremove:
1315 msg = "Removing %d stale sstate objects for arch %s" % (len(toremove), a)
1316 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
1317
1318 removed = 0
1319 for (manname, tid, stamps) in toremove:
1320 sstate_clean_manifest(manname, d)
1321 for stamp in stamps:
1322 bb.utils.remove(stamp)
1323 removed = removed + 1
1324 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1325
1326 bb.event.fire(bb.event.ProcessFinished(msg), d)
1327}