blob: 2175ace4c40114c49a9373e3dfb7636a819343b3 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001SSTATE_VERSION = "3"
2
3SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
4SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
5
Andrew Geissler82c905d2020-04-13 13:39:40 -05006def generate_sstatefn(spec, hash, taskname, siginfo, d):
7 if taskname is None:
8 return ""
9 extension = ".tgz"
10 # 8 chars reserved for siginfo
11 limit = 254 - 8
12 if siginfo:
13 limit = 254
14 extension = ".tgz.siginfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015 if not hash:
16 hash = "INVALID"
Andrew Geissler82c905d2020-04-13 13:39:40 -050017 fn = spec + hash + "_" + taskname + extension
18 # If the filename is too long, attempt to reduce it
19 if len(fn) > limit:
20 components = spec.split(":")
21 # Fields 0,5,6 are mandatory, 1 is most useful, 2,3,4 are just for information
22 # 7 is for the separators
23 avail = (254 - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3
24 components[2] = components[2][:avail]
25 components[3] = components[3][:avail]
26 components[4] = components[4][:avail]
27 spec = ":".join(components)
28 fn = spec + hash + "_" + taskname + extension
29 if len(fn) > limit:
30 bb.fatal("Unable to reduce sstate name to less than 255 chararacters")
31 return hash[:2] + "/" + hash[2:4] + "/" + fn
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032
33SSTATE_PKGARCH = "${PACKAGE_ARCH}"
34SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050035SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
Andrew Geissler82c905d2020-04-13 13:39:40 -050036SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d.getVar('SSTATE_CURRTASK'), False, d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
38SSTATE_EXTRAPATH = ""
39SSTATE_EXTRAPATHWILDCARD = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -050040SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041
Patrick Williamsc0f7c042017-02-23 20:41:17 -060042# explicitly make PV to depend on evaluated value of PV variable
43PV[vardepvalue] = "${PV}"
44
Patrick Williamsc124f4f2015-09-15 14:41:29 -050045# We don't want the sstate to depend on things like the distro string
46# of the system, we let the sstate paths take care of this.
47SSTATE_EXTRAPATH[vardepvalue] = ""
Brad Bishop19323692019-04-05 15:28:33 -040048SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049
50# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080051SSTATE_DUPWHITELIST = "${DEPLOY_DIR}/licenses/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052# Avoid docbook/sgml catalog warnings for now
53SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
Brad Bishop316dfdd2018-06-25 12:45:53 -040054# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
55SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/"
56SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080057# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used
58SSTATE_DUPWHITELIST += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059# Archive the sources for many architectures in one deploy folder
60SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080061# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files
62SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/ovmf"
63SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/grub-efi"
64SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/systemd-boot"
65SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/microcode"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066
Brad Bishop6e60e8b2018-02-01 10:27:11 -050067SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
68SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
69SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050071BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
73SSTATE_ARCHS = " \
74 ${BUILD_ARCH} \
Andrew Geissler6ce62a22020-11-30 19:58:47 -060075 ${BUILD_ARCH}_${ORIGNATIVELSBSTRING} \
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
77 ${BUILD_ARCH}_${TARGET_ARCH} \
78 ${SDK_ARCH}_${SDK_OS} \
79 ${SDK_ARCH}_${PACKAGE_ARCH} \
80 allarch \
81 ${PACKAGE_ARCH} \
Brad Bishop316dfdd2018-06-25 12:45:53 -040082 ${PACKAGE_EXTRA_ARCHS} \
83 ${MACHINE_ARCH}"
Andrew Geissler6ce62a22020-11-30 19:58:47 -060084SSTATE_ARCHS[vardepsexclude] = "ORIGNATIVELSBSTRING"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
86SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
87
88SSTATECREATEFUNCS = "sstate_hardcode_path"
Brad Bishop19323692019-04-05 15:28:33 -040089SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090SSTATEPOSTCREATEFUNCS = ""
91SSTATEPREINSTFUNCS = ""
92SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
93SSTATEPOSTINSTFUNCS = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050094EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050096# Check whether sstate exists for tasks that support sstate and are in the
97# locked signatures file.
98SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
99
100# Check whether the task's computed hash matches the task's hash in the
101# locked signatures file.
102SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500103
104# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
105# not sign)
106SSTATE_SIG_KEY ?= ""
107SSTATE_SIG_PASSPHRASE ?= ""
108# Whether to verify the GnUPG signatures when extracting sstate archives
109SSTATE_VERIFY_SIG ?= "0"
110
Brad Bishop19323692019-04-05 15:28:33 -0400111SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
112SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
113 the output hash for a task, which in turn is used to determine equivalency. \
114 "
115
Brad Bishop19323692019-04-05 15:28:33 -0400116SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
117SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
118 hash equivalency server, such as PN, PV, taskname, etc. This information \
119 is very useful for developers looking at task data, but may leak sensitive \
120 data if the equivalence server is public. \
121 "
122
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123python () {
124 if bb.data.inherits_class('native', d):
125 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
126 elif bb.data.inherits_class('crosssdk', d):
127 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
128 elif bb.data.inherits_class('cross', d):
129 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TARGET_ARCH}"))
130 elif bb.data.inherits_class('nativesdk', d):
131 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
132 elif bb.data.inherits_class('cross-canadian', d):
133 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500134 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135 d.setVar('SSTATE_PKGARCH', "allarch")
136 else:
137 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
138
139 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
140 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500141 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
Brad Bishop19323692019-04-05 15:28:33 -0400142 d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500144 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 d.setVar('SSTATETASKS', " ".join(unique_tasks))
146 for task in unique_tasks:
147 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
148 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
149}
150
151def sstate_init(task, d):
152 ss = {}
153 ss['task'] = task
154 ss['dirs'] = []
155 ss['plaindirs'] = []
156 ss['lockfiles'] = []
157 ss['lockfiles-shared'] = []
158 return ss
159
160def sstate_state_fromvars(d, task = None):
161 if task is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 task = d.getVar('BB_CURRENTTASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500163 if not task:
164 bb.fatal("sstate code running without task context?!")
165 task = task.replace("_setscene", "")
166
167 if task.startswith("do_"):
168 task = task[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500169 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
170 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
171 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
172 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
173 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
174 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
175 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176 if not task or len(inputs) != len(outputs):
177 bb.fatal("sstate variables not setup correctly?!")
178
179 if task == "populate_lic":
180 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
181 d.setVar("SSTATE_EXTRAPATH", "")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500182 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183
184 ss = sstate_init(task, d)
185 for i in range(len(inputs)):
186 sstate_add(ss, inputs[i], outputs[i], d)
187 ss['lockfiles'] = lockfiles
188 ss['lockfiles-shared'] = lockfilesshared
189 ss['plaindirs'] = plaindirs
190 ss['interceptfuncs'] = interceptfuncs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500191 ss['fixmedir'] = fixmedir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192 return ss
193
194def sstate_add(ss, source, dest, d):
195 if not source.endswith("/"):
196 source = source + "/"
197 if not dest.endswith("/"):
198 dest = dest + "/"
199 source = os.path.normpath(source)
200 dest = os.path.normpath(dest)
201 srcbase = os.path.basename(source)
202 ss['dirs'].append([srcbase, source, dest])
203 return ss
204
205def sstate_install(ss, d):
206 import oe.path
207 import oe.sstatesig
208 import subprocess
209
210 sharedfiles = []
211 shareddirs = []
212 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
213
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500214 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
215
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
217
218 if os.access(manifest, os.R_OK):
219 bb.fatal("Package already staged (%s)?!" % manifest)
220
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600221 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
222
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223 locks = []
224 for lock in ss['lockfiles-shared']:
225 locks.append(bb.utils.lockfile(lock, True))
226 for lock in ss['lockfiles']:
227 locks.append(bb.utils.lockfile(lock))
228
229 for state in ss['dirs']:
230 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
231 for walkroot, dirs, files in os.walk(state[1]):
232 for file in files:
233 srcpath = os.path.join(walkroot, file)
234 dstpath = srcpath.replace(state[1], state[2])
235 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
236 sharedfiles.append(dstpath)
237 for dir in dirs:
238 srcdir = os.path.join(walkroot, dir)
239 dstdir = srcdir.replace(state[1], state[2])
240 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500241 if os.path.islink(srcdir):
242 sharedfiles.append(dstdir)
243 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 if not dstdir.endswith("/"):
245 dstdir = dstdir + "/"
246 shareddirs.append(dstdir)
247
248 # Check the file list for conflicts against files which already exist
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500249 whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250 match = []
251 for f in sharedfiles:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500252 if os.path.exists(f) and not os.path.islink(f):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500253 f = os.path.normpath(f)
254 realmatch = True
255 for w in whitelist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 w = os.path.normpath(w)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 if f.startswith(w):
258 realmatch = False
259 break
260 if realmatch:
261 match.append(f)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500264 if search_output:
265 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
266 else:
267 match.append(" (not matched to any task)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 if match:
269 bb.error("The recipe %s is trying to install files into a shared " \
270 "area when those files already exist. Those files and their manifest " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 "location are:\n %s\nPlease verify which recipe should provide the " \
272 "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \
273 "break things - if not now, possibly in the future (we've seen builds fail " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 "several months later). If the system knew how to recover from this " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500275 "automatically it would, however there are several different scenarios " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500276 "which can result in this and we don't know which one this is. It may be " \
277 "you have switched providers of something like virtual/kernel (e.g. from " \
278 "linux-yocto to linux-yocto-dev), in that case you need to execute the " \
279 "clean task for both recipes and it will resolve this error. It may be " \
280 "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500281 "those recipes should again resolve this error, however switching " \
282 "DISTRO_FEATURES on an existing build directory is not supported - you " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500283 "should really clean out tmp and rebuild (reusing sstate should be safe). " \
284 "It could be the overlapping files detected are harmless in which case " \
285 "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \
286 "also be your build is including two different conflicting versions of " \
287 "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \
288 "be to resolve the conflict. If in doubt, please ask on the mailing list, " \
289 "sharing the error and filelist above." % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500290 (d.getVar('PN'), "\n ".join(match)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500291 bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.")
292
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500293 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
294 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
295 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
296
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297 # Write out the manifest
298 f = open(manifest, "w")
299 for file in sharedfiles:
300 f.write(file + "\n")
301
302 # We want to ensure that directories appear at the end of the manifest
303 # so that when we test to see if they should be deleted any contents
304 # added by the task will have been removed first.
305 dirs = sorted(shareddirs, key=len)
306 # Must remove children first, which will have a longer path than the parent
307 for di in reversed(dirs):
308 f.write(di + "\n")
309 f.close()
310
311 # Append to the list of manifests for this PACKAGE_ARCH
312
313 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
314 l = bb.utils.lockfile(i + ".lock")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500315 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 manifests = []
317 if os.path.exists(i):
318 with open(i, "r") as f:
319 manifests = f.readlines()
William A. Kennington IIIac69b482021-06-02 12:28:27 -0700320 # We append new entries, we don't remove older entries which may have the same
321 # manifest name but different versions from stamp/workdir. See below.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322 if filedata not in manifests:
323 with open(i, "a+") as f:
324 f.write(filedata)
325 bb.utils.unlockfile(l)
326
327 # Run the actual file install
328 for state in ss['dirs']:
329 if os.path.exists(state[1]):
330 oe.path.copyhardlinktree(state[1], state[2])
331
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500332 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500333 # All hooks should run in the SSTATE_INSTDIR
334 bb.build.exec_func(postinst, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335
336 for lock in locks:
337 bb.utils.unlockfile(lock)
338
339sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX"
340sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
341
342def sstate_installpkg(ss, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500343 from oe.gpg_sign import get_signer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500345 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
Andrew Geissler82c905d2020-04-13 13:39:40 -0500346 d.setVar("SSTATE_CURRTASK", ss['task'])
347 sstatefetch = d.getVar('SSTATE_PKGNAME')
348 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500349
350 if not os.path.exists(sstatepkg):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800351 pstaging_fetch(sstatefetch, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
353 if not os.path.isfile(sstatepkg):
Brad Bishop08902b02019-08-20 09:16:51 -0400354 bb.note("Sstate package %s does not exist" % sstatepkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 return False
356
357 sstate_clean(ss, d)
358
359 d.setVar('SSTATE_INSTDIR', sstateinst)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500361 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500362 if not os.path.isfile(sstatepkg + '.sig'):
363 bb.warn("No signature file for sstate package %s, skipping acceleration..." % sstatepkg)
364 return False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500365 signer = get_signer(d, 'local')
366 if not signer.verify(sstatepkg + '.sig'):
Brad Bishop08902b02019-08-20 09:16:51 -0400367 bb.warn("Cannot verify signature on sstate package %s, skipping acceleration..." % sstatepkg)
368 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500370 # Empty sstateinst directory, ensure its clean
371 if os.path.exists(sstateinst):
372 oe.path.remove(sstateinst)
373 bb.utils.mkdirhier(sstateinst)
374
375 sstateinst = d.getVar("SSTATE_INSTDIR")
376 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
377
378 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500379 # All hooks should run in the SSTATE_INSTDIR
380 bb.build.exec_func(f, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500382 return sstate_installpkgdir(ss, d)
383
384def sstate_installpkgdir(ss, d):
385 import oe.path
386 import subprocess
387
388 sstateinst = d.getVar("SSTATE_INSTDIR")
389 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
390
391 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
392 # All hooks should run in the SSTATE_INSTDIR
393 bb.build.exec_func(f, d, (sstateinst,))
394
395 def prepdir(dir):
396 # remove dir if it exists, ensure any parent directories do exist
397 if os.path.exists(dir):
398 oe.path.remove(dir)
399 bb.utils.mkdirhier(dir)
400 oe.path.remove(dir)
401
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 for state in ss['dirs']:
403 prepdir(state[1])
Andrew Geisslerc926e172021-05-07 16:11:35 -0500404 bb.utils.rename(sstateinst + state[0], state[1])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500405 sstate_install(ss, d)
406
407 for plain in ss['plaindirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500408 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500409 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500410 src = sstateinst + "/" + plain.replace(workdir, '')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500411 if sharedworkdir in plain:
412 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413 dest = plain
414 bb.utils.mkdirhier(src)
415 prepdir(dest)
Andrew Geisslerc926e172021-05-07 16:11:35 -0500416 bb.utils.rename(src, dest)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417
418 return True
419
420python sstate_hardcode_path_unpack () {
421 # Fixup hardcoded paths
422 #
423 # Note: The logic below must match the reverse logic in
424 # sstate_hardcode_path(d)
425 import subprocess
426
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500427 sstateinst = d.getVar('SSTATE_INSTDIR')
428 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
429 fixmefn = sstateinst + "fixmepath"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500430 if os.path.isfile(fixmefn):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500431 staging_target = d.getVar('RECIPE_SYSROOT')
432 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500433
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500434 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500436 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
437 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
438 else:
439 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500441 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500442 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500443 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
445
446 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
447 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
448
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500449 # Defer do_populate_sysroot relocation command
450 if sstatefixmedir:
451 bb.utils.mkdirhier(sstatefixmedir)
452 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
453 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
454 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
455 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
456 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
457 f.write(sstate_hardcode_cmd)
458 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
459 return
460
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500461 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500462 subprocess.check_call(sstate_hardcode_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800464 # Need to remove this or we'd copy it into the target directory and may
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500465 # conflict with another writer
466 os.remove(fixmefn)
467}
468
469def sstate_clean_cachefile(ss, d):
470 import oe.path
471
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300472 if d.getVarFlag('do_%s' % ss['task'], 'task'):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500473 d.setVar("SSTATE_PATH_CURRTASK", ss['task'])
474 sstatepkgfile = d.getVar('SSTATE_PATHSPEC')
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300475 bb.note("Removing %s" % sstatepkgfile)
476 oe.path.remove(sstatepkgfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500477
478def sstate_clean_cachefiles(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500479 for task in (d.getVar('SSTATETASKS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480 ld = d.createCopy()
481 ss = sstate_state_fromvars(ld, task)
482 sstate_clean_cachefile(ss, ld)
483
Andrew Geissler5f350902021-07-23 13:09:54 -0400484def sstate_clean_manifest(manifest, d, canrace=False, prefix=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500485 import oe.path
486
487 mfile = open(manifest)
488 entries = mfile.readlines()
489 mfile.close()
490
491 for entry in entries:
492 entry = entry.strip()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500493 if prefix and not entry.startswith("/"):
494 entry = prefix + "/" + entry
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 bb.debug(2, "Removing manifest: %s" % entry)
496 # We can race against another package populating directories as we're removing them
497 # so we ignore errors here.
498 try:
499 if entry.endswith("/"):
500 if os.path.islink(entry[:-1]):
501 os.remove(entry[:-1])
Andrew Geissler5f350902021-07-23 13:09:54 -0400502 elif os.path.exists(entry) and len(os.listdir(entry)) == 0 and not canrace:
503 # Removing directories whilst builds are in progress exposes a race. Only
504 # do it in contexts where it is safe to do so.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505 os.rmdir(entry[:-1])
506 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500507 os.remove(entry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500508 except OSError:
509 pass
510
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600511 postrm = manifest + ".postrm"
512 if os.path.exists(manifest + ".postrm"):
513 import subprocess
514 os.chmod(postrm, 0o755)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500515 subprocess.check_call(postrm, shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600516 oe.path.remove(postrm)
517
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500518 oe.path.remove(manifest)
519
520def sstate_clean(ss, d):
521 import oe.path
522 import glob
523
524 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500525 stamp_clean = d.getVar("STAMPCLEAN")
526 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500527 if extrainf:
528 d2.setVar("SSTATE_MANMACH", extrainf)
529 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
530 else:
531 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
532
533 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
534
535 if os.path.exists(manifest):
536 locks = []
537 for lock in ss['lockfiles-shared']:
538 locks.append(bb.utils.lockfile(lock))
539 for lock in ss['lockfiles']:
540 locks.append(bb.utils.lockfile(lock))
541
Andrew Geissler5f350902021-07-23 13:09:54 -0400542 sstate_clean_manifest(manifest, d, canrace=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500543
544 for lock in locks:
545 bb.utils.unlockfile(lock)
546
547 # Remove the current and previous stamps, but keep the sigdata.
548 #
549 # The glob() matches do_task* which may match multiple tasks, for
550 # example: do_package and do_package_write_ipk, so we need to
551 # exactly match *.do_task.* and *.do_task_setscene.*
552 rm_stamp = '.do_%s.' % ss['task']
553 rm_setscene = '.do_%s_setscene.' % ss['task']
554 # For BB_SIGNATURE_HANDLER = "noop"
555 rm_nohash = ".do_%s" % ss['task']
556 for stfile in glob.glob(wildcard_stfile):
557 # Keep the sigdata
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500558 if ".sigdata." in stfile or ".sigbasedata." in stfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500559 continue
560 # Preserve taint files in the stamps directory
561 if stfile.endswith('.taint'):
562 continue
563 if rm_stamp in stfile or rm_setscene in stfile or \
564 stfile.endswith(rm_nohash):
565 oe.path.remove(stfile)
566
567sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
568
569CLEANFUNCS += "sstate_cleanall"
570
571python sstate_cleanall() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500572 bb.note("Removing shared state for package %s" % d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500573
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500574 manifest_dir = d.getVar('SSTATE_MANIFESTS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575 if not os.path.exists(manifest_dir):
576 return
577
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500578 tasks = d.getVar('SSTATETASKS').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579 for name in tasks:
580 ld = d.createCopy()
581 shared_state = sstate_state_fromvars(ld, name)
582 sstate_clean(shared_state, ld)
583}
584
585python sstate_hardcode_path () {
586 import subprocess, platform
587
588 # Need to remove hardcoded paths and fix these when we install the
589 # staging packages.
590 #
591 # Note: the logic in this function needs to match the reverse logic
592 # in sstate_installpkg(ss, d)
593
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500594 staging_target = d.getVar('RECIPE_SYSROOT')
595 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
596 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500597
Brad Bishop316dfdd2018-06-25 12:45:53 -0400598 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500599 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500600 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500601 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
602 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400603 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500604 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400605 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
606 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500608 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500609 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500610 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500612 sstate_grep_cmd += " -e '%s'" % (fixme_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500613
614 fixmefn = sstate_builddir + "fixmepath"
615
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500616 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500617 sstate_filelist_cmd = "tee %s" % (fixmefn)
618
619 # fixmepath file needs relative paths, drop sstate_builddir prefix
620 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
621
622 xargs_no_empty_run_cmd = '--no-run-if-empty'
623 if platform.system() == 'Darwin':
624 xargs_no_empty_run_cmd = ''
625
626 # Limit the fixpaths and sed operations based on the initial grep search
627 # This has the side effect of making sure the vfs cache is hot
628 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
629
630 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500631 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500632
633 # If the fixmefn is empty, remove it..
634 if os.stat(fixmefn).st_size == 0:
635 os.remove(fixmefn)
636 else:
637 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500638 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500639}
640
641def sstate_package(ss, d):
642 import oe.path
643
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500644 tmpdir = d.getVar('TMPDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500645
646 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
Andrew Geissler82c905d2020-04-13 13:39:40 -0500647 d.setVar("SSTATE_CURRTASK", ss['task'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500648 bb.utils.remove(sstatebuild, recurse=True)
649 bb.utils.mkdirhier(sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500650 for state in ss['dirs']:
651 if not os.path.exists(state[1]):
652 continue
653 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500654 # Find and error for absolute symlinks. We could attempt to relocate but its not
655 # clear where the symlink is relative to in this context. We could add that markup
656 # to sstate tasks but there aren't many of these so better just avoid them entirely.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500657 for walkroot, dirs, files in os.walk(state[1]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500658 for file in files + dirs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500659 srcpath = os.path.join(walkroot, file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500660 if not os.path.islink(srcpath):
661 continue
662 link = os.readlink(srcpath)
663 if not os.path.isabs(link):
664 continue
665 if not link.startswith(tmpdir):
666 continue
667 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500668 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
Andrew Geisslerc926e172021-05-07 16:11:35 -0500669 bb.utils.rename(state[1], sstatebuild + state[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500670
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500671 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500672 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500673 for plain in ss['plaindirs']:
674 pdir = plain.replace(workdir, sstatebuild)
Brad Bishop977dc1a2019-02-06 16:01:43 -0500675 if sharedworkdir in plain:
676 pdir = plain.replace(sharedworkdir, sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500677 bb.utils.mkdirhier(plain)
678 bb.utils.mkdirhier(pdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -0500679 bb.utils.rename(plain, pdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500680
681 d.setVar('SSTATE_BUILDDIR', sstatebuild)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500682 d.setVar('SSTATE_INSTDIR', sstatebuild)
683
684 if d.getVar('SSTATE_SKIP_CREATION') == '1':
685 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500686
Brad Bishop08902b02019-08-20 09:16:51 -0400687 sstate_create_package = ['sstate_report_unihash', 'sstate_create_package']
688 if d.getVar('SSTATE_SIG_KEY'):
689 sstate_create_package.append('sstate_sign_package')
690
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500691 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
Brad Bishop08902b02019-08-20 09:16:51 -0400692 sstate_create_package + \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500693 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500694 # All hooks should run in SSTATE_BUILDDIR.
695 bb.build.exec_func(f, d, (sstatebuild,))
696
Andrew Geissler82c905d2020-04-13 13:39:40 -0500697 # SSTATE_PKG may have been changed by sstate_report_unihash
698 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
699 if not os.path.exists(siginfo):
700 bb.siggen.dump_this_task(siginfo, d)
701 else:
Andrew Geisslerc182c622020-05-15 14:13:32 -0500702 try:
703 os.utime(siginfo, None)
704 except PermissionError:
705 pass
Andrew Geissler5f350902021-07-23 13:09:54 -0400706 except OSError as e:
707 # Handle read-only file systems gracefully
Patrick Williams0ca19cc2021-08-16 14:03:13 -0500708 import errno
Andrew Geissler5f350902021-07-23 13:09:54 -0400709 if e.errno != errno.EROFS:
710 raise e
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500711
712 return
713
William A. Kennington IIIac69b482021-06-02 12:28:27 -0700714sstate_package[vardepsexclude] += "SSTATE_SIG_KEY"
715
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800716def pstaging_fetch(sstatefetch, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500717 import bb.fetch2
718
719 # Only try and fetch if the user has configured a mirror
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500720 mirrors = d.getVar('SSTATE_MIRRORS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500721 if not mirrors:
722 return
723
724 # Copy the data object and override DL_DIR and SRC_URI
725 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500726
727 dldir = localdata.expand("${SSTATE_DIR}")
728 bb.utils.mkdirhier(dldir)
729
730 localdata.delVar('MIRRORS')
731 localdata.setVar('FILESPATH', dldir)
732 localdata.setVar('DL_DIR', dldir)
733 localdata.setVar('PREMIRRORS', mirrors)
734
735 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
736 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400737 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
738 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500739 localdata.delVar('BB_NO_NETWORK')
740
741 # Try a fetch from the sstate mirror, if it fails just return and
742 # we will build the package
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600743 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
744 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500745 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600746 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500747
748 for srcuri in uris:
749 localdata.setVar('SRC_URI', srcuri)
750 try:
751 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500752 fetcher.checkstatus()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500753 fetcher.download()
754
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500755 except bb.fetch2.BBFetchException:
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500756 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500757
758def sstate_setscene(d):
759 shared_state = sstate_state_fromvars(d)
760 accelerate = sstate_installpkg(shared_state, d)
761 if not accelerate:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600762 bb.fatal("No suitable staging package found")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500763
764python sstate_task_prefunc () {
765 shared_state = sstate_state_fromvars(d)
766 sstate_clean(shared_state, d)
767}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500768sstate_task_prefunc[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500769
770python sstate_task_postfunc () {
771 shared_state = sstate_state_fromvars(d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500772
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500773 for intercept in shared_state['interceptfuncs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500774 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
775
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600776 omask = os.umask(0o002)
777 if omask != 0o002:
778 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500779 sstate_package(shared_state, d)
780 os.umask(omask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500781
782 sstateinst = d.getVar("SSTATE_INSTDIR")
783 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
784
785 sstate_installpkgdir(shared_state, d)
786
787 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500788}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500789sstate_task_postfunc[dirs] = "${WORKDIR}"
790
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500791
792#
793# Shell function to generate a sstate package from a directory
794# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
795#
796sstate_create_package () {
Andrew Geissler82c905d2020-04-13 13:39:40 -0500797 # Exit early if it already exists
Brad Bishop08902b02019-08-20 09:16:51 -0400798 if [ -e ${SSTATE_PKG} ]; then
Andrew Geisslerc182c622020-05-15 14:13:32 -0500799 [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
Brad Bishop08902b02019-08-20 09:16:51 -0400800 return
801 fi
802
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500803 mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
Andrew Geissler82c905d2020-04-13 13:39:40 -0500804 TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
805
806 # Use pigz if available
807 OPT="-czS"
808 if [ -x "$(command -v pigz)" ]; then
809 OPT="-I pigz -cS"
810 fi
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800811
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500812 # Need to handle empty directories
813 if [ "$(ls -A)" ]; then
814 set +e
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800815 tar $OPT -f $TFILE *
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500816 ret=$?
817 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
818 exit 1
819 fi
820 set -e
821 else
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800822 tar $OPT --file=$TFILE --files-from=/dev/null
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500823 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500824 chmod 0664 $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400825 # Skip if it was already created by some other process
826 if [ ! -e ${SSTATE_PKG} ]; then
Andrew Geissler82c905d2020-04-13 13:39:40 -0500827 # Move into place using ln to attempt an atomic op.
828 # Abort if it already exists
829 ln $TFILE ${SSTATE_PKG} && rm $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400830 else
831 rm $TFILE
832 fi
Andrew Geisslerc182c622020-05-15 14:13:32 -0500833 [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500834}
835
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500836python sstate_sign_package () {
837 from oe.gpg_sign import get_signer
838
Brad Bishop08902b02019-08-20 09:16:51 -0400839
840 signer = get_signer(d, 'local')
841 sstate_pkg = d.getVar('SSTATE_PKG')
842 if os.path.exists(sstate_pkg + '.sig'):
843 os.unlink(sstate_pkg + '.sig')
844 signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
845 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500846}
847
Brad Bishop19323692019-04-05 15:28:33 -0400848python sstate_report_unihash() {
849 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
850
851 if report_unihash:
852 ss = sstate_state_fromvars(d)
853 report_unihash(os.getcwd(), ss['task'], d)
854}
855
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500856#
857# Shell function to decompress and prepare a package for installation
858# Will be run from within SSTATE_INSTDIR.
859#
860sstate_unpack_package () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500861 tar -xvzf ${SSTATE_PKG}
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500862 # update .siginfo atime on local/NFS mirror
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500863 [ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500864 # Use "! -w ||" to return true for read only files
865 [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500866 [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
867 [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500868}
869
870BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
871
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500872def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
Brad Bishop08902b02019-08-20 09:16:51 -0400873 found = set()
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600874 foundLocal = set()
875 foundNet = set()
Brad Bishop08902b02019-08-20 09:16:51 -0400876 missed = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500877
Brad Bishop19323692019-04-05 15:28:33 -0400878 def gethash(task):
Brad Bishop08902b02019-08-20 09:16:51 -0400879 return sq_data['unihash'][task]
Brad Bishop19323692019-04-05 15:28:33 -0400880
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500881 def getpathcomponents(task, d):
882 # Magic data from BB_HASHFILENAME
Brad Bishop08902b02019-08-20 09:16:51 -0400883 splithashfn = sq_data['hashfn'][task].split(" ")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500884 spec = splithashfn[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500885 if splithashfn[0] == "True":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500886 extrapath = d.getVar("NATIVELSBSTRING") + "/"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500887 else:
888 extrapath = ""
Brad Bishop08902b02019-08-20 09:16:51 -0400889
890 tname = bb.runqueue.taskname_from_tid(task)[3:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500891
892 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
893 spec = splithashfn[2]
894 extrapath = ""
895
896 return spec, extrapath, tname
897
898
Brad Bishop08902b02019-08-20 09:16:51 -0400899 for tid in sq_data['hash']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500900
Brad Bishop08902b02019-08-20 09:16:51 -0400901 spec, extrapath, tname = getpathcomponents(tid, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500902
Andrew Geissler82c905d2020-04-13 13:39:40 -0500903 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500904
905 if os.path.exists(sstatefile):
906 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
Brad Bishop08902b02019-08-20 09:16:51 -0400907 found.add(tid)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600908 foundLocal.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500909 continue
910 else:
Brad Bishop08902b02019-08-20 09:16:51 -0400911 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500912 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
913
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500914 mirrors = d.getVar("SSTATE_MIRRORS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500915 if mirrors:
916 # Copy the data object and override DL_DIR and SRC_URI
917 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500918
919 dldir = localdata.expand("${SSTATE_DIR}")
920 localdata.delVar('MIRRORS')
921 localdata.setVar('FILESPATH', dldir)
922 localdata.setVar('DL_DIR', dldir)
923 localdata.setVar('PREMIRRORS', mirrors)
924
925 bb.debug(2, "SState using premirror of: %s" % mirrors)
926
927 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
928 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400929 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
930 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500931 localdata.delVar('BB_NO_NETWORK')
932
933 from bb.fetch2 import FetchConnectionCache
934 def checkstatus_init(thread_worker):
935 thread_worker.connection_cache = FetchConnectionCache()
936
937 def checkstatus_end(thread_worker):
938 thread_worker.connection_cache.close_connections()
939
940 def checkstatus(thread_worker, arg):
Brad Bishop08902b02019-08-20 09:16:51 -0400941 (tid, sstatefile) = arg
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500942
943 localdata2 = bb.data.createCopy(localdata)
944 srcuri = "file://" + sstatefile
945 localdata.setVar('SRC_URI', srcuri)
946 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
947
948 try:
949 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
950 connection_cache=thread_worker.connection_cache)
951 fetcher.checkstatus()
952 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
Brad Bishop08902b02019-08-20 09:16:51 -0400953 found.add(tid)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600954 foundNet.add(tid)
Brad Bishop08902b02019-08-20 09:16:51 -0400955 if tid in missed:
956 missed.remove(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500957 except:
Brad Bishop08902b02019-08-20 09:16:51 -0400958 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500959 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600960 pass
Andrew Geissler82c905d2020-04-13 13:39:40 -0500961 if len(tasklist) >= min_tasks:
962 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500963
964 tasklist = []
Andrew Geissler82c905d2020-04-13 13:39:40 -0500965 min_tasks = 100
Brad Bishop08902b02019-08-20 09:16:51 -0400966 for tid in sq_data['hash']:
967 if tid in found:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500968 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400969 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500970 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
Brad Bishop08902b02019-08-20 09:16:51 -0400971 tasklist.append((tid, sstatefile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500972
973 if tasklist:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500974 if len(tasklist) >= min_tasks:
975 msg = "Checking sstate mirror object availability"
976 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600977
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500978 import multiprocessing
979 nproc = min(multiprocessing.cpu_count(), len(tasklist))
980
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600981 bb.event.enable_threadlock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500982 pool = oe.utils.ThreadedPool(nproc, len(tasklist),
983 worker_init=checkstatus_init, worker_end=checkstatus_end)
984 for t in tasklist:
985 pool.add_task(checkstatus, t)
986 pool.start()
987 pool.wait_completion()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600988 bb.event.disable_threadlock()
989
Andrew Geissler82c905d2020-04-13 13:39:40 -0500990 if len(tasklist) >= min_tasks:
991 bb.event.fire(bb.event.ProcessFinished(msg), d)
Brad Bishop96ff1982019-08-19 13:50:42 -0400992
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500993 inheritlist = d.getVar("INHERIT")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500994 if "toaster" in inheritlist:
995 evdata = {'missed': [], 'found': []};
Brad Bishop08902b02019-08-20 09:16:51 -0400996 for tid in missed:
997 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500998 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
Brad Bishop08902b02019-08-20 09:16:51 -0400999 evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
1000 for tid in found:
1001 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001002 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
Brad Bishop08902b02019-08-20 09:16:51 -04001003 evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001004 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
1005
Brad Bishop1d80a2e2019-11-15 16:35:03 -05001006 if summary:
1007 # Print some summary statistics about the current task completion and how much sstate
1008 # reuse there was. Avoid divide by zero errors.
1009 total = len(sq_data['hash'])
1010 complete = 0
1011 if currentcount:
1012 complete = (len(found) + currentcount) / (total + currentcount) * 100
1013 match = 0
1014 if total:
1015 match = len(found) / total * 100
Andrew Geissler9b4d8b02021-02-19 12:26:16 -06001016 bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001017
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001018 if hasattr(bb.parse.siggen, "checkhashes"):
Brad Bishop08902b02019-08-20 09:16:51 -04001019 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001020
Brad Bishop08902b02019-08-20 09:16:51 -04001021 return found
Patrick Williams213cb262021-08-07 19:21:33 -05001022setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001023
1024BB_SETSCENE_DEPVALID = "setscene_depvalid"
1025
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001026def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001027 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
1028 # task is included in taskdependees too
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001029 # Return - False - We need this dependency
1030 # - True - We can skip this dependency
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001031 import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001032
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001033 def logit(msg, log):
1034 if log is not None:
1035 log.append(msg)
1036 else:
1037 bb.debug(2, msg)
1038
1039 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001040
1041 def isNativeCross(x):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001042 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001043
1044 # We only need to trigger populate_lic through direct dependencies
1045 if taskdependees[task][1] == "do_populate_lic":
1046 return True
1047
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001048 # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
1049 if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
1050 return True
1051
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001052 # We only need to trigger packagedata through direct dependencies
1053 # but need to preserve packagedata on packagedata links
1054 if taskdependees[task][1] == "do_packagedata":
1055 for dep in taskdependees:
1056 if taskdependees[dep][1] == "do_packagedata":
1057 return False
1058 return True
1059
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001060 for dep in taskdependees:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001061 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001062 if task == dep:
1063 continue
1064 if dep in notneeded:
1065 continue
1066 # do_package_write_* and do_package doesn't need do_package
1067 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
1068 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001069 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
1070 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
1071 return False
1072 # do_package/packagedata/package_qa don't need do_populate_sysroot
1073 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001074 continue
1075 # Native/Cross packages don't exist and are noexec anyway
1076 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
1077 continue
1078
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001079 # This is due to the [depends] in useradd.bbclass complicating matters
1080 # The logic *is* reversed here due to the way hard setscene dependencies are injected
1081 if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
1082 continue
1083
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001084 # Consider sysroot depending on sysroot tasks
1085 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001086 # Allow excluding certain recursive dependencies. If a recipe needs it should add a
1087 # specific dependency itself, rather than relying on one of its dependees to pull
1088 # them in.
1089 # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
1090 not_needed = False
1091 excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT')
1092 if excludedeps is None:
1093 # Cache the regular expressions for speed
1094 excludedeps = []
1095 for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split():
1096 excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))
1097 d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps)
1098 for excl in excludedeps:
1099 if excl[0].match(taskdependees[dep][0]):
1100 if excl[1].match(taskdependees[task][0]):
1101 not_needed = True
1102 break
1103 if not_needed:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001104 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001105 # For meta-extsdk-toolchain we want all sysroot dependencies
1106 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
1107 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001108 # Native/Cross populate_sysroot need their dependencies
1109 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
1110 return False
1111 # Target populate_sysroot depended on by cross tools need to be installed
1112 if isNativeCross(taskdependees[dep][0]):
1113 return False
1114 # Native/cross tools depended upon by target sysroot are not needed
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001115 # Add an exception for shadow-native as required by useradd.bbclass
1116 if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001117 continue
1118 # Target populate_sysroot need their dependencies
1119 return False
1120
1121 if taskdependees[task][1] == 'do_shared_workdir':
1122 continue
1123
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001124 if taskdependees[dep][1] == "do_populate_lic":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001125 continue
1126
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001127
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001128 # Safe fallthrough default
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001129 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001130 return False
1131 return True
1132
1133addhandler sstate_eventhandler
1134sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
1135python sstate_eventhandler() {
1136 d = e.data
Andrew Geissler82c905d2020-04-13 13:39:40 -05001137 writtensstate = d.getVar('SSTATE_CURRTASK')
1138 if not writtensstate:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001139 taskname = d.getVar("BB_RUNTASK")[3:]
1140 spec = d.getVar('SSTATE_PKGSPEC')
1141 swspec = d.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001142 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1143 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1144 d.setVar("SSTATE_EXTRAPATH", "")
Andrew Geissler82c905d2020-04-13 13:39:40 -05001145 d.setVar("SSTATE_CURRTASK", taskname)
1146 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
1147 if not os.path.exists(siginfo):
1148 bb.siggen.dump_this_task(siginfo, d)
1149 else:
Andrew Geisslerc182c622020-05-15 14:13:32 -05001150 try:
1151 os.utime(siginfo, None)
1152 except PermissionError:
1153 pass
Andrew Geissler5f350902021-07-23 13:09:54 -04001154 except OSError as e:
1155 # Handle read-only file systems gracefully
Patrick Williams0ca19cc2021-08-16 14:03:13 -05001156 import errno
Andrew Geissler5f350902021-07-23 13:09:54 -04001157 if e.errno != errno.EROFS:
1158 raise e
Andrew Geisslerc182c622020-05-15 14:13:32 -05001159
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001160}
1161
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001162SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001163
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001164#
1165# Event handler which removes manifests and stamps file for recipes which are no
1166# longer 'reachable' in a build where they once were. 'Reachable' refers to
1167# whether a recipe is parsed so recipes in a layer which was removed would no
1168# longer be reachable. Switching between systemd and sysvinit where recipes
1169# became skipped would be another example.
1170#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001171# Also optionally removes the workdir of those tasks/recipes
1172#
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001173addhandler sstate_eventhandler_reachablestamps
1174sstate_eventhandler_reachablestamps[eventmask] = "bb.event.ReachableStamps"
1175python sstate_eventhandler_reachablestamps() {
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001176 import glob
1177 d = e.data
1178 stamps = e.stamps.values()
1179 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001180 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1181 preservestamps = []
1182 if os.path.exists(preservestampfile):
1183 with open(preservestampfile, 'r') as f:
1184 preservestamps = f.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001185 seen = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001186
1187 # The machine index contains all the stamps this machine has ever seen in this build directory.
1188 # We should only remove things which this machine once accessed but no longer does.
1189 machineindex = set()
1190 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1191 mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
1192 if os.path.exists(mi):
1193 with open(mi, "r") as f:
1194 machineindex = set(line.strip() for line in f.readlines())
1195
Brad Bishop316dfdd2018-06-25 12:45:53 -04001196 for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001197 toremove = []
1198 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1199 if not os.path.exists(i):
1200 continue
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001201 manseen = set()
1202 ignore = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001203 with open(i, "r") as f:
1204 lines = f.readlines()
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001205 for l in reversed(lines):
Brad Bishop19323692019-04-05 15:28:33 -04001206 try:
1207 (stamp, manifest, workdir) = l.split()
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001208 # The index may have multiple entries for the same manifest as the code above only appends
1209 # new entries and there may be an entry with matching manifest but differing version in stamp/workdir.
1210 # The last entry in the list is the valid one, any earlier entries with matching manifests
1211 # should be ignored.
1212 if manifest in manseen:
1213 ignore.append(l)
1214 continue
1215 manseen.add(manifest)
Brad Bishop19323692019-04-05 15:28:33 -04001216 if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
1217 toremove.append(l)
1218 if stamp not in seen:
1219 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
1220 seen.append(stamp)
1221 except ValueError:
1222 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001223
1224 if toremove:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001225 msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a)
1226 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001227
Brad Bishop316dfdd2018-06-25 12:45:53 -04001228 removed = 0
1229 for r in toremove:
1230 (stamp, manifest, workdir) = r.split()
1231 for m in glob.glob(manifest + ".*"):
1232 if m.endswith(".postrm"):
1233 continue
1234 sstate_clean_manifest(m, d)
1235 bb.utils.remove(stamp + "*")
1236 if removeworkdir:
1237 bb.utils.remove(workdir, recurse = True)
1238 lines.remove(r)
1239 removed = removed + 1
1240 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1241
1242 bb.event.fire(bb.event.ProcessFinished(msg), d)
1243
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001244 with open(i, "w") as f:
1245 for l in lines:
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001246 if l in ignore:
1247 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001248 f.write(l)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001249 machineindex |= set(stamps)
1250 with open(mi, "w") as f:
1251 for l in machineindex:
1252 f.write(l + "\n")
1253
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001254 if preservestamps:
1255 os.remove(preservestampfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001256}
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001257
1258
1259#
1260# Bitbake can generate an event showing which setscene tasks are 'stale',
1261# i.e. which ones will be rerun. These are ones where a stamp file is present but
1262# it is stable (e.g. taskhash doesn't match). With that list we can go through
1263# the manifests for matching tasks and "uninstall" those manifests now. We do
1264# this now rather than mid build since the distribution of files between sstate
1265# objects may have changed, new tasks may run first and if those new tasks overlap
1266# with the stale tasks, we'd see overlapping files messages and failures. Thankfully
1267# removing these files is fast.
1268#
1269addhandler sstate_eventhandler_stalesstate
1270sstate_eventhandler_stalesstate[eventmask] = "bb.event.StaleSetSceneTasks"
1271python sstate_eventhandler_stalesstate() {
1272 d = e.data
1273 tasks = e.tasks
1274
1275 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1276
1277 for a in list(set(d.getVar("SSTATE_ARCHS").split())):
1278 toremove = []
1279 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1280 if not os.path.exists(i):
1281 continue
1282 with open(i, "r") as f:
1283 lines = f.readlines()
1284 for l in lines:
1285 try:
1286 (stamp, manifest, workdir) = l.split()
1287 for tid in tasks:
1288 for s in tasks[tid]:
1289 if s.startswith(stamp):
1290 taskname = bb.runqueue.taskname_from_tid(tid)[3:]
1291 manname = manifest + "." + taskname
1292 if os.path.exists(manname):
1293 bb.debug(2, "Sstate for %s is stale, removing related manifest %s" % (tid, manname))
1294 toremove.append((manname, tid, tasks[tid]))
1295 break
1296 except ValueError:
1297 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
1298
1299 if toremove:
1300 msg = "Removing %d stale sstate objects for arch %s" % (len(toremove), a)
1301 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
1302
1303 removed = 0
1304 for (manname, tid, stamps) in toremove:
1305 sstate_clean_manifest(manname, d)
1306 for stamp in stamps:
1307 bb.utils.remove(stamp)
1308 removed = removed + 1
1309 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1310
1311 bb.event.fire(bb.event.ProcessFinished(msg), d)
1312}