blob: 554e401ee25203b8502b50b28d4221bca6183be1 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001SSTATE_VERSION = "3"
2
3SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
4SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
5
Andrew Geissler82c905d2020-04-13 13:39:40 -05006def generate_sstatefn(spec, hash, taskname, siginfo, d):
7 if taskname is None:
8 return ""
9 extension = ".tgz"
10 # 8 chars reserved for siginfo
11 limit = 254 - 8
12 if siginfo:
13 limit = 254
14 extension = ".tgz.siginfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015 if not hash:
16 hash = "INVALID"
Andrew Geissler82c905d2020-04-13 13:39:40 -050017 fn = spec + hash + "_" + taskname + extension
18 # If the filename is too long, attempt to reduce it
19 if len(fn) > limit:
20 components = spec.split(":")
21 # Fields 0,5,6 are mandatory, 1 is most useful, 2,3,4 are just for information
22 # 7 is for the separators
23 avail = (254 - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3
24 components[2] = components[2][:avail]
25 components[3] = components[3][:avail]
26 components[4] = components[4][:avail]
27 spec = ":".join(components)
28 fn = spec + hash + "_" + taskname + extension
29 if len(fn) > limit:
30 bb.fatal("Unable to reduce sstate name to less than 255 chararacters")
31 return hash[:2] + "/" + hash[2:4] + "/" + fn
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032
33SSTATE_PKGARCH = "${PACKAGE_ARCH}"
34SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050035SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
Andrew Geissler82c905d2020-04-13 13:39:40 -050036SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d.getVar('SSTATE_CURRTASK'), False, d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
38SSTATE_EXTRAPATH = ""
39SSTATE_EXTRAPATHWILDCARD = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -050040SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041
Patrick Williamsc0f7c042017-02-23 20:41:17 -060042# explicitly make PV to depend on evaluated value of PV variable
43PV[vardepvalue] = "${PV}"
44
Patrick Williamsc124f4f2015-09-15 14:41:29 -050045# We don't want the sstate to depend on things like the distro string
46# of the system, we let the sstate paths take care of this.
47SSTATE_EXTRAPATH[vardepvalue] = ""
Brad Bishop19323692019-04-05 15:28:33 -040048SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049
50# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080051SSTATE_DUPWHITELIST = "${DEPLOY_DIR}/licenses/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052# Avoid docbook/sgml catalog warnings for now
53SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
Brad Bishop316dfdd2018-06-25 12:45:53 -040054# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
55SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/"
56SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080057# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used
58SSTATE_DUPWHITELIST += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059# Archive the sources for many architectures in one deploy folder
60SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080061# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files
62SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/ovmf"
63SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/grub-efi"
64SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/systemd-boot"
65SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/microcode"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066
Brad Bishop6e60e8b2018-02-01 10:27:11 -050067SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
68SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
69SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050071BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
73SSTATE_ARCHS = " \
74 ${BUILD_ARCH} \
Andrew Geissler6ce62a22020-11-30 19:58:47 -060075 ${BUILD_ARCH}_${ORIGNATIVELSBSTRING} \
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
77 ${BUILD_ARCH}_${TARGET_ARCH} \
78 ${SDK_ARCH}_${SDK_OS} \
79 ${SDK_ARCH}_${PACKAGE_ARCH} \
80 allarch \
81 ${PACKAGE_ARCH} \
Brad Bishop316dfdd2018-06-25 12:45:53 -040082 ${PACKAGE_EXTRA_ARCHS} \
83 ${MACHINE_ARCH}"
Andrew Geissler6ce62a22020-11-30 19:58:47 -060084SSTATE_ARCHS[vardepsexclude] = "ORIGNATIVELSBSTRING"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
86SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
87
88SSTATECREATEFUNCS = "sstate_hardcode_path"
Brad Bishop19323692019-04-05 15:28:33 -040089SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090SSTATEPOSTCREATEFUNCS = ""
91SSTATEPREINSTFUNCS = ""
92SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
93SSTATEPOSTINSTFUNCS = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050094EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050096# Check whether sstate exists for tasks that support sstate and are in the
97# locked signatures file.
98SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
99
100# Check whether the task's computed hash matches the task's hash in the
101# locked signatures file.
102SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500103
104# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
105# not sign)
106SSTATE_SIG_KEY ?= ""
107SSTATE_SIG_PASSPHRASE ?= ""
108# Whether to verify the GnUPG signatures when extracting sstate archives
109SSTATE_VERIFY_SIG ?= "0"
110
Brad Bishop19323692019-04-05 15:28:33 -0400111SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
112SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
113 the output hash for a task, which in turn is used to determine equivalency. \
114 "
115
Brad Bishop19323692019-04-05 15:28:33 -0400116SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
117SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
118 hash equivalency server, such as PN, PV, taskname, etc. This information \
119 is very useful for developers looking at task data, but may leak sensitive \
120 data if the equivalence server is public. \
121 "
122
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123python () {
124 if bb.data.inherits_class('native', d):
125 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
126 elif bb.data.inherits_class('crosssdk', d):
127 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
128 elif bb.data.inherits_class('cross', d):
129 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TARGET_ARCH}"))
130 elif bb.data.inherits_class('nativesdk', d):
131 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
132 elif bb.data.inherits_class('cross-canadian', d):
133 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500134 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135 d.setVar('SSTATE_PKGARCH', "allarch")
136 else:
137 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
138
139 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
140 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500141 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
Brad Bishop19323692019-04-05 15:28:33 -0400142 d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500144 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 d.setVar('SSTATETASKS', " ".join(unique_tasks))
146 for task in unique_tasks:
147 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
148 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
149}
150
151def sstate_init(task, d):
152 ss = {}
153 ss['task'] = task
154 ss['dirs'] = []
155 ss['plaindirs'] = []
156 ss['lockfiles'] = []
157 ss['lockfiles-shared'] = []
158 return ss
159
160def sstate_state_fromvars(d, task = None):
161 if task is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 task = d.getVar('BB_CURRENTTASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500163 if not task:
164 bb.fatal("sstate code running without task context?!")
165 task = task.replace("_setscene", "")
166
167 if task.startswith("do_"):
168 task = task[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500169 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
170 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
171 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
172 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
173 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
174 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
175 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176 if not task or len(inputs) != len(outputs):
177 bb.fatal("sstate variables not setup correctly?!")
178
179 if task == "populate_lic":
180 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
181 d.setVar("SSTATE_EXTRAPATH", "")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500182 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183
184 ss = sstate_init(task, d)
185 for i in range(len(inputs)):
186 sstate_add(ss, inputs[i], outputs[i], d)
187 ss['lockfiles'] = lockfiles
188 ss['lockfiles-shared'] = lockfilesshared
189 ss['plaindirs'] = plaindirs
190 ss['interceptfuncs'] = interceptfuncs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500191 ss['fixmedir'] = fixmedir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192 return ss
193
194def sstate_add(ss, source, dest, d):
195 if not source.endswith("/"):
196 source = source + "/"
197 if not dest.endswith("/"):
198 dest = dest + "/"
199 source = os.path.normpath(source)
200 dest = os.path.normpath(dest)
201 srcbase = os.path.basename(source)
202 ss['dirs'].append([srcbase, source, dest])
203 return ss
204
205def sstate_install(ss, d):
206 import oe.path
207 import oe.sstatesig
208 import subprocess
209
210 sharedfiles = []
211 shareddirs = []
212 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
213
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500214 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
215
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
217
218 if os.access(manifest, os.R_OK):
219 bb.fatal("Package already staged (%s)?!" % manifest)
220
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600221 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
222
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223 locks = []
224 for lock in ss['lockfiles-shared']:
225 locks.append(bb.utils.lockfile(lock, True))
226 for lock in ss['lockfiles']:
227 locks.append(bb.utils.lockfile(lock))
228
229 for state in ss['dirs']:
230 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
231 for walkroot, dirs, files in os.walk(state[1]):
232 for file in files:
233 srcpath = os.path.join(walkroot, file)
234 dstpath = srcpath.replace(state[1], state[2])
235 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
236 sharedfiles.append(dstpath)
237 for dir in dirs:
238 srcdir = os.path.join(walkroot, dir)
239 dstdir = srcdir.replace(state[1], state[2])
240 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500241 if os.path.islink(srcdir):
242 sharedfiles.append(dstdir)
243 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 if not dstdir.endswith("/"):
245 dstdir = dstdir + "/"
246 shareddirs.append(dstdir)
247
248 # Check the file list for conflicts against files which already exist
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500249 whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500250 match = []
251 for f in sharedfiles:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500252 if os.path.exists(f) and not os.path.islink(f):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500253 f = os.path.normpath(f)
254 realmatch = True
255 for w in whitelist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600256 w = os.path.normpath(w)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 if f.startswith(w):
258 realmatch = False
259 break
260 if realmatch:
261 match.append(f)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500264 if search_output:
265 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
266 else:
267 match.append(" (not matched to any task)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 if match:
269 bb.error("The recipe %s is trying to install files into a shared " \
270 "area when those files already exist. Those files and their manifest " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 "location are:\n %s\nPlease verify which recipe should provide the " \
272 "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \
273 "break things - if not now, possibly in the future (we've seen builds fail " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500274 "several months later). If the system knew how to recover from this " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500275 "automatically it would, however there are several different scenarios " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500276 "which can result in this and we don't know which one this is. It may be " \
277 "you have switched providers of something like virtual/kernel (e.g. from " \
278 "linux-yocto to linux-yocto-dev), in that case you need to execute the " \
279 "clean task for both recipes and it will resolve this error. It may be " \
280 "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500281 "those recipes should again resolve this error, however switching " \
282 "DISTRO_FEATURES on an existing build directory is not supported - you " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500283 "should really clean out tmp and rebuild (reusing sstate should be safe). " \
284 "It could be the overlapping files detected are harmless in which case " \
285 "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \
286 "also be your build is including two different conflicting versions of " \
287 "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \
288 "be to resolve the conflict. If in doubt, please ask on the mailing list, " \
289 "sharing the error and filelist above." % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500290 (d.getVar('PN'), "\n ".join(match)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500291 bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.")
292
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500293 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
294 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
295 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
296
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297 # Write out the manifest
298 f = open(manifest, "w")
299 for file in sharedfiles:
300 f.write(file + "\n")
301
302 # We want to ensure that directories appear at the end of the manifest
303 # so that when we test to see if they should be deleted any contents
304 # added by the task will have been removed first.
305 dirs = sorted(shareddirs, key=len)
306 # Must remove children first, which will have a longer path than the parent
307 for di in reversed(dirs):
308 f.write(di + "\n")
309 f.close()
310
311 # Append to the list of manifests for this PACKAGE_ARCH
312
313 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
314 l = bb.utils.lockfile(i + ".lock")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500315 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 manifests = []
317 if os.path.exists(i):
318 with open(i, "r") as f:
319 manifests = f.readlines()
William A. Kennington IIIac69b482021-06-02 12:28:27 -0700320 # We append new entries, we don't remove older entries which may have the same
321 # manifest name but different versions from stamp/workdir. See below.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322 if filedata not in manifests:
323 with open(i, "a+") as f:
324 f.write(filedata)
325 bb.utils.unlockfile(l)
326
327 # Run the actual file install
328 for state in ss['dirs']:
329 if os.path.exists(state[1]):
330 oe.path.copyhardlinktree(state[1], state[2])
331
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500332 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500333 # All hooks should run in the SSTATE_INSTDIR
334 bb.build.exec_func(postinst, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335
336 for lock in locks:
337 bb.utils.unlockfile(lock)
338
339sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX"
340sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
341
342def sstate_installpkg(ss, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500343 from oe.gpg_sign import get_signer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500345 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
Andrew Geissler82c905d2020-04-13 13:39:40 -0500346 d.setVar("SSTATE_CURRTASK", ss['task'])
347 sstatefetch = d.getVar('SSTATE_PKGNAME')
348 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500349
350 if not os.path.exists(sstatepkg):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800351 pstaging_fetch(sstatefetch, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
353 if not os.path.isfile(sstatepkg):
Brad Bishop08902b02019-08-20 09:16:51 -0400354 bb.note("Sstate package %s does not exist" % sstatepkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 return False
356
357 sstate_clean(ss, d)
358
359 d.setVar('SSTATE_INSTDIR', sstateinst)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500361 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500362 if not os.path.isfile(sstatepkg + '.sig'):
363 bb.warn("No signature file for sstate package %s, skipping acceleration..." % sstatepkg)
364 return False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500365 signer = get_signer(d, 'local')
366 if not signer.verify(sstatepkg + '.sig'):
Brad Bishop08902b02019-08-20 09:16:51 -0400367 bb.warn("Cannot verify signature on sstate package %s, skipping acceleration..." % sstatepkg)
368 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500370 # Empty sstateinst directory, ensure its clean
371 if os.path.exists(sstateinst):
372 oe.path.remove(sstateinst)
373 bb.utils.mkdirhier(sstateinst)
374
375 sstateinst = d.getVar("SSTATE_INSTDIR")
376 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
377
378 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500379 # All hooks should run in the SSTATE_INSTDIR
380 bb.build.exec_func(f, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500382 return sstate_installpkgdir(ss, d)
383
384def sstate_installpkgdir(ss, d):
385 import oe.path
386 import subprocess
387
388 sstateinst = d.getVar("SSTATE_INSTDIR")
389 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
390
391 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
392 # All hooks should run in the SSTATE_INSTDIR
393 bb.build.exec_func(f, d, (sstateinst,))
394
395 def prepdir(dir):
396 # remove dir if it exists, ensure any parent directories do exist
397 if os.path.exists(dir):
398 oe.path.remove(dir)
399 bb.utils.mkdirhier(dir)
400 oe.path.remove(dir)
401
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 for state in ss['dirs']:
403 prepdir(state[1])
Andrew Geisslerc926e172021-05-07 16:11:35 -0500404 bb.utils.rename(sstateinst + state[0], state[1])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500405 sstate_install(ss, d)
406
407 for plain in ss['plaindirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500408 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500409 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500410 src = sstateinst + "/" + plain.replace(workdir, '')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500411 if sharedworkdir in plain:
412 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413 dest = plain
414 bb.utils.mkdirhier(src)
415 prepdir(dest)
Andrew Geisslerc926e172021-05-07 16:11:35 -0500416 bb.utils.rename(src, dest)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417
418 return True
419
420python sstate_hardcode_path_unpack () {
421 # Fixup hardcoded paths
422 #
423 # Note: The logic below must match the reverse logic in
424 # sstate_hardcode_path(d)
425 import subprocess
426
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500427 sstateinst = d.getVar('SSTATE_INSTDIR')
428 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
429 fixmefn = sstateinst + "fixmepath"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500430 if os.path.isfile(fixmefn):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500431 staging_target = d.getVar('RECIPE_SYSROOT')
432 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500433
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500434 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500436 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
437 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
438 else:
439 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500441 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500442 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500443 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
445
446 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
447 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
448
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500449 # Defer do_populate_sysroot relocation command
450 if sstatefixmedir:
451 bb.utils.mkdirhier(sstatefixmedir)
452 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
453 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
454 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
455 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
456 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
457 f.write(sstate_hardcode_cmd)
458 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
459 return
460
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500461 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500462 subprocess.check_call(sstate_hardcode_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800464 # Need to remove this or we'd copy it into the target directory and may
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500465 # conflict with another writer
466 os.remove(fixmefn)
467}
468
469def sstate_clean_cachefile(ss, d):
470 import oe.path
471
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300472 if d.getVarFlag('do_%s' % ss['task'], 'task'):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500473 d.setVar("SSTATE_PATH_CURRTASK", ss['task'])
474 sstatepkgfile = d.getVar('SSTATE_PATHSPEC')
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300475 bb.note("Removing %s" % sstatepkgfile)
476 oe.path.remove(sstatepkgfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500477
478def sstate_clean_cachefiles(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500479 for task in (d.getVar('SSTATETASKS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480 ld = d.createCopy()
481 ss = sstate_state_fromvars(ld, task)
482 sstate_clean_cachefile(ss, ld)
483
Andrew Geissler5f350902021-07-23 13:09:54 -0400484def sstate_clean_manifest(manifest, d, canrace=False, prefix=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500485 import oe.path
486
487 mfile = open(manifest)
488 entries = mfile.readlines()
489 mfile.close()
490
491 for entry in entries:
492 entry = entry.strip()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500493 if prefix and not entry.startswith("/"):
494 entry = prefix + "/" + entry
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 bb.debug(2, "Removing manifest: %s" % entry)
496 # We can race against another package populating directories as we're removing them
497 # so we ignore errors here.
498 try:
499 if entry.endswith("/"):
500 if os.path.islink(entry[:-1]):
501 os.remove(entry[:-1])
Andrew Geissler5f350902021-07-23 13:09:54 -0400502 elif os.path.exists(entry) and len(os.listdir(entry)) == 0 and not canrace:
503 # Removing directories whilst builds are in progress exposes a race. Only
504 # do it in contexts where it is safe to do so.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505 os.rmdir(entry[:-1])
506 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500507 os.remove(entry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500508 except OSError:
509 pass
510
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600511 postrm = manifest + ".postrm"
512 if os.path.exists(manifest + ".postrm"):
513 import subprocess
514 os.chmod(postrm, 0o755)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500515 subprocess.check_call(postrm, shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600516 oe.path.remove(postrm)
517
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500518 oe.path.remove(manifest)
519
520def sstate_clean(ss, d):
521 import oe.path
522 import glob
523
524 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500525 stamp_clean = d.getVar("STAMPCLEAN")
526 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500527 if extrainf:
528 d2.setVar("SSTATE_MANMACH", extrainf)
529 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
530 else:
531 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
532
533 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
534
535 if os.path.exists(manifest):
536 locks = []
537 for lock in ss['lockfiles-shared']:
538 locks.append(bb.utils.lockfile(lock))
539 for lock in ss['lockfiles']:
540 locks.append(bb.utils.lockfile(lock))
541
Andrew Geissler5f350902021-07-23 13:09:54 -0400542 sstate_clean_manifest(manifest, d, canrace=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500543
544 for lock in locks:
545 bb.utils.unlockfile(lock)
546
547 # Remove the current and previous stamps, but keep the sigdata.
548 #
549 # The glob() matches do_task* which may match multiple tasks, for
550 # example: do_package and do_package_write_ipk, so we need to
551 # exactly match *.do_task.* and *.do_task_setscene.*
552 rm_stamp = '.do_%s.' % ss['task']
553 rm_setscene = '.do_%s_setscene.' % ss['task']
554 # For BB_SIGNATURE_HANDLER = "noop"
555 rm_nohash = ".do_%s" % ss['task']
556 for stfile in glob.glob(wildcard_stfile):
557 # Keep the sigdata
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500558 if ".sigdata." in stfile or ".sigbasedata." in stfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500559 continue
560 # Preserve taint files in the stamps directory
561 if stfile.endswith('.taint'):
562 continue
563 if rm_stamp in stfile or rm_setscene in stfile or \
564 stfile.endswith(rm_nohash):
565 oe.path.remove(stfile)
566
567sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
568
569CLEANFUNCS += "sstate_cleanall"
570
571python sstate_cleanall() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500572 bb.note("Removing shared state for package %s" % d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500573
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500574 manifest_dir = d.getVar('SSTATE_MANIFESTS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575 if not os.path.exists(manifest_dir):
576 return
577
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500578 tasks = d.getVar('SSTATETASKS').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579 for name in tasks:
580 ld = d.createCopy()
581 shared_state = sstate_state_fromvars(ld, name)
582 sstate_clean(shared_state, ld)
583}
584
585python sstate_hardcode_path () {
586 import subprocess, platform
587
588 # Need to remove hardcoded paths and fix these when we install the
589 # staging packages.
590 #
591 # Note: the logic in this function needs to match the reverse logic
592 # in sstate_installpkg(ss, d)
593
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500594 staging_target = d.getVar('RECIPE_SYSROOT')
595 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
596 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500597
Brad Bishop316dfdd2018-06-25 12:45:53 -0400598 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500599 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500600 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500601 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
602 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400603 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500604 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400605 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
606 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500608 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500609 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500610 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500612 sstate_grep_cmd += " -e '%s'" % (fixme_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500613
614 fixmefn = sstate_builddir + "fixmepath"
615
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500616 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500617 sstate_filelist_cmd = "tee %s" % (fixmefn)
618
619 # fixmepath file needs relative paths, drop sstate_builddir prefix
620 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
621
622 xargs_no_empty_run_cmd = '--no-run-if-empty'
623 if platform.system() == 'Darwin':
624 xargs_no_empty_run_cmd = ''
625
626 # Limit the fixpaths and sed operations based on the initial grep search
627 # This has the side effect of making sure the vfs cache is hot
628 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
629
630 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500631 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500632
633 # If the fixmefn is empty, remove it..
634 if os.stat(fixmefn).st_size == 0:
635 os.remove(fixmefn)
636 else:
637 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500638 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500639}
640
641def sstate_package(ss, d):
642 import oe.path
643
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500644 tmpdir = d.getVar('TMPDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500645
646 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
Andrew Geissler82c905d2020-04-13 13:39:40 -0500647 d.setVar("SSTATE_CURRTASK", ss['task'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500648 bb.utils.remove(sstatebuild, recurse=True)
649 bb.utils.mkdirhier(sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500650 for state in ss['dirs']:
651 if not os.path.exists(state[1]):
652 continue
653 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500654 # Find and error for absolute symlinks. We could attempt to relocate but its not
655 # clear where the symlink is relative to in this context. We could add that markup
656 # to sstate tasks but there aren't many of these so better just avoid them entirely.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500657 for walkroot, dirs, files in os.walk(state[1]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500658 for file in files + dirs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500659 srcpath = os.path.join(walkroot, file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500660 if not os.path.islink(srcpath):
661 continue
662 link = os.readlink(srcpath)
663 if not os.path.isabs(link):
664 continue
665 if not link.startswith(tmpdir):
666 continue
667 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500668 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
Andrew Geisslerc926e172021-05-07 16:11:35 -0500669 bb.utils.rename(state[1], sstatebuild + state[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500670
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500671 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500672 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500673 for plain in ss['plaindirs']:
674 pdir = plain.replace(workdir, sstatebuild)
Brad Bishop977dc1a2019-02-06 16:01:43 -0500675 if sharedworkdir in plain:
676 pdir = plain.replace(sharedworkdir, sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500677 bb.utils.mkdirhier(plain)
678 bb.utils.mkdirhier(pdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -0500679 bb.utils.rename(plain, pdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500680
681 d.setVar('SSTATE_BUILDDIR', sstatebuild)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500682 d.setVar('SSTATE_INSTDIR', sstatebuild)
683
684 if d.getVar('SSTATE_SKIP_CREATION') == '1':
685 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500686
Brad Bishop08902b02019-08-20 09:16:51 -0400687 sstate_create_package = ['sstate_report_unihash', 'sstate_create_package']
688 if d.getVar('SSTATE_SIG_KEY'):
689 sstate_create_package.append('sstate_sign_package')
690
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500691 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
Brad Bishop08902b02019-08-20 09:16:51 -0400692 sstate_create_package + \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500693 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500694 # All hooks should run in SSTATE_BUILDDIR.
695 bb.build.exec_func(f, d, (sstatebuild,))
696
Andrew Geissler82c905d2020-04-13 13:39:40 -0500697 # SSTATE_PKG may have been changed by sstate_report_unihash
698 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
699 if not os.path.exists(siginfo):
700 bb.siggen.dump_this_task(siginfo, d)
701 else:
Andrew Geisslerc182c622020-05-15 14:13:32 -0500702 try:
703 os.utime(siginfo, None)
704 except PermissionError:
705 pass
Andrew Geissler5f350902021-07-23 13:09:54 -0400706 except OSError as e:
707 # Handle read-only file systems gracefully
708 if e.errno != errno.EROFS:
709 raise e
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500710
711 return
712
William A. Kennington IIIac69b482021-06-02 12:28:27 -0700713sstate_package[vardepsexclude] += "SSTATE_SIG_KEY"
714
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800715def pstaging_fetch(sstatefetch, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500716 import bb.fetch2
717
718 # Only try and fetch if the user has configured a mirror
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500719 mirrors = d.getVar('SSTATE_MIRRORS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500720 if not mirrors:
721 return
722
723 # Copy the data object and override DL_DIR and SRC_URI
724 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500725
726 dldir = localdata.expand("${SSTATE_DIR}")
727 bb.utils.mkdirhier(dldir)
728
729 localdata.delVar('MIRRORS')
730 localdata.setVar('FILESPATH', dldir)
731 localdata.setVar('DL_DIR', dldir)
732 localdata.setVar('PREMIRRORS', mirrors)
733
734 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
735 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400736 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
737 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500738 localdata.delVar('BB_NO_NETWORK')
739
740 # Try a fetch from the sstate mirror, if it fails just return and
741 # we will build the package
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600742 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
743 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500744 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600745 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500746
747 for srcuri in uris:
748 localdata.setVar('SRC_URI', srcuri)
749 try:
750 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500751 fetcher.checkstatus()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500752 fetcher.download()
753
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500754 except bb.fetch2.BBFetchException:
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500755 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500756
757def sstate_setscene(d):
758 shared_state = sstate_state_fromvars(d)
759 accelerate = sstate_installpkg(shared_state, d)
760 if not accelerate:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600761 bb.fatal("No suitable staging package found")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500762
763python sstate_task_prefunc () {
764 shared_state = sstate_state_fromvars(d)
765 sstate_clean(shared_state, d)
766}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500767sstate_task_prefunc[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500768
769python sstate_task_postfunc () {
770 shared_state = sstate_state_fromvars(d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500771
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500772 for intercept in shared_state['interceptfuncs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500773 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
774
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600775 omask = os.umask(0o002)
776 if omask != 0o002:
777 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500778 sstate_package(shared_state, d)
779 os.umask(omask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500780
781 sstateinst = d.getVar("SSTATE_INSTDIR")
782 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
783
784 sstate_installpkgdir(shared_state, d)
785
786 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500787}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500788sstate_task_postfunc[dirs] = "${WORKDIR}"
789
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500790
791#
792# Shell function to generate a sstate package from a directory
793# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
794#
795sstate_create_package () {
Andrew Geissler82c905d2020-04-13 13:39:40 -0500796 # Exit early if it already exists
Brad Bishop08902b02019-08-20 09:16:51 -0400797 if [ -e ${SSTATE_PKG} ]; then
Andrew Geisslerc182c622020-05-15 14:13:32 -0500798 [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
Brad Bishop08902b02019-08-20 09:16:51 -0400799 return
800 fi
801
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500802 mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
Andrew Geissler82c905d2020-04-13 13:39:40 -0500803 TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
804
805 # Use pigz if available
806 OPT="-czS"
807 if [ -x "$(command -v pigz)" ]; then
808 OPT="-I pigz -cS"
809 fi
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800810
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500811 # Need to handle empty directories
812 if [ "$(ls -A)" ]; then
813 set +e
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800814 tar $OPT -f $TFILE *
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500815 ret=$?
816 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
817 exit 1
818 fi
819 set -e
820 else
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800821 tar $OPT --file=$TFILE --files-from=/dev/null
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500822 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500823 chmod 0664 $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400824 # Skip if it was already created by some other process
825 if [ ! -e ${SSTATE_PKG} ]; then
Andrew Geissler82c905d2020-04-13 13:39:40 -0500826 # Move into place using ln to attempt an atomic op.
827 # Abort if it already exists
828 ln $TFILE ${SSTATE_PKG} && rm $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400829 else
830 rm $TFILE
831 fi
Andrew Geisslerc182c622020-05-15 14:13:32 -0500832 [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500833}
834
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500835python sstate_sign_package () {
836 from oe.gpg_sign import get_signer
837
Brad Bishop08902b02019-08-20 09:16:51 -0400838
839 signer = get_signer(d, 'local')
840 sstate_pkg = d.getVar('SSTATE_PKG')
841 if os.path.exists(sstate_pkg + '.sig'):
842 os.unlink(sstate_pkg + '.sig')
843 signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
844 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500845}
846
Brad Bishop19323692019-04-05 15:28:33 -0400847python sstate_report_unihash() {
848 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
849
850 if report_unihash:
851 ss = sstate_state_fromvars(d)
852 report_unihash(os.getcwd(), ss['task'], d)
853}
854
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500855#
856# Shell function to decompress and prepare a package for installation
857# Will be run from within SSTATE_INSTDIR.
858#
859sstate_unpack_package () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500860 tar -xvzf ${SSTATE_PKG}
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500861 # update .siginfo atime on local/NFS mirror
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500862 [ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500863 # Use "! -w ||" to return true for read only files
864 [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500865 [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
866 [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500867}
868
869BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
870
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500871def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
Brad Bishop08902b02019-08-20 09:16:51 -0400872 found = set()
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600873 foundLocal = set()
874 foundNet = set()
Brad Bishop08902b02019-08-20 09:16:51 -0400875 missed = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500876
Brad Bishop19323692019-04-05 15:28:33 -0400877 def gethash(task):
Brad Bishop08902b02019-08-20 09:16:51 -0400878 return sq_data['unihash'][task]
Brad Bishop19323692019-04-05 15:28:33 -0400879
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500880 def getpathcomponents(task, d):
881 # Magic data from BB_HASHFILENAME
Brad Bishop08902b02019-08-20 09:16:51 -0400882 splithashfn = sq_data['hashfn'][task].split(" ")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500883 spec = splithashfn[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500884 if splithashfn[0] == "True":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500885 extrapath = d.getVar("NATIVELSBSTRING") + "/"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500886 else:
887 extrapath = ""
Brad Bishop08902b02019-08-20 09:16:51 -0400888
889 tname = bb.runqueue.taskname_from_tid(task)[3:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500890
891 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
892 spec = splithashfn[2]
893 extrapath = ""
894
895 return spec, extrapath, tname
896
897
Brad Bishop08902b02019-08-20 09:16:51 -0400898 for tid in sq_data['hash']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500899
Brad Bishop08902b02019-08-20 09:16:51 -0400900 spec, extrapath, tname = getpathcomponents(tid, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500901
Andrew Geissler82c905d2020-04-13 13:39:40 -0500902 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500903
904 if os.path.exists(sstatefile):
905 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
Brad Bishop08902b02019-08-20 09:16:51 -0400906 found.add(tid)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600907 foundLocal.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500908 continue
909 else:
Brad Bishop08902b02019-08-20 09:16:51 -0400910 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500911 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
912
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500913 mirrors = d.getVar("SSTATE_MIRRORS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500914 if mirrors:
915 # Copy the data object and override DL_DIR and SRC_URI
916 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500917
918 dldir = localdata.expand("${SSTATE_DIR}")
919 localdata.delVar('MIRRORS')
920 localdata.setVar('FILESPATH', dldir)
921 localdata.setVar('DL_DIR', dldir)
922 localdata.setVar('PREMIRRORS', mirrors)
923
924 bb.debug(2, "SState using premirror of: %s" % mirrors)
925
926 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
927 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400928 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
929 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500930 localdata.delVar('BB_NO_NETWORK')
931
932 from bb.fetch2 import FetchConnectionCache
933 def checkstatus_init(thread_worker):
934 thread_worker.connection_cache = FetchConnectionCache()
935
936 def checkstatus_end(thread_worker):
937 thread_worker.connection_cache.close_connections()
938
939 def checkstatus(thread_worker, arg):
Brad Bishop08902b02019-08-20 09:16:51 -0400940 (tid, sstatefile) = arg
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500941
942 localdata2 = bb.data.createCopy(localdata)
943 srcuri = "file://" + sstatefile
944 localdata.setVar('SRC_URI', srcuri)
945 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
946
947 try:
948 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
949 connection_cache=thread_worker.connection_cache)
950 fetcher.checkstatus()
951 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
Brad Bishop08902b02019-08-20 09:16:51 -0400952 found.add(tid)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600953 foundNet.add(tid)
Brad Bishop08902b02019-08-20 09:16:51 -0400954 if tid in missed:
955 missed.remove(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500956 except:
Brad Bishop08902b02019-08-20 09:16:51 -0400957 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500958 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600959 pass
Andrew Geissler82c905d2020-04-13 13:39:40 -0500960 if len(tasklist) >= min_tasks:
961 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500962
963 tasklist = []
Andrew Geissler82c905d2020-04-13 13:39:40 -0500964 min_tasks = 100
Brad Bishop08902b02019-08-20 09:16:51 -0400965 for tid in sq_data['hash']:
966 if tid in found:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500967 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400968 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500969 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
Brad Bishop08902b02019-08-20 09:16:51 -0400970 tasklist.append((tid, sstatefile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500971
972 if tasklist:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500973 if len(tasklist) >= min_tasks:
974 msg = "Checking sstate mirror object availability"
975 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600976
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500977 import multiprocessing
978 nproc = min(multiprocessing.cpu_count(), len(tasklist))
979
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600980 bb.event.enable_threadlock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500981 pool = oe.utils.ThreadedPool(nproc, len(tasklist),
982 worker_init=checkstatus_init, worker_end=checkstatus_end)
983 for t in tasklist:
984 pool.add_task(checkstatus, t)
985 pool.start()
986 pool.wait_completion()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600987 bb.event.disable_threadlock()
988
Andrew Geissler82c905d2020-04-13 13:39:40 -0500989 if len(tasklist) >= min_tasks:
990 bb.event.fire(bb.event.ProcessFinished(msg), d)
Brad Bishop96ff1982019-08-19 13:50:42 -0400991
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500992 inheritlist = d.getVar("INHERIT")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500993 if "toaster" in inheritlist:
994 evdata = {'missed': [], 'found': []};
Brad Bishop08902b02019-08-20 09:16:51 -0400995 for tid in missed:
996 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500997 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
Brad Bishop08902b02019-08-20 09:16:51 -0400998 evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
999 for tid in found:
1000 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001001 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
Brad Bishop08902b02019-08-20 09:16:51 -04001002 evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001003 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
1004
Brad Bishop1d80a2e2019-11-15 16:35:03 -05001005 if summary:
1006 # Print some summary statistics about the current task completion and how much sstate
1007 # reuse there was. Avoid divide by zero errors.
1008 total = len(sq_data['hash'])
1009 complete = 0
1010 if currentcount:
1011 complete = (len(found) + currentcount) / (total + currentcount) * 100
1012 match = 0
1013 if total:
1014 match = len(found) / total * 100
Andrew Geissler9b4d8b02021-02-19 12:26:16 -06001015 bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001016
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001017 if hasattr(bb.parse.siggen, "checkhashes"):
Brad Bishop08902b02019-08-20 09:16:51 -04001018 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001019
Brad Bishop08902b02019-08-20 09:16:51 -04001020 return found
Patrick Williams213cb262021-08-07 19:21:33 -05001021setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001022
1023BB_SETSCENE_DEPVALID = "setscene_depvalid"
1024
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001025def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001026 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
1027 # task is included in taskdependees too
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001028 # Return - False - We need this dependency
1029 # - True - We can skip this dependency
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001030 import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001031
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001032 def logit(msg, log):
1033 if log is not None:
1034 log.append(msg)
1035 else:
1036 bb.debug(2, msg)
1037
1038 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001039
1040 def isNativeCross(x):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001041 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001042
1043 # We only need to trigger populate_lic through direct dependencies
1044 if taskdependees[task][1] == "do_populate_lic":
1045 return True
1046
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001047 # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
1048 if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
1049 return True
1050
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001051 # We only need to trigger packagedata through direct dependencies
1052 # but need to preserve packagedata on packagedata links
1053 if taskdependees[task][1] == "do_packagedata":
1054 for dep in taskdependees:
1055 if taskdependees[dep][1] == "do_packagedata":
1056 return False
1057 return True
1058
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001059 for dep in taskdependees:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001060 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001061 if task == dep:
1062 continue
1063 if dep in notneeded:
1064 continue
1065 # do_package_write_* and do_package doesn't need do_package
1066 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
1067 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001068 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
1069 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
1070 return False
1071 # do_package/packagedata/package_qa don't need do_populate_sysroot
1072 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001073 continue
1074 # Native/Cross packages don't exist and are noexec anyway
1075 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
1076 continue
1077
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001078 # This is due to the [depends] in useradd.bbclass complicating matters
1079 # The logic *is* reversed here due to the way hard setscene dependencies are injected
1080 if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
1081 continue
1082
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001083 # Consider sysroot depending on sysroot tasks
1084 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001085 # Allow excluding certain recursive dependencies. If a recipe needs it should add a
1086 # specific dependency itself, rather than relying on one of its dependees to pull
1087 # them in.
1088 # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
1089 not_needed = False
1090 excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT')
1091 if excludedeps is None:
1092 # Cache the regular expressions for speed
1093 excludedeps = []
1094 for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split():
1095 excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))
1096 d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps)
1097 for excl in excludedeps:
1098 if excl[0].match(taskdependees[dep][0]):
1099 if excl[1].match(taskdependees[task][0]):
1100 not_needed = True
1101 break
1102 if not_needed:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001103 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001104 # For meta-extsdk-toolchain we want all sysroot dependencies
1105 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
1106 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001107 # Native/Cross populate_sysroot need their dependencies
1108 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
1109 return False
1110 # Target populate_sysroot depended on by cross tools need to be installed
1111 if isNativeCross(taskdependees[dep][0]):
1112 return False
1113 # Native/cross tools depended upon by target sysroot are not needed
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001114 # Add an exception for shadow-native as required by useradd.bbclass
1115 if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001116 continue
1117 # Target populate_sysroot need their dependencies
1118 return False
1119
1120 if taskdependees[task][1] == 'do_shared_workdir':
1121 continue
1122
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001123 if taskdependees[dep][1] == "do_populate_lic":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001124 continue
1125
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001126
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001127 # Safe fallthrough default
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001128 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001129 return False
1130 return True
1131
1132addhandler sstate_eventhandler
1133sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
1134python sstate_eventhandler() {
1135 d = e.data
Andrew Geissler82c905d2020-04-13 13:39:40 -05001136 writtensstate = d.getVar('SSTATE_CURRTASK')
1137 if not writtensstate:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001138 taskname = d.getVar("BB_RUNTASK")[3:]
1139 spec = d.getVar('SSTATE_PKGSPEC')
1140 swspec = d.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001141 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1142 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1143 d.setVar("SSTATE_EXTRAPATH", "")
Andrew Geissler82c905d2020-04-13 13:39:40 -05001144 d.setVar("SSTATE_CURRTASK", taskname)
1145 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
1146 if not os.path.exists(siginfo):
1147 bb.siggen.dump_this_task(siginfo, d)
1148 else:
Andrew Geisslerc182c622020-05-15 14:13:32 -05001149 try:
1150 os.utime(siginfo, None)
1151 except PermissionError:
1152 pass
Andrew Geissler5f350902021-07-23 13:09:54 -04001153 except OSError as e:
1154 # Handle read-only file systems gracefully
1155 if e.errno != errno.EROFS:
1156 raise e
Andrew Geisslerc182c622020-05-15 14:13:32 -05001157
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001158}
1159
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001160SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001161
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001162#
1163# Event handler which removes manifests and stamps file for recipes which are no
1164# longer 'reachable' in a build where they once were. 'Reachable' refers to
1165# whether a recipe is parsed so recipes in a layer which was removed would no
1166# longer be reachable. Switching between systemd and sysvinit where recipes
1167# became skipped would be another example.
1168#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001169# Also optionally removes the workdir of those tasks/recipes
1170#
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001171addhandler sstate_eventhandler_reachablestamps
1172sstate_eventhandler_reachablestamps[eventmask] = "bb.event.ReachableStamps"
1173python sstate_eventhandler_reachablestamps() {
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001174 import glob
1175 d = e.data
1176 stamps = e.stamps.values()
1177 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001178 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1179 preservestamps = []
1180 if os.path.exists(preservestampfile):
1181 with open(preservestampfile, 'r') as f:
1182 preservestamps = f.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001183 seen = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001184
1185 # The machine index contains all the stamps this machine has ever seen in this build directory.
1186 # We should only remove things which this machine once accessed but no longer does.
1187 machineindex = set()
1188 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1189 mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
1190 if os.path.exists(mi):
1191 with open(mi, "r") as f:
1192 machineindex = set(line.strip() for line in f.readlines())
1193
Brad Bishop316dfdd2018-06-25 12:45:53 -04001194 for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001195 toremove = []
1196 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1197 if not os.path.exists(i):
1198 continue
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001199 manseen = set()
1200 ignore = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001201 with open(i, "r") as f:
1202 lines = f.readlines()
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001203 for l in reversed(lines):
Brad Bishop19323692019-04-05 15:28:33 -04001204 try:
1205 (stamp, manifest, workdir) = l.split()
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001206 # The index may have multiple entries for the same manifest as the code above only appends
1207 # new entries and there may be an entry with matching manifest but differing version in stamp/workdir.
1208 # The last entry in the list is the valid one, any earlier entries with matching manifests
1209 # should be ignored.
1210 if manifest in manseen:
1211 ignore.append(l)
1212 continue
1213 manseen.add(manifest)
Brad Bishop19323692019-04-05 15:28:33 -04001214 if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
1215 toremove.append(l)
1216 if stamp not in seen:
1217 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
1218 seen.append(stamp)
1219 except ValueError:
1220 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001221
1222 if toremove:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001223 msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a)
1224 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001225
Brad Bishop316dfdd2018-06-25 12:45:53 -04001226 removed = 0
1227 for r in toremove:
1228 (stamp, manifest, workdir) = r.split()
1229 for m in glob.glob(manifest + ".*"):
1230 if m.endswith(".postrm"):
1231 continue
1232 sstate_clean_manifest(m, d)
1233 bb.utils.remove(stamp + "*")
1234 if removeworkdir:
1235 bb.utils.remove(workdir, recurse = True)
1236 lines.remove(r)
1237 removed = removed + 1
1238 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1239
1240 bb.event.fire(bb.event.ProcessFinished(msg), d)
1241
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001242 with open(i, "w") as f:
1243 for l in lines:
William A. Kennington IIIac69b482021-06-02 12:28:27 -07001244 if l in ignore:
1245 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001246 f.write(l)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001247 machineindex |= set(stamps)
1248 with open(mi, "w") as f:
1249 for l in machineindex:
1250 f.write(l + "\n")
1251
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001252 if preservestamps:
1253 os.remove(preservestampfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001254}
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001255
1256
1257#
1258# Bitbake can generate an event showing which setscene tasks are 'stale',
1259# i.e. which ones will be rerun. These are ones where a stamp file is present but
1260# it is stable (e.g. taskhash doesn't match). With that list we can go through
1261# the manifests for matching tasks and "uninstall" those manifests now. We do
1262# this now rather than mid build since the distribution of files between sstate
1263# objects may have changed, new tasks may run first and if those new tasks overlap
1264# with the stale tasks, we'd see overlapping files messages and failures. Thankfully
1265# removing these files is fast.
1266#
1267addhandler sstate_eventhandler_stalesstate
1268sstate_eventhandler_stalesstate[eventmask] = "bb.event.StaleSetSceneTasks"
1269python sstate_eventhandler_stalesstate() {
1270 d = e.data
1271 tasks = e.tasks
1272
1273 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1274
1275 for a in list(set(d.getVar("SSTATE_ARCHS").split())):
1276 toremove = []
1277 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1278 if not os.path.exists(i):
1279 continue
1280 with open(i, "r") as f:
1281 lines = f.readlines()
1282 for l in lines:
1283 try:
1284 (stamp, manifest, workdir) = l.split()
1285 for tid in tasks:
1286 for s in tasks[tid]:
1287 if s.startswith(stamp):
1288 taskname = bb.runqueue.taskname_from_tid(tid)[3:]
1289 manname = manifest + "." + taskname
1290 if os.path.exists(manname):
1291 bb.debug(2, "Sstate for %s is stale, removing related manifest %s" % (tid, manname))
1292 toremove.append((manname, tid, tasks[tid]))
1293 break
1294 except ValueError:
1295 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
1296
1297 if toremove:
1298 msg = "Removing %d stale sstate objects for arch %s" % (len(toremove), a)
1299 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
1300
1301 removed = 0
1302 for (manname, tid, stamps) in toremove:
1303 sstate_clean_manifest(manname, d)
1304 for stamp in stamps:
1305 bb.utils.remove(stamp)
1306 removed = removed + 1
1307 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1308
1309 bb.event.fire(bb.event.ProcessFinished(msg), d)
1310}