blob: 8e8efd18d58dea51eb28928ede07182d4f0fc7b2 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001SSTATE_VERSION = "3"
2
3SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
4SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
5
Andrew Geissler82c905d2020-04-13 13:39:40 -05006def generate_sstatefn(spec, hash, taskname, siginfo, d):
7 if taskname is None:
8 return ""
9 extension = ".tgz"
10 # 8 chars reserved for siginfo
11 limit = 254 - 8
12 if siginfo:
13 limit = 254
14 extension = ".tgz.siginfo"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015 if not hash:
16 hash = "INVALID"
Andrew Geissler82c905d2020-04-13 13:39:40 -050017 fn = spec + hash + "_" + taskname + extension
18 # If the filename is too long, attempt to reduce it
19 if len(fn) > limit:
20 components = spec.split(":")
21 # Fields 0,5,6 are mandatory, 1 is most useful, 2,3,4 are just for information
22 # 7 is for the separators
23 avail = (254 - len(hash + "_" + taskname + extension) - len(components[0]) - len(components[1]) - len(components[5]) - len(components[6]) - 7) // 3
24 components[2] = components[2][:avail]
25 components[3] = components[3][:avail]
26 components[4] = components[4][:avail]
27 spec = ":".join(components)
28 fn = spec + hash + "_" + taskname + extension
29 if len(fn) > limit:
30 bb.fatal("Unable to reduce sstate name to less than 255 chararacters")
31 return hash[:2] + "/" + hash[2:4] + "/" + fn
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032
33SSTATE_PKGARCH = "${PACKAGE_ARCH}"
34SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050035SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
Andrew Geissler82c905d2020-04-13 13:39:40 -050036SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d.getVar('SSTATE_CURRTASK'), False, d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
38SSTATE_EXTRAPATH = ""
39SSTATE_EXTRAPATHWILDCARD = ""
Andrew Geissler82c905d2020-04-13 13:39:40 -050040SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/*/${SSTATE_PKGSPEC}*_${SSTATE_PATH_CURRTASK}.tgz*"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041
Patrick Williamsc0f7c042017-02-23 20:41:17 -060042# explicitly make PV to depend on evaluated value of PV variable
43PV[vardepvalue] = "${PV}"
44
Patrick Williamsc124f4f2015-09-15 14:41:29 -050045# We don't want the sstate to depend on things like the distro string
46# of the system, we let the sstate paths take care of this.
47SSTATE_EXTRAPATH[vardepvalue] = ""
Brad Bishop19323692019-04-05 15:28:33 -040048SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050049
50# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080051SSTATE_DUPWHITELIST = "${DEPLOY_DIR}/licenses/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052# Avoid docbook/sgml catalog warnings for now
53SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
Brad Bishop316dfdd2018-06-25 12:45:53 -040054# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
55SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/"
56SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080057# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used
58SSTATE_DUPWHITELIST += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059# Archive the sources for many architectures in one deploy folder
60SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080061# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files
62SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/ovmf"
63SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/grub-efi"
64SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/systemd-boot"
65SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/microcode"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066
Brad Bishop6e60e8b2018-02-01 10:27:11 -050067SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
68SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
69SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050071BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
73SSTATE_ARCHS = " \
74 ${BUILD_ARCH} \
Andrew Geissler6ce62a22020-11-30 19:58:47 -060075 ${BUILD_ARCH}_${ORIGNATIVELSBSTRING} \
Patrick Williamsc124f4f2015-09-15 14:41:29 -050076 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
77 ${BUILD_ARCH}_${TARGET_ARCH} \
78 ${SDK_ARCH}_${SDK_OS} \
79 ${SDK_ARCH}_${PACKAGE_ARCH} \
80 allarch \
81 ${PACKAGE_ARCH} \
Brad Bishop316dfdd2018-06-25 12:45:53 -040082 ${PACKAGE_EXTRA_ARCHS} \
83 ${MACHINE_ARCH}"
Andrew Geissler6ce62a22020-11-30 19:58:47 -060084SSTATE_ARCHS[vardepsexclude] = "ORIGNATIVELSBSTRING"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
86SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
87
88SSTATECREATEFUNCS = "sstate_hardcode_path"
Brad Bishop19323692019-04-05 15:28:33 -040089SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090SSTATEPOSTCREATEFUNCS = ""
91SSTATEPREINSTFUNCS = ""
92SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
93SSTATEPOSTINSTFUNCS = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050094EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050096# Check whether sstate exists for tasks that support sstate and are in the
97# locked signatures file.
98SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
99
100# Check whether the task's computed hash matches the task's hash in the
101# locked signatures file.
102SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500103
104# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
105# not sign)
106SSTATE_SIG_KEY ?= ""
107SSTATE_SIG_PASSPHRASE ?= ""
108# Whether to verify the GnUPG signatures when extracting sstate archives
109SSTATE_VERIFY_SIG ?= "0"
110
Brad Bishop19323692019-04-05 15:28:33 -0400111SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
112SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
113 the output hash for a task, which in turn is used to determine equivalency. \
114 "
115
Brad Bishop19323692019-04-05 15:28:33 -0400116SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
117SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
118 hash equivalency server, such as PN, PV, taskname, etc. This information \
119 is very useful for developers looking at task data, but may leak sensitive \
120 data if the equivalence server is public. \
121 "
122
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123python () {
124 if bb.data.inherits_class('native', d):
125 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
Andrew Geissler6ce62a22020-11-30 19:58:47 -0600126 if d.getVar("PN") == "pseudo-native":
127 d.appendVar('SSTATE_PKGARCH', '_${ORIGNATIVELSBSTRING}')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 elif bb.data.inherits_class('crosssdk', d):
129 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
130 elif bb.data.inherits_class('cross', d):
131 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TARGET_ARCH}"))
132 elif bb.data.inherits_class('nativesdk', d):
133 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
134 elif bb.data.inherits_class('cross-canadian', d):
135 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500136 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500137 d.setVar('SSTATE_PKGARCH', "allarch")
138 else:
139 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
140
141 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
142 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500143 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
Brad Bishop19323692019-04-05 15:28:33 -0400144 d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500146 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500147 d.setVar('SSTATETASKS', " ".join(unique_tasks))
148 for task in unique_tasks:
149 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
150 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
151}
152
153def sstate_init(task, d):
154 ss = {}
155 ss['task'] = task
156 ss['dirs'] = []
157 ss['plaindirs'] = []
158 ss['lockfiles'] = []
159 ss['lockfiles-shared'] = []
160 return ss
161
162def sstate_state_fromvars(d, task = None):
163 if task is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500164 task = d.getVar('BB_CURRENTTASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500165 if not task:
166 bb.fatal("sstate code running without task context?!")
167 task = task.replace("_setscene", "")
168
169 if task.startswith("do_"):
170 task = task[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500171 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
172 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
173 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
174 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
175 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
176 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
177 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178 if not task or len(inputs) != len(outputs):
179 bb.fatal("sstate variables not setup correctly?!")
180
181 if task == "populate_lic":
182 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
183 d.setVar("SSTATE_EXTRAPATH", "")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500184 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185
186 ss = sstate_init(task, d)
187 for i in range(len(inputs)):
188 sstate_add(ss, inputs[i], outputs[i], d)
189 ss['lockfiles'] = lockfiles
190 ss['lockfiles-shared'] = lockfilesshared
191 ss['plaindirs'] = plaindirs
192 ss['interceptfuncs'] = interceptfuncs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500193 ss['fixmedir'] = fixmedir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500194 return ss
195
196def sstate_add(ss, source, dest, d):
197 if not source.endswith("/"):
198 source = source + "/"
199 if not dest.endswith("/"):
200 dest = dest + "/"
201 source = os.path.normpath(source)
202 dest = os.path.normpath(dest)
203 srcbase = os.path.basename(source)
204 ss['dirs'].append([srcbase, source, dest])
205 return ss
206
207def sstate_install(ss, d):
208 import oe.path
209 import oe.sstatesig
210 import subprocess
211
212 sharedfiles = []
213 shareddirs = []
214 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
215
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500216 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
217
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
219
220 if os.access(manifest, os.R_OK):
221 bb.fatal("Package already staged (%s)?!" % manifest)
222
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600223 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
224
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225 locks = []
226 for lock in ss['lockfiles-shared']:
227 locks.append(bb.utils.lockfile(lock, True))
228 for lock in ss['lockfiles']:
229 locks.append(bb.utils.lockfile(lock))
230
231 for state in ss['dirs']:
232 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
233 for walkroot, dirs, files in os.walk(state[1]):
234 for file in files:
235 srcpath = os.path.join(walkroot, file)
236 dstpath = srcpath.replace(state[1], state[2])
237 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
238 sharedfiles.append(dstpath)
239 for dir in dirs:
240 srcdir = os.path.join(walkroot, dir)
241 dstdir = srcdir.replace(state[1], state[2])
242 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500243 if os.path.islink(srcdir):
244 sharedfiles.append(dstdir)
245 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500246 if not dstdir.endswith("/"):
247 dstdir = dstdir + "/"
248 shareddirs.append(dstdir)
249
250 # Check the file list for conflicts against files which already exist
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500251 whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500252 match = []
253 for f in sharedfiles:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500254 if os.path.exists(f) and not os.path.islink(f):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 f = os.path.normpath(f)
256 realmatch = True
257 for w in whitelist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600258 w = os.path.normpath(w)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259 if f.startswith(w):
260 realmatch = False
261 break
262 if realmatch:
263 match.append(f)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500264 sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500265 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500266 if search_output:
267 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
268 else:
269 match.append(" (not matched to any task)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500270 if match:
271 bb.error("The recipe %s is trying to install files into a shared " \
272 "area when those files already exist. Those files and their manifest " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500273 "location are:\n %s\nPlease verify which recipe should provide the " \
274 "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \
275 "break things - if not now, possibly in the future (we've seen builds fail " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500276 "several months later). If the system knew how to recover from this " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500277 "automatically it would, however there are several different scenarios " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278 "which can result in this and we don't know which one this is. It may be " \
279 "you have switched providers of something like virtual/kernel (e.g. from " \
280 "linux-yocto to linux-yocto-dev), in that case you need to execute the " \
281 "clean task for both recipes and it will resolve this error. It may be " \
282 "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500283 "those recipes should again resolve this error, however switching " \
284 "DISTRO_FEATURES on an existing build directory is not supported - you " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500285 "should really clean out tmp and rebuild (reusing sstate should be safe). " \
286 "It could be the overlapping files detected are harmless in which case " \
287 "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \
288 "also be your build is including two different conflicting versions of " \
289 "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \
290 "be to resolve the conflict. If in doubt, please ask on the mailing list, " \
291 "sharing the error and filelist above." % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500292 (d.getVar('PN'), "\n ".join(match)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500293 bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.")
294
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500295 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
296 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
297 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
298
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500299 # Write out the manifest
300 f = open(manifest, "w")
301 for file in sharedfiles:
302 f.write(file + "\n")
303
304 # We want to ensure that directories appear at the end of the manifest
305 # so that when we test to see if they should be deleted any contents
306 # added by the task will have been removed first.
307 dirs = sorted(shareddirs, key=len)
308 # Must remove children first, which will have a longer path than the parent
309 for di in reversed(dirs):
310 f.write(di + "\n")
311 f.close()
312
313 # Append to the list of manifests for this PACKAGE_ARCH
314
315 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
316 l = bb.utils.lockfile(i + ".lock")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500317 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500318 manifests = []
319 if os.path.exists(i):
320 with open(i, "r") as f:
321 manifests = f.readlines()
322 if filedata not in manifests:
323 with open(i, "a+") as f:
324 f.write(filedata)
325 bb.utils.unlockfile(l)
326
327 # Run the actual file install
328 for state in ss['dirs']:
329 if os.path.exists(state[1]):
330 oe.path.copyhardlinktree(state[1], state[2])
331
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500332 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500333 # All hooks should run in the SSTATE_INSTDIR
334 bb.build.exec_func(postinst, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500335
336 for lock in locks:
337 bb.utils.unlockfile(lock)
338
339sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX"
340sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
341
342def sstate_installpkg(ss, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500343 from oe.gpg_sign import get_signer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500344
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500345 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
Andrew Geissler82c905d2020-04-13 13:39:40 -0500346 d.setVar("SSTATE_CURRTASK", ss['task'])
347 sstatefetch = d.getVar('SSTATE_PKGNAME')
348 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500349
350 if not os.path.exists(sstatepkg):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800351 pstaging_fetch(sstatefetch, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
353 if not os.path.isfile(sstatepkg):
Brad Bishop08902b02019-08-20 09:16:51 -0400354 bb.note("Sstate package %s does not exist" % sstatepkg)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500355 return False
356
357 sstate_clean(ss, d)
358
359 d.setVar('SSTATE_INSTDIR', sstateinst)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500361 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500362 if not os.path.isfile(sstatepkg + '.sig'):
363 bb.warn("No signature file for sstate package %s, skipping acceleration..." % sstatepkg)
364 return False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500365 signer = get_signer(d, 'local')
366 if not signer.verify(sstatepkg + '.sig'):
Brad Bishop08902b02019-08-20 09:16:51 -0400367 bb.warn("Cannot verify signature on sstate package %s, skipping acceleration..." % sstatepkg)
368 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500370 # Empty sstateinst directory, ensure its clean
371 if os.path.exists(sstateinst):
372 oe.path.remove(sstateinst)
373 bb.utils.mkdirhier(sstateinst)
374
375 sstateinst = d.getVar("SSTATE_INSTDIR")
376 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
377
378 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500379 # All hooks should run in the SSTATE_INSTDIR
380 bb.build.exec_func(f, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500382 return sstate_installpkgdir(ss, d)
383
384def sstate_installpkgdir(ss, d):
385 import oe.path
386 import subprocess
387
388 sstateinst = d.getVar("SSTATE_INSTDIR")
389 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
390
391 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
392 # All hooks should run in the SSTATE_INSTDIR
393 bb.build.exec_func(f, d, (sstateinst,))
394
395 def prepdir(dir):
396 # remove dir if it exists, ensure any parent directories do exist
397 if os.path.exists(dir):
398 oe.path.remove(dir)
399 bb.utils.mkdirhier(dir)
400 oe.path.remove(dir)
401
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500402 for state in ss['dirs']:
403 prepdir(state[1])
404 os.rename(sstateinst + state[0], state[1])
405 sstate_install(ss, d)
406
407 for plain in ss['plaindirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500408 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500409 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500410 src = sstateinst + "/" + plain.replace(workdir, '')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500411 if sharedworkdir in plain:
412 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413 dest = plain
414 bb.utils.mkdirhier(src)
415 prepdir(dest)
416 os.rename(src, dest)
417
418 return True
419
420python sstate_hardcode_path_unpack () {
421 # Fixup hardcoded paths
422 #
423 # Note: The logic below must match the reverse logic in
424 # sstate_hardcode_path(d)
425 import subprocess
426
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500427 sstateinst = d.getVar('SSTATE_INSTDIR')
428 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
429 fixmefn = sstateinst + "fixmepath"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500430 if os.path.isfile(fixmefn):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500431 staging_target = d.getVar('RECIPE_SYSROOT')
432 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500433
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500434 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500435 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500436 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
437 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
438 else:
439 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500441 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500442 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500443 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
445
446 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
447 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
448
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500449 # Defer do_populate_sysroot relocation command
450 if sstatefixmedir:
451 bb.utils.mkdirhier(sstatefixmedir)
452 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
453 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
454 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
455 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
456 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
457 f.write(sstate_hardcode_cmd)
458 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
459 return
460
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500461 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500462 subprocess.check_call(sstate_hardcode_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500463
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800464 # Need to remove this or we'd copy it into the target directory and may
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500465 # conflict with another writer
466 os.remove(fixmefn)
467}
468
469def sstate_clean_cachefile(ss, d):
470 import oe.path
471
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300472 if d.getVarFlag('do_%s' % ss['task'], 'task'):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500473 d.setVar("SSTATE_PATH_CURRTASK", ss['task'])
474 sstatepkgfile = d.getVar('SSTATE_PATHSPEC')
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300475 bb.note("Removing %s" % sstatepkgfile)
476 oe.path.remove(sstatepkgfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500477
478def sstate_clean_cachefiles(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500479 for task in (d.getVar('SSTATETASKS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480 ld = d.createCopy()
481 ss = sstate_state_fromvars(ld, task)
482 sstate_clean_cachefile(ss, ld)
483
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500484def sstate_clean_manifest(manifest, d, prefix=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500485 import oe.path
486
487 mfile = open(manifest)
488 entries = mfile.readlines()
489 mfile.close()
490
491 for entry in entries:
492 entry = entry.strip()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500493 if prefix and not entry.startswith("/"):
494 entry = prefix + "/" + entry
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500495 bb.debug(2, "Removing manifest: %s" % entry)
496 # We can race against another package populating directories as we're removing them
497 # so we ignore errors here.
498 try:
499 if entry.endswith("/"):
500 if os.path.islink(entry[:-1]):
501 os.remove(entry[:-1])
502 elif os.path.exists(entry) and len(os.listdir(entry)) == 0:
503 os.rmdir(entry[:-1])
504 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500505 os.remove(entry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500506 except OSError:
507 pass
508
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600509 postrm = manifest + ".postrm"
510 if os.path.exists(manifest + ".postrm"):
511 import subprocess
512 os.chmod(postrm, 0o755)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500513 subprocess.check_call(postrm, shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600514 oe.path.remove(postrm)
515
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500516 oe.path.remove(manifest)
517
518def sstate_clean(ss, d):
519 import oe.path
520 import glob
521
522 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500523 stamp_clean = d.getVar("STAMPCLEAN")
524 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500525 if extrainf:
526 d2.setVar("SSTATE_MANMACH", extrainf)
527 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
528 else:
529 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
530
531 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
532
533 if os.path.exists(manifest):
534 locks = []
535 for lock in ss['lockfiles-shared']:
536 locks.append(bb.utils.lockfile(lock))
537 for lock in ss['lockfiles']:
538 locks.append(bb.utils.lockfile(lock))
539
540 sstate_clean_manifest(manifest, d)
541
542 for lock in locks:
543 bb.utils.unlockfile(lock)
544
545 # Remove the current and previous stamps, but keep the sigdata.
546 #
547 # The glob() matches do_task* which may match multiple tasks, for
548 # example: do_package and do_package_write_ipk, so we need to
549 # exactly match *.do_task.* and *.do_task_setscene.*
550 rm_stamp = '.do_%s.' % ss['task']
551 rm_setscene = '.do_%s_setscene.' % ss['task']
552 # For BB_SIGNATURE_HANDLER = "noop"
553 rm_nohash = ".do_%s" % ss['task']
554 for stfile in glob.glob(wildcard_stfile):
555 # Keep the sigdata
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500556 if ".sigdata." in stfile or ".sigbasedata." in stfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500557 continue
558 # Preserve taint files in the stamps directory
559 if stfile.endswith('.taint'):
560 continue
561 if rm_stamp in stfile or rm_setscene in stfile or \
562 stfile.endswith(rm_nohash):
563 oe.path.remove(stfile)
564
565sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
566
567CLEANFUNCS += "sstate_cleanall"
568
569python sstate_cleanall() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500570 bb.note("Removing shared state for package %s" % d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500571
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500572 manifest_dir = d.getVar('SSTATE_MANIFESTS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500573 if not os.path.exists(manifest_dir):
574 return
575
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500576 tasks = d.getVar('SSTATETASKS').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500577 for name in tasks:
578 ld = d.createCopy()
579 shared_state = sstate_state_fromvars(ld, name)
580 sstate_clean(shared_state, ld)
581}
582
583python sstate_hardcode_path () {
584 import subprocess, platform
585
586 # Need to remove hardcoded paths and fix these when we install the
587 # staging packages.
588 #
589 # Note: the logic in this function needs to match the reverse logic
590 # in sstate_installpkg(ss, d)
591
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500592 staging_target = d.getVar('RECIPE_SYSROOT')
593 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
594 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500595
Brad Bishop316dfdd2018-06-25 12:45:53 -0400596 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500597 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500598 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500599 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
600 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400601 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500602 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400603 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
604 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500605
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500606 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500608 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500609 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500610 sstate_grep_cmd += " -e '%s'" % (fixme_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611
612 fixmefn = sstate_builddir + "fixmepath"
613
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500614 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615 sstate_filelist_cmd = "tee %s" % (fixmefn)
616
617 # fixmepath file needs relative paths, drop sstate_builddir prefix
618 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
619
620 xargs_no_empty_run_cmd = '--no-run-if-empty'
621 if platform.system() == 'Darwin':
622 xargs_no_empty_run_cmd = ''
623
624 # Limit the fixpaths and sed operations based on the initial grep search
625 # This has the side effect of making sure the vfs cache is hot
626 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
627
628 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500629 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500630
631 # If the fixmefn is empty, remove it..
632 if os.stat(fixmefn).st_size == 0:
633 os.remove(fixmefn)
634 else:
635 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500636 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500637}
638
639def sstate_package(ss, d):
640 import oe.path
641
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500642 tmpdir = d.getVar('TMPDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500643
644 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
Andrew Geissler82c905d2020-04-13 13:39:40 -0500645 d.setVar("SSTATE_CURRTASK", ss['task'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500646 bb.utils.remove(sstatebuild, recurse=True)
647 bb.utils.mkdirhier(sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500648 for state in ss['dirs']:
649 if not os.path.exists(state[1]):
650 continue
651 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500652 # Find and error for absolute symlinks. We could attempt to relocate but its not
653 # clear where the symlink is relative to in this context. We could add that markup
654 # to sstate tasks but there aren't many of these so better just avoid them entirely.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500655 for walkroot, dirs, files in os.walk(state[1]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500656 for file in files + dirs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500657 srcpath = os.path.join(walkroot, file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500658 if not os.path.islink(srcpath):
659 continue
660 link = os.readlink(srcpath)
661 if not os.path.isabs(link):
662 continue
663 if not link.startswith(tmpdir):
664 continue
665 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500666 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500667 os.rename(state[1], sstatebuild + state[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500668
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500669 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500670 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500671 for plain in ss['plaindirs']:
672 pdir = plain.replace(workdir, sstatebuild)
Brad Bishop977dc1a2019-02-06 16:01:43 -0500673 if sharedworkdir in plain:
674 pdir = plain.replace(sharedworkdir, sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500675 bb.utils.mkdirhier(plain)
676 bb.utils.mkdirhier(pdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500677 os.rename(plain, pdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500678
679 d.setVar('SSTATE_BUILDDIR', sstatebuild)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500680 d.setVar('SSTATE_INSTDIR', sstatebuild)
681
682 if d.getVar('SSTATE_SKIP_CREATION') == '1':
683 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500684
Brad Bishop08902b02019-08-20 09:16:51 -0400685 sstate_create_package = ['sstate_report_unihash', 'sstate_create_package']
686 if d.getVar('SSTATE_SIG_KEY'):
687 sstate_create_package.append('sstate_sign_package')
688
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500689 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
Brad Bishop08902b02019-08-20 09:16:51 -0400690 sstate_create_package + \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500691 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500692 # All hooks should run in SSTATE_BUILDDIR.
693 bb.build.exec_func(f, d, (sstatebuild,))
694
Andrew Geissler82c905d2020-04-13 13:39:40 -0500695 # SSTATE_PKG may have been changed by sstate_report_unihash
696 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
697 if not os.path.exists(siginfo):
698 bb.siggen.dump_this_task(siginfo, d)
699 else:
Andrew Geisslerc182c622020-05-15 14:13:32 -0500700 try:
701 os.utime(siginfo, None)
702 except PermissionError:
703 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500704
705 return
706
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800707def pstaging_fetch(sstatefetch, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500708 import bb.fetch2
709
710 # Only try and fetch if the user has configured a mirror
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500711 mirrors = d.getVar('SSTATE_MIRRORS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500712 if not mirrors:
713 return
714
715 # Copy the data object and override DL_DIR and SRC_URI
716 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500717
718 dldir = localdata.expand("${SSTATE_DIR}")
719 bb.utils.mkdirhier(dldir)
720
721 localdata.delVar('MIRRORS')
722 localdata.setVar('FILESPATH', dldir)
723 localdata.setVar('DL_DIR', dldir)
724 localdata.setVar('PREMIRRORS', mirrors)
725
726 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
727 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400728 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
729 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500730 localdata.delVar('BB_NO_NETWORK')
731
732 # Try a fetch from the sstate mirror, if it fails just return and
733 # we will build the package
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600734 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
735 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500736 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600737 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500738
739 for srcuri in uris:
740 localdata.setVar('SRC_URI', srcuri)
741 try:
742 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500743 fetcher.checkstatus()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500744 fetcher.download()
745
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500746 except bb.fetch2.BBFetchException:
Andrew Geissler4ed12e12020-06-05 18:00:41 -0500747 pass
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500748
749def sstate_setscene(d):
750 shared_state = sstate_state_fromvars(d)
751 accelerate = sstate_installpkg(shared_state, d)
752 if not accelerate:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600753 bb.fatal("No suitable staging package found")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500754
755python sstate_task_prefunc () {
756 shared_state = sstate_state_fromvars(d)
757 sstate_clean(shared_state, d)
758}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500759sstate_task_prefunc[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500760
761python sstate_task_postfunc () {
762 shared_state = sstate_state_fromvars(d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500763
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500764 for intercept in shared_state['interceptfuncs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500765 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
766
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600767 omask = os.umask(0o002)
768 if omask != 0o002:
769 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500770 sstate_package(shared_state, d)
771 os.umask(omask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500772
773 sstateinst = d.getVar("SSTATE_INSTDIR")
774 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
775
776 sstate_installpkgdir(shared_state, d)
777
778 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500779}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500780sstate_task_postfunc[dirs] = "${WORKDIR}"
781
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500782
783#
784# Shell function to generate a sstate package from a directory
785# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
786#
787sstate_create_package () {
Andrew Geissler82c905d2020-04-13 13:39:40 -0500788 # Exit early if it already exists
Brad Bishop08902b02019-08-20 09:16:51 -0400789 if [ -e ${SSTATE_PKG} ]; then
Andrew Geisslerc182c622020-05-15 14:13:32 -0500790 [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
Brad Bishop08902b02019-08-20 09:16:51 -0400791 return
792 fi
793
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500794 mkdir --mode=0775 -p `dirname ${SSTATE_PKG}`
Andrew Geissler82c905d2020-04-13 13:39:40 -0500795 TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
796
797 # Use pigz if available
798 OPT="-czS"
799 if [ -x "$(command -v pigz)" ]; then
800 OPT="-I pigz -cS"
801 fi
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800802
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500803 # Need to handle empty directories
804 if [ "$(ls -A)" ]; then
805 set +e
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800806 tar $OPT -f $TFILE *
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500807 ret=$?
808 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
809 exit 1
810 fi
811 set -e
812 else
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800813 tar $OPT --file=$TFILE --files-from=/dev/null
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500814 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500815 chmod 0664 $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400816 # Skip if it was already created by some other process
817 if [ ! -e ${SSTATE_PKG} ]; then
Andrew Geissler82c905d2020-04-13 13:39:40 -0500818 # Move into place using ln to attempt an atomic op.
819 # Abort if it already exists
820 ln $TFILE ${SSTATE_PKG} && rm $TFILE
Brad Bishop08902b02019-08-20 09:16:51 -0400821 else
822 rm $TFILE
823 fi
Andrew Geisslerc182c622020-05-15 14:13:32 -0500824 [ ! -w ${SSTATE_PKG} ] || touch ${SSTATE_PKG}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500825}
826
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500827python sstate_sign_package () {
828 from oe.gpg_sign import get_signer
829
Brad Bishop08902b02019-08-20 09:16:51 -0400830
831 signer = get_signer(d, 'local')
832 sstate_pkg = d.getVar('SSTATE_PKG')
833 if os.path.exists(sstate_pkg + '.sig'):
834 os.unlink(sstate_pkg + '.sig')
835 signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
836 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500837}
838
Brad Bishop19323692019-04-05 15:28:33 -0400839python sstate_report_unihash() {
840 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
841
842 if report_unihash:
843 ss = sstate_state_fromvars(d)
844 report_unihash(os.getcwd(), ss['task'], d)
845}
846
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500847#
848# Shell function to decompress and prepare a package for installation
849# Will be run from within SSTATE_INSTDIR.
850#
851sstate_unpack_package () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500852 tar -xvzf ${SSTATE_PKG}
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500853 # update .siginfo atime on local/NFS mirror
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500854 [ -O ${SSTATE_PKG}.siginfo ] && [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500855 # Use "! -w ||" to return true for read only files
856 [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500857 [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
858 [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500859}
860
861BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
862
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500863def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
Brad Bishop08902b02019-08-20 09:16:51 -0400864 found = set()
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600865 foundLocal = set()
866 foundNet = set()
Brad Bishop08902b02019-08-20 09:16:51 -0400867 missed = set()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500868
Brad Bishop19323692019-04-05 15:28:33 -0400869 def gethash(task):
Brad Bishop08902b02019-08-20 09:16:51 -0400870 return sq_data['unihash'][task]
Brad Bishop19323692019-04-05 15:28:33 -0400871
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500872 def getpathcomponents(task, d):
873 # Magic data from BB_HASHFILENAME
Brad Bishop08902b02019-08-20 09:16:51 -0400874 splithashfn = sq_data['hashfn'][task].split(" ")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500875 spec = splithashfn[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500876 if splithashfn[0] == "True":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500877 extrapath = d.getVar("NATIVELSBSTRING") + "/"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500878 else:
879 extrapath = ""
Brad Bishop08902b02019-08-20 09:16:51 -0400880
881 tname = bb.runqueue.taskname_from_tid(task)[3:]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500882
883 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
884 spec = splithashfn[2]
885 extrapath = ""
886
887 return spec, extrapath, tname
888
889
Brad Bishop08902b02019-08-20 09:16:51 -0400890 for tid in sq_data['hash']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500891
Brad Bishop08902b02019-08-20 09:16:51 -0400892 spec, extrapath, tname = getpathcomponents(tid, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500893
Andrew Geissler82c905d2020-04-13 13:39:40 -0500894 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500895
896 if os.path.exists(sstatefile):
897 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
Brad Bishop08902b02019-08-20 09:16:51 -0400898 found.add(tid)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600899 foundLocal.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500900 continue
901 else:
Brad Bishop08902b02019-08-20 09:16:51 -0400902 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500903 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
904
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500905 mirrors = d.getVar("SSTATE_MIRRORS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500906 if mirrors:
907 # Copy the data object and override DL_DIR and SRC_URI
908 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500909
910 dldir = localdata.expand("${SSTATE_DIR}")
911 localdata.delVar('MIRRORS')
912 localdata.setVar('FILESPATH', dldir)
913 localdata.setVar('DL_DIR', dldir)
914 localdata.setVar('PREMIRRORS', mirrors)
915
916 bb.debug(2, "SState using premirror of: %s" % mirrors)
917
918 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
919 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400920 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
921 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500922 localdata.delVar('BB_NO_NETWORK')
923
924 from bb.fetch2 import FetchConnectionCache
925 def checkstatus_init(thread_worker):
926 thread_worker.connection_cache = FetchConnectionCache()
927
928 def checkstatus_end(thread_worker):
929 thread_worker.connection_cache.close_connections()
930
931 def checkstatus(thread_worker, arg):
Brad Bishop08902b02019-08-20 09:16:51 -0400932 (tid, sstatefile) = arg
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500933
934 localdata2 = bb.data.createCopy(localdata)
935 srcuri = "file://" + sstatefile
936 localdata.setVar('SRC_URI', srcuri)
937 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
938
939 try:
940 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
941 connection_cache=thread_worker.connection_cache)
942 fetcher.checkstatus()
943 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
Brad Bishop08902b02019-08-20 09:16:51 -0400944 found.add(tid)
Andrew Geissler9b4d8b02021-02-19 12:26:16 -0600945 foundNet.add(tid)
Brad Bishop08902b02019-08-20 09:16:51 -0400946 if tid in missed:
947 missed.remove(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500948 except:
Brad Bishop08902b02019-08-20 09:16:51 -0400949 missed.add(tid)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500950 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600951 pass
Andrew Geissler82c905d2020-04-13 13:39:40 -0500952 if len(tasklist) >= min_tasks:
953 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500954
955 tasklist = []
Andrew Geissler82c905d2020-04-13 13:39:40 -0500956 min_tasks = 100
Brad Bishop08902b02019-08-20 09:16:51 -0400957 for tid in sq_data['hash']:
958 if tid in found:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500959 continue
Brad Bishop08902b02019-08-20 09:16:51 -0400960 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500961 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
Brad Bishop08902b02019-08-20 09:16:51 -0400962 tasklist.append((tid, sstatefile))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500963
964 if tasklist:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500965 if len(tasklist) >= min_tasks:
966 msg = "Checking sstate mirror object availability"
967 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600968
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500969 import multiprocessing
970 nproc = min(multiprocessing.cpu_count(), len(tasklist))
971
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600972 bb.event.enable_threadlock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500973 pool = oe.utils.ThreadedPool(nproc, len(tasklist),
974 worker_init=checkstatus_init, worker_end=checkstatus_end)
975 for t in tasklist:
976 pool.add_task(checkstatus, t)
977 pool.start()
978 pool.wait_completion()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600979 bb.event.disable_threadlock()
980
Andrew Geissler82c905d2020-04-13 13:39:40 -0500981 if len(tasklist) >= min_tasks:
982 bb.event.fire(bb.event.ProcessFinished(msg), d)
Brad Bishop96ff1982019-08-19 13:50:42 -0400983
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500984 inheritlist = d.getVar("INHERIT")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500985 if "toaster" in inheritlist:
986 evdata = {'missed': [], 'found': []};
Brad Bishop08902b02019-08-20 09:16:51 -0400987 for tid in missed:
988 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500989 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
Brad Bishop08902b02019-08-20 09:16:51 -0400990 evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
991 for tid in found:
992 spec, extrapath, tname = getpathcomponents(tid, d)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500993 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, False, d))
Brad Bishop08902b02019-08-20 09:16:51 -0400994 evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500995 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
996
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500997 if summary:
998 # Print some summary statistics about the current task completion and how much sstate
999 # reuse there was. Avoid divide by zero errors.
1000 total = len(sq_data['hash'])
1001 complete = 0
1002 if currentcount:
1003 complete = (len(found) + currentcount) / (total + currentcount) * 100
1004 match = 0
1005 if total:
1006 match = len(found) / total * 100
Andrew Geissler9b4d8b02021-02-19 12:26:16 -06001007 bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001008
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001009 if hasattr(bb.parse.siggen, "checkhashes"):
Brad Bishop08902b02019-08-20 09:16:51 -04001010 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001011
Brad Bishop08902b02019-08-20 09:16:51 -04001012 return found
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001013
1014BB_SETSCENE_DEPVALID = "setscene_depvalid"
1015
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001016def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001017 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
1018 # task is included in taskdependees too
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001019 # Return - False - We need this dependency
1020 # - True - We can skip this dependency
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001021 import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001022
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001023 def logit(msg, log):
1024 if log is not None:
1025 log.append(msg)
1026 else:
1027 bb.debug(2, msg)
1028
1029 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001030
1031 def isNativeCross(x):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001032 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001033
1034 # We only need to trigger populate_lic through direct dependencies
1035 if taskdependees[task][1] == "do_populate_lic":
1036 return True
1037
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001038 # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
1039 if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
1040 return True
1041
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001042 # We only need to trigger packagedata through direct dependencies
1043 # but need to preserve packagedata on packagedata links
1044 if taskdependees[task][1] == "do_packagedata":
1045 for dep in taskdependees:
1046 if taskdependees[dep][1] == "do_packagedata":
1047 return False
1048 return True
1049
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001050 for dep in taskdependees:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001051 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001052 if task == dep:
1053 continue
1054 if dep in notneeded:
1055 continue
1056 # do_package_write_* and do_package doesn't need do_package
1057 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
1058 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001059 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
1060 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
1061 return False
1062 # do_package/packagedata/package_qa don't need do_populate_sysroot
1063 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001064 continue
1065 # Native/Cross packages don't exist and are noexec anyway
1066 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
1067 continue
1068
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001069 # This is due to the [depends] in useradd.bbclass complicating matters
1070 # The logic *is* reversed here due to the way hard setscene dependencies are injected
1071 if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
1072 continue
1073
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001074 # Consider sysroot depending on sysroot tasks
1075 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001076 # Allow excluding certain recursive dependencies. If a recipe needs it should add a
1077 # specific dependency itself, rather than relying on one of its dependees to pull
1078 # them in.
1079 # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
1080 not_needed = False
1081 excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT')
1082 if excludedeps is None:
1083 # Cache the regular expressions for speed
1084 excludedeps = []
1085 for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split():
1086 excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))
1087 d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps)
1088 for excl in excludedeps:
1089 if excl[0].match(taskdependees[dep][0]):
1090 if excl[1].match(taskdependees[task][0]):
1091 not_needed = True
1092 break
1093 if not_needed:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001094 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001095 # For meta-extsdk-toolchain we want all sysroot dependencies
1096 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
1097 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001098 # Native/Cross populate_sysroot need their dependencies
1099 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
1100 return False
1101 # Target populate_sysroot depended on by cross tools need to be installed
1102 if isNativeCross(taskdependees[dep][0]):
1103 return False
1104 # Native/cross tools depended upon by target sysroot are not needed
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001105 # Add an exception for shadow-native as required by useradd.bbclass
1106 if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001107 continue
1108 # Target populate_sysroot need their dependencies
1109 return False
1110
1111 if taskdependees[task][1] == 'do_shared_workdir':
1112 continue
1113
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001114 if taskdependees[dep][1] == "do_populate_lic":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001115 continue
1116
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001117
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001118 # Safe fallthrough default
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001119 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001120 return False
1121 return True
1122
1123addhandler sstate_eventhandler
1124sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
1125python sstate_eventhandler() {
1126 d = e.data
Andrew Geissler82c905d2020-04-13 13:39:40 -05001127 writtensstate = d.getVar('SSTATE_CURRTASK')
1128 if not writtensstate:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001129 taskname = d.getVar("BB_RUNTASK")[3:]
1130 spec = d.getVar('SSTATE_PKGSPEC')
1131 swspec = d.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001132 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1133 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1134 d.setVar("SSTATE_EXTRAPATH", "")
Andrew Geissler82c905d2020-04-13 13:39:40 -05001135 d.setVar("SSTATE_CURRTASK", taskname)
1136 siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
1137 if not os.path.exists(siginfo):
1138 bb.siggen.dump_this_task(siginfo, d)
1139 else:
Andrew Geisslerc182c622020-05-15 14:13:32 -05001140 try:
1141 os.utime(siginfo, None)
1142 except PermissionError:
1143 pass
1144
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001145}
1146
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001147SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001148
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001149#
1150# Event handler which removes manifests and stamps file for recipes which are no
1151# longer 'reachable' in a build where they once were. 'Reachable' refers to
1152# whether a recipe is parsed so recipes in a layer which was removed would no
1153# longer be reachable. Switching between systemd and sysvinit where recipes
1154# became skipped would be another example.
1155#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001156# Also optionally removes the workdir of those tasks/recipes
1157#
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001158addhandler sstate_eventhandler_reachablestamps
1159sstate_eventhandler_reachablestamps[eventmask] = "bb.event.ReachableStamps"
1160python sstate_eventhandler_reachablestamps() {
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001161 import glob
1162 d = e.data
1163 stamps = e.stamps.values()
1164 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001165 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1166 preservestamps = []
1167 if os.path.exists(preservestampfile):
1168 with open(preservestampfile, 'r') as f:
1169 preservestamps = f.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001170 seen = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001171
1172 # The machine index contains all the stamps this machine has ever seen in this build directory.
1173 # We should only remove things which this machine once accessed but no longer does.
1174 machineindex = set()
1175 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1176 mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
1177 if os.path.exists(mi):
1178 with open(mi, "r") as f:
1179 machineindex = set(line.strip() for line in f.readlines())
1180
Brad Bishop316dfdd2018-06-25 12:45:53 -04001181 for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001182 toremove = []
1183 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1184 if not os.path.exists(i):
1185 continue
1186 with open(i, "r") as f:
1187 lines = f.readlines()
1188 for l in lines:
Brad Bishop19323692019-04-05 15:28:33 -04001189 try:
1190 (stamp, manifest, workdir) = l.split()
1191 if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
1192 toremove.append(l)
1193 if stamp not in seen:
1194 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
1195 seen.append(stamp)
1196 except ValueError:
1197 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001198
1199 if toremove:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001200 msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a)
1201 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001202
Brad Bishop316dfdd2018-06-25 12:45:53 -04001203 removed = 0
1204 for r in toremove:
1205 (stamp, manifest, workdir) = r.split()
1206 for m in glob.glob(manifest + ".*"):
1207 if m.endswith(".postrm"):
1208 continue
1209 sstate_clean_manifest(m, d)
1210 bb.utils.remove(stamp + "*")
1211 if removeworkdir:
1212 bb.utils.remove(workdir, recurse = True)
1213 lines.remove(r)
1214 removed = removed + 1
1215 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1216
1217 bb.event.fire(bb.event.ProcessFinished(msg), d)
1218
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001219 with open(i, "w") as f:
1220 for l in lines:
1221 f.write(l)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001222 machineindex |= set(stamps)
1223 with open(mi, "w") as f:
1224 for l in machineindex:
1225 f.write(l + "\n")
1226
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001227 if preservestamps:
1228 os.remove(preservestampfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001229}
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001230
1231
1232#
1233# Bitbake can generate an event showing which setscene tasks are 'stale',
1234# i.e. which ones will be rerun. These are ones where a stamp file is present but
1235# it is stable (e.g. taskhash doesn't match). With that list we can go through
1236# the manifests for matching tasks and "uninstall" those manifests now. We do
1237# this now rather than mid build since the distribution of files between sstate
1238# objects may have changed, new tasks may run first and if those new tasks overlap
1239# with the stale tasks, we'd see overlapping files messages and failures. Thankfully
1240# removing these files is fast.
1241#
1242addhandler sstate_eventhandler_stalesstate
1243sstate_eventhandler_stalesstate[eventmask] = "bb.event.StaleSetSceneTasks"
1244python sstate_eventhandler_stalesstate() {
1245 d = e.data
1246 tasks = e.tasks
1247
1248 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1249
1250 for a in list(set(d.getVar("SSTATE_ARCHS").split())):
1251 toremove = []
1252 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1253 if not os.path.exists(i):
1254 continue
1255 with open(i, "r") as f:
1256 lines = f.readlines()
1257 for l in lines:
1258 try:
1259 (stamp, manifest, workdir) = l.split()
1260 for tid in tasks:
1261 for s in tasks[tid]:
1262 if s.startswith(stamp):
1263 taskname = bb.runqueue.taskname_from_tid(tid)[3:]
1264 manname = manifest + "." + taskname
1265 if os.path.exists(manname):
1266 bb.debug(2, "Sstate for %s is stale, removing related manifest %s" % (tid, manname))
1267 toremove.append((manname, tid, tasks[tid]))
1268 break
1269 except ValueError:
1270 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
1271
1272 if toremove:
1273 msg = "Removing %d stale sstate objects for arch %s" % (len(toremove), a)
1274 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
1275
1276 removed = 0
1277 for (manname, tid, stamps) in toremove:
1278 sstate_clean_manifest(manname, d)
1279 for stamp in stamps:
1280 bb.utils.remove(stamp)
1281 removed = removed + 1
1282 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1283
1284 bb.event.fire(bb.event.ProcessFinished(msg), d)
1285}