blob: b604729d84a184c6342f217a55fc9e09433d2316 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001SSTATE_VERSION = "3"
2
3SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
4SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}"
5
6def generate_sstatefn(spec, hash, d):
7 if not hash:
8 hash = "INVALID"
9 return hash[:2] + "/" + spec + hash
10
11SSTATE_PKGARCH = "${PACKAGE_ARCH}"
12SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050013SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
Brad Bishop19323692019-04-05 15:28:33 -040014SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
16SSTATE_EXTRAPATH = ""
17SSTATE_EXTRAPATHWILDCARD = ""
18SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
19
Patrick Williamsc0f7c042017-02-23 20:41:17 -060020# explicitly make PV to depend on evaluated value of PV variable
21PV[vardepvalue] = "${PV}"
22
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023# We don't want the sstate to depend on things like the distro string
24# of the system, we let the sstate paths take care of this.
25SSTATE_EXTRAPATH[vardepvalue] = ""
Brad Bishop19323692019-04-05 15:28:33 -040026SSTATE_EXTRAPATHWILDCARD[vardepvalue] = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027
28# For multilib rpm the allarch packagegroup files can overwrite (in theory they're identical)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080029SSTATE_DUPWHITELIST = "${DEPLOY_DIR}/licenses/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050030# Avoid docbook/sgml catalog warnings for now
31SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml"
Brad Bishop316dfdd2018-06-25 12:45:53 -040032# sdk-provides-dummy-nativesdk and nativesdk-buildtools-perl-dummy overlap for different SDKMACHINE
33SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/sdk_provides_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-nativesdk/"
34SSTATE_DUPWHITELIST += "${DEPLOY_DIR_RPM}/buildtools_dummy_nativesdk/ ${DEPLOY_DIR_IPK}/buildtools-dummy-nativesdk/"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080035# target-sdk-provides-dummy overlaps that allarch is disabled when multilib is used
36SSTATE_DUPWHITELIST += "${COMPONENTS_DIR}/sdk-provides-dummy-target/ ${DEPLOY_DIR_RPM}/sdk_provides_dummy_target/ ${DEPLOY_DIR_IPK}/sdk-provides-dummy-target/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037# Archive the sources for many architectures in one deploy folder
38SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080039# ovmf/grub-efi/systemd-boot/intel-microcode multilib recipes can generate identical overlapping files
40SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/ovmf"
41SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/grub-efi"
42SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/systemd-boot"
43SSTATE_DUPWHITELIST += "${DEPLOY_DIR_IMAGE}/microcode"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050044
Brad Bishop6e60e8b2018-02-01 10:27:11 -050045SSTATE_SCAN_FILES ?= "*.la *-config *_config postinst-*"
46SSTATE_SCAN_CMD ??= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES").split())}" \) -type f'
47SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_NATIVE} -e ${HOSTTOOLS_DIR} ${SSTATE_BUILDDIR}'
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050049BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050050
51SSTATE_ARCHS = " \
52 ${BUILD_ARCH} \
53 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
54 ${BUILD_ARCH}_${TARGET_ARCH} \
55 ${SDK_ARCH}_${SDK_OS} \
56 ${SDK_ARCH}_${PACKAGE_ARCH} \
57 allarch \
58 ${PACKAGE_ARCH} \
Brad Bishop316dfdd2018-06-25 12:45:53 -040059 ${PACKAGE_EXTRA_ARCHS} \
60 ${MACHINE_ARCH}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061
62SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
63
64SSTATECREATEFUNCS = "sstate_hardcode_path"
Brad Bishop19323692019-04-05 15:28:33 -040065SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066SSTATEPOSTCREATEFUNCS = ""
67SSTATEPREINSTFUNCS = ""
68SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
69SSTATEPOSTINSTFUNCS = ""
Brad Bishop6e60e8b2018-02-01 10:27:11 -050070EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050072# Check whether sstate exists for tasks that support sstate and are in the
73# locked signatures file.
74SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error'
75
76# Check whether the task's computed hash matches the task's hash in the
77# locked signatures file.
78SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079
80# The GnuPG key ID and passphrase to use to sign sstate archives (or unset to
81# not sign)
82SSTATE_SIG_KEY ?= ""
83SSTATE_SIG_PASSPHRASE ?= ""
84# Whether to verify the GnUPG signatures when extracting sstate archives
85SSTATE_VERIFY_SIG ?= "0"
86
Brad Bishop19323692019-04-05 15:28:33 -040087SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
88SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
89 the output hash for a task, which in turn is used to determine equivalency. \
90 "
91
92SSTATE_HASHEQUIV_SERVER ?= ""
93SSTATE_HASHEQUIV_SERVER[doc] = "The hash equivalence sever. For example, \
94 'http://192.168.0.1:5000'. Do not include a trailing slash \
95 "
96
97SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
98SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
99 hash equivalency server, such as PN, PV, taskname, etc. This information \
100 is very useful for developers looking at task data, but may leak sensitive \
101 data if the equivalence server is public. \
102 "
103
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500104python () {
105 if bb.data.inherits_class('native', d):
106 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
107 elif bb.data.inherits_class('crosssdk', d):
108 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
109 elif bb.data.inherits_class('cross', d):
110 d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TARGET_ARCH}"))
111 elif bb.data.inherits_class('nativesdk', d):
112 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${SDK_OS}"))
113 elif bb.data.inherits_class('cross-canadian', d):
114 d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500115 elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH") == "all":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500116 d.setVar('SSTATE_PKGARCH', "allarch")
117 else:
118 d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}"))
119
120 if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d):
121 d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500122 d.setVar('BB_HASHFILENAME', "True ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}")
Brad Bishop19323692019-04-05 15:28:33 -0400123 d.setVar('SSTATE_EXTRAPATHWILDCARD', "${NATIVELSBSTRING}/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500124
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500125 unique_tasks = sorted(set((d.getVar('SSTATETASKS') or "").split()))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500126 d.setVar('SSTATETASKS', " ".join(unique_tasks))
127 for task in unique_tasks:
128 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
129 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc")
130}
131
132def sstate_init(task, d):
133 ss = {}
134 ss['task'] = task
135 ss['dirs'] = []
136 ss['plaindirs'] = []
137 ss['lockfiles'] = []
138 ss['lockfiles-shared'] = []
139 return ss
140
141def sstate_state_fromvars(d, task = None):
142 if task is None:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 task = d.getVar('BB_CURRENTTASK')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144 if not task:
145 bb.fatal("sstate code running without task context?!")
146 task = task.replace("_setscene", "")
147
148 if task.startswith("do_"):
149 task = task[3:]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500150 inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs') or "").split()
151 outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs') or "").split()
152 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
153 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
154 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
155 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
156 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500157 if not task or len(inputs) != len(outputs):
158 bb.fatal("sstate variables not setup correctly?!")
159
160 if task == "populate_lic":
161 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
162 d.setVar("SSTATE_EXTRAPATH", "")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500163 d.setVar('SSTATE_EXTRAPATHWILDCARD', "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500164
165 ss = sstate_init(task, d)
166 for i in range(len(inputs)):
167 sstate_add(ss, inputs[i], outputs[i], d)
168 ss['lockfiles'] = lockfiles
169 ss['lockfiles-shared'] = lockfilesshared
170 ss['plaindirs'] = plaindirs
171 ss['interceptfuncs'] = interceptfuncs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500172 ss['fixmedir'] = fixmedir
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173 return ss
174
175def sstate_add(ss, source, dest, d):
176 if not source.endswith("/"):
177 source = source + "/"
178 if not dest.endswith("/"):
179 dest = dest + "/"
180 source = os.path.normpath(source)
181 dest = os.path.normpath(dest)
182 srcbase = os.path.basename(source)
183 ss['dirs'].append([srcbase, source, dest])
184 return ss
185
186def sstate_install(ss, d):
187 import oe.path
188 import oe.sstatesig
189 import subprocess
190
191 sharedfiles = []
192 shareddirs = []
193 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
194
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500195 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
196
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500197 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
198
199 if os.access(manifest, os.R_OK):
200 bb.fatal("Package already staged (%s)?!" % manifest)
201
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600202 d.setVar("SSTATE_INST_POSTRM", manifest + ".postrm")
203
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 locks = []
205 for lock in ss['lockfiles-shared']:
206 locks.append(bb.utils.lockfile(lock, True))
207 for lock in ss['lockfiles']:
208 locks.append(bb.utils.lockfile(lock))
209
210 for state in ss['dirs']:
211 bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
212 for walkroot, dirs, files in os.walk(state[1]):
213 for file in files:
214 srcpath = os.path.join(walkroot, file)
215 dstpath = srcpath.replace(state[1], state[2])
216 #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
217 sharedfiles.append(dstpath)
218 for dir in dirs:
219 srcdir = os.path.join(walkroot, dir)
220 dstdir = srcdir.replace(state[1], state[2])
221 #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500222 if os.path.islink(srcdir):
223 sharedfiles.append(dstdir)
224 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500225 if not dstdir.endswith("/"):
226 dstdir = dstdir + "/"
227 shareddirs.append(dstdir)
228
229 # Check the file list for conflicts against files which already exist
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500230 whitelist = (d.getVar("SSTATE_DUPWHITELIST") or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231 match = []
232 for f in sharedfiles:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500233 if os.path.exists(f) and not os.path.islink(f):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500234 f = os.path.normpath(f)
235 realmatch = True
236 for w in whitelist:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600237 w = os.path.normpath(w)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238 if f.startswith(w):
239 realmatch = False
240 break
241 if realmatch:
242 match.append(f)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500243 sstate_search_cmd = "grep -rlF '%s' %s --exclude=master.list | sed -e 's:^.*/::'" % (f, d.expand("${SSTATE_MANIFESTS}"))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500245 if search_output:
246 match.append(" (matched in %s)" % search_output.decode('utf-8').rstrip())
247 else:
248 match.append(" (not matched to any task)")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500249 if match:
250 bb.error("The recipe %s is trying to install files into a shared " \
251 "area when those files already exist. Those files and their manifest " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500252 "location are:\n %s\nPlease verify which recipe should provide the " \
253 "above files.\n\nThe build has stopped, as continuing in this scenario WILL " \
254 "break things - if not now, possibly in the future (we've seen builds fail " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 "several months later). If the system knew how to recover from this " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500256 "automatically it would, however there are several different scenarios " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 "which can result in this and we don't know which one this is. It may be " \
258 "you have switched providers of something like virtual/kernel (e.g. from " \
259 "linux-yocto to linux-yocto-dev), in that case you need to execute the " \
260 "clean task for both recipes and it will resolve this error. It may be " \
261 "you changed DISTRO_FEATURES from systemd to udev or vice versa. Cleaning " \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500262 "those recipes should again resolve this error, however switching " \
263 "DISTRO_FEATURES on an existing build directory is not supported - you " \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500264 "should really clean out tmp and rebuild (reusing sstate should be safe). " \
265 "It could be the overlapping files detected are harmless in which case " \
266 "adding them to SSTATE_DUPWHITELIST may be the correct solution. It could " \
267 "also be your build is including two different conflicting versions of " \
268 "things (e.g. bluez 4 and bluez 5 and the correct solution for that would " \
269 "be to resolve the conflict. If in doubt, please ask on the mailing list, " \
270 "sharing the error and filelist above." % \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500271 (d.getVar('PN'), "\n ".join(match)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 bb.fatal("If the above message is too much, the simpler version is you're advised to wipe out tmp and rebuild (reusing sstate is fine). That will likely fix things in most (but not all) cases.")
273
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500274 if ss['fixmedir'] and os.path.exists(ss['fixmedir'] + "/fixmepath.cmd"):
275 sharedfiles.append(ss['fixmedir'] + "/fixmepath.cmd")
276 sharedfiles.append(ss['fixmedir'] + "/fixmepath")
277
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278 # Write out the manifest
279 f = open(manifest, "w")
280 for file in sharedfiles:
281 f.write(file + "\n")
282
283 # We want to ensure that directories appear at the end of the manifest
284 # so that when we test to see if they should be deleted any contents
285 # added by the task will have been removed first.
286 dirs = sorted(shareddirs, key=len)
287 # Must remove children first, which will have a longer path than the parent
288 for di in reversed(dirs):
289 f.write(di + "\n")
290 f.close()
291
292 # Append to the list of manifests for this PACKAGE_ARCH
293
294 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
295 l = bb.utils.lockfile(i + ".lock")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500296 filedata = d.getVar("STAMP") + " " + d2.getVar("SSTATE_MANFILEPREFIX") + " " + d.getVar("WORKDIR") + "\n"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500297 manifests = []
298 if os.path.exists(i):
299 with open(i, "r") as f:
300 manifests = f.readlines()
301 if filedata not in manifests:
302 with open(i, "a+") as f:
303 f.write(filedata)
304 bb.utils.unlockfile(l)
305
306 # Run the actual file install
307 for state in ss['dirs']:
308 if os.path.exists(state[1]):
309 oe.path.copyhardlinktree(state[1], state[2])
310
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500311 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500312 # All hooks should run in the SSTATE_INSTDIR
313 bb.build.exec_func(postinst, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500314
315 for lock in locks:
316 bb.utils.unlockfile(lock)
317
318sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX"
319sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
320
321def sstate_installpkg(ss, d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500322 from oe.gpg_sign import get_signer
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500323
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500325 sstatefetch = d.getVar('SSTATE_PKGNAME') + '_' + ss['task'] + ".tgz"
326 sstatepkg = d.getVar('SSTATE_PKG') + '_' + ss['task'] + ".tgz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500327
328 if not os.path.exists(sstatepkg):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800329 pstaging_fetch(sstatefetch, d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500330
331 if not os.path.isfile(sstatepkg):
332 bb.note("Staging package %s does not exist" % sstatepkg)
333 return False
334
335 sstate_clean(ss, d)
336
337 d.setVar('SSTATE_INSTDIR', sstateinst)
338 d.setVar('SSTATE_PKG', sstatepkg)
339
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500340 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500341 signer = get_signer(d, 'local')
342 if not signer.verify(sstatepkg + '.sig'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500343 bb.warn("Cannot verify signature on sstate package %s" % sstatepkg)
344
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500345 # Empty sstateinst directory, ensure its clean
346 if os.path.exists(sstateinst):
347 oe.path.remove(sstateinst)
348 bb.utils.mkdirhier(sstateinst)
349
350 sstateinst = d.getVar("SSTATE_INSTDIR")
351 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
352
353 for f in (d.getVar('SSTATEPREINSTFUNCS') or '').split() + ['sstate_unpack_package']:
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500354 # All hooks should run in the SSTATE_INSTDIR
355 bb.build.exec_func(f, d, (sstateinst,))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500356
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500357 return sstate_installpkgdir(ss, d)
358
359def sstate_installpkgdir(ss, d):
360 import oe.path
361 import subprocess
362
363 sstateinst = d.getVar("SSTATE_INSTDIR")
364 d.setVar('SSTATE_FIXMEDIR', ss['fixmedir'])
365
366 for f in (d.getVar('SSTATEPOSTUNPACKFUNCS') or '').split():
367 # All hooks should run in the SSTATE_INSTDIR
368 bb.build.exec_func(f, d, (sstateinst,))
369
370 def prepdir(dir):
371 # remove dir if it exists, ensure any parent directories do exist
372 if os.path.exists(dir):
373 oe.path.remove(dir)
374 bb.utils.mkdirhier(dir)
375 oe.path.remove(dir)
376
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500377 for state in ss['dirs']:
378 prepdir(state[1])
379 os.rename(sstateinst + state[0], state[1])
380 sstate_install(ss, d)
381
382 for plain in ss['plaindirs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500383 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500384 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385 src = sstateinst + "/" + plain.replace(workdir, '')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500386 if sharedworkdir in plain:
387 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500388 dest = plain
389 bb.utils.mkdirhier(src)
390 prepdir(dest)
391 os.rename(src, dest)
392
393 return True
394
395python sstate_hardcode_path_unpack () {
396 # Fixup hardcoded paths
397 #
398 # Note: The logic below must match the reverse logic in
399 # sstate_hardcode_path(d)
400 import subprocess
401
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500402 sstateinst = d.getVar('SSTATE_INSTDIR')
403 sstatefixmedir = d.getVar('SSTATE_FIXMEDIR')
404 fixmefn = sstateinst + "fixmepath"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500405 if os.path.isfile(fixmefn):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500406 staging_target = d.getVar('RECIPE_SYSROOT')
407 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500408
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500409 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500410 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500411 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
412 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (staging_target, staging_host)
413 else:
414 sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g'" % (staging_target)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500416 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500417 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500418 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500419 sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
420
421 # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed
422 sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd)
423
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500424 # Defer do_populate_sysroot relocation command
425 if sstatefixmedir:
426 bb.utils.mkdirhier(sstatefixmedir)
427 with open(sstatefixmedir + "/fixmepath.cmd", "w") as f:
428 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(fixmefn, sstatefixmedir + "/fixmepath")
429 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(sstateinst, "FIXMEFINALSSTATEINST")
430 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_host, "FIXMEFINALSSTATEHOST")
431 sstate_hardcode_cmd = sstate_hardcode_cmd.replace(staging_target, "FIXMEFINALSSTATETARGET")
432 f.write(sstate_hardcode_cmd)
433 bb.utils.copyfile(fixmefn, sstatefixmedir + "/fixmepath")
434 return
435
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436 bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500437 subprocess.check_call(sstate_hardcode_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800439 # Need to remove this or we'd copy it into the target directory and may
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440 # conflict with another writer
441 os.remove(fixmefn)
442}
443
444def sstate_clean_cachefile(ss, d):
445 import oe.path
446
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500447 sstatepkgfile = d.getVar('SSTATE_PATHSPEC') + "*_" + ss['task'] + ".tgz*"
Brad Bishopa5c52ff2018-11-23 10:55:50 +1300448 if d.getVarFlag('do_%s' % ss['task'], 'task'):
449 bb.note("Removing %s" % sstatepkgfile)
450 oe.path.remove(sstatepkgfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500451
452def sstate_clean_cachefiles(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500453 for task in (d.getVar('SSTATETASKS') or "").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500454 ld = d.createCopy()
455 ss = sstate_state_fromvars(ld, task)
456 sstate_clean_cachefile(ss, ld)
457
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500458def sstate_clean_manifest(manifest, d, prefix=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500459 import oe.path
460
461 mfile = open(manifest)
462 entries = mfile.readlines()
463 mfile.close()
464
465 for entry in entries:
466 entry = entry.strip()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500467 if prefix and not entry.startswith("/"):
468 entry = prefix + "/" + entry
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500469 bb.debug(2, "Removing manifest: %s" % entry)
470 # We can race against another package populating directories as we're removing them
471 # so we ignore errors here.
472 try:
473 if entry.endswith("/"):
474 if os.path.islink(entry[:-1]):
475 os.remove(entry[:-1])
476 elif os.path.exists(entry) and len(os.listdir(entry)) == 0:
477 os.rmdir(entry[:-1])
478 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500479 os.remove(entry)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500480 except OSError:
481 pass
482
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600483 postrm = manifest + ".postrm"
484 if os.path.exists(manifest + ".postrm"):
485 import subprocess
486 os.chmod(postrm, 0o755)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500487 subprocess.check_call(postrm, shell=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600488 oe.path.remove(postrm)
489
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500490 oe.path.remove(manifest)
491
492def sstate_clean(ss, d):
493 import oe.path
494 import glob
495
496 d2 = d.createCopy()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500497 stamp_clean = d.getVar("STAMPCLEAN")
498 extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500499 if extrainf:
500 d2.setVar("SSTATE_MANMACH", extrainf)
501 wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf)
502 else:
503 wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task'])
504
505 manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task'])
506
507 if os.path.exists(manifest):
508 locks = []
509 for lock in ss['lockfiles-shared']:
510 locks.append(bb.utils.lockfile(lock))
511 for lock in ss['lockfiles']:
512 locks.append(bb.utils.lockfile(lock))
513
514 sstate_clean_manifest(manifest, d)
515
516 for lock in locks:
517 bb.utils.unlockfile(lock)
518
519 # Remove the current and previous stamps, but keep the sigdata.
520 #
521 # The glob() matches do_task* which may match multiple tasks, for
522 # example: do_package and do_package_write_ipk, so we need to
523 # exactly match *.do_task.* and *.do_task_setscene.*
524 rm_stamp = '.do_%s.' % ss['task']
525 rm_setscene = '.do_%s_setscene.' % ss['task']
526 # For BB_SIGNATURE_HANDLER = "noop"
527 rm_nohash = ".do_%s" % ss['task']
528 for stfile in glob.glob(wildcard_stfile):
529 # Keep the sigdata
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500530 if ".sigdata." in stfile or ".sigbasedata." in stfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500531 continue
532 # Preserve taint files in the stamps directory
533 if stfile.endswith('.taint'):
534 continue
535 if rm_stamp in stfile or rm_setscene in stfile or \
536 stfile.endswith(rm_nohash):
537 oe.path.remove(stfile)
538
539sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX"
540
541CLEANFUNCS += "sstate_cleanall"
542
543python sstate_cleanall() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500544 bb.note("Removing shared state for package %s" % d.getVar('PN'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500545
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500546 manifest_dir = d.getVar('SSTATE_MANIFESTS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500547 if not os.path.exists(manifest_dir):
548 return
549
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500550 tasks = d.getVar('SSTATETASKS').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500551 for name in tasks:
552 ld = d.createCopy()
553 shared_state = sstate_state_fromvars(ld, name)
554 sstate_clean(shared_state, ld)
555}
556
557python sstate_hardcode_path () {
558 import subprocess, platform
559
560 # Need to remove hardcoded paths and fix these when we install the
561 # staging packages.
562 #
563 # Note: the logic in this function needs to match the reverse logic
564 # in sstate_installpkg(ss, d)
565
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500566 staging_target = d.getVar('RECIPE_SYSROOT')
567 staging_host = d.getVar('RECIPE_SYSROOT_NATIVE')
568 sstate_builddir = d.getVar('SSTATE_BUILDDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500569
Brad Bishop316dfdd2018-06-25 12:45:53 -0400570 sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % staging_host
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500571 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross-canadian', d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500572 sstate_grep_cmd = "grep -l -e '%s'" % (staging_host)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500573 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
574 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400575 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500576 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400577 sstate_grep_cmd = "grep -l -e '%s' -e '%s'" % (staging_target, staging_host)
578 sstate_sed_cmd += " -e 's:%s:FIXMESTAGINGDIRTARGET:g'" % staging_target
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500579
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500580 extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES') or ''
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500581 for fixmevar in extra_staging_fixmes.split():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500582 fixme_path = d.getVar(fixmevar)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500583 sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500584 sstate_grep_cmd += " -e '%s'" % (fixme_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500585
586 fixmefn = sstate_builddir + "fixmepath"
587
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500588 sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500589 sstate_filelist_cmd = "tee %s" % (fixmefn)
590
591 # fixmepath file needs relative paths, drop sstate_builddir prefix
592 sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn)
593
594 xargs_no_empty_run_cmd = '--no-run-if-empty'
595 if platform.system() == 'Darwin':
596 xargs_no_empty_run_cmd = ''
597
598 # Limit the fixpaths and sed operations based on the initial grep search
599 # This has the side effect of making sure the vfs cache is hot
600 sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd)
601
602 bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500603 subprocess.check_output(sstate_hardcode_cmd, shell=True, cwd=sstate_builddir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500604
605 # If the fixmefn is empty, remove it..
606 if os.stat(fixmefn).st_size == 0:
607 os.remove(fixmefn)
608 else:
609 bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500610 subprocess.check_output(sstate_filelist_relative_cmd, shell=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500611}
612
613def sstate_package(ss, d):
614 import oe.path
615
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500616 tmpdir = d.getVar('TMPDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500617
618 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500619 sstatepkg = d.getVar('SSTATE_PKG') + '_'+ ss['task'] + ".tgz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500620 bb.utils.remove(sstatebuild, recurse=True)
621 bb.utils.mkdirhier(sstatebuild)
622 bb.utils.mkdirhier(os.path.dirname(sstatepkg))
623 for state in ss['dirs']:
624 if not os.path.exists(state[1]):
625 continue
626 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500627 # Find and error for absolute symlinks. We could attempt to relocate but its not
628 # clear where the symlink is relative to in this context. We could add that markup
629 # to sstate tasks but there aren't many of these so better just avoid them entirely.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500630 for walkroot, dirs, files in os.walk(state[1]):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500631 for file in files + dirs:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500632 srcpath = os.path.join(walkroot, file)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500633 if not os.path.islink(srcpath):
634 continue
635 link = os.readlink(srcpath)
636 if not os.path.isabs(link):
637 continue
638 if not link.startswith(tmpdir):
639 continue
640 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500641 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500642 os.rename(state[1], sstatebuild + state[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500643
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500644 workdir = d.getVar('WORKDIR')
Brad Bishop977dc1a2019-02-06 16:01:43 -0500645 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500646 for plain in ss['plaindirs']:
647 pdir = plain.replace(workdir, sstatebuild)
Brad Bishop977dc1a2019-02-06 16:01:43 -0500648 if sharedworkdir in plain:
649 pdir = plain.replace(sharedworkdir, sstatebuild)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500650 bb.utils.mkdirhier(plain)
651 bb.utils.mkdirhier(pdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500652 os.rename(plain, pdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500653
654 d.setVar('SSTATE_BUILDDIR', sstatebuild)
655 d.setVar('SSTATE_PKG', sstatepkg)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500656 d.setVar('SSTATE_INSTDIR', sstatebuild)
657
658 if d.getVar('SSTATE_SKIP_CREATION') == '1':
659 return
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500660
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500661 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
Brad Bishop19323692019-04-05 15:28:33 -0400662 ['sstate_report_unihash', 'sstate_create_package', 'sstate_sign_package'] + \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500663 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500664 # All hooks should run in SSTATE_BUILDDIR.
665 bb.build.exec_func(f, d, (sstatebuild,))
666
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500667 bb.siggen.dump_this_task(sstatepkg + ".siginfo", d)
668
669 return
670
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800671def pstaging_fetch(sstatefetch, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672 import bb.fetch2
673
674 # Only try and fetch if the user has configured a mirror
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500675 mirrors = d.getVar('SSTATE_MIRRORS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500676 if not mirrors:
677 return
678
679 # Copy the data object and override DL_DIR and SRC_URI
680 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500681
682 dldir = localdata.expand("${SSTATE_DIR}")
683 bb.utils.mkdirhier(dldir)
684
685 localdata.delVar('MIRRORS')
686 localdata.setVar('FILESPATH', dldir)
687 localdata.setVar('DL_DIR', dldir)
688 localdata.setVar('PREMIRRORS', mirrors)
689
690 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
691 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400692 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
693 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500694 localdata.delVar('BB_NO_NETWORK')
695
696 # Try a fetch from the sstate mirror, if it fails just return and
697 # we will build the package
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600698 uris = ['file://{0};downloadfilename={0}'.format(sstatefetch),
699 'file://{0}.siginfo;downloadfilename={0}.siginfo'.format(sstatefetch)]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500700 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600701 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500702
703 for srcuri in uris:
704 localdata.setVar('SRC_URI', srcuri)
705 try:
706 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
707 fetcher.download()
708
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500709 except bb.fetch2.BBFetchException:
710 break
711
712def sstate_setscene(d):
713 shared_state = sstate_state_fromvars(d)
714 accelerate = sstate_installpkg(shared_state, d)
715 if not accelerate:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600716 bb.fatal("No suitable staging package found")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500717
718python sstate_task_prefunc () {
719 shared_state = sstate_state_fromvars(d)
720 sstate_clean(shared_state, d)
721}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500722sstate_task_prefunc[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500723
724python sstate_task_postfunc () {
725 shared_state = sstate_state_fromvars(d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500726
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500727 for intercept in shared_state['interceptfuncs']:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500728 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),))
729
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600730 omask = os.umask(0o002)
731 if omask != 0o002:
732 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500733 sstate_package(shared_state, d)
734 os.umask(omask)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500735
736 sstateinst = d.getVar("SSTATE_INSTDIR")
737 d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
738
739 sstate_installpkgdir(shared_state, d)
740
741 bb.utils.remove(d.getVar("SSTATE_BUILDDIR"), recurse=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500742}
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500743sstate_task_postfunc[dirs] = "${WORKDIR}"
744
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500745
746#
747# Shell function to generate a sstate package from a directory
748# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
749#
750sstate_create_package () {
751 TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800752
753 # Use pigz if available
754 OPT="-czS"
755 if [ -x "$(command -v pigz)" ]; then
756 OPT="-I pigz -cS"
757 fi
758
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500759 # Need to handle empty directories
760 if [ "$(ls -A)" ]; then
761 set +e
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800762 tar $OPT -f $TFILE *
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500763 ret=$?
764 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then
765 exit 1
766 fi
767 set -e
768 else
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800769 tar $OPT --file=$TFILE --files-from=/dev/null
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500770 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500771 chmod 0664 $TFILE
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500772 mv -f $TFILE ${SSTATE_PKG}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500773}
774
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500775python sstate_sign_package () {
776 from oe.gpg_sign import get_signer
777
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500778 if d.getVar('SSTATE_SIG_KEY'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500779 signer = get_signer(d, 'local')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500780 sstate_pkg = d.getVar('SSTATE_PKG')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500781 if os.path.exists(sstate_pkg + '.sig'):
782 os.unlink(sstate_pkg + '.sig')
783 signer.detach_sign(sstate_pkg, d.getVar('SSTATE_SIG_KEY', False), None,
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500784 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500785}
786
Brad Bishop19323692019-04-05 15:28:33 -0400787python sstate_report_unihash() {
788 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
789
790 if report_unihash:
791 ss = sstate_state_fromvars(d)
792 report_unihash(os.getcwd(), ss['task'], d)
793}
794
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500795#
796# Shell function to decompress and prepare a package for installation
797# Will be run from within SSTATE_INSTDIR.
798#
799sstate_unpack_package () {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500800 tar -xvzf ${SSTATE_PKG}
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500801 # update .siginfo atime on local/NFS mirror
802 [ -w ${SSTATE_PKG}.siginfo ] && [ -h ${SSTATE_PKG}.siginfo ] && touch -a ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500803 # Use "! -w ||" to return true for read only files
804 [ ! -w ${SSTATE_PKG} ] || touch --no-dereference ${SSTATE_PKG}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500805 [ ! -w ${SSTATE_PKG}.sig ] || [ ! -e ${SSTATE_PKG}.sig ] || touch --no-dereference ${SSTATE_PKG}.sig
806 [ ! -w ${SSTATE_PKG}.siginfo ] || [ ! -e ${SSTATE_PKG}.siginfo ] || touch --no-dereference ${SSTATE_PKG}.siginfo
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500807}
808
809BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
810
Brad Bishop19323692019-04-05 15:28:33 -0400811def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, sq_unihash=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500812
813 ret = []
814 missed = []
815 extension = ".tgz"
816 if siginfo:
817 extension = extension + ".siginfo"
818
Brad Bishop19323692019-04-05 15:28:33 -0400819 def gethash(task):
820 if sq_unihash is not None:
821 return sq_unihash[task]
822 return sq_hash[task]
823
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500824 def getpathcomponents(task, d):
825 # Magic data from BB_HASHFILENAME
826 splithashfn = sq_hashfn[task].split(" ")
827 spec = splithashfn[1]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500828 if splithashfn[0] == "True":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500829 extrapath = d.getVar("NATIVELSBSTRING") + "/"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500830 else:
831 extrapath = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500832
833 tname = sq_task[task][3:]
834
835 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
836 spec = splithashfn[2]
837 extrapath = ""
838
839 return spec, extrapath, tname
840
841
842 for task in range(len(sq_fn)):
843
844 spec, extrapath, tname = getpathcomponents(task, d)
845
Brad Bishop19323692019-04-05 15:28:33 -0400846 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500847
848 if os.path.exists(sstatefile):
849 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
850 ret.append(task)
851 continue
852 else:
853 missed.append(task)
854 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
855
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500856 mirrors = d.getVar("SSTATE_MIRRORS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500857 if mirrors:
858 # Copy the data object and override DL_DIR and SRC_URI
859 localdata = bb.data.createCopy(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500860
861 dldir = localdata.expand("${SSTATE_DIR}")
862 localdata.delVar('MIRRORS')
863 localdata.setVar('FILESPATH', dldir)
864 localdata.setVar('DL_DIR', dldir)
865 localdata.setVar('PREMIRRORS', mirrors)
866
867 bb.debug(2, "SState using premirror of: %s" % mirrors)
868
869 # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK,
870 # we'll want to allow network access for the current set of fetches.
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400871 if bb.utils.to_boolean(localdata.getVar('BB_NO_NETWORK')) and \
872 bb.utils.to_boolean(localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500873 localdata.delVar('BB_NO_NETWORK')
874
875 from bb.fetch2 import FetchConnectionCache
876 def checkstatus_init(thread_worker):
877 thread_worker.connection_cache = FetchConnectionCache()
878
879 def checkstatus_end(thread_worker):
880 thread_worker.connection_cache.close_connections()
881
882 def checkstatus(thread_worker, arg):
883 (task, sstatefile) = arg
884
885 localdata2 = bb.data.createCopy(localdata)
886 srcuri = "file://" + sstatefile
887 localdata.setVar('SRC_URI', srcuri)
888 bb.debug(2, "SState: Attempting to fetch %s" % srcuri)
889
890 try:
891 fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2,
892 connection_cache=thread_worker.connection_cache)
893 fetcher.checkstatus()
894 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
895 ret.append(task)
896 if task in missed:
897 missed.remove(task)
898 except:
899 missed.append(task)
900 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600901 pass
Brad Bishop316dfdd2018-06-25 12:45:53 -0400902 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500903
904 tasklist = []
905 for task in range(len(sq_fn)):
906 if task in ret:
907 continue
908 spec, extrapath, tname = getpathcomponents(task, d)
Brad Bishop19323692019-04-05 15:28:33 -0400909 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500910 tasklist.append((task, sstatefile))
911
912 if tasklist:
Brad Bishop316dfdd2018-06-25 12:45:53 -0400913 msg = "Checking sstate mirror object availability"
914 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600915
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500916 import multiprocessing
917 nproc = min(multiprocessing.cpu_count(), len(tasklist))
918
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600919 bb.event.enable_threadlock()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500920 pool = oe.utils.ThreadedPool(nproc, len(tasklist),
921 worker_init=checkstatus_init, worker_end=checkstatus_end)
922 for t in tasklist:
923 pool.add_task(checkstatus, t)
924 pool.start()
925 pool.wait_completion()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600926 bb.event.disable_threadlock()
927
Brad Bishop316dfdd2018-06-25 12:45:53 -0400928 bb.event.fire(bb.event.ProcessFinished(msg), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500929
Brad Bishop96ff1982019-08-19 13:50:42 -0400930 # Likely checking an individual task hash again for multiconfig sharing of sstate tasks so skip reporting
931 if len(sq_fn) == 1:
932 return ret
933
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500934 inheritlist = d.getVar("INHERIT")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500935 if "toaster" in inheritlist:
936 evdata = {'missed': [], 'found': []};
937 for task in missed:
938 spec, extrapath, tname = getpathcomponents(task, d)
Brad Bishop19323692019-04-05 15:28:33 -0400939 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz")
940 evdata['missed'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500941 for task in ret:
942 spec, extrapath, tname = getpathcomponents(task, d)
Brad Bishop19323692019-04-05 15:28:33 -0400943 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz")
944 evdata['found'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500945 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
946
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800947 # Print some summary statistics about the current task completion and how much sstate
948 # reuse there was. Avoid divide by zero errors.
949 total = len(sq_fn)
950 currentcount = d.getVar("BB_SETSCENE_STAMPCURRENT_COUNT") or 0
951 complete = 0
952 if currentcount:
953 complete = (len(ret) + currentcount) / (total + currentcount) * 100
954 match = 0
955 if total:
956 match = len(ret) / total * 100
957 bb.plain("Sstate summary: Wanted %d Found %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(ret), len(missed), currentcount, match, complete))
958
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500959 if hasattr(bb.parse.siggen, "checkhashes"):
960 bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d)
961
962 return ret
963
964BB_SETSCENE_DEPVALID = "setscene_depvalid"
965
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500966def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500967 # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
968 # task is included in taskdependees too
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500969 # Return - False - We need this dependency
970 # - True - We can skip this dependency
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800971 import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500972
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500973 def logit(msg, log):
974 if log is not None:
975 log.append(msg)
976 else:
977 bb.debug(2, msg)
978
979 logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500980
981 def isNativeCross(x):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500982 return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500983
984 # We only need to trigger populate_lic through direct dependencies
985 if taskdependees[task][1] == "do_populate_lic":
986 return True
987
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500988 # stash_locale and gcc_stash_builddir are never needed as a dependency for built objects
989 if taskdependees[task][1] == "do_stash_locale" or taskdependees[task][1] == "do_gcc_stash_builddir":
990 return True
991
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500992 # We only need to trigger packagedata through direct dependencies
993 # but need to preserve packagedata on packagedata links
994 if taskdependees[task][1] == "do_packagedata":
995 for dep in taskdependees:
996 if taskdependees[dep][1] == "do_packagedata":
997 return False
998 return True
999
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001000 for dep in taskdependees:
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001001 logit(" considering dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001002 if task == dep:
1003 continue
1004 if dep in notneeded:
1005 continue
1006 # do_package_write_* and do_package doesn't need do_package
1007 if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
1008 continue
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001009 # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
1010 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']:
1011 return False
1012 # do_package/packagedata/package_qa don't need do_populate_sysroot
1013 if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001014 continue
1015 # Native/Cross packages don't exist and are noexec anyway
1016 if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
1017 continue
1018
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001019 # This is due to the [depends] in useradd.bbclass complicating matters
1020 # The logic *is* reversed here due to the way hard setscene dependencies are injected
1021 if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
1022 continue
1023
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001024 # Consider sysroot depending on sysroot tasks
1025 if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001026 # Allow excluding certain recursive dependencies. If a recipe needs it should add a
1027 # specific dependency itself, rather than relying on one of its dependees to pull
1028 # them in.
1029 # See also http://lists.openembedded.org/pipermail/openembedded-core/2018-January/146324.html
1030 not_needed = False
1031 excludedeps = d.getVar('_SSTATE_EXCLUDEDEPS_SYSROOT')
1032 if excludedeps is None:
1033 # Cache the regular expressions for speed
1034 excludedeps = []
1035 for excl in (d.getVar('SSTATE_EXCLUDEDEPS_SYSROOT') or "").split():
1036 excludedeps.append((re.compile(excl.split('->', 1)[0]), re.compile(excl.split('->', 1)[1])))
1037 d.setVar('_SSTATE_EXCLUDEDEPS_SYSROOT', excludedeps)
1038 for excl in excludedeps:
1039 if excl[0].match(taskdependees[dep][0]):
1040 if excl[1].match(taskdependees[task][0]):
1041 not_needed = True
1042 break
1043 if not_needed:
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001044 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001045 # For meta-extsdk-toolchain we want all sysroot dependencies
1046 if taskdependees[dep][0] == 'meta-extsdk-toolchain':
1047 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001048 # Native/Cross populate_sysroot need their dependencies
1049 if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
1050 return False
1051 # Target populate_sysroot depended on by cross tools need to be installed
1052 if isNativeCross(taskdependees[dep][0]):
1053 return False
1054 # Native/cross tools depended upon by target sysroot are not needed
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001055 # Add an exception for shadow-native as required by useradd.bbclass
1056 if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001057 continue
1058 # Target populate_sysroot need their dependencies
1059 return False
1060
1061 if taskdependees[task][1] == 'do_shared_workdir':
1062 continue
1063
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001064 if taskdependees[dep][1] == "do_populate_lic":
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001065 continue
1066
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001067
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001068 # Safe fallthrough default
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001069 logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001070 return False
1071 return True
1072
1073addhandler sstate_eventhandler
1074sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded"
1075python sstate_eventhandler() {
1076 d = e.data
1077 # When we write an sstate package we rewrite the SSTATE_PKG
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001078 spkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001079 if not spkg.endswith(".tgz"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001080 taskname = d.getVar("BB_RUNTASK")[3:]
1081 spec = d.getVar('SSTATE_PKGSPEC')
1082 swspec = d.getVar('SSTATE_SWSPEC')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001083 if taskname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and swspec:
1084 d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}")
1085 d.setVar("SSTATE_EXTRAPATH", "")
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001086 sstatepkg = d.getVar('SSTATE_PKG')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001087 bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d)
1088}
1089
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001090SSTATE_PRUNE_OBSOLETEWORKDIR ?= "1"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001091
1092# Event handler which removes manifests and stamps file for
1093# recipes which are no longer reachable in a build where they
1094# once were.
1095# Also optionally removes the workdir of those tasks/recipes
1096#
1097addhandler sstate_eventhandler2
1098sstate_eventhandler2[eventmask] = "bb.event.ReachableStamps"
1099python sstate_eventhandler2() {
1100 import glob
1101 d = e.data
1102 stamps = e.stamps.values()
1103 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR", False) == "1")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001104 preservestampfile = d.expand('${SSTATE_MANIFESTS}/preserve-stamps')
1105 preservestamps = []
1106 if os.path.exists(preservestampfile):
1107 with open(preservestampfile, 'r') as f:
1108 preservestamps = f.readlines()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001109 seen = []
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001110
1111 # The machine index contains all the stamps this machine has ever seen in this build directory.
1112 # We should only remove things which this machine once accessed but no longer does.
1113 machineindex = set()
1114 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
1115 mi = d.expand("${SSTATE_MANIFESTS}/index-machine-${MACHINE}")
1116 if os.path.exists(mi):
1117 with open(mi, "r") as f:
1118 machineindex = set(line.strip() for line in f.readlines())
1119
Brad Bishop316dfdd2018-06-25 12:45:53 -04001120 for a in sorted(list(set(d.getVar("SSTATE_ARCHS").split()))):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001121 toremove = []
1122 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
1123 if not os.path.exists(i):
1124 continue
1125 with open(i, "r") as f:
1126 lines = f.readlines()
1127 for l in lines:
Brad Bishop19323692019-04-05 15:28:33 -04001128 try:
1129 (stamp, manifest, workdir) = l.split()
1130 if stamp not in stamps and stamp not in preservestamps and stamp in machineindex:
1131 toremove.append(l)
1132 if stamp not in seen:
1133 bb.debug(2, "Stamp %s is not reachable, removing related manifests" % stamp)
1134 seen.append(stamp)
1135 except ValueError:
1136 bb.fatal("Invalid line '%s' in sstate manifest '%s'" % (l, i))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001137
1138 if toremove:
Brad Bishop316dfdd2018-06-25 12:45:53 -04001139 msg = "Removing %d recipes from the %s sysroot" % (len(toremove), a)
1140 bb.event.fire(bb.event.ProcessStarted(msg, len(toremove)), d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001141
Brad Bishop316dfdd2018-06-25 12:45:53 -04001142 removed = 0
1143 for r in toremove:
1144 (stamp, manifest, workdir) = r.split()
1145 for m in glob.glob(manifest + ".*"):
1146 if m.endswith(".postrm"):
1147 continue
1148 sstate_clean_manifest(m, d)
1149 bb.utils.remove(stamp + "*")
1150 if removeworkdir:
1151 bb.utils.remove(workdir, recurse = True)
1152 lines.remove(r)
1153 removed = removed + 1
1154 bb.event.fire(bb.event.ProcessProgress(msg, removed), d)
1155
1156 bb.event.fire(bb.event.ProcessFinished(msg), d)
1157
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001158 with open(i, "w") as f:
1159 for l in lines:
1160 f.write(l)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001161 machineindex |= set(stamps)
1162 with open(mi, "w") as f:
1163 for l in machineindex:
1164 f.write(l + "\n")
1165
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001166 if preservestamps:
1167 os.remove(preservestampfile)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001168}